From e29fb01f4cf1835b933374515d6a30950f784ec1 Mon Sep 17 00:00:00 2001 From: John Mitchell Date: Tue, 7 Mar 2017 15:20:13 -0500 Subject: [PATCH 01/58] make host config required when allow callbacks checkbox is checked --- awx/ui/client/src/forms/JobTemplates.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/awx/ui/client/src/forms/JobTemplates.js b/awx/ui/client/src/forms/JobTemplates.js index 63369d320a..0156b47770 100644 --- a/awx/ui/client/src/forms/JobTemplates.js +++ b/awx/ui/client/src/forms/JobTemplates.js @@ -349,7 +349,11 @@ export default dataPlacement: 'right', dataTitle: i18n._("Host Config Key"), dataContainer: "body", - ngDisabled: '!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate)' + ngDisabled: '!(job_template_obj.summary_fields.user_capabilities.edit || canAddJobTemplate)', + awRequiredWhen: { + reqExpression: 'allow_callbacks', + alwaysShowAsterisk: true + } }, labels: { label: i18n._('Labels'), From d243f13c1c1307137d894c6b6ea0ca0787c3be1b Mon Sep 17 00:00:00 2001 From: John Mitchell Date: Tue, 7 Mar 2017 11:23:53 -0500 Subject: [PATCH 02/58] remove schedule button for manual projects --- awx/ui/client/src/lists/Projects.js | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/ui/client/src/lists/Projects.js b/awx/ui/client/src/lists/Projects.js index 80bcaab1f2..11a036542a 100644 --- a/awx/ui/client/src/lists/Projects.js +++ b/awx/ui/client/src/lists/Projects.js @@ -97,6 +97,7 @@ export default awToolTip: "{{ project.scm_schedule_tooltip }}", ngClass: "project.scm_type_class", dataPlacement: 'top', + ngShow: "project.summary_fields.user_capabilities.schedule" }, edit: { ngClick: "editProject(project.id)", From f4d4c43d94b7e0b60276df21ff7d0856cec8c840 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 23 Feb 2017 15:22:08 -0500 Subject: [PATCH 03/58] prohibit `order_by=` for sensitive fields see: #5526 --- awx/api/filters.py | 15 +++++++++++++++ awx/main/tests/functional/api/test_credential.py | 15 +++++++++++++++ awx/main/tests/functional/api/test_inventory.py | 15 +++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/awx/api/filters.py b/awx/api/filters.py index 41bbc3bba8..fbd7a0ba64 100644 --- a/awx/api/filters.py +++ b/awx/api/filters.py @@ -311,6 +311,8 @@ class OrderByBackend(BaseFilterBackend): else: order_by = (value,) if order_by: + order_by = self._strip_sensitive_model_fields(queryset.model, order_by) + # Special handling of the type field for ordering. In this # case, we're not sorting exactly on the type field, but # given the limited number of views with multiple types, @@ -333,3 +335,16 @@ class OrderByBackend(BaseFilterBackend): except FieldError as e: # Return a 400 for invalid field names. raise ParseError(*e.args) + + def _strip_sensitive_model_fields(self, model, order_by): + for field_name in order_by: + # strip off the negation prefix `-` if it exists + field_name = field_name.split('-')[-1] + try: + # if the field name is encrypted/sensitive, don't sort on it + if field_name in getattr(model, 'PASSWORD_FIELDS', ()) or \ + getattr(model._meta.get_field(field_name), '__prevent_search__', False): + raise ParseError(_('cannot order by field %s') % field_name) + except FieldDoesNotExist: + pass + yield field_name diff --git a/awx/main/tests/functional/api/test_credential.py b/awx/main/tests/functional/api/test_credential.py index 8f596cdac9..458a629cce 100644 --- a/awx/main/tests/functional/api/test_credential.py +++ b/awx/main/tests/functional/api/test_credential.py @@ -339,6 +339,21 @@ def test_list_created_org_credentials(post, get, organization, org_admin, org_me assert response.data['count'] == 0 +@pytest.mark.parametrize('order_by', ('password', '-password', 'password,pk', '-password,pk')) +@pytest.mark.django_db +def test_list_cannot_order_by_encrypted_field(post, get, organization, org_admin, order_by): + for i, password in enumerate(('abc', 'def', 'xyz')): + response = post(reverse('api:credential_list'), { + 'organization': organization.id, + 'name': 'C%d' % i, + 'password': password + }, org_admin) + + response = get(reverse('api:credential_list'), org_admin, + QUERY_STRING='order_by=%s' % order_by, status=400) + assert response.status_code == 400 + + # # Openstack Credentials # diff --git a/awx/main/tests/functional/api/test_inventory.py b/awx/main/tests/functional/api/test_inventory.py index 925d7352fd..839cffab9e 100644 --- a/awx/main/tests/functional/api/test_inventory.py +++ b/awx/main/tests/functional/api/test_inventory.py @@ -35,6 +35,21 @@ def test_edit_inventory(put, inventory, alice, role_field, expected_status_code) put(reverse('api:inventory_detail', args=(inventory.id,)), data, alice, expect=expected_status_code) +@pytest.mark.parametrize('order_by', ('script', '-script', 'script,pk', '-script,pk')) +@pytest.mark.django_db +def test_list_cannot_order_by_unsearchable_field(get, organization, alice, order_by): + for i, script in enumerate(('#!/bin/a', '#!/bin/b', '#!/bin/c')): + custom_script = organization.custom_inventory_scripts.create( + name="I%d" % i, + script=script + ) + custom_script.admin_role.members.add(alice) + + response = get(reverse('api:inventory_script_list'), alice, + QUERY_STRING='order_by=%s' % order_by, status=400) + assert response.status_code == 400 + + @pytest.mark.parametrize("role_field,expected_status_code", [ (None, 403), ('admin_role', 201), From 8a8901afe4a7ab8c43386b07bebbd7f68a2596ca Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Fri, 10 Mar 2017 11:53:21 -0500 Subject: [PATCH 04/58] Default image tag for unit test container image. --- tools/docker-compose/unit-tests/docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/docker-compose/unit-tests/docker-compose.yml b/tools/docker-compose/unit-tests/docker-compose.yml index 76a4a40342..6dff70fb8d 100644 --- a/tools/docker-compose/unit-tests/docker-compose.yml +++ b/tools/docker-compose/unit-tests/docker-compose.yml @@ -1,11 +1,11 @@ --- -version: '2' +version: '3' services: unit-tests: build: context: ../../../ dockerfile: tools/docker-compose/unit-tests/Dockerfile - image: gcr.io/ansible-tower-engineering/unit-test-runner:${GIT_BRANCH} + image: gcr.io/ansible-tower-engineering/unit-test-runner:${GIT_BRANCH:-latest} environment: SWIG_FEATURES: "-cpperraswarn -includeall -I/usr/include/openssl" TEST_DIRS: awx/main/tests/functional awx/main/tests/unit awx/conf/tests awx/sso/tests From 5098d0e99c594e322836675822e7333e5a2d6834 Mon Sep 17 00:00:00 2001 From: Graham Mainwaring Date: Fri, 10 Mar 2017 14:53:05 -0500 Subject: [PATCH 05/58] Fix case where Get-ChildItems does not return any results --- awx/plugins/library/win_scan_packages.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/plugins/library/win_scan_packages.ps1 b/awx/plugins/library/win_scan_packages.ps1 index 2c9455d154..2ab3fdbec6 100644 --- a/awx/plugins/library/win_scan_packages.ps1 +++ b/awx/plugins/library/win_scan_packages.ps1 @@ -25,7 +25,7 @@ if ([System.IntPtr]::Size -eq 4) { # This is a 32-bit Windows system, so we only check for 32-bit programs, which will be # at the native registry location. - $packages = Get-ChildItem -Path $uninstall_native_path | + [PSObject []]$packages = Get-ChildItem -Path $uninstall_native_path | Get-ItemProperty | Select-Object -Property @{Name="name"; Expression={$_."DisplayName"}}, @{Name="version"; Expression={$_."DisplayVersion"}}, @@ -38,7 +38,7 @@ if ([System.IntPtr]::Size -eq 4) { # This is a 64-bit Windows system, so we check for 64-bit programs in the native # registry location, and also for 32-bit programs under Wow6432Node. - $packages = Get-ChildItem -Path $uninstall_native_path | + [PSObject []]$packages = Get-ChildItem -Path $uninstall_native_path | Get-ItemProperty | Select-Object -Property @{Name="name"; Expression={$_."DisplayName"}}, @{Name="version"; Expression={$_."DisplayVersion"}}, From d4cd9d355fad50e5ae1cf71c809ad9f217d4f6c9 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 10 Mar 2017 15:08:06 -0500 Subject: [PATCH 06/58] add ui support for specifying extra vars for ad-hoc commands see: #1744 --- .../manage/adhoc/adhoc.controller.js | 11 ++++++++-- .../inventories/manage/adhoc/adhoc.form.js | 21 +++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/awx/ui/client/src/inventories/manage/adhoc/adhoc.controller.js b/awx/ui/client/src/inventories/manage/adhoc/adhoc.controller.js index dec6b6397a..e264f83ba7 100644 --- a/awx/ui/client/src/inventories/manage/adhoc/adhoc.controller.js +++ b/awx/ui/client/src/inventories/manage/adhoc/adhoc.controller.js @@ -12,7 +12,7 @@ function adhocController($q, $scope, $location, $stateParams, $state, CheckPasswords, PromptForPasswords, CreateLaunchDialog, CreateSelect2, adhocForm, GenerateForm, Rest, ProcessErrors, ClearScope, GetBasePath, GetChoices, - KindChange, CredentialList, Empty, Wait) { + KindChange, CredentialList, ParseTypeChange, Empty, Wait) { ClearScope(); @@ -162,6 +162,12 @@ function adhocController($q, $scope, $location, $stateParams, privateFn.initializeForm(id, urls, hostPattern); + // init codemirror + $scope.extra_vars = '---'; + $scope.parseType = 'yaml'; + $scope.envParseType = 'yaml'; + ParseTypeChange({ scope: $scope, field_id: 'adhoc_extra_vars' , variable: "extra_vars"}); + $scope.formCancel = function(){ $state.go('inventoryManage'); }; @@ -199,6 +205,7 @@ function adhocController($q, $scope, $location, $stateParams, "module_args": "", "forks": 0, "verbosity": 0, + "extra_vars": "", "privilege_escalation": "" }; @@ -297,5 +304,5 @@ function adhocController($q, $scope, $location, $stateParams, export default ['$q', '$scope', '$location', '$stateParams', '$state', 'CheckPasswords', 'PromptForPasswords', 'CreateLaunchDialog', 'CreateSelect2', 'adhocForm', 'GenerateForm', 'Rest', 'ProcessErrors', 'ClearScope', 'GetBasePath', - 'GetChoices', 'KindChange', 'CredentialList', 'Empty', 'Wait', + 'GetChoices', 'KindChange', 'CredentialList', 'ParseTypeChange', 'Empty', 'Wait', adhocController]; diff --git a/awx/ui/client/src/inventories/manage/adhoc/adhoc.form.js b/awx/ui/client/src/inventories/manage/adhoc/adhoc.form.js index f3233cac50..c58068deb6 100644 --- a/awx/ui/client/src/inventories/manage/adhoc/adhoc.form.js +++ b/awx/ui/client/src/inventories/manage/adhoc/adhoc.form.js @@ -10,7 +10,7 @@ * @description This form is for executing an adhoc command */ -export default function() { +export default ['i18n', function(i18n) { return { addTitle: 'EXECUTE COMMAND', name: 'adhoc', @@ -121,6 +121,23 @@ export default function() { dataPlacement: 'right', dataContainer: "body" }, + extra_vars: { + label: i18n._('Extra Variables'), + type: 'textarea', + class: 'Form-textAreaLabel Form-formGroup--fullWidth', + rows: 6, + "default": "---", + column: 2, + awPopOver: "

" + i18n.sprintf(i18n._("Pass extra command line variables. This is the %s or %s command line parameter " + + "for %s. Provide key/value pairs using either YAML or JSON."), '-e', '--extra-vars', 'ansible') + "

" + + "JSON:
\n" + + "
{
 \"somevar\": \"somevalue\",
 \"password\": \"magic\"
}
\n" + + "YAML:
\n" + + "
---
somevar: somevalue
password: magic
\n", + dataTitle: i18n._('Extra Variables'), + dataPlacement: 'right', + dataContainer: "body" + } }, buttons: { reset: { @@ -139,4 +156,4 @@ export default function() { related: {} }; -} +}]; From 6af9638d8197a5a8dcafe6ade876f44a2cb5ef93 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 10 Mar 2017 15:46:42 -0500 Subject: [PATCH 07/58] job event data marked with _ansible_no_log should be censored see: #5691 --- awx/lib/tower_display_callback/module.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/awx/lib/tower_display_callback/module.py b/awx/lib/tower_display_callback/module.py index c40c94ec5a..b69238026f 100644 --- a/awx/lib/tower_display_callback/module.py +++ b/awx/lib/tower_display_callback/module.py @@ -310,11 +310,15 @@ class BaseCallbackModule(CallbackBase): if result._task.get_name() == 'setup': result._result.get('ansible_facts', {}).pop('ansible_env', None) + res = result._result + if res.get('_ansible_no_log', False): + res = {'censored': "the output has been hidden due to the fact that 'no_log: true' was specified for this result"} + event_data = dict( host=result._host.get_name(), remote_addr=result._host.address, task=result._task, - res=result._result, + res=res, event_loop=result._task.loop if hasattr(result._task, 'loop') else None, ) with self.capture_event_data('runner_on_ok', **event_data): From 388b0fdf0493efb977cb0f52cf293d531d7cf783 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sat, 11 Mar 2017 19:05:46 -0500 Subject: [PATCH 08/58] Force reinstall of setuptools and pip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For some reason -I wasn't cutting it. Even though we were trying to install pip 8.1.2, this was happening: Collecting pip==8.1.2 Using cached pip-8.1.2-py2.py3-none-any.whl Installing collected packages: pip Successfully installed pip-9.0.1 ¯\_(ツ)_/¯ --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c3511d5dbf..2f12ea110a 100644 --- a/Makefile +++ b/Makefile @@ -264,8 +264,8 @@ virtualenv_ansible: fi; \ if [ ! -d "$(VENV_BASE)/ansible" ]; then \ virtualenv --system-site-packages --setuptools $(VENV_BASE)/ansible && \ - $(VENV_BASE)/ansible/bin/pip install -I setuptools==23.0.0 && \ - $(VENV_BASE)/ansible/bin/pip install -I pip==8.1.2; \ + $(VENV_BASE)/ansible/bin/pip install --force-reinstall setuptools==23.0.0 && \ + $(VENV_BASE)/ansible/bin/pip install --force-reinstall pip==8.1.2; \ fi; \ fi @@ -276,8 +276,8 @@ virtualenv_tower: fi; \ if [ ! -d "$(VENV_BASE)/tower" ]; then \ virtualenv --system-site-packages --setuptools $(VENV_BASE)/tower && \ - $(VENV_BASE)/tower/bin/pip install -I setuptools==23.0.0 && \ - $(VENV_BASE)/tower/bin/pip install -I pip==8.1.2; \ + $(VENV_BASE)/tower/bin/pip install --force-reinstall setuptools==23.0.0 && \ + $(VENV_BASE)/tower/bin/pip install --force-reinstall pip==8.1.2; \ fi; \ fi From c255c07ea5b35f07fefafe273daf33f7fabe9ede Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 6 Mar 2017 15:56:39 -0500 Subject: [PATCH 09/58] pass ansible the "raw" git repo url * I explored the |urlencode() filter instead. Ansible and git didn't like this. * Quoting doesn't seemed to be required. Ansible and the git module are smart enough to do the right thing. I tested with a space in the repo url path and it works. --- awx/playbooks/project_update.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml index 8fdd3349c3..eb09cd3657 100644 --- a/awx/playbooks/project_update.yml +++ b/awx/playbooks/project_update.yml @@ -25,7 +25,7 @@ - name: update project using git and accept hostkey git: dest: "{{project_path|quote}}" - repo: "{{scm_url|quote}}" + repo: "{{scm_url}}" version: "{{scm_branch|quote}}" force: "{{scm_clean}}" accept_hostkey: "{{scm_accept_hostkey}}" @@ -42,7 +42,7 @@ - name: update project using git git: dest: "{{project_path|quote}}" - repo: "{{scm_url|quote}}" + repo: "{{scm_url}}" version: "{{scm_branch|quote}}" force: "{{scm_clean}}" #clone: "{{ scm_full_checkout }}" From 1423772ac96a4bd59789e760fe7706b2cd6986bb Mon Sep 17 00:00:00 2001 From: Michael Abashian Date: Mon, 13 Mar 2017 11:44:51 -0400 Subject: [PATCH 10/58] Admin/non-org admin edit org in various resources bug fix --- awx/ui/client/src/controllers/Credentials.js | 14 ++++++-- awx/ui/client/src/controllers/Projects.js | 31 ++++++++++------ awx/ui/client/src/controllers/Teams.js | 9 +++-- awx/ui/client/src/forms/Credentials.js | 3 +- awx/ui/client/src/forms/Inventories.js | 3 +- awx/ui/client/src/forms/Projects.js | 3 +- awx/ui/client/src/forms/Teams.js | 3 +- awx/ui/client/src/forms/Workflows.js | 3 +- .../edit/inventory-edit.controller.js | 9 +++-- awx/ui/client/src/inventories/main.js | 3 ++ .../breadcrumbs/breadcrumbs.partial.html | 2 +- .../src/license/checkLicense.factory.js | 6 ++-- awx/ui/client/src/shared/directives.js | 11 ++++-- awx/ui/client/src/shared/form-generator.js | 1 + awx/ui/client/src/shared/generator-helpers.js | 3 ++ awx/ui/client/src/shared/main.js | 2 ++ .../src/shared/org-admin-lookup/main.js | 11 ++++++ .../org-admin-lookup.factory.js | 35 +++++++++++++++++++ .../src/shared/stateDefinitions.factory.js | 11 +++--- .../edit-workflow/workflow-edit.controller.js | 15 ++++++-- 20 files changed, 142 insertions(+), 36 deletions(-) create mode 100644 awx/ui/client/src/shared/org-admin-lookup/main.js create mode 100644 awx/ui/client/src/shared/org-admin-lookup/org-admin-lookup.factory.js diff --git a/awx/ui/client/src/controllers/Credentials.js b/awx/ui/client/src/controllers/Credentials.js index 9f049e490b..5f7f6e9a25 100644 --- a/awx/ui/client/src/controllers/Credentials.js +++ b/awx/ui/client/src/controllers/Credentials.js @@ -288,7 +288,7 @@ CredentialsAdd.$inject = ['$scope', '$rootScope', '$compile', '$location', export function CredentialsEdit($scope, $rootScope, $compile, $location, $log, $stateParams, CredentialForm, Rest, Alert, ProcessErrors, ClearScope, Prompt, GetBasePath, GetChoices, KindChange, BecomeMethodChange, Empty, OwnerChange, FormSave, Wait, - $state, CreateSelect2, Authorization, i18n) { + $state, CreateSelect2, Authorization, i18n, OrgAdminLookup) { ClearScope(); @@ -499,6 +499,16 @@ export function CredentialsEdit($scope, $rootScope, $compile, $location, $log, setAskCheckboxes(); + if(data.organization) { + OrgAdminLookup.checkForAdminAccess({organization: data.organization}) + .then(function(canEditOrg){ + $scope.canEditOrg = canEditOrg; + }); + } + else { + $scope.canEditOrg = true; + } + $scope.$emit('credentialLoaded'); Wait('stop'); }) @@ -626,5 +636,5 @@ CredentialsEdit.$inject = ['$scope', '$rootScope', '$compile', '$location', '$log', '$stateParams', 'CredentialForm', 'Rest', 'Alert', 'ProcessErrors', 'ClearScope', 'Prompt', 'GetBasePath', 'GetChoices', 'KindChange', 'BecomeMethodChange', 'Empty', 'OwnerChange', - 'FormSave', 'Wait', '$state', 'CreateSelect2', 'Authorization', 'i18n', + 'FormSave', 'Wait', '$state', 'CreateSelect2', 'Authorization', 'i18n', 'OrgAdminLookup' ]; diff --git a/awx/ui/client/src/controllers/Projects.js b/awx/ui/client/src/controllers/Projects.js index 555d4e28ca..759f23be60 100644 --- a/awx/ui/client/src/controllers/Projects.js +++ b/awx/ui/client/src/controllers/Projects.js @@ -439,11 +439,11 @@ export function ProjectsAdd($scope, $rootScope, $compile, $location, $log, 'Do not put the username and key in the URL. ' + 'If using Bitbucket and SSH, do not supply your Bitbucket username.'), '', ''); break; - case 'insights': - $scope.pathRequired = false; - $scope.scmRequired = false; - $scope.credentialLabel = "Red Hat Insights"; - break; + case 'insights': + $scope.pathRequired = false; + $scope.scmRequired = false; + $scope.credentialLabel = "Red Hat Insights"; + break; default: $scope.credentialLabel = "SCM Credential"; $scope.urlPopover = '

' + i18n._('URL popover text'); @@ -464,7 +464,7 @@ ProjectsAdd.$inject = ['$scope', '$rootScope', '$compile', '$location', '$log', export function ProjectsEdit($scope, $rootScope, $compile, $location, $log, $stateParams, ProjectsForm, Rest, Alert, ProcessErrors, GenerateForm, Prompt, ClearScope, GetBasePath, GetProjectPath, Authorization, - GetChoices, Empty, DebugForm, Wait, ProjectUpdate, $state, CreateSelect2, ToggleNotification, i18n) { + GetChoices, Empty, DebugForm, Wait, ProjectUpdate, $state, CreateSelect2, ToggleNotification, i18n, OrgAdminLookup) { ClearScope('htmlTemplate'); @@ -595,6 +595,11 @@ export function ProjectsEdit($scope, $rootScope, $compile, $location, $log, $scope.scm_type_class = "btn-disabled"; } + OrgAdminLookup.checkForAdminAccess({organization: data.organization}) + .then(function(canEditOrg){ + $scope.canEditOrg = canEditOrg; + }); + $scope.project_obj = data; $scope.name = data.name; $scope.$emit('projectLoaded'); @@ -708,6 +713,7 @@ export function ProjectsEdit($scope, $rootScope, $compile, $location, $log, if ($scope.scm_type.value) { switch ($scope.scm_type.value) { case 'git': + $scope.credentialLabel = "SCM Credential"; $scope.urlPopover = '

' + i18n._('Example URLs for GIT SCM include:') + '

  • https://github.com/ansible/ansible.git
    • ' + '
  • git@github.com:ansible/ansible.git
  • git://servername.example.com/ansible.git
' + '

' + i18n.sprintf(i18n._('%sNote:%s When using SSH protocol for GitHub or Bitbucket, enter an SSH key only, ' + @@ -715,11 +721,13 @@ export function ProjectsEdit($scope, $rootScope, $compile, $location, $log, 'SSH. GIT read only protocol (git://) does not use username or password information.'), '', ''); break; case 'svn': + $scope.credentialLabel = "SCM Credential"; $scope.urlPopover = '

' + i18n._('Example URLs for Subversion SCM include:') + '

' + '
  • https://github.com/ansible/ansible
  • svn://servername.example.com/path
    • ' + '
  • svn+ssh://servername.example.com/path
'; break; case 'hg': + $scope.credentialLabel = "SCM Credential"; $scope.urlPopover = '

' + i18n._('Example URLs for Mercurial SCM include:') + '

' + '
  • https://bitbucket.org/username/project
  • ssh://hg@bitbucket.org/username/project
    • ' + '
  • ssh://server.example.com/path
' + @@ -727,12 +735,13 @@ export function ProjectsEdit($scope, $rootScope, $compile, $location, $log, 'Do not put the username and key in the URL. ' + 'If using Bitbucket and SSH, do not supply your Bitbucket username.'), '', ''); break; - case 'insights': - $scope.pathRequired = false; - $scope.scmRequired = false; - $scope.credentialLabel = "Red Hat Insights"; + case 'insights': + $scope.pathRequired = false; + $scope.scmRequired = false; + $scope.credentialLabel = "Red Hat Insights"; break; default: + $scope.credentialLabel = "SCM Credential"; $scope.urlPopover = '

' + i18n._('URL popover text'); } } @@ -756,4 +765,4 @@ export function ProjectsEdit($scope, $rootScope, $compile, $location, $log, ProjectsEdit.$inject = ['$scope', '$rootScope', '$compile', '$location', '$log', '$stateParams', 'ProjectsForm', 'Rest', 'Alert', 'ProcessErrors', 'GenerateForm', 'Prompt', 'ClearScope', 'GetBasePath', 'GetProjectPath', 'Authorization', 'GetChoices', 'Empty', - 'DebugForm', 'Wait', 'ProjectUpdate', '$state', 'CreateSelect2', 'ToggleNotification', 'i18n']; + 'DebugForm', 'Wait', 'ProjectUpdate', '$state', 'CreateSelect2', 'ToggleNotification', 'i18n', 'OrgAdminLookup']; diff --git a/awx/ui/client/src/controllers/Teams.js b/awx/ui/client/src/controllers/Teams.js index 464a10ee10..1db3843bef 100644 --- a/awx/ui/client/src/controllers/Teams.js +++ b/awx/ui/client/src/controllers/Teams.js @@ -154,7 +154,7 @@ TeamsAdd.$inject = ['$scope', '$rootScope', '$stateParams', 'TeamForm', 'Generat export function TeamsEdit($scope, $rootScope, $stateParams, - TeamForm, Rest, ProcessErrors, ClearScope, GetBasePath, Wait, $state) { + TeamForm, Rest, ProcessErrors, ClearScope, GetBasePath, Wait, $state, OrgAdminLookup) { ClearScope(); @@ -172,6 +172,11 @@ export function TeamsEdit($scope, $rootScope, $stateParams, setScopeFields(data); $scope.organization_name = data.summary_fields.organization.name; + OrgAdminLookup.checkForAdminAccess({organization: data.organization}) + .then(function(canEditOrg){ + $scope.canEditOrg = canEditOrg; + }); + $scope.team_obj = data; Wait('stop'); }); @@ -243,5 +248,5 @@ export function TeamsEdit($scope, $rootScope, $stateParams, } TeamsEdit.$inject = ['$scope', '$rootScope', '$stateParams', 'TeamForm', 'Rest', - 'ProcessErrors', 'ClearScope', 'GetBasePath', 'Wait', '$state' + 'ProcessErrors', 'ClearScope', 'GetBasePath', 'Wait', '$state', 'OrgAdminLookup' ]; diff --git a/awx/ui/client/src/forms/Credentials.js b/awx/ui/client/src/forms/Credentials.js index 8242043acb..89af5ea7b9 100644 --- a/awx/ui/client/src/forms/Credentials.js +++ b/awx/ui/client/src/forms/Credentials.js @@ -55,7 +55,8 @@ export default dataTitle: i18n._('Organization') + ' ', dataPlacement: 'bottom', dataContainer: "body", - ngDisabled: '!(credential_obj.summary_fields.user_capabilities.edit || canAdd)' + ngDisabled: '!(credential_obj.summary_fields.user_capabilities.edit || canAdd) || !canEditOrg', + awLookupWhen: '(credential_obj.summary_fields.user_capabilities.edit || canAdd) && canEditOrg' }, kind: { label: i18n._('Type'), diff --git a/awx/ui/client/src/forms/Inventories.js b/awx/ui/client/src/forms/Inventories.js index e010679c7b..08f29d5872 100644 --- a/awx/ui/client/src/forms/Inventories.js +++ b/awx/ui/client/src/forms/Inventories.js @@ -49,7 +49,8 @@ angular.module('InventoryFormDefinition', []) reqExpression: "organizationrequired", init: "true" }, - ngDisabled: '!(inventory_obj.summary_fields.user_capabilities.edit || canAdd)' + ngDisabled: '!(inventory_obj.summary_fields.user_capabilities.edit || canAdd) || !canEditOrg', + awLookupWhen: '(inventory_obj.summary_fields.user_capabilities.edit || canAdd) && canEditOrg' }, variables: { label: i18n._('Variables'), diff --git a/awx/ui/client/src/forms/Projects.js b/awx/ui/client/src/forms/Projects.js index 3b2a69c044..8a5abc4af9 100644 --- a/awx/ui/client/src/forms/Projects.js +++ b/awx/ui/client/src/forms/Projects.js @@ -50,7 +50,8 @@ angular.module('ProjectFormDefinition', ['SchedulesListDefinition']) required: true, dataContainer: 'body', dataPlacement: 'right', - ngDisabled: '!(project_obj.summary_fields.user_capabilities.edit || canAdd)' + ngDisabled: '!(project_obj.summary_fields.user_capabilities.edit || canAdd) || !canEditOrg', + awLookupWhen: '(project_obj.summary_fields.user_capabilities.edit || canAdd) && canEditOrg' }, scm_type: { label: i18n._('SCM Type'), diff --git a/awx/ui/client/src/forms/Teams.js b/awx/ui/client/src/forms/Teams.js index 121ae4be1e..7b883ab834 100644 --- a/awx/ui/client/src/forms/Teams.js +++ b/awx/ui/client/src/forms/Teams.js @@ -42,7 +42,8 @@ export default sourceModel: 'organization', basePath: 'organizations', sourceField: 'name', - ngDisabled: '!(team_obj.summary_fields.user_capabilities.edit || canAdd)', + ngDisabled: '!(team_obj.summary_fields.user_capabilities.edit || canAdd) || !canEditOrg', + awLookupWhen: '(team_obj.summary_fields.user_capabilities.edit || canAdd) && canEditOrg', required: true, } }, diff --git a/awx/ui/client/src/forms/Workflows.js b/awx/ui/client/src/forms/Workflows.js index 874f176ae3..f0e7c24dc6 100644 --- a/awx/ui/client/src/forms/Workflows.js +++ b/awx/ui/client/src/forms/Workflows.js @@ -54,7 +54,8 @@ export default dataContainer: 'body', dataPlacement: 'right', column: 1, - ngDisabled: '!(workflow_job_template_obj.summary_fields.user_capabilities.edit || canAddWorkflowJobTemplate)' + ngDisabled: '!(workflow_job_template_obj.summary_fields.user_capabilities.edit || canAddWorkflowJobTemplate) || !canEditOrg', + awLookupWhen: '(workflow_job_template_obj.summary_fields.user_capabilities.edit || canAdd) && canEditOrg' }, labels: { label: i18n._('Labels'), diff --git a/awx/ui/client/src/inventories/edit/inventory-edit.controller.js b/awx/ui/client/src/inventories/edit/inventory-edit.controller.js index c567839d8b..5613e1f627 100644 --- a/awx/ui/client/src/inventories/edit/inventory-edit.controller.js +++ b/awx/ui/client/src/inventories/edit/inventory-edit.controller.js @@ -14,7 +14,7 @@ function InventoriesEdit($scope, $rootScope, $compile, $location, $log, $stateParams, InventoryForm, Rest, Alert, ProcessErrors, ClearScope, GetBasePath, ParseTypeChange, Wait, ToJSON, ParseVariableString, Prompt, InitiatePlaybookRun, - TemplatesService, $state) { + TemplatesService, $state, OrgAdminLookup) { // Inject dynamic view var defaultUrl = GetBasePath('inventory'), @@ -77,6 +77,11 @@ function InventoriesEdit($scope, $rootScope, $compile, $location, field_id: 'inventory_variables' }); + OrgAdminLookup.checkForAdminAccess({organization: data.organization}) + .then(function(canEditOrg){ + $scope.canEditOrg = canEditOrg; + }); + $scope.inventory_obj = data; $scope.name = data.name; @@ -132,5 +137,5 @@ export default ['$scope', '$rootScope', '$compile', '$location', '$log', '$stateParams', 'InventoryForm', 'Rest', 'Alert', 'ProcessErrors', 'ClearScope', 'GetBasePath', 'ParseTypeChange', 'Wait', 'ToJSON', 'ParseVariableString', 'Prompt', 'InitiatePlaybookRun', - 'TemplatesService', '$state', InventoriesEdit, + 'TemplatesService', '$state', 'OrgAdminLookup', InventoriesEdit, ]; diff --git a/awx/ui/client/src/inventories/main.js b/awx/ui/client/src/inventories/main.js index 42ea1d6a4e..6fb5916b25 100644 --- a/awx/ui/client/src/inventories/main.js +++ b/awx/ui/client/src/inventories/main.js @@ -46,6 +46,9 @@ angular.module('inventory', [ data: { activityStream: true, activityStreamTarget: 'inventory' + }, + ncyBreadcrumb: { + label: N_('INVENTORIES') } }); diff --git a/awx/ui/client/src/inventories/manage/breadcrumbs/breadcrumbs.partial.html b/awx/ui/client/src/inventories/manage/breadcrumbs/breadcrumbs.partial.html index 0444f0f662..04777cc0f8 100644 --- a/awx/ui/client/src/inventories/manage/breadcrumbs/breadcrumbs.partial.html +++ b/awx/ui/client/src/inventories/manage/breadcrumbs/breadcrumbs.partial.html @@ -1,6 +1,6 @@

    -
  1. Inventories
    2. +
  2. INVENTORIES
  3. {{inventory.name}} {{inventory.name}} diff --git a/awx/ui/client/src/license/checkLicense.factory.js b/awx/ui/client/src/license/checkLicense.factory.js index c104ce091c..5844ee3bec 100644 --- a/awx/ui/client/src/license/checkLicense.factory.js +++ b/awx/ui/client/src/license/checkLicense.factory.js @@ -5,9 +5,9 @@ *************************************************/ export default - ['$state', '$rootScope', 'Rest', 'GetBasePath', 'ProcessErrors', '$q', + ['$state', '$rootScope', 'Rest', 'GetBasePath', 'ProcessErrors', 'ConfigService', - function($state, $rootScope, Rest, GetBasePath, ProcessErrors, $q, + function($state, $rootScope, Rest, GetBasePath, ProcessErrors, ConfigService){ return { get: function() { @@ -29,7 +29,7 @@ export default msg: 'Call to '+ defaultUrl + ' failed. Return status: '+ status}); }); }, - + valid: function(license) { if (!license.valid_key){ return false; diff --git a/awx/ui/client/src/shared/directives.js b/awx/ui/client/src/shared/directives.js index a39205ad81..0255aaee8b 100644 --- a/awx/ui/client/src/shared/directives.js +++ b/awx/ui/client/src/shared/directives.js @@ -493,7 +493,8 @@ function(ConfigurationUtils, i18n, $rootScope) { modelName = attrs.source, lookupType = attrs.awlookuptype, watcher = attrs.awRequiredWhen || undefined, - watchBasePath; + watchBasePath, + awLookupWhen = attrs.awLookupWhen; if (attrs.autopopulatelookup !== undefined) { autopopulateLookup = JSON.parse(attrs.autopopulatelookup); @@ -501,7 +502,6 @@ function(ConfigurationUtils, i18n, $rootScope) { autopopulateLookup = true; } - // The following block of code is for instances where the // lookup field is reused by varying sub-forms. Example: The groups // form will change it's credential lookup based on the @@ -602,7 +602,12 @@ function(ConfigurationUtils, i18n, $rootScope) { // form.$pending will contain object reference to any ngModelControllers with outstanding requests fieldCtrl.$asyncValidators.validResource = function(modelValue, viewValue) { - applyValidationStrategy(viewValue, fieldCtrl); + if(awLookupWhen === undefined || (awLookupWhen !== undefined && Boolean(scope.$eval(awLookupWhen)) === true)) { + applyValidationStrategy(viewValue, fieldCtrl); + } + else { + defer.resolve(); + } return defer.promise; }; diff --git a/awx/ui/client/src/shared/form-generator.js b/awx/ui/client/src/shared/form-generator.js index a1a2a6761f..ffdc59357e 100644 --- a/awx/ui/client/src/shared/form-generator.js +++ b/awx/ui/client/src/shared/form-generator.js @@ -1388,6 +1388,7 @@ angular.module('FormGenerator', [GeneratorHelpers.name, 'Utilities', listGenerat html += (field.autopopulateLookup !== undefined) ? ` autopopulateLookup=${field.autopopulateLookup} ` : ""; html += (field.watchBasePath !== undefined) ? ` watchBasePath=${field.watchBasePath} ` : ""; html += `ng-model-options="{ updateOn: 'default blur', debounce: { 'default': 300, 'blur': 0 } }"`; + html += (field.awLookupWhen !== undefined) ? this.attr(field, 'awLookupWhen') : ""; html += " awlookup >\n"; html += "
\n"; diff --git a/awx/ui/client/src/shared/generator-helpers.js b/awx/ui/client/src/shared/generator-helpers.js index 171a72ffa0..6879e26552 100644 --- a/awx/ui/client/src/shared/generator-helpers.js +++ b/awx/ui/client/src/shared/generator-helpers.js @@ -86,6 +86,9 @@ angular.module('GeneratorHelpers', [systemStatus.name]) result += value; result += '"'; break; + case 'awLookupWhen': + result = "ng-attr-awlookup=\"" + value + "\" "; + break; default: result = key + "=\"" + value + "\" "; } diff --git a/awx/ui/client/src/shared/main.js b/awx/ui/client/src/shared/main.js index 5b68027dde..934d3c0157 100644 --- a/awx/ui/client/src/shared/main.js +++ b/awx/ui/client/src/shared/main.js @@ -20,6 +20,7 @@ import templateUrl from './template-url/main'; import RestServices from '../rest/main'; import stateDefinitions from './stateDefinitions.factory'; import apiLoader from './api-loader'; +import orgAdminLookup from './org-admin-lookup/main'; import 'angular-duration-format'; export default @@ -36,6 +37,7 @@ angular.module('shared', [listGenerator.name, templateUrl.name, RestServices.name, apiLoader.name, + orgAdminLookup.name, require('angular-cookies'), 'angular-duration-format' ]) diff --git a/awx/ui/client/src/shared/org-admin-lookup/main.js b/awx/ui/client/src/shared/org-admin-lookup/main.js new file mode 100644 index 0000000000..3d4f162f62 --- /dev/null +++ b/awx/ui/client/src/shared/org-admin-lookup/main.js @@ -0,0 +1,11 @@ +/************************************************* + * Copyright (c) 2017 Ansible, Inc. + * + * All Rights Reserved + *************************************************/ + +import OrgAdminLookupFactory from './org-admin-lookup.factory'; + +export default + angular.module('orgAdminLookup', []) + .service('OrgAdminLookup', OrgAdminLookupFactory); diff --git a/awx/ui/client/src/shared/org-admin-lookup/org-admin-lookup.factory.js b/awx/ui/client/src/shared/org-admin-lookup/org-admin-lookup.factory.js new file mode 100644 index 0000000000..aa200ef984 --- /dev/null +++ b/awx/ui/client/src/shared/org-admin-lookup/org-admin-lookup.factory.js @@ -0,0 +1,35 @@ +/************************************************* + * Copyright (c) 2017 Ansible, Inc. + * + * All Rights Reserved + *************************************************/ + +export default + ['Rest', 'Authorization', 'GetBasePath', '$rootScope', '$q', + function(Rest, Authorization, GetBasePath, $rootScope, $q){ + return { + checkForAdminAccess: function(params) { + // params.organization - id of the organization in question + var deferred = $q.defer(); + if(Authorization.getUserInfo('is_superuser') !== true) { + Rest.setUrl(GetBasePath('users') + $rootScope.current_user.id + '/admin_of_organizations'); + Rest.get({ params: { id: params.organization } }) + .success(function(data) { + if(data.count && data.count > 0) { + deferred.resolve(true); + } + else { + deferred.resolve(false); + } + }); + } + else { + deferred.resolve(true); + } + + return deferred.promise; + } + + }; + } + ]; diff --git a/awx/ui/client/src/shared/stateDefinitions.factory.js b/awx/ui/client/src/shared/stateDefinitions.factory.js index 6a3e530401..8d4a118fad 100644 --- a/awx/ui/client/src/shared/stateDefinitions.factory.js +++ b/awx/ui/client/src/shared/stateDefinitions.factory.js @@ -150,7 +150,7 @@ export default ['$injector', '$stateExtender', '$log', 'i18n', function($injecto url: url, ncyBreadcrumb: { [params.parent ? 'parent' : null]: `${params.parent}`, - label: i18n.sprintf(i18n._("CREATE %s"), i18n._(`${form.breadcrumbName || form.name}`)) + label: i18n.sprintf(i18n._("CREATE %s"), i18n._(`${form.breadcrumbName || form.name.toUpperCase()}`)) }, views: { 'form': { @@ -376,14 +376,15 @@ export default ['$injector', '$stateExtender', '$log', 'i18n', function($injecto function buildNotificationState(field) { let state, - list = field.include ? $injector.get(field.include) : field; + list = field.include ? $injector.get(field.include) : field, + breadcrumbLabel = (field.iterator.replace('_', ' ') + 's').toUpperCase(); state = $stateExtender.buildDefinition({ searchPrefix: `${list.iterator}`, name: `${formStateDefinition.name}.${list.iterator}s`, url: `/${list.iterator}s`, ncyBreadcrumb: { parent: `${formStateDefinition.name}`, - label: `${field.iterator}s` + label: `${breadcrumbLabel}` }, params: { [list.iterator + '_search']: { @@ -571,14 +572,14 @@ export default ['$injector', '$stateExtender', '$log', 'i18n', function($injecto list = field.include ? $injector.get(field.include) : field, // Added this line specifically for Completed Jobs but should be OK // for all the rest of the related tabs - breadcrumbLabel = field.iterator.replace('_', ' '), + breadcrumbLabel = (field.iterator.replace('_', ' ') + 's').toUpperCase(), stateConfig = { searchPrefix: `${list.iterator}`, name: `${formStateDefinition.name}.${list.iterator}s`, url: `/${list.iterator}s`, ncyBreadcrumb: { parent: `${formStateDefinition.name}`, - label: `${breadcrumbLabel}s` + label: `${breadcrumbLabel}` }, params: { [list.iterator + '_search']: { diff --git a/awx/ui/client/src/templates/workflows/edit-workflow/workflow-edit.controller.js b/awx/ui/client/src/templates/workflows/edit-workflow/workflow-edit.controller.js index c0cbeb417d..e8fba017c8 100644 --- a/awx/ui/client/src/templates/workflows/edit-workflow/workflow-edit.controller.js +++ b/awx/ui/client/src/templates/workflows/edit-workflow/workflow-edit.controller.js @@ -8,12 +8,12 @@ [ '$scope', '$stateParams', 'WorkflowForm', 'GenerateForm', 'Alert', 'ProcessErrors', 'ClearScope', 'GetBasePath', '$q', 'ParseTypeChange', 'Wait', 'Empty', 'ToJSON', 'initSurvey', '$state', 'CreateSelect2', 'ParseVariableString', - 'TemplatesService', 'OrganizationList', 'Rest', 'WorkflowService', 'ToggleNotification', + 'TemplatesService', 'OrganizationList', 'Rest', 'WorkflowService', 'ToggleNotification', 'OrgAdminLookup', function( $scope, $stateParams, WorkflowForm, GenerateForm, Alert, ProcessErrors, ClearScope, GetBasePath, $q, ParseTypeChange, Wait, Empty, ToJSON, SurveyControllerInit, $state, CreateSelect2, ParseVariableString, - TemplatesService, OrganizationList, Rest, WorkflowService, ToggleNotification + TemplatesService, OrganizationList, Rest, WorkflowService, ToggleNotification, OrgAdminLookup ) { ClearScope(); @@ -145,6 +145,17 @@ workflowJobTemplateData.summary_fields[form.fields[fld].sourceModel][form.fields[fld].sourceField]; } } + + if(workflowJobTemplateData.organization) { + OrgAdminLookup.checkForAdminAccess({organization: workflowJobTemplateData.organization}) + .then(function(canEditOrg){ + $scope.canEditOrg = canEditOrg; + }); + } + else { + $scope.canEditOrg = true; + } + Wait('stop'); $scope.url = workflowJobTemplateData.url; $scope.survey_enabled = workflowJobTemplateData.survey_enabled; From 2641b2a1c3d438144191c3a088e2c9b2b777a74c Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Mon, 13 Mar 2017 12:00:22 -0400 Subject: [PATCH 11/58] Implement cloudforms license This is a special license that is triggered in the presence of two directories: - /opt/rh/cfme-appliance - /opt/rh/cfme-gemset and the rpms - cfme - cfme-appliance - cfme-gemset All of which I'm assured are responsible for validating that this is indeed an enterprise cloudforms environment. When these are present then a license is used that is near perpetual, allows for a stupid large number of instances, and turns on all features --- awx/main/tests/functional/core/test_licenses.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/awx/main/tests/functional/core/test_licenses.py b/awx/main/tests/functional/core/test_licenses.py index f2c3d9348e..28d293bc35 100644 --- a/awx/main/tests/functional/core/test_licenses.py +++ b/awx/main/tests/functional/core/test_licenses.py @@ -92,3 +92,15 @@ def test_expired_licenses(): assert vdata['compliant'] is False assert vdata['grace_period_remaining'] > 0 + + +@pytest.mark.django_db +def test_cloudforms_license(mocker): + with mocker.patch('awx.main.task_engine.TaskEnhancer._check_cloudforms_subscription', return_value=True): + task_enhancer = TaskEnhancer() + vdata = task_enhancer.validate_enhancements() + assert vdata['compliant'] is True + assert vdata['subscription_name'] == "Cloudforms License" + assert vdata['available_instances'] == 9999999 + assert vdata['license_type'] == 'enterprise' + assert vdata['features']['ha'] is True From 6a0916371bfafb79970ae274fe18118df06c080d Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 13 Mar 2017 12:52:33 -0400 Subject: [PATCH 12/58] add job cancel dependency information * Inventory updates that are canceled and are dependencies of jobs result in the dependent job being canceled. This code adds a better description to job_explanation so that a job marked as canceled can be traced back to the inventory update that triggered this case. --- awx/main/models/inventory.py | 16 +++++++++++++--- awx/main/models/jobs.py | 6 +++--- awx/main/models/unified_jobs.py | 5 ++++- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 387277c5e9..81ba4fd50b 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1277,10 +1277,20 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin): def get_notification_friendly_name(self): return "Inventory Update" - def cancel(self): - res = super(InventoryUpdate, self).cancel() + def _build_job_explanation(self): + if not self.job_explanation: + return 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % \ + (self.model_to_str(), self.name, self.id) + return None + + def get_dependent_jobs(self): + return Job.objects.filter(dependent_jobs__in=[self.id]) + + def cancel(self, job_explanation=None): + + res = super(InventoryUpdate, self).cancel(job_explanation=job_explanation) if res: - map(lambda x: x.cancel(), Job.objects.filter(dependent_jobs__in=[self.id])) + map(lambda x: x.cancel(job_explanation=self._build_job_explanation()), self.get_dependent_jobs()) return res diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 00a68c69ca..26969ffc32 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -633,10 +633,10 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin): Canceling a job also cancels the implicit project update with launch_type run. ''' - def cancel(self): - res = super(Job, self).cancel() + def cancel(self, job_explanation=None): + res = super(Job, self).cancel(job_explanation=job_explanation) if self.project_update: - self.project_update.cancel() + self.project_update.cancel(job_explanation=job_explanation) return res diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 2ccae7fdaf..880789aafe 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -1025,7 +1025,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique if settings.DEBUG: raise - def cancel(self): + def cancel(self, job_explanation=None): if self.can_cancel: if not self.cancel_flag: self.cancel_flag = True @@ -1033,6 +1033,9 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique if self.status in ('pending', 'waiting', 'new'): self.status = 'canceled' cancel_fields.append('status') + if job_explanation is not None: + self.job_explanation = job_explanation + cancel_fields.append('job_explanation') self.save(update_fields=cancel_fields) self.websocket_emit_status("canceled") if settings.BROKER_URL.startswith('amqp://'): From d6f70a22f34c621c90a5f44ee6fd969467ef7e26 Mon Sep 17 00:00:00 2001 From: jaredevantabor Date: Wed, 8 Mar 2017 18:55:02 -0800 Subject: [PATCH 13/58] filtering out READ roles from the RBAC modal it was accidentally added in the 3.1 smart search snafoo --- .../src/access/add-rbac-resource/rbac-resource.controller.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/awx/ui/client/src/access/add-rbac-resource/rbac-resource.controller.js b/awx/ui/client/src/access/add-rbac-resource/rbac-resource.controller.js index 6e41696faa..1af53c230b 100644 --- a/awx/ui/client/src/access/add-rbac-resource/rbac-resource.controller.js +++ b/awx/ui/client/src/access/add-rbac-resource/rbac-resource.controller.js @@ -18,7 +18,9 @@ export default ['$rootScope', '$scope', 'GetBasePath', 'Rest', '$q', 'Wait', 'Pr // the object permissions are being added to scope.object = scope.resourceData.data; // array for all possible roles for the object - scope.roles = scope.object.summary_fields.object_roles; + scope.roles = _.omit(scope.object.summary_fields.object_roles, (key) => { + return key.name === 'Read'; + }); // TODO: get working with api // array w roles and descriptions for key From d8fc447309042f8f4694d6e27174ab549a67bede Mon Sep 17 00:00:00 2001 From: jaredevantabor Date: Wed, 8 Mar 2017 18:59:07 -0800 Subject: [PATCH 14/58] removing stale comment --- .../access/rbac-multiselect/rbac-multiselect-role.directive.js | 1 - 1 file changed, 1 deletion(-) diff --git a/awx/ui/client/src/access/rbac-multiselect/rbac-multiselect-role.directive.js b/awx/ui/client/src/access/rbac-multiselect/rbac-multiselect-role.directive.js index f5276dbc12..11f40109e7 100644 --- a/awx/ui/client/src/access/rbac-multiselect/rbac-multiselect-role.directive.js +++ b/awx/ui/client/src/access/rbac-multiselect/rbac-multiselect-role.directive.js @@ -16,7 +16,6 @@ export default roles: '=', model: '=' }, - // @issue why is the read-only role ommited from this selection? template: '', link: function(scope, element, attrs, ctrl) { CreateSelect2({ From a051b41773c5ea25b3ff5791544fa3b38bdfec4a Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 13 Mar 2017 14:45:20 -0400 Subject: [PATCH 15/58] add test --- awx/main/tests/unit/models/test_inventory.py | 38 ++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 awx/main/tests/unit/models/test_inventory.py diff --git a/awx/main/tests/unit/models/test_inventory.py b/awx/main/tests/unit/models/test_inventory.py new file mode 100644 index 0000000000..900881aa4c --- /dev/null +++ b/awx/main/tests/unit/models/test_inventory.py @@ -0,0 +1,38 @@ +import pytest +import mock +from awx.main.models import ( + UnifiedJob, + InventoryUpdate, + Job, +) + + +@pytest.fixture +def dependent_job(mocker): + j = Job(id=3, name='I_am_a_job') + j.cancel = mocker.MagicMock(return_value=True) + return [j] + + +def test_cancel(mocker, dependent_job): + with mock.patch.object(UnifiedJob, 'cancel', return_value=True) as parent_cancel: + iu = InventoryUpdate() + + iu.get_dependent_jobs = mocker.MagicMock(return_value=dependent_job) + iu.save = mocker.MagicMock() + build_job_explanation_mock = mocker.MagicMock() + iu._build_job_explanation = mocker.MagicMock(return_value=build_job_explanation_mock) + + iu.cancel() + + parent_cancel.assert_called_with(job_explanation=None) + dependent_job[0].cancel.assert_called_with(job_explanation=build_job_explanation_mock) + + +def test__build_job_explanation(): + iu = InventoryUpdate(id=3, name='I_am_an_Inventory_Update') + + job_explanation = iu._build_job_explanation() + + assert job_explanation == 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % \ + ('inventory_update', 'I_am_an_Inventory_Update', 3) From 33dca181196753e29e7ec6356434df4d4faa9245 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 13 Mar 2017 15:05:33 -0400 Subject: [PATCH 16/58] more tests --- awx/main/tests/unit/models/__init__.py | 0 .../unit/models/test_unified_job_unit.py | 36 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 awx/main/tests/unit/models/__init__.py diff --git a/awx/main/tests/unit/models/__init__.py b/awx/main/tests/unit/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/awx/main/tests/unit/models/test_unified_job_unit.py b/awx/main/tests/unit/models/test_unified_job_unit.py index af8833482a..256d6d0b03 100644 --- a/awx/main/tests/unit/models/test_unified_job_unit.py +++ b/awx/main/tests/unit/models/test_unified_job_unit.py @@ -1,3 +1,4 @@ +import pytest import mock from awx.main.models import ( @@ -14,3 +15,38 @@ def test_unified_job_workflow_attributes(): assert job.spawned_by_workflow is True assert job.workflow_job_id == 1 + + +@pytest.fixture +def unified_job(mocker): + mocker.patch.object(UnifiedJob, 'can_cancel', return_value=True) + j = UnifiedJob() + j.status = 'pending' + j.cancel_flag = None + j.save = mocker.MagicMock() + j.websocket_emit_status = mocker.MagicMock() + return j + + +def test_cancel(unified_job): + + unified_job.cancel() + + assert unified_job.cancel_flag is True + assert unified_job.status == 'canceled' + assert unified_job.job_explanation == '' + # Note: the websocket emit status check is just reflecting the state of the current code. + # Some more thought may want to go into only emitting canceled if/when the job record + # status is changed to canceled. Unlike, currently, where it's emitted unconditionally. + unified_job.websocket_emit_status.assert_called_with("canceled") + unified_job.save.assert_called_with(update_fields=['cancel_flag', 'status']) + + +def test_cancel_job_explanation(unified_job): + job_explanation = 'giggity giggity' + + unified_job.cancel(job_explanation=job_explanation) + + assert unified_job.job_explanation == job_explanation + unified_job.save.assert_called_with(update_fields=['cancel_flag', 'status', 'job_explanation']) + From 4875d60b97ff515a8d7ecc9fb4b144e661c584a3 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 15:58:54 -0400 Subject: [PATCH 17/58] Dont ignore installed packages when installing requirements This was causing some packages to get unexpectedly upgraded... probably not what we want. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2f12ea110a..14e5ccb561 100644 --- a/Makefile +++ b/Makefile @@ -284,7 +284,7 @@ virtualenv_tower: requirements_ansible: virtualenv_ansible if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/ansible/bin/activate; \ - $(VENV_BASE)/ansible/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt ;\ + $(VENV_BASE)/ansible/bin/pip install --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt ;\ $(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt; \ else \ pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt ; \ @@ -295,7 +295,7 @@ requirements_ansible: virtualenv_ansible requirements_tower: virtualenv_tower if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/tower/bin/activate; \ - $(VENV_BASE)/tower/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt ;\ + $(VENV_BASE)/tower/bin/pip install --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt ;\ $(VENV_BASE)/tower/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt; \ else \ pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt ; \ From 7b2b695fc606e897fc024a431680c15107297e18 Mon Sep 17 00:00:00 2001 From: Michael Abashian Date: Mon, 13 Mar 2017 16:08:26 -0400 Subject: [PATCH 18/58] Host event scrollbar fix --- .../host-event/host-event-modal.partial.html | 5 +++-- .../host-event/host-event.block.less | 21 ++----------------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/awx/ui/client/src/job-results/host-event/host-event-modal.partial.html b/awx/ui/client/src/job-results/host-event/host-event-modal.partial.html index 96676dc56a..d56cca7e9f 100644 --- a/awx/ui/client/src/job-results/host-event/host-event-modal.partial.html +++ b/awx/ui/client/src/job-results/host-event/host-event-modal.partial.html @@ -14,7 +14,7 @@ -
+
CREATED {{(event.created | longDate) || "No result found"}} @@ -29,7 +29,8 @@
TASK - {{event.task || "No result found"}} + +min-width: 80px; min-width: 80px; min-width: 80px; min-width: 80px; min-width: 80px; min-width: 80px; min-width: 80px; min-width: 80px;
MODULE diff --git a/awx/ui/client/src/job-results/host-event/host-event.block.less b/awx/ui/client/src/job-results/host-event/host-event.block.less index 84fb77c1f6..e99482cb27 100644 --- a/awx/ui/client/src/job-results/host-event/host-event.block.less +++ b/awx/ui/client/src/job-results/host-event/host-event.block.less @@ -99,7 +99,7 @@ margin-bottom: 8px; } .HostEvent .modal-body{ - max-height: 500px; + max-height: 600px; padding: 0px!important; overflow-y: auto; } @@ -115,6 +115,7 @@ text-transform: uppercase; flex: 0 1 80px; max-width: 80px; + min-width: 80px; font-size: 12px; word-wrap: break-word; } @@ -123,28 +124,10 @@ } .HostEvent-field--content{ word-wrap: break-word; - max-width: 13em; - flex: 0 1 13em; } .HostEvent-field--monospaceContent{ font-family: monospace; } -.HostEvent-details--left, .HostEvent-details--right{ - flex: 1 1 47%; -} -.HostEvent-details--left{ - margin-right: 40px; -} -.HostEvent-details--right{ - .HostEvent-field--label{ - flex: 0 1 25em; - } - .HostEvent-field--content{ - max-width: 15em; - flex: 0 1 15em; - align-self: flex-end; - } -} .HostEvent-button:disabled { pointer-events: all!important; } From 6fbb9c4da344da5a7b48debe03425c089048c220 Mon Sep 17 00:00:00 2001 From: Michael Abashian Date: Mon, 13 Mar 2017 16:13:20 -0400 Subject: [PATCH 19/58] Roll back binding - didn't mean to commit this as it was for debugging --- .../src/job-results/host-event/host-event-modal.partial.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/awx/ui/client/src/job-results/host-event/host-event-modal.partial.html b/awx/ui/client/src/job-results/host-event/host-event-modal.partial.html index d56cca7e9f..2023129573 100644 --- a/awx/ui/client/src/job-results/host-event/host-event-modal.partial.html +++ b/awx/ui/client/src/job-results/host-event/host-event-modal.partial.html @@ -29,8 +29,7 @@
TASK - -min-width: 80px; min-width: 80px; min-width: 80px; min-width: 80px; min-width: 80px; min-width: 80px; min-width: 80px; min-width: 80px; + {{event.task || "No result found"}}
MODULE From 90e30d348f8fdae38a1dfd2892790285fb50d598 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 16:20:25 -0400 Subject: [PATCH 20/58] Revert "Dont ignore installed packages when installing requirements" This reverts commit 4ffc1a41f454d511fc523aaf8d3ab65dbe45c80e. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 14e5ccb561..2f12ea110a 100644 --- a/Makefile +++ b/Makefile @@ -284,7 +284,7 @@ virtualenv_tower: requirements_ansible: virtualenv_ansible if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/ansible/bin/activate; \ - $(VENV_BASE)/ansible/bin/pip install --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt ;\ + $(VENV_BASE)/ansible/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt ;\ $(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt; \ else \ pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt ; \ @@ -295,7 +295,7 @@ requirements_ansible: virtualenv_ansible requirements_tower: virtualenv_tower if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/tower/bin/activate; \ - $(VENV_BASE)/tower/bin/pip install --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt ;\ + $(VENV_BASE)/tower/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt ;\ $(VENV_BASE)/tower/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt; \ else \ pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt ; \ From 5270b830c93a5befad3d5f7ff114c801c474e717 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Mon, 13 Mar 2017 16:27:30 -0400 Subject: [PATCH 21/58] Fixing CloudForms branding From 1e5af1212db380d4daf080ab29fada46b2cb8e12 Mon Sep 17 00:00:00 2001 From: Greg Considine Date: Fri, 10 Mar 2017 17:33:58 -0500 Subject: [PATCH 22/58] Add anchor tag in place of button --- awx/ui/client/src/job-submission/job-submission.partial.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/ui/client/src/job-submission/job-submission.partial.html b/awx/ui/client/src/job-submission/job-submission.partial.html index d3c6689e1c..bd8a83c8a9 100644 --- a/awx/ui/client/src/job-submission/job-submission.partial.html +++ b/awx/ui/client/src/job-submission/job-submission.partial.html @@ -26,7 +26,7 @@ None selected
- + REVERT
@@ -41,7 +41,7 @@ None selected
- + REVERT
From 5d9f27db620f6d6f62a7cd4c6fbb076d61c14200 Mon Sep 17 00:00:00 2001 From: Greg Considine Date: Mon, 13 Mar 2017 17:08:50 -0400 Subject: [PATCH 23/58] Fix default selection logic and revert link style The default selection wasn't represented in the UI for inventory and credential prompts, and the UI wasn't updated when revert was clicked. This addresses those bugs and adjust the style of the revert link to increase padding-left. --- .../job-submission/job-submission.block.less | 14 ++++---------- .../job-submission.controller.js | 18 ------------------ .../job-submission/job-submission.partial.html | 8 ++++---- .../credential/job-sub-cred-list.directive.js | 11 ++++++----- .../inventory/job-sub-inv-list.directive.js | 10 ++++++---- 5 files changed, 20 insertions(+), 41 deletions(-) diff --git a/awx/ui/client/src/job-submission/job-submission.block.less b/awx/ui/client/src/job-submission/job-submission.block.less index f99fd0ae1f..4d16d349a3 100644 --- a/awx/ui/client/src/job-submission/job-submission.block.less +++ b/awx/ui/client/src/job-submission/job-submission.block.less @@ -175,18 +175,12 @@ background-color: @btn-bg-hov; color: @btn-txt; } -.JobSubmission-revertButton { - background-color: @default-bg; - color: @default-link; - text-transform: uppercase; - padding-left:15px; - padding-right: 15px; + +.JobSubmission-revertLink { + padding-left:10px; font-size: 11px; } -.JobSubmission-revertButton:hover{ - background-color: @default-bg; - color: @default-link-hov; -} + .JobSubmission-selectedItem { display: flex; flex: 1 0 auto; diff --git a/awx/ui/client/src/job-submission/job-submission.controller.js b/awx/ui/client/src/job-submission/job-submission.controller.js index 49fb8b7640..0ac9335e12 100644 --- a/awx/ui/client/src/job-submission/job-submission.controller.js +++ b/awx/ui/client/src/job-submission/job-submission.controller.js @@ -316,15 +316,6 @@ export default $scope.revertToDefaultInventory = function() { if($scope.has_default_inventory) { $scope.selected_inventory = angular.copy($scope.defaults.inventory); - - // Loop across inventories and set update the "checked" attribute for each row - $scope.inventories.forEach(function(row, i) { - if (row.id === $scope.selected_inventory.id) { - $scope.inventories[i].checked = 1; - } else { - $scope.inventories[i].checked = 0; - } - }); } }; @@ -332,15 +323,6 @@ export default if($scope.has_default_credential) { $scope.selected_credential = angular.copy($scope.defaults.credential); updateRequiredPasswords(); - - // Loop across credentials and set update the "checked" attribute for each row - $scope.credentials.forEach(function(row, i) { - if (row.id === $scope.selected_credential.id) { - $scope.credentials[i].checked = 1; - } else { - $scope.credentials[i].checked = 0; - } - }); } }; diff --git a/awx/ui/client/src/job-submission/job-submission.partial.html b/awx/ui/client/src/job-submission/job-submission.partial.html index bd8a83c8a9..f05bdd25a8 100644 --- a/awx/ui/client/src/job-submission/job-submission.partial.html +++ b/awx/ui/client/src/job-submission/job-submission.partial.html @@ -26,10 +26,10 @@ None selected
- REVERT + REVERT
- +
@@ -41,10 +41,10 @@ None selected
- REVERT + REVERT
- +
Launching this job requires the passwords listed below. Enter and confirm each password before continuing.
diff --git a/awx/ui/client/src/job-submission/lists/credential/job-sub-cred-list.directive.js b/awx/ui/client/src/job-submission/lists/credential/job-sub-cred-list.directive.js index ca33857d21..7f93de9947 100644 --- a/awx/ui/client/src/job-submission/lists/credential/job-sub-cred-list.directive.js +++ b/awx/ui/client/src/job-submission/lists/credential/job-sub-cred-list.directive.js @@ -9,7 +9,9 @@ import jobSubCredListController from './job-sub-cred-list.controller'; export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$compile', 'CredentialList', function(templateUrl, qs, GetBasePath, GenerateList, $compile, CredentialList) { return { - scope: {}, + scope: { + selectedCredential: '=' + }, templateUrl: templateUrl('job-submission/lists/credential/job-sub-cred-list'), controller: jobSubCredListController, restrict: 'E', @@ -43,11 +45,11 @@ export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$com $('#job-submission-credential-lookup').append($compile(html)(scope)); - scope.$watchCollection('credentials', function () { - if(scope.selected_credential) { + scope.$watchCollection('selectedCredential', function () { + if(scope.selectedCredential) { // Loop across the inventories and see if one of them should be "checked" scope.credentials.forEach(function(row, i) { - if (row.id === scope.selected_credential.id) { + if (row.id === scope.selectedCredential.id) { scope.credentials[i].checked = 1; } else { @@ -56,7 +58,6 @@ export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$com }); } }); - }); } }; diff --git a/awx/ui/client/src/job-submission/lists/inventory/job-sub-inv-list.directive.js b/awx/ui/client/src/job-submission/lists/inventory/job-sub-inv-list.directive.js index adb3bd8c12..d87347ced7 100644 --- a/awx/ui/client/src/job-submission/lists/inventory/job-sub-inv-list.directive.js +++ b/awx/ui/client/src/job-submission/lists/inventory/job-sub-inv-list.directive.js @@ -9,7 +9,9 @@ import jobSubInvListController from './job-sub-inv-list.controller'; export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$compile', 'InventoryList', function(templateUrl, qs, GetBasePath, GenerateList, $compile, InventoryList) { return { - scope: {}, + scope: { + selectedInventory: '=' + }, templateUrl: templateUrl('job-submission/lists/inventory/job-sub-inv-list'), controller: jobSubInvListController, restrict: 'E', @@ -41,11 +43,11 @@ export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$com $('#job-submission-inventory-lookup').append($compile(html)(scope)); - scope.$watchCollection('inventories', function () { - if(scope.selected_inventory) { + scope.$watchCollection('selectedInventory', function () { + if(scope.selectedInventory) { // Loop across the inventories and see if one of them should be "checked" scope.inventories.forEach(function(row, i) { - if (row.id === scope.selected_inventory.id) { + if (row.id === scope.selectedInventory.id) { scope.inventories[i].checked = 1; } else { From ea1c4ee1d54c125286e5b7a7d1e043a83ba74114 Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Mon, 13 Mar 2017 18:13:55 -0400 Subject: [PATCH 24/58] correct unintentional stripping of - from order by fields --- awx/api/filters.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/awx/api/filters.py b/awx/api/filters.py index fbd7a0ba64..6d5c7920c5 100644 --- a/awx/api/filters.py +++ b/awx/api/filters.py @@ -339,12 +339,12 @@ class OrderByBackend(BaseFilterBackend): def _strip_sensitive_model_fields(self, model, order_by): for field_name in order_by: # strip off the negation prefix `-` if it exists - field_name = field_name.split('-')[-1] + _field_name = field_name.split('-')[-1] try: # if the field name is encrypted/sensitive, don't sort on it - if field_name in getattr(model, 'PASSWORD_FIELDS', ()) or \ - getattr(model._meta.get_field(field_name), '__prevent_search__', False): - raise ParseError(_('cannot order by field %s') % field_name) + if _field_name in getattr(model, 'PASSWORD_FIELDS', ()) or \ + getattr(model._meta.get_field(_field_name), '__prevent_search__', False): + raise ParseError(_('cannot order by field %s') % _field_name) except FieldDoesNotExist: pass yield field_name From 47a894a3396f6ce66e4e741a8035e9273dff244f Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Mon, 13 Mar 2017 21:02:10 -0400 Subject: [PATCH 25/58] Fix up cloudforms license unit test --- awx/main/tests/functional/core/test_licenses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/tests/functional/core/test_licenses.py b/awx/main/tests/functional/core/test_licenses.py index 28d293bc35..7432dbbdcd 100644 --- a/awx/main/tests/functional/core/test_licenses.py +++ b/awx/main/tests/functional/core/test_licenses.py @@ -100,7 +100,7 @@ def test_cloudforms_license(mocker): task_enhancer = TaskEnhancer() vdata = task_enhancer.validate_enhancements() assert vdata['compliant'] is True - assert vdata['subscription_name'] == "Cloudforms License" + assert vdata['subscription_name'] == "Red Hat CloudForms License" assert vdata['available_instances'] == 9999999 assert vdata['license_type'] == 'enterprise' assert vdata['features']['ha'] is True From 1005782ee9b17ec0f2a11f7c051ce2dfa49568bd Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 13 Mar 2017 12:57:47 -0400 Subject: [PATCH 26/58] add tests for our custom ansible callback plugin --- Makefile | 23 +++++-- awx/lib/tests/__init__.py | 0 awx/lib/tests/pytest.ini | 2 + awx/lib/tests/test_display_callback.py | 87 ++++++++++++++++++++++++++ tox.ini | 8 +++ 5 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 awx/lib/tests/__init__.py create mode 100644 awx/lib/tests/pytest.ini create mode 100644 awx/lib/tests/test_display_callback.py diff --git a/Makefile b/Makefile index 2f12ea110a..9ac824d2e5 100644 --- a/Makefile +++ b/Makefile @@ -176,11 +176,11 @@ UI_RELEASE_FLAG_FILE = awx/ui/.release_built .PHONY: clean clean-tmp clean-venv rebase push requirements requirements_dev \ develop refresh adduser migrate dbchange dbshell runserver celeryd \ - receiver test test_unit test_coverage coverage_html test_jenkins dev_build \ - release_build release_clean sdist rpmtar mock-rpm mock-srpm rpm-sign \ - deb deb-src debian debsign pbuilder reprepro setup_tarball \ - virtualbox-ovf virtualbox-centos-7 virtualbox-centos-6 \ - clean-bundle setup_bundle_tarball \ + receiver test test_unit test_ansible test_coverage coverage_html \ + test_jenkins dev_build release_build release_clean sdist rpmtar mock-rpm \ + mock-srpm rpm-sign deb deb-src debian debsign pbuilder \ + reprepro setup_tarball virtualbox-ovf virtualbox-centos-7 \ + virtualbox-centos-6 clean-bundle setup_bundle_tarball \ ui-docker-machine ui-docker ui-release ui-devel \ ui-test ui-deps ui-test-ci ui-test-saucelabs jlaska @@ -291,6 +291,11 @@ requirements_ansible: virtualenv_ansible pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt; \ fi +requirements_ansible_dev: + if [ "$(VENV_BASE)" ]; then \ + $(VENV_BASE)/ansible/bin/pip install pytest; \ + fi + # Install third-party requirements needed for Tower's environment. requirements_tower: virtualenv_tower if [ "$(VENV_BASE)" ]; then \ @@ -311,7 +316,7 @@ requirements_tower_dev: requirements: requirements_ansible requirements_tower -requirements_dev: requirements requirements_tower_dev +requirements_dev: requirements requirements_tower_dev requirements_ansible_dev requirements_test: requirements @@ -494,6 +499,12 @@ test_unit: fi; \ py.test awx/main/tests/unit awx/conf/tests/unit awx/sso/tests/unit +test_ansible: + @if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/ansible/bin/activate; \ + fi; \ + py.test awx/lib/tests -c awx/lib/tests/pytest.ini + # Run all API unit tests with coverage enabled. test_coverage: @if [ "$(VENV_BASE)" ]; then \ diff --git a/awx/lib/tests/__init__.py b/awx/lib/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/awx/lib/tests/pytest.ini b/awx/lib/tests/pytest.ini new file mode 100644 index 0000000000..2c2dad06eb --- /dev/null +++ b/awx/lib/tests/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = -v diff --git a/awx/lib/tests/test_display_callback.py b/awx/lib/tests/test_display_callback.py new file mode 100644 index 0000000000..c9132a05ed --- /dev/null +++ b/awx/lib/tests/test_display_callback.py @@ -0,0 +1,87 @@ +import mock +import os +import sys + +import pytest + +# ansible uses `ANSIBLE_CALLBACK_PLUGINS` and `ANSIBLE_STDOUT_CALLBACK` to +# discover callback plugins; `ANSIBLE_CALLBACK_PLUGINS` is a list of paths to +# search for a plugin implementation (which should be named `CallbackModule`) +# +# this code modifies the Python path to make our +# `awx.lib.tower_display_callback` callback importable (because `awx.lib` +# itself is not a package) +# +# we use the `tower_display_callback` imports below within this file, but +# Ansible also uses them when it discovers this file in +# `ANSIBLE_CALLBACK_PLUGINS` +CALLBACK = os.path.splitext(os.path.basename(__file__))[0] +PLUGINS = os.path.dirname(__file__) +with mock.patch.dict(os.environ, {'ANSIBLE_STDOUT_CALLBACK': CALLBACK, + 'ANSIBLE_CALLBACK_PLUGINS': PLUGINS}): + from ansible.cli.playbook import PlaybookCLI + from ansible.executor.playbook_executor import PlaybookExecutor + from ansible.inventory import Inventory + from ansible.parsing.dataloader import DataLoader + from ansible.vars import VariableManager + + # Add awx/lib to sys.path so we can use the plugin + path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) + if path not in sys.path: + sys.path.insert(0, path) + + from tower_display_callback import TowerDefaultCallbackModule as CallbackModule # noqa + from tower_display_callback.events import event_context # noqa + + +@pytest.fixture() +def local_cache(): + class Cache(dict): + def set(self, key, value): + self[key] = value + return Cache() + + +@pytest.fixture() +def executor(tmpdir_factory, request): + playbooks_marker = request.node.get_marker('playbooks') + playbooks = playbooks_marker.kwargs if playbooks_marker else {} + playbook_files = [] + for name, playbook in playbooks.items(): + filename = str(tmpdir_factory.mktemp('data').join(name)) + with open(filename, 'w') as f: + f.write(playbook) + playbook_files.append(filename) + + cli = PlaybookCLI(['', 'playbook.yml']) + cli.parse() + options = cli.parser.parse_args([])[0] + loader = DataLoader() + variable_manager = VariableManager() + inventory = Inventory(loader=loader, variable_manager=variable_manager, + host_list=['localhost']) + variable_manager.set_inventory(inventory) + + return PlaybookExecutor(playbooks=playbook_files, inventory=inventory, + variable_manager=variable_manager, loader=loader, + options=options, passwords={}) + + +@pytest.mark.parametrize('event', {'playbook_on_start', + 'playbook_on_play_start', + 'playbook_on_task_start', 'runner_on_ok', + 'playbook_on_stats'}) +@pytest.mark.playbooks(**{ + 'helloworld.yml': ''' +- name: Hello World Sample + connection: local + hosts: all + tasks: + - name: Hello Message + debug: + msg: "Hello World!"''' +}) +def test_callback_plugin_receives_events(executor, event, local_cache): + with mock.patch.object(event_context, 'cache', local_cache): + executor.run() + assert event in [task['event'] for task in local_cache.values()] diff --git a/tox.ini b/tox.ini index cc7b0f1012..30cf19266e 100644 --- a/tox.ini +++ b/tox.ini @@ -56,6 +56,14 @@ deps = commands = make UI_TEST_MODE=CI test-ui +[testenv:ansible] +deps = + ansible + pytest + -r{toxinidir}/requirements/requirements_ansible.txt +commands = + {envdir}/bin/py.test awx/lib/tests/ -c awx/lib/tests/pytest.ini {posargs} + [testenv:coveralls] commands= coverage combine From 5bbb8248651d7805900bd2a302de857c8e557dc9 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 14 Mar 2017 11:13:42 -0400 Subject: [PATCH 27/58] add tests for no_log filtering see: #5691 --- awx/lib/tests/test_display_callback.py | 68 +++++++++++++++++++++--- awx/lib/tower_display_callback/module.py | 7 ++- 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/awx/lib/tests/test_display_callback.py b/awx/lib/tests/test_display_callback.py index c9132a05ed..25d12774d3 100644 --- a/awx/lib/tests/test_display_callback.py +++ b/awx/lib/tests/test_display_callback.py @@ -1,3 +1,5 @@ +from collections import OrderedDict +import json import mock import os import sys @@ -36,7 +38,7 @@ with mock.patch.dict(os.environ, {'ANSIBLE_STDOUT_CALLBACK': CALLBACK, @pytest.fixture() def local_cache(): - class Cache(dict): + class Cache(OrderedDict): def set(self, key, value): self[key] = value return Cache() @@ -44,8 +46,7 @@ def local_cache(): @pytest.fixture() def executor(tmpdir_factory, request): - playbooks_marker = request.node.get_marker('playbooks') - playbooks = playbooks_marker.kwargs if playbooks_marker else {} + playbooks = request.node.callspec.params.get('playbook') playbook_files = [] for name, playbook in playbooks.items(): filename = str(tmpdir_factory.mktemp('data').join(name)) @@ -71,17 +72,68 @@ def executor(tmpdir_factory, request): 'playbook_on_play_start', 'playbook_on_task_start', 'runner_on_ok', 'playbook_on_stats'}) -@pytest.mark.playbooks(**{ - 'helloworld.yml': ''' +@pytest.mark.parametrize('playbook', [ +{'helloworld.yml': ''' - name: Hello World Sample connection: local hosts: all + gather_facts: no tasks: - name: Hello Message debug: - msg: "Hello World!"''' -}) -def test_callback_plugin_receives_events(executor, event, local_cache): + msg: "Hello World!" +'''} # noqa +]) +def test_callback_plugin_receives_events(executor, local_cache, event, + playbook): with mock.patch.object(event_context, 'cache', local_cache): executor.run() assert event in [task['event'] for task in local_cache.values()] + + +@pytest.mark.parametrize('playbook', [ +{'no_log_on_ok.yml': ''' +- name: args should not be logged when task-level no_log is set + connection: local + hosts: all + gather_facts: no + tasks: + - shell: echo "SENSITIVE" + no_log: true +'''}, # noqa +{'no_log_on_fail.yml': ''' +- name: failed args should not be logged when task-level no_log is set + connection: local + hosts: all + gather_facts: no + tasks: + - shell: echo "SENSITIVE" + no_log: true + failed_when: true + ignore_errors: true +'''}, # noqa +{'no_log_on_skip.yml': ''' +- name: skipped task args should be suppressed with no_log + connection: local + hosts: all + gather_facts: no + tasks: + - shell: echo "SENSITIVE" + no_log: true + when: false +'''}, # noqa +{'no_log_on_play.yml': ''' +- name: play-level no_log set + connection: local + hosts: all + gather_facts: no + no_log: true + tasks: + - name: args should not be logged when play-level no_log set + shell: echo "SENSITIVE" +'''}, # noqa +]) +def test_callback_plugin_no_log_filters(executor, local_cache, playbook): + with mock.patch.object(event_context, 'cache', local_cache): + executor.run() + assert 'SENSITIVE' not in json.dumps(local_cache.items()) diff --git a/awx/lib/tower_display_callback/module.py b/awx/lib/tower_display_callback/module.py index b69238026f..74b6fec9d6 100644 --- a/awx/lib/tower_display_callback/module.py +++ b/awx/lib/tower_display_callback/module.py @@ -326,10 +326,15 @@ class BaseCallbackModule(CallbackBase): def v2_runner_on_failed(self, result, ignore_errors=False): # FIXME: Add verbosity for exception/results output. + + res = result._result + if res.get('_ansible_no_log', False): + res = {'censored': "the output has been hidden due to the fact that 'no_log: true' was specified for this result"} + event_data = dict( host=result._host.get_name(), remote_addr=result._host.address, - res=result._result, + res=res, task=result._task, ignore_errors=ignore_errors, event_loop=result._task.loop if hasattr(result._task, 'loop') else None, From a5ea468bfea3e1dc772c960329278814782b9e1e Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 14 Mar 2017 12:54:14 -0400 Subject: [PATCH 28/58] add a test to ensure that job events properly strip out environment vars see: #5601 --- awx/lib/tests/test_display_callback.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/awx/lib/tests/test_display_callback.py b/awx/lib/tests/test_display_callback.py index 25d12774d3..13822eb7b8 100644 --- a/awx/lib/tests/test_display_callback.py +++ b/awx/lib/tests/test_display_callback.py @@ -137,3 +137,21 @@ def test_callback_plugin_no_log_filters(executor, local_cache, playbook): with mock.patch.object(event_context, 'cache', local_cache): executor.run() assert 'SENSITIVE' not in json.dumps(local_cache.items()) + + +@pytest.mark.parametrize('playbook', [ +{'strip_env_vars.yml': ''' +- name: sensitive environment variables should be stripped from events + connection: local + hosts: all + tasks: + - shell: echo "Hello, World!" +'''}, # noqa +]) +def test_callback_plugin_strips_task_environ_variables(executor, local_cache, + playbook): + with mock.patch.object(event_context, 'cache', local_cache): + executor.run() + for event in local_cache.values(): + if event['event_data'].get('task') == 'setup': + assert os.environ['VIRTUAL_ENV'] not in json.dumps(event) From d4addb83dfe862ea0749395f525c89065da6e266 Mon Sep 17 00:00:00 2001 From: Greg Considine Date: Tue, 14 Mar 2017 13:22:47 -0400 Subject: [PATCH 29/58] Add logic to unset watchers using scope on destroy --- .../credential/job-sub-cred-list.directive.js | 18 +++++++++++------- .../inventory/job-sub-inv-list.directive.js | 18 +++++++++++------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/awx/ui/client/src/job-submission/lists/credential/job-sub-cred-list.directive.js b/awx/ui/client/src/job-submission/lists/credential/job-sub-cred-list.directive.js index 7f93de9947..93964e6774 100644 --- a/awx/ui/client/src/job-submission/lists/credential/job-sub-cred-list.directive.js +++ b/awx/ui/client/src/job-submission/lists/credential/job-sub-cred-list.directive.js @@ -7,7 +7,7 @@ import jobSubCredListController from './job-sub-cred-list.controller'; export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$compile', 'CredentialList', - function(templateUrl, qs, GetBasePath, GenerateList, $compile, CredentialList) { + (templateUrl, qs, GetBasePath, GenerateList, $compile, CredentialList) => { return { scope: { selectedCredential: '=' @@ -15,7 +15,9 @@ export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$com templateUrl: templateUrl('job-submission/lists/credential/job-sub-cred-list'), controller: jobSubCredListController, restrict: 'E', - link: function(scope) { + link: scope => { + let toDestroy = []; + scope.credential_default_params = { order_by: 'name', page_size: 5, @@ -30,11 +32,11 @@ export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$com // Fire off the initial search qs.search(GetBasePath('credentials'), scope.credential_default_params) - .then(function(res) { + .then(res => { scope.credential_dataset = res.data; scope.credentials = scope.credential_dataset.results; - var credList = _.cloneDeep(CredentialList); + let credList = _.cloneDeep(CredentialList); let html = GenerateList.build({ list: credList, input_type: 'radio', @@ -45,10 +47,10 @@ export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$com $('#job-submission-credential-lookup').append($compile(html)(scope)); - scope.$watchCollection('selectedCredential', function () { + toDestroy.push(scope.$watchCollection('selectedCredential', () => { if(scope.selectedCredential) { // Loop across the inventories and see if one of them should be "checked" - scope.credentials.forEach(function(row, i) { + scope.credentials.forEach((row, i) => { if (row.id === scope.selectedCredential.id) { scope.credentials[i].checked = 1; } @@ -57,8 +59,10 @@ export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$com } }); } - }); + })); }); + + scope.$on('$destroy', () => toDestroy.forEach(watcher => watcher())); } }; }]; diff --git a/awx/ui/client/src/job-submission/lists/inventory/job-sub-inv-list.directive.js b/awx/ui/client/src/job-submission/lists/inventory/job-sub-inv-list.directive.js index d87347ced7..2025a6b213 100644 --- a/awx/ui/client/src/job-submission/lists/inventory/job-sub-inv-list.directive.js +++ b/awx/ui/client/src/job-submission/lists/inventory/job-sub-inv-list.directive.js @@ -7,7 +7,7 @@ import jobSubInvListController from './job-sub-inv-list.controller'; export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$compile', 'InventoryList', - function(templateUrl, qs, GetBasePath, GenerateList, $compile, InventoryList) { + (templateUrl, qs, GetBasePath, GenerateList, $compile, InventoryList) => { return { scope: { selectedInventory: '=' @@ -15,7 +15,9 @@ export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$com templateUrl: templateUrl('job-submission/lists/inventory/job-sub-inv-list'), controller: jobSubInvListController, restrict: 'E', - link: function(scope) { + link: scope => { + let toDestroy = []; + scope.inventory_default_params = { order_by: 'name', page_size: 5 @@ -28,11 +30,11 @@ export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$com // Fire off the initial search qs.search(GetBasePath('inventory'), scope.inventory_default_params) - .then(function(res) { + .then(res => { scope.inventory_dataset = res.data; scope.inventories = scope.inventory_dataset.results; - var invList = _.cloneDeep(InventoryList); + let invList = _.cloneDeep(InventoryList); let html = GenerateList.build({ list: invList, input_type: 'radio', @@ -43,10 +45,10 @@ export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$com $('#job-submission-inventory-lookup').append($compile(html)(scope)); - scope.$watchCollection('selectedInventory', function () { + toDestroy.push(scope.$watchCollection('selectedInventory', () => { if(scope.selectedInventory) { // Loop across the inventories and see if one of them should be "checked" - scope.inventories.forEach(function(row, i) { + scope.inventories.forEach((row, i) => { if (row.id === scope.selectedInventory.id) { scope.inventories[i].checked = 1; } @@ -55,8 +57,10 @@ export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$com } }); } - }); + })); }); + + scope.$on('$destory', () => toDestroy.forEach(watcher => watcher())); } }; }]; From bc4174adf8bd73e815ebc37aec61fdfd380e815a Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Tue, 14 Mar 2017 13:23:14 -0400 Subject: [PATCH 30/58] Ignore .retry files for project updates --- awx/main/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 6a448a1b7b..1ad050a993 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1160,6 +1160,7 @@ class RunProjectUpdate(BaseTask): ''' env = super(RunProjectUpdate, self).build_env(project_update, **kwargs) env = self.add_ansible_venv(env) + env['ANSIBLE_RETRY_FILES_ENABLED'] = str(False) env['ANSIBLE_ASK_PASS'] = str(False) env['ANSIBLE_ASK_SUDO_PASS'] = str(False) env['DISPLAY'] = '' # Prevent stupid password popup when running tests. From 081a545158e170868fbf8484fe20ac1603029311 Mon Sep 17 00:00:00 2001 From: jaredevantabor Date: Tue, 14 Mar 2017 10:49:46 -0700 Subject: [PATCH 31/58] making insights credential required on projects page and changing label to "Credential" instead of "Red Hat Insights" --- awx/ui/client/src/controllers/Projects.js | 10 ++++++++-- awx/ui/client/src/forms/Projects.js | 5 +++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/awx/ui/client/src/controllers/Projects.js b/awx/ui/client/src/controllers/Projects.js index 759f23be60..afae415a97 100644 --- a/awx/ui/client/src/controllers/Projects.js +++ b/awx/ui/client/src/controllers/Projects.js @@ -348,6 +348,7 @@ export function ProjectsAdd($scope, $rootScope, $compile, $location, $log, }); $scope.scmRequired = false; + $scope.credRequired = false; master.scm_type = $scope.scm_type; }); @@ -408,6 +409,7 @@ export function ProjectsAdd($scope, $rootScope, $compile, $location, $log, if ($scope.scm_type) { $scope.pathRequired = ($scope.scm_type.value === 'manual') ? true : false; $scope.scmRequired = ($scope.scm_type.value !== 'manual') ? true : false; + $scope.credRequired = ($scope.scm_type.value === 'insights') ? true : false; $scope.scmBranchLabel = ($scope.scm_type.value === 'svn') ? 'Revision #' : 'SCM Branch'; } @@ -442,7 +444,8 @@ export function ProjectsAdd($scope, $rootScope, $compile, $location, $log, case 'insights': $scope.pathRequired = false; $scope.scmRequired = false; - $scope.credentialLabel = "Red Hat Insights"; + $scope.credRequired = true; + $scope.credentialLabel = "Credential"; break; default: $scope.credentialLabel = "SCM Credential"; @@ -518,6 +521,7 @@ export function ProjectsEdit($scope, $rootScope, $compile, $location, $log, $scope.pathRequired = ($scope.scm_type.value === 'manual') ? true : false; $scope.scmRequired = ($scope.scm_type.value !== 'manual') ? true : false; + $scope.credRequired = ($scope.scm_type.value === 'insights') ? true : false; $scope.scmBranchLabel = ($scope.scm_type.value === 'svn') ? 'Revision #' : 'SCM Branch'; Wait('stop'); @@ -706,6 +710,7 @@ export function ProjectsEdit($scope, $rootScope, $compile, $location, $log, if ($scope.scm_type) { $scope.pathRequired = ($scope.scm_type.value === 'manual') ? true : false; $scope.scmRequired = ($scope.scm_type.value !== 'manual') ? true : false; + $scope.credRequired = ($scope.scm_type.value === 'insights') ? true : false; $scope.scmBranchLabel = ($scope.scm_type.value === 'svn') ? i18n._('Revision #') : i18n._('SCM Branch'); } @@ -738,7 +743,8 @@ export function ProjectsEdit($scope, $rootScope, $compile, $location, $log, case 'insights': $scope.pathRequired = false; $scope.scmRequired = false; - $scope.credentialLabel = "Red Hat Insights"; + $scope.credRequired = true; + $scope.credentialLabel = "Credential"; break; default: $scope.credentialLabel = "SCM Credential"; diff --git a/awx/ui/client/src/forms/Projects.js b/awx/ui/client/src/forms/Projects.js index 8a5abc4af9..3960888166 100644 --- a/awx/ui/client/src/forms/Projects.js +++ b/awx/ui/client/src/forms/Projects.js @@ -136,6 +136,11 @@ angular.module('ProjectFormDefinition', ['SchedulesListDefinition']) search: { kind: 'scm' }, + autopopulateLookup: false, + awRequiredWhen: { + reqExpression: "credRequired", + init: false + }, ngShow: "scm_type && scm_type.value !== 'manual'", sourceModel: 'credential', awLookupType: 'scm_credential', From 66c786b97110a7765779428c7b9bbb438a8d599c Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 14 Mar 2017 13:39:35 -0400 Subject: [PATCH 32/58] simplify memcache patching in our callback plugin tests --- awx/lib/tests/test_display_callback.py | 51 ++++++++++++++++---------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/awx/lib/tests/test_display_callback.py b/awx/lib/tests/test_display_callback.py index 13822eb7b8..2e3b58bdfc 100644 --- a/awx/lib/tests/test_display_callback.py +++ b/awx/lib/tests/test_display_callback.py @@ -28,7 +28,7 @@ with mock.patch.dict(os.environ, {'ANSIBLE_STDOUT_CALLBACK': CALLBACK, from ansible.vars import VariableManager # Add awx/lib to sys.path so we can use the plugin - path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) + path = os.path.abspath(os.path.join(PLUGINS, '..', '..')) if path not in sys.path: sys.path.insert(0, path) @@ -37,11 +37,15 @@ with mock.patch.dict(os.environ, {'ANSIBLE_STDOUT_CALLBACK': CALLBACK, @pytest.fixture() -def local_cache(): +def cache(request): class Cache(OrderedDict): def set(self, key, value): self[key] = value - return Cache() + local_cache = Cache() + patch = mock.patch.object(event_context, 'cache', local_cache) + patch.start() + request.addfinalizer(patch.stop) + return local_cache @pytest.fixture() @@ -84,11 +88,10 @@ def executor(tmpdir_factory, request): msg: "Hello World!" '''} # noqa ]) -def test_callback_plugin_receives_events(executor, local_cache, event, - playbook): - with mock.patch.object(event_context, 'cache', local_cache): - executor.run() - assert event in [task['event'] for task in local_cache.values()] +def test_callback_plugin_receives_events(executor, cache, event, playbook): + executor.run() + assert len(cache) + assert event in [task['event'] for task in cache.values()] @pytest.mark.parametrize('playbook', [ @@ -132,11 +135,23 @@ def test_callback_plugin_receives_events(executor, local_cache, event, - name: args should not be logged when play-level no_log set shell: echo "SENSITIVE" '''}, # noqa +{'async_no_log.yml': ''' +- name: async task args should suppressed with no_log + connection: local + hosts: all + gather_facts: no + no_log: true + tasks: + - async: 10 + poll: 1 + shell: echo "SENSITIVE" + no_log: true +'''}, # noqa ]) -def test_callback_plugin_no_log_filters(executor, local_cache, playbook): - with mock.patch.object(event_context, 'cache', local_cache): - executor.run() - assert 'SENSITIVE' not in json.dumps(local_cache.items()) +def test_callback_plugin_no_log_filters(executor, cache, playbook): + executor.run() + assert len(cache) + assert 'SENSITIVE' not in json.dumps(cache.items()) @pytest.mark.parametrize('playbook', [ @@ -148,10 +163,8 @@ def test_callback_plugin_no_log_filters(executor, local_cache, playbook): - shell: echo "Hello, World!" '''}, # noqa ]) -def test_callback_plugin_strips_task_environ_variables(executor, local_cache, - playbook): - with mock.patch.object(event_context, 'cache', local_cache): - executor.run() - for event in local_cache.values(): - if event['event_data'].get('task') == 'setup': - assert os.environ['VIRTUAL_ENV'] not in json.dumps(event) +def test_callback_plugin_strips_task_environ_variables(executor, cache, playbook): + executor.run() + assert len(cache) + for event in cache.values(): + assert os.environ['VIRTUAL_ENV'] not in json.dumps(event) From ba9ce56c357b6844c499302430d5bfd801950294 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 14 Mar 2017 13:47:44 -0400 Subject: [PATCH 33/58] add callback plugin tests to CI --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9ac824d2e5..541a04a571 100644 --- a/Makefile +++ b/Makefile @@ -487,7 +487,7 @@ check: flake8 pep8 # pyflakes pylint TEST_DIRS ?= awx/main/tests awx/conf/tests awx/sso/tests # Run all API unit tests. -test: +test: test_ansible @if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/tower/bin/activate; \ fi; \ From 377b55dd2fa198b7bde2855aae1f035ddcaab7a4 Mon Sep 17 00:00:00 2001 From: Greg Considine Date: Tue, 14 Mar 2017 14:51:20 -0400 Subject: [PATCH 34/58] Fix typo in reference to --- .../lists/inventory/job-sub-inv-list.directive.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/ui/client/src/job-submission/lists/inventory/job-sub-inv-list.directive.js b/awx/ui/client/src/job-submission/lists/inventory/job-sub-inv-list.directive.js index 2025a6b213..305643e0e3 100644 --- a/awx/ui/client/src/job-submission/lists/inventory/job-sub-inv-list.directive.js +++ b/awx/ui/client/src/job-submission/lists/inventory/job-sub-inv-list.directive.js @@ -60,7 +60,7 @@ export default [ 'templateUrl', 'QuerySet', 'GetBasePath', 'generateList', '$com })); }); - scope.$on('$destory', () => toDestroy.forEach(watcher => watcher())); + scope.$on('$destroy', () => toDestroy.forEach(watcher => watcher())); } }; }]; From 5041fe5bf750fe1d38875c403cfe2cab1f420e4e Mon Sep 17 00:00:00 2001 From: jaredevantabor Date: Tue, 14 Mar 2017 13:45:39 -0700 Subject: [PATCH 35/58] changing readonlye UX for extra vars on workflow results and job results by changing background and cursor type --- .../src/job-results/job-results.block.less | 17 +---------------- .../workflow-results.block.less | 17 +---------------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/awx/ui/client/src/job-results/job-results.block.less b/awx/ui/client/src/job-results/job-results.block.less index f9d0718b10..e13afcfee3 100644 --- a/awx/ui/client/src/job-results/job-results.block.less +++ b/awx/ui/client/src/job-results/job-results.block.less @@ -213,23 +213,8 @@ job-results-standard-out { color: @default-icon; } -.JobResults .CodeMirror.cm-s-default, -.JobResults .CodeMirror-line { - background-color: #f6f6f6; -} - -.JobResults .CodeMirror-gutter.CodeMirror-lint-markers, -.JobResults .CodeMirror-gutter.CodeMirror-linenumbers { - background-color: #ebebeb; - color: @b7grey; -} - .JobResults .CodeMirror-lines { - cursor: default; -} - -.JobResults .CodeMirror-cursors { - display: none; + cursor: not-allowed; } .JobResults-downloadTooLarge { diff --git a/awx/ui/client/src/workflow-results/workflow-results.block.less b/awx/ui/client/src/workflow-results/workflow-results.block.less index 9134948c38..dacc21fad7 100644 --- a/awx/ui/client/src/workflow-results/workflow-results.block.less +++ b/awx/ui/client/src/workflow-results/workflow-results.block.less @@ -146,21 +146,6 @@ color: @default-icon; } -.WorkflowResults .CodeMirror.cm-s-default, -.WorkflowResults .CodeMirror-line { - background-color: #f6f6f6; -} - -.WorkflowResults .CodeMirror-gutter.CodeMirror-lint-markers, -.WorkflowResults .CodeMirror-gutter.CodeMirror-linenumbers { - background-color: #ebebeb; - color: @b7grey; -} - .WorkflowResults .CodeMirror-lines { - cursor: default; -} - -.WorkflowResults .CodeMirror-cursors { - display: none; + cursor: not-allowed; } From 1c2535f0505a725e8f11312144ce4933e69c43f6 Mon Sep 17 00:00:00 2001 From: Michael Abashian Date: Tue, 14 Mar 2017 16:11:11 -0400 Subject: [PATCH 36/58] Users can interact with organization field when adding --- awx/ui/client/src/controllers/Credentials.js | 1 + awx/ui/client/src/controllers/Projects.js | 1 + awx/ui/client/src/controllers/Teams.js | 1 + awx/ui/client/src/forms/Workflows.js | 2 +- awx/ui/client/src/inventories/add/inventory-add.controller.js | 1 + .../templates/workflows/add-workflow/workflow-add.controller.js | 1 + 6 files changed, 6 insertions(+), 1 deletion(-) diff --git a/awx/ui/client/src/controllers/Credentials.js b/awx/ui/client/src/controllers/Credentials.js index 5f7f6e9a25..83f19d2bc1 100644 --- a/awx/ui/client/src/controllers/Credentials.js +++ b/awx/ui/client/src/controllers/Credentials.js @@ -126,6 +126,7 @@ export function CredentialsAdd($scope, $rootScope, $compile, $location, $log, init(); function init() { + $scope.canEditOrg = true; // Load the list of options for Kind GetChoices({ scope: $scope, diff --git a/awx/ui/client/src/controllers/Projects.js b/awx/ui/client/src/controllers/Projects.js index 759f23be60..7c1b98d17d 100644 --- a/awx/ui/client/src/controllers/Projects.js +++ b/awx/ui/client/src/controllers/Projects.js @@ -314,6 +314,7 @@ export function ProjectsAdd($scope, $rootScope, $compile, $location, $log, init(); function init() { + $scope.canEditOrg = true; Rest.setUrl(GetBasePath('projects')); Rest.options() .success(function(data) { diff --git a/awx/ui/client/src/controllers/Teams.js b/awx/ui/client/src/controllers/Teams.js index 1db3843bef..f6f3dc44db 100644 --- a/awx/ui/client/src/controllers/Teams.js +++ b/awx/ui/client/src/controllers/Teams.js @@ -110,6 +110,7 @@ export function TeamsAdd($scope, $rootScope, $stateParams, TeamForm, GenerateFor init(); function init() { + $scope.canEditOrg = true; // apply form definition's default field values GenerateForm.applyDefaults(form, $scope); diff --git a/awx/ui/client/src/forms/Workflows.js b/awx/ui/client/src/forms/Workflows.js index f0e7c24dc6..821a3ab072 100644 --- a/awx/ui/client/src/forms/Workflows.js +++ b/awx/ui/client/src/forms/Workflows.js @@ -55,7 +55,7 @@ export default dataPlacement: 'right', column: 1, ngDisabled: '!(workflow_job_template_obj.summary_fields.user_capabilities.edit || canAddWorkflowJobTemplate) || !canEditOrg', - awLookupWhen: '(workflow_job_template_obj.summary_fields.user_capabilities.edit || canAdd) && canEditOrg' + awLookupWhen: '(workflow_job_template_obj.summary_fields.user_capabilities.edit || canAddWorkflowJobTemplate) && canEditOrg' }, labels: { label: i18n._('Labels'), diff --git a/awx/ui/client/src/inventories/add/inventory-add.controller.js b/awx/ui/client/src/inventories/add/inventory-add.controller.js index a20fb44891..a6effbbb81 100644 --- a/awx/ui/client/src/inventories/add/inventory-add.controller.js +++ b/awx/ui/client/src/inventories/add/inventory-add.controller.js @@ -39,6 +39,7 @@ function InventoriesAdd($scope, $rootScope, $compile, $location, $log, init(); function init() { + $scope.canEditOrg = true; form.formLabelSize = null; form.formFieldSize = null; diff --git a/awx/ui/client/src/templates/workflows/add-workflow/workflow-add.controller.js b/awx/ui/client/src/templates/workflows/add-workflow/workflow-add.controller.js index 11c7e35a5c..036ff6b21f 100644 --- a/awx/ui/client/src/templates/workflows/add-workflow/workflow-add.controller.js +++ b/awx/ui/client/src/templates/workflows/add-workflow/workflow-add.controller.js @@ -29,6 +29,7 @@ generator = GenerateForm; function init() { + $scope.canEditOrg = true; $scope.parseType = 'yaml'; $scope.can_edit = true; // apply form definition's default field values From 4742cede0ddd240b9577a2e82d69f5387ceca2f4 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 16:45:20 -0400 Subject: [PATCH 37/58] Add pip and setuptools to requirements files These packages were getting silently updated, making vendoring these dependencies difficult. This ensures that the expected versions of these packages are installed in the virtualenvs. --- requirements/requirements.in | 2 ++ requirements/requirements.txt | 3 ++- requirements/requirements_ansible.in | 2 ++ requirements/requirements_ansible.txt | 3 ++- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index 5d3b4c0e45..19d222e315 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -49,3 +49,5 @@ slackclient==1.0.2 twilio==5.6.0 uWSGI==2.0.14 xmltodict==0.10.2 +pip==8.1.2 +setuptools==23.0.0 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 0012ad8330..1ff0a862cb 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -197,4 +197,5 @@ xmltodict==0.10.2 zope.interface==4.3.3 # via twisted # The following packages are considered to be unsafe in a requirements file: -# setuptools # via cryptography, django-polymorphic, python-ldap, zope.interface +pip==8.1.2 +setuptools==23.0.0 diff --git a/requirements/requirements_ansible.in b/requirements/requirements_ansible.in index 671eaf1496..c5479b85d9 100644 --- a/requirements/requirements_ansible.in +++ b/requirements/requirements_ansible.in @@ -11,3 +11,5 @@ pyvmomi==6.5 pywinrm[kerberos]==0.2.2 secretstorage==2.3.1 shade==1.13.1 +setuptools==23.0.0 +pip==8.1.2 diff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt index 687378c9aa..763dc872cb 100644 --- a/requirements/requirements_ansible.txt +++ b/requirements/requirements_ansible.txt @@ -128,4 +128,5 @@ wrapt==1.10.8 # via debtcollector, positional xmltodict==0.10.2 # via pywinrm # The following packages are considered to be unsafe in a requirements file: -# setuptools # via cryptography +pip==8.1.2 +setuptools==23.0.0 From 092c0b739e1cc51f3cb58289327855ca77c9cde4 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 17:40:32 -0400 Subject: [PATCH 38/58] Simplify requirements targets AFAIK, there is never a situation where VENV is going to be unset. --- Makefile | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 541a04a571..f56e2eade6 100644 --- a/Makefile +++ b/Makefile @@ -282,14 +282,8 @@ virtualenv_tower: fi requirements_ansible: virtualenv_ansible - if [ "$(VENV_BASE)" ]; then \ - . $(VENV_BASE)/ansible/bin/activate; \ - $(VENV_BASE)/ansible/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt ;\ - $(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt; \ - else \ - pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt ; \ - pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt; \ - fi + $(VENV_BASE)/ansible/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt + $(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt requirements_ansible_dev: if [ "$(VENV_BASE)" ]; then \ @@ -298,21 +292,12 @@ requirements_ansible_dev: # Install third-party requirements needed for Tower's environment. requirements_tower: virtualenv_tower - if [ "$(VENV_BASE)" ]; then \ - . $(VENV_BASE)/tower/bin/activate; \ - $(VENV_BASE)/tower/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt ;\ - $(VENV_BASE)/tower/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt; \ - else \ - pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt ; \ - pip uninstall --yes -r requirements/requirements_tower_uninstall.txt; \ - fi + $(VENV_BASE)/tower/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt + $(VENV_BASE)/tower/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt requirements_tower_dev: - if [ "$(VENV_BASE)" ]; then \ - . $(VENV_BASE)/tower/bin/activate; \ - $(VENV_BASE)/tower/bin/pip install -r requirements/requirements_dev.txt; \ - $(VENV_BASE)/tower/bin/pip uninstall --yes -r requirements/requirements_dev_uninstall.txt; \ - fi + $(VENV_BASE)/tower/bin/pip install -r requirements/requirements_dev.txt + $(VENV_BASE)/tower/bin/pip uninstall --yes -r requirements/requirements_dev_uninstall.txt requirements: requirements_ansible requirements_tower From fbb6650ad0ce18eaabc8e823750416f172e91645 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 17:56:46 -0400 Subject: [PATCH 39/58] Add PIP_OPTIONS This will allow us to conditionally pass in things like --no-index and --find-links, which will be necessary for building in Brew. --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index f56e2eade6..5cfbcde4b5 100644 --- a/Makefile +++ b/Makefile @@ -264,8 +264,8 @@ virtualenv_ansible: fi; \ if [ ! -d "$(VENV_BASE)/ansible" ]; then \ virtualenv --system-site-packages --setuptools $(VENV_BASE)/ansible && \ - $(VENV_BASE)/ansible/bin/pip install --force-reinstall setuptools==23.0.0 && \ - $(VENV_BASE)/ansible/bin/pip install --force-reinstall pip==8.1.2; \ + $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --force-reinstall setuptools==23.0.0 && \ + $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --force-reinstall pip==8.1.2; \ fi; \ fi @@ -276,13 +276,13 @@ virtualenv_tower: fi; \ if [ ! -d "$(VENV_BASE)/tower" ]; then \ virtualenv --system-site-packages --setuptools $(VENV_BASE)/tower && \ - $(VENV_BASE)/tower/bin/pip install --force-reinstall setuptools==23.0.0 && \ - $(VENV_BASE)/tower/bin/pip install --force-reinstall pip==8.1.2; \ + $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --force-reinstall setuptools==23.0.0 && \ + $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --force-reinstall pip==8.1.2; \ fi; \ fi requirements_ansible: virtualenv_ansible - $(VENV_BASE)/ansible/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt + $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt $(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt requirements_ansible_dev: @@ -292,7 +292,7 @@ requirements_ansible_dev: # Install third-party requirements needed for Tower's environment. requirements_tower: virtualenv_tower - $(VENV_BASE)/tower/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt + $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt $(VENV_BASE)/tower/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt requirements_tower_dev: From c94f294d068ca37384d168d0b088e2d24c165549 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 17:58:41 -0400 Subject: [PATCH 40/58] Use --ignore-installed when creating virtualenvs --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 5cfbcde4b5..8801f80bc9 100644 --- a/Makefile +++ b/Makefile @@ -264,8 +264,8 @@ virtualenv_ansible: fi; \ if [ ! -d "$(VENV_BASE)/ansible" ]; then \ virtualenv --system-site-packages --setuptools $(VENV_BASE)/ansible && \ - $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --force-reinstall setuptools==23.0.0 && \ - $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --force-reinstall pip==8.1.2; \ + $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed setuptools==23.0.0 && \ + $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed pip==8.1.2; \ fi; \ fi @@ -276,8 +276,8 @@ virtualenv_tower: fi; \ if [ ! -d "$(VENV_BASE)/tower" ]; then \ virtualenv --system-site-packages --setuptools $(VENV_BASE)/tower && \ - $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --force-reinstall setuptools==23.0.0 && \ - $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --force-reinstall pip==8.1.2; \ + $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --ignore-installed setuptools==23.0.0 && \ + $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --ignore-installed pip==8.1.2; \ fi; \ fi From f79cd028f06d831877a573ff172c828ef8641dc5 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 19:40:46 -0400 Subject: [PATCH 41/58] Add separate requirements file for setup_requires stuff pip doesnt know how to resolve setup_requires dependencies. we need this for offline pip installs when building in brew --- requirements/requirements_setup_requires.txt | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 requirements/requirements_setup_requires.txt diff --git a/requirements/requirements_setup_requires.txt b/requirements/requirements_setup_requires.txt new file mode 100644 index 0000000000..f9857bb2a6 --- /dev/null +++ b/requirements/requirements_setup_requires.txt @@ -0,0 +1,5 @@ +pbr>=1.8 +setuptools_scm>=1.15.0 +vcversioner>=2.16.0.0 +pytest-runner +isort From 7e1eeb994b84679be23579e93700c23f84b4bda8 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 18:07:36 -0400 Subject: [PATCH 42/58] Add make targets for vendoring python requirements --- Makefile | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Makefile b/Makefile index 8801f80bc9..579e451b93 100644 --- a/Makefile +++ b/Makefile @@ -708,6 +708,30 @@ rpm-build/$(SDIST_TAR_FILE): rpm-build dist/$(SDIST_TAR_FILE) rpmtar: sdist rpm-build/$(SDIST_TAR_FILE) +brewrpmtar: rpm-build/python-deps.tar.gz rpmtar + +rpm-build/python-deps.tar.gz: requirements/vendor rpm-build + tar czf rpm-build/python-deps.tar.gz requirements/vendor + +requirements/vendor: + pip download \ + --no-binary=:all: \ + --requirement=requirements/requirements_ansible.txt \ + --dest=$@ \ + --exists-action=i + + pip download \ + --no-binary=:all: \ + --requirement=requirements/requirements.txt \ + --dest=$@ \ + --exists-action=i + + pip download \ + --no-binary=:all: \ + --requirement=requirements/requirements_setup_requires.txt \ + --dest=$@ \ + --exists-action=i + rpm-build/$(RPM_NVR).src.rpm: /etc/mock/$(MOCK_CFG).cfg $(MOCK_BIN) -r $(MOCK_CFG) --resultdir rpm-build --buildsrpm --spec rpm-build/$(NAME).spec --sources rpm-build \ --define "tower_version $(VERSION)" --define "tower_release $(RELEASE)" $(SCL_DEFINES) From d76d55adafe6313fc264167065ce9c5e3f4072c0 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 18:15:32 -0400 Subject: [PATCH 43/58] Add brew-srpm target --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 579e451b93..0d933cff61 100644 --- a/Makefile +++ b/Makefile @@ -742,6 +742,8 @@ mock-srpm: rpmtar rpm-build/$(RPM_NVR).src.rpm @echo rpm-build/$(RPM_NVR).src.rpm @echo "#############################################" +brew-srpm: brewrpmtar mock-srpm + rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm: rpm-build/$(RPM_NVR).src.rpm $(MOCK_BIN) -r $(MOCK_CFG) --resultdir rpm-build --rebuild rpm-build/$(RPM_NVR).src.rpm \ --define "tower_version $(VERSION)" --define "tower_release $(RELEASE)" $(SCL_DEFINES) From d69242254e320ed1c8d9adddcc798f77b0504127 Mon Sep 17 00:00:00 2001 From: Michael Abashian Date: Wed, 15 Mar 2017 12:03:23 -0400 Subject: [PATCH 44/58] Fix tooltip/link overlay issue on workflow nodes --- .../workflow-chart/workflow-chart.directive.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/awx/ui/client/src/templates/workflows/workflow-chart/workflow-chart.directive.js b/awx/ui/client/src/templates/workflows/workflow-chart/workflow-chart.directive.js index fe62994d7b..aea7cc0d62 100644 --- a/awx/ui/client/src/templates/workflows/workflow-chart/workflow-chart.directive.js +++ b/awx/ui/client/src/templates/workflows/workflow-chart/workflow-chart.directive.js @@ -347,6 +347,20 @@ export default [ '$state','moment', '$timeout', '$window', if(!d.isStartNode) { let resourceName = (d.unifiedJobTemplate && d.unifiedJobTemplate.name) ? d.unifiedJobTemplate.name : ""; if(resourceName && resourceName.length > maxNodeTextLength) { + // When the graph is initially rendered all the links come after the nodes (when you look at the dom). + // SVG components are painted in order of appearance. There is no concept of z-index, only the order. + // As such, we need to move the nodes after the links so that when the tooltip renders it shows up on top + // of the links and not underneath them. I tried rendering the links before the nodes but that lead to + // some weird link animation that I didn't care to try to fix. + svgGroup.selectAll("g.node").each(function() { + this.parentNode.appendChild(this); + }); + // After the nodes have been properly placed after the links, we need to make sure that the node that + // the user is hovering over is at the very end of the list. This way the tooltip will appear on top + // of all other nodes. + svgGroup.selectAll("g.node").sort(function (a) { + return (a.id !== d.id) ? -1 : 1; + }); // Render the tooltip quickly in the dom and then remove. This lets us know how big the tooltip is so that we can place // it properly on the workflow let tooltipDimensionChecker = $(""); From b3733d233e28caf807e83602d0e350ffe4361975 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 15 Mar 2017 10:46:08 -0400 Subject: [PATCH 45/58] always use async http logging - even in celery workers additionally, add a timeout to requests to avoid thread starvation see: #5718 see: #5729 --- awx/main/tasks.py | 2 +- awx/main/tests/unit/utils/test_handlers.py | 38 ++++++++-------------- awx/main/utils/handlers.py | 32 +++++------------- awx/settings/defaults.py | 1 + 4 files changed, 24 insertions(+), 49 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 1ad050a993..d60d936531 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -90,7 +90,7 @@ def celery_startup(conf=None, **kwargs): @worker_process_init.connect def task_set_logger_pre_run(*args, **kwargs): cache.close() - configure_external_logger(settings, async_flag=False, is_startup=False) + configure_external_logger(settings, is_startup=False) def _clear_cache_keys(set_of_keys): diff --git a/awx/main/tests/unit/utils/test_handlers.py b/awx/main/tests/unit/utils/test_handlers.py index 3de3b2e7b7..ba9ffc3e33 100644 --- a/awx/main/tests/unit/utils/test_handlers.py +++ b/awx/main/tests/unit/utils/test_handlers.py @@ -2,6 +2,7 @@ import base64 import json import logging +from django.conf import settings from django.conf import LazySettings import pytest import requests @@ -40,17 +41,16 @@ def ok200_adapter(): return OK200Adapter() -def test_https_logging_handler_requests_sync_implementation(): - handler = HTTPSHandler(async=False) - assert not isinstance(handler.session, FuturesSession) - assert isinstance(handler.session, requests.Session) - - def test_https_logging_handler_requests_async_implementation(): - handler = HTTPSHandler(async=True) + handler = HTTPSHandler() assert isinstance(handler.session, FuturesSession) +def test_https_logging_handler_has_default_http_timeout(): + handler = HTTPSHandler.from_django_settings(settings) + assert handler.http_timeout == 5 + + @pytest.mark.parametrize('param', PARAM_NAMES.keys()) def test_https_logging_handler_defaults(param): handler = HTTPSHandler() @@ -114,18 +114,12 @@ def test_https_logging_handler_skip_log(params, logger_name, expected): assert handler.skip_log(logger_name) is expected -@pytest.mark.parametrize('message_type, async', [ - ('logstash', False), - ('logstash', True), - ('splunk', False), - ('splunk', True), -]) +@pytest.mark.parametrize('message_type', ['logstash', 'splunk']) def test_https_logging_handler_emit(ok200_adapter, dummy_log_record, - message_type, async): + message_type): handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True, message_type=message_type, - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'], - async=async) + enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) handler.setFormatter(LogstashFormatter()) handler.session.mount('http://', ok200_adapter) async_futures = handler.emit(dummy_log_record) @@ -151,14 +145,12 @@ def test_https_logging_handler_emit(ok200_adapter, dummy_log_record, assert body['message'] == 'User joe logged in' -@pytest.mark.parametrize('async', (True, False)) def test_https_logging_handler_emit_logstash_with_creds(ok200_adapter, - dummy_log_record, async): + dummy_log_record): handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True, username='user', password='pass', message_type='logstash', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'], - async=async) + enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) handler.setFormatter(LogstashFormatter()) handler.session.mount('http://', ok200_adapter) async_futures = handler.emit(dummy_log_record) @@ -169,13 +161,11 @@ def test_https_logging_handler_emit_logstash_with_creds(ok200_adapter, assert request.headers['Authorization'] == 'Basic %s' % base64.b64encode("user:pass") -@pytest.mark.parametrize('async', (True, False)) def test_https_logging_handler_emit_splunk_with_creds(ok200_adapter, - dummy_log_record, async): + dummy_log_record): handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True, password='pass', message_type='splunk', - enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'], - async=async) + enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) handler.setFormatter(LogstashFormatter()) handler.session.mount('http://', ok200_adapter) async_futures = handler.emit(dummy_log_record) diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py index fe2fb87228..c65cfb7cdd 100644 --- a/awx/main/utils/handlers.py +++ b/awx/main/utils/handlers.py @@ -33,6 +33,7 @@ PARAM_NAMES = { 'enabled_loggers': 'LOG_AGGREGATOR_LOGGERS', 'indv_facts': 'LOG_AGGREGATOR_INDIVIDUAL_FACTS', 'enabled_flag': 'LOG_AGGREGATOR_ENABLED', + 'http_timeout': 'LOG_AGGREGATOR_HTTP_TIMEOUT', } @@ -51,13 +52,9 @@ class BaseHTTPSHandler(logging.Handler): def __init__(self, fqdn=False, **kwargs): super(BaseHTTPSHandler, self).__init__() self.fqdn = fqdn - self.async = kwargs.get('async', True) for fd in PARAM_NAMES: setattr(self, fd, kwargs.get(fd, None)) - if self.async: - self.session = FuturesSession() - else: - self.session = requests.Session() + self.session = FuturesSession() self.add_auth_information() @classmethod @@ -105,10 +102,8 @@ class BaseHTTPSHandler(logging.Handler): payload_str = json.dumps(payload_input) else: payload_str = payload_input - if self.async: - return dict(data=payload_str, background_callback=unused_callback) - else: - return dict(data=payload_str) + return dict(data=payload_str, background_callback=unused_callback, + timeout=self.http_timeout) def skip_log(self, logger_name): if self.host == '' or (not self.enabled_flag): @@ -123,10 +118,6 @@ class BaseHTTPSHandler(logging.Handler): Emit a log record. Returns a list of zero or more ``concurrent.futures.Future`` objects. - When ``self.async`` is True, the list will contain one - Future object for each HTTP request made. When ``self.async`` is - False, the list will be empty. - See: https://docs.python.org/3/library/concurrent.futures.html#future-objects http://pythonhosted.org/futures/ @@ -147,17 +138,10 @@ class BaseHTTPSHandler(logging.Handler): for key in facts_dict: fact_payload = copy(payload_data) fact_payload.update(facts_dict[key]) - if self.async: - async_futures.append(self._send(fact_payload)) - else: - self._send(fact_payload) + async_futures.append(self._send(fact_payload)) return async_futures - if self.async: - return [self._send(payload)] - - self._send(payload) - return [] + return [self._send(payload)] except (KeyboardInterrupt, SystemExit): raise except: @@ -179,7 +163,7 @@ def add_or_remove_logger(address, instance): specific_logger.handlers.append(instance) -def configure_external_logger(settings_module, async_flag=True, is_startup=True): +def configure_external_logger(settings_module, is_startup=True): is_enabled = settings_module.LOG_AGGREGATOR_ENABLED if is_startup and (not is_enabled): @@ -188,7 +172,7 @@ def configure_external_logger(settings_module, async_flag=True, is_startup=True) instance = None if is_enabled: - instance = BaseHTTPSHandler.from_django_settings(settings_module, async=async_flag) + instance = BaseHTTPSHandler.from_django_settings(settings_module) instance.setFormatter(LogstashFormatter(settings_module=settings_module)) awx_logger_instance = instance if is_enabled and 'awx' not in settings_module.LOG_AGGREGATOR_LOGGERS: diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index e673e347a7..e3af1781db 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -867,6 +867,7 @@ INSIGHTS_URL_BASE = "https://access.redhat.com" TOWER_SETTINGS_MANIFEST = {} LOG_AGGREGATOR_ENABLED = False +LOG_AGGREGATOR_HTTP_TIMEOUT = 5 # The number of retry attempts for websocket session establishment # If you're encountering issues establishing websockets in clustered Tower, From dd8b9ec96cebe887e970ce78cb698850d14816e2 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 15 Mar 2017 14:01:10 -0400 Subject: [PATCH 46/58] Merge pull request #5738 from chrismeyersfsu/improvement-let_migrate_error_show_through if tower-manage migrate fails, let the error show From 98ad2684d36fc08ccb98db99866f015fc38bb3b6 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 15 Mar 2017 17:23:05 -0400 Subject: [PATCH 47/58] when async log shipping fails, log an explanation --- awx/main/tests/unit/utils/test_handlers.py | 39 ++++++++++++++++++++++ awx/main/utils/handlers.py | 34 ++++++++++++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/awx/main/tests/unit/utils/test_handlers.py b/awx/main/tests/unit/utils/test_handlers.py index ba9ffc3e33..d7b7f5aed9 100644 --- a/awx/main/tests/unit/utils/test_handlers.py +++ b/awx/main/tests/unit/utils/test_handlers.py @@ -1,4 +1,5 @@ import base64 +import cStringIO import json import logging @@ -41,6 +42,17 @@ def ok200_adapter(): return OK200Adapter() +@pytest.fixture() +def connection_error_adapter(): + class ConnectionErrorAdapter(requests.adapters.HTTPAdapter): + + def send(self, request, **kwargs): + err = requests.packages.urllib3.exceptions.SSLError() + raise requests.exceptions.ConnectionError(err, request=request) + + return ConnectionErrorAdapter() + + def test_https_logging_handler_requests_async_implementation(): handler = HTTPSHandler() assert isinstance(handler.session, FuturesSession) @@ -114,6 +126,33 @@ def test_https_logging_handler_skip_log(params, logger_name, expected): assert handler.skip_log(logger_name) is expected +def test_https_logging_handler_connection_error(connection_error_adapter, + dummy_log_record): + handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True, + message_type='logstash', + enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking']) + handler.setFormatter(LogstashFormatter()) + handler.session.mount('http://', connection_error_adapter) + + buff = cStringIO.StringIO() + logging.getLogger('awx.main.utils.handlers').addHandler( + logging.StreamHandler(buff) + ) + + async_futures = handler.emit(dummy_log_record) + with pytest.raises(requests.exceptions.ConnectionError): + [future.result() for future in async_futures] + assert 'failed to emit log to external aggregator\nTraceback' in buff.getvalue() + + # we should only log failures *periodically*, so causing *another* + # immediate failure shouldn't report a second ConnectionError + buff.truncate(0) + async_futures = handler.emit(dummy_log_record) + with pytest.raises(requests.exceptions.ConnectionError): + [future.result() for future in async_futures] + assert buff.getvalue() == '' + + @pytest.mark.parametrize('message_type', ['logstash', 'splunk']) def test_https_logging_handler_emit(ok200_adapter, dummy_log_record, message_type): diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py index c65cfb7cdd..b1b96b492f 100644 --- a/awx/main/utils/handlers.py +++ b/awx/main/utils/handlers.py @@ -5,6 +5,8 @@ import logging import json import requests +import time +from concurrent.futures import ThreadPoolExecutor from copy import copy # loggly @@ -18,6 +20,8 @@ from awx.main.utils.formatters import LogstashFormatter __all__ = ['HTTPSNullHandler', 'BaseHTTPSHandler', 'configure_external_logger'] +logger = logging.getLogger('awx.main.utils.handlers') + # AWX external logging handler, generally designed to be used # with the accompanying LogstashHandler, derives from python-logstash library # Non-blocking request accomplished by FuturesSession, similar @@ -48,13 +52,41 @@ class HTTPSNullHandler(logging.NullHandler): return super(HTTPSNullHandler, self).__init__() +class VerboseThreadPoolExecutor(ThreadPoolExecutor): + + last_log_emit = 0 + + def submit(self, func, *args, **kwargs): + def _wrapped(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception: + # If an exception occurs in a concurrent thread worker (like + # a ConnectionError or a read timeout), periodically log + # that failure. + # + # This approach isn't really thread-safe, so we could + # potentially log once per thread every 10 seconds, but it + # beats logging *every* failed HTTP request in a scenario where + # you've typo'd your log aggregator hostname. + now = time.time() + if now - self.last_log_emit > 10: + logger.exception('failed to emit log to external aggregator') + self.last_log_emit = now + raise + return super(VerboseThreadPoolExecutor, self).submit(_wrapped, *args, + **kwargs) + + class BaseHTTPSHandler(logging.Handler): def __init__(self, fqdn=False, **kwargs): super(BaseHTTPSHandler, self).__init__() self.fqdn = fqdn for fd in PARAM_NAMES: setattr(self, fd, kwargs.get(fd, None)) - self.session = FuturesSession() + self.session = FuturesSession(executor=VerboseThreadPoolExecutor( + max_workers=2 # this is the default used by requests_futures + )) self.add_auth_information() @classmethod From 95e41e078c5f2528ab93ff4faebaeeec52f45814 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Wed, 15 Mar 2017 17:42:59 -0400 Subject: [PATCH 48/58] Handle can_start_without_user_input cornercase. --- awx/main/models/jobs.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 26969ffc32..b960b0fbd4 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -310,8 +310,10 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour elif self.variables_needed_to_start: variables_needed = True prompting_needed = False - for value in self._ask_for_vars_dict().values(): - if value: + for key, value in self._ask_for_vars_dict().iteritems(): + if value and not (key == 'extra_vars' + and callback_extra_vars is not None + and not variables_needed): prompting_needed = True return (not prompting_needed and not self.passwords_needed_to_start and @@ -1139,7 +1141,7 @@ class JobEvent(CreatedModifiedModel): # Save artifact data to parent job (if provided). if artifact_dict: if event_data and isinstance(event_data, dict): - # Note: Core has not added support for marking artifacts as + # Note: Core has not added support for marking artifacts as # sensitive yet. Going forward, core will not use # _ansible_no_log to denote sensitive set_stats calls. # Instead, they plan to add a flag outside of the traditional From 90bcc3d6ab1405f2ba4a691bb4667bbacba76ee3 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Thu, 16 Mar 2017 11:08:19 -0400 Subject: [PATCH 49/58] Unit test added. --- awx/main/models/jobs.py | 5 ++--- .../tests/unit/models/test_job_template_unit.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index b960b0fbd4..992217ec33 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -311,9 +311,8 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour variables_needed = True prompting_needed = False for key, value in self._ask_for_vars_dict().iteritems(): - if value and not (key == 'extra_vars' - and callback_extra_vars is not None - and not variables_needed): + if value and not (key == 'extra_vars' and + callback_extra_vars is not None): prompting_needed = True return (not prompting_needed and not self.passwords_needed_to_start and diff --git a/awx/main/tests/unit/models/test_job_template_unit.py b/awx/main/tests/unit/models/test_job_template_unit.py index 194ce68cef..a6086b7b9d 100644 --- a/awx/main/tests/unit/models/test_job_template_unit.py +++ b/awx/main/tests/unit/models/test_job_template_unit.py @@ -115,3 +115,16 @@ def test_job_template_survey_mixin_length(job_template_factory): {'type':'password', 'variable':'my_other_variable'}]} kwargs = obj._update_unified_job_kwargs(extra_vars={'my_variable':'$encrypted$'}) assert kwargs['extra_vars'] == '{"my_variable": "my_default"}' + + +def test_job_template_can_start_with_callback_extra_vars_provided(job_template_factory): + objects = job_template_factory( + 'callback_extra_vars_test', + organization='org1', + inventory='inventory1', + credential='cred1', + persisted=False, + ) + obj = objects.job_template + obj.ask_variables_on_launch = True + assert obj.can_start_without_user_input(callback_extra_vars='{"foo": "bar"}') is True From 75ba0208ada55d3201024de620cc8070444154c1 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 16 Mar 2017 12:16:09 -0400 Subject: [PATCH 50/58] output logs to /logstash.conf in our default compose file --- tools/docker-compose/Dockerfile-logstash | 2 ++ tools/docker-compose/logstash.conf | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/docker-compose/Dockerfile-logstash b/tools/docker-compose/Dockerfile-logstash index 7117b90847..e105713035 100644 --- a/tools/docker-compose/Dockerfile-logstash +++ b/tools/docker-compose/Dockerfile-logstash @@ -1,3 +1,5 @@ FROM logstash:5-alpine COPY logstash.conf / +RUN touch /logstash.log +RUN chown logstash:logstash /logstash.log CMD ["-f", "/logstash.conf"] diff --git a/tools/docker-compose/logstash.conf b/tools/docker-compose/logstash.conf index 9ab5145642..b340bfe195 100644 --- a/tools/docker-compose/logstash.conf +++ b/tools/docker-compose/logstash.conf @@ -15,5 +15,8 @@ filter { } output { - stdout { codec => rubydebug } + stdout { codec => rubydebug } + file { + path => "/logstash.log" + } } From 999a304c3b148ee938b4edf4dbe118d52b92491f Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 17 Mar 2017 09:54:09 -0400 Subject: [PATCH 51/58] add no_log filtering for runner_item_on_xyz events see: #5691 --- awx/lib/tests/test_display_callback.py | 20 ++++++++++++++++---- awx/lib/tower_display_callback/module.py | 22 +++++++++++++--------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/awx/lib/tests/test_display_callback.py b/awx/lib/tests/test_display_callback.py index 2e3b58bdfc..1cd2ec613c 100644 --- a/awx/lib/tests/test_display_callback.py +++ b/awx/lib/tests/test_display_callback.py @@ -126,14 +126,13 @@ def test_callback_plugin_receives_events(executor, cache, event, playbook): when: false '''}, # noqa {'no_log_on_play.yml': ''' -- name: play-level no_log set +- name: args should not be logged when play-level no_log set connection: local hosts: all gather_facts: no no_log: true tasks: - - name: args should not be logged when play-level no_log set - shell: echo "SENSITIVE" + - shell: echo "SENSITIVE" '''}, # noqa {'async_no_log.yml': ''' - name: async task args should suppressed with no_log @@ -147,6 +146,19 @@ def test_callback_plugin_receives_events(executor, cache, event, playbook): shell: echo "SENSITIVE" no_log: true '''}, # noqa +{'with_items.yml': ''' +- name: with_items tasks should be suppressed with no_log + connection: local + hosts: all + gather_facts: no + tasks: + - shell: echo {{ item }} + no_log: true + with_items: [ "SENSITIVE", "SENSITIVE-SKIPPED", "SENSITIVE-FAILED" ] + when: item != "SENSITIVE-SKIPPED" + failed_when: item == "SENSITIVE-FAILED" + ignore_errors: yes +'''}, # noqa ]) def test_callback_plugin_no_log_filters(executor, cache, playbook): executor.run() @@ -167,4 +179,4 @@ def test_callback_plugin_strips_task_environ_variables(executor, cache, playbook executor.run() assert len(cache) for event in cache.values(): - assert os.environ['VIRTUAL_ENV'] not in json.dumps(event) + assert os.environ['PATH'] not in json.dumps(event) diff --git a/awx/lib/tower_display_callback/module.py b/awx/lib/tower_display_callback/module.py index 74b6fec9d6..9d8cf24fb9 100644 --- a/awx/lib/tower_display_callback/module.py +++ b/awx/lib/tower_display_callback/module.py @@ -75,6 +75,11 @@ class BaseCallbackModule(CallbackBase): super(BaseCallbackModule, self).__init__() self.task_uuids = set() + def censor_result(self, res): + if res.get('_ansible_no_log', False): + return {'censored': "the output has been hidden due to the fact that 'no_log: true' was specified for this result"} # noqa + return res + @contextlib.contextmanager def capture_event_data(self, event, **event_data): @@ -310,9 +315,7 @@ class BaseCallbackModule(CallbackBase): if result._task.get_name() == 'setup': result._result.get('ansible_facts', {}).pop('ansible_env', None) - res = result._result - if res.get('_ansible_no_log', False): - res = {'censored': "the output has been hidden due to the fact that 'no_log: true' was specified for this result"} + res = self.censor_result(result._result) event_data = dict( host=result._host.get_name(), @@ -327,9 +330,7 @@ class BaseCallbackModule(CallbackBase): def v2_runner_on_failed(self, result, ignore_errors=False): # FIXME: Add verbosity for exception/results output. - res = result._result - if res.get('_ansible_no_log', False): - res = {'censored': "the output has been hidden due to the fact that 'no_log: true' was specified for this result"} + res = self.censor_result(result._result) event_data = dict( host=result._host.get_name(), @@ -424,28 +425,31 @@ class BaseCallbackModule(CallbackBase): super(BaseCallbackModule, self).v2_on_file_diff(result) def v2_runner_item_on_ok(self, result): + res = self.censor_result(result._result) event_data = dict( host=result._host.get_name(), task=result._task, - res=result._result, + res=res, ) with self.capture_event_data('runner_item_on_ok', **event_data): super(BaseCallbackModule, self).v2_runner_item_on_ok(result) def v2_runner_item_on_failed(self, result): + res = self.censor_result(result._result) event_data = dict( host=result._host.get_name(), task=result._task, - res=result._result, + res=res, ) with self.capture_event_data('runner_item_on_failed', **event_data): super(BaseCallbackModule, self).v2_runner_item_on_failed(result) def v2_runner_item_on_skipped(self, result): + res = self.censor_result(result._result) event_data = dict( host=result._host.get_name(), task=result._task, - res=result._result, + res=res, ) with self.capture_event_data('runner_item_on_skipped', **event_data): super(BaseCallbackModule, self).v2_runner_item_on_skipped(result) From 03593b957b78c5d192e095b12c88a37127f1bc8f Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Fri, 17 Mar 2017 10:45:55 -0400 Subject: [PATCH 52/58] Parse the mercurial revision correctly in setup playbook Originally the value was coming out as: xxxxxx default tip When we want xxxxxx In order for the subsequent revision request to work correctly --- awx/playbooks/project_update.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml index eb09cd3657..c3803c7118 100644 --- a/awx/playbooks/project_update.yml +++ b/awx/playbooks/project_update.yml @@ -160,6 +160,11 @@ scm_version: "{{scm_version|regex_replace('^.*Revision: ([0-9]+).*$', '\\1')}}" when: scm_type == 'svn' + - name: parse hg version string properly + set_fact: + scm_version: "{{scm_version|regex_replace('^([A-Za-z0-9]+).*$', '\\1')}}" + when: scm_type == 'hg' + - name: Repository Version debug: msg="Repository Version {{ scm_version }}" when: scm_version is defined From 6083e9482e33282ff07252bffd4e11f4e2d23b92 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Fri, 17 Mar 2017 12:51:19 -0400 Subject: [PATCH 53/58] Refactor job template callback post to mimic the behavior of normal jt launch. --- awx/api/views.py | 10 +++++----- awx/main/models/jobs.py | 13 +++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/awx/api/views.py b/awx/api/views.py index 4b13a8b2e9..5d7891665d 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -2727,14 +2727,14 @@ class JobTemplateCallback(GenericAPIView): return Response(data, status=status.HTTP_400_BAD_REQUEST) # Everything is fine; actually create the job. + kv = {"limit": limit, "launch_type": 'callback'} + if extra_vars is not None and job_template.ask_variables_on_launch: + kv['extra_vars'] = callback_filter_out_ansible_extra_vars(extra_vars) with transaction.atomic(): - job = job_template.create_job(limit=limit, launch_type='callback') + job = job_template.create_job(**kv) # Send a signal to celery that the job should be started. - kv = {"inventory_sources_already_updated": inventory_sources_already_updated} - if extra_vars is not None: - kv['extra_vars'] = callback_filter_out_ansible_extra_vars(extra_vars) - result = job.signal_start(**kv) + result = job.signal_start(inventory_sources_already_updated=inventory_sources_already_updated) if not result: data = dict(msg=_('Error starting job!')) return Response(data, status=status.HTTP_400_BAD_REQUEST) diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 992217ec33..fd7718514b 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -300,6 +300,8 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour Return whether job template can be used to start a new job without requiring any user input. ''' + # It is worthwhile to find out if this function is now only used by + # provisioning callback. variables_needed = False if callback_extra_vars: extra_vars_dict = parse_yaml_or_json(callback_extra_vars) @@ -310,10 +312,13 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour elif self.variables_needed_to_start: variables_needed = True prompting_needed = False - for key, value in self._ask_for_vars_dict().iteritems(): - if value and not (key == 'extra_vars' and - callback_extra_vars is not None): - prompting_needed = True + # The behavior of provisioning callback should mimic + # that of job template launch, so prompting_needed should + # not block a provisioning callback from creating/launching jobs. + if callback_extra_vars is None: + for value in self._ask_for_vars_dict().values(): + if value: + prompting_needed = True return (not prompting_needed and not self.passwords_needed_to_start and not variables_needed) From 2edd4b338de56d7943730b01ac3ec5cb5143dfd3 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Fri, 17 Mar 2017 15:40:55 -0400 Subject: [PATCH 54/58] Add functional test to gurarantee consistent behavior of provisioning callback with jt launch. --- awx/api/views.py | 3 ++- .../functional/api/test_job_runtime_params.py | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/awx/api/views.py b/awx/api/views.py index 5d7891665d..3c5d5b9dc9 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -2678,7 +2678,8 @@ class JobTemplateCallback(GenericAPIView): def post(self, request, *args, **kwargs): extra_vars = None - if request.content_type == "application/json": + # Be careful here: content_type can look like '; charset=blar' + if request.content_type.startswith("application/json"): extra_vars = request.data.get("extra_vars", None) # Permission class should have already validated host_config_key. job_template = self.get_object() diff --git a/awx/main/tests/functional/api/test_job_runtime_params.py b/awx/main/tests/functional/api/test_job_runtime_params.py index af8bd659fd..4412325043 100644 --- a/awx/main/tests/functional/api/test_job_runtime_params.py +++ b/awx/main/tests/functional/api/test_job_runtime_params.py @@ -344,3 +344,30 @@ def test_job_launch_unprompted_vars_with_survey(mocker, survey_spec_factory, job # Check that the survey variable is accepted and the job variable isn't mock_job.signal_start.assert_called_once() + + +@pytest.mark.django_db +@pytest.mark.job_runtime_vars +def test_jt_provisioning_callback(mocker, survey_spec_factory, job_template_prompts, post, admin_user, host): + job_template = job_template_prompts(True) + job_template.host_config_key = "foo" + job_template.survey_enabled = True + job_template.survey_spec = survey_spec_factory('survey_var') + job_template.save() + + with mocker.patch('awx.main.access.BaseAccess.check_license'): + mock_job = mocker.MagicMock(spec=Job, id=968, extra_vars={"job_launch_var": 3, "survey_var": 4}) + with mocker.patch.object(JobTemplate, 'create_unified_job', return_value=mock_job): + with mocker.patch('awx.api.serializers.JobSerializer.to_representation', return_value={}): + with mocker.patch('awx.api.views.JobTemplateCallback.find_matching_hosts', return_value=[host]): + response = post( + reverse('api:job_template_callback', args=[job_template.pk]), + dict(extra_vars={"job_launch_var": 3, "survey_var": 4}, host_config_key="foo"), + admin_user, expect=201, format='json') + assert JobTemplate.create_unified_job.called + assert JobTemplate.create_unified_job.call_args == ({'extra_vars': {'survey_var': 4, + 'job_launch_var': 3}, + 'launch_type': 'callback', + 'limit': 'single-host'},) + + mock_job.signal_start.assert_called_once() From 6a7743b274ef7c65b425fa583f4aa0b60cabdb5f Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 17 Mar 2017 16:55:34 -0400 Subject: [PATCH 55/58] fix a callback bug that causes a task_args leak between job events see: #5802 --- awx/lib/tests/test_display_callback.py | 31 ++++++++++++++++ awx/lib/tower_display_callback/module.py | 46 ++++++------------------ 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/awx/lib/tests/test_display_callback.py b/awx/lib/tests/test_display_callback.py index 1cd2ec613c..d0c53f4fc0 100644 --- a/awx/lib/tests/test_display_callback.py +++ b/awx/lib/tests/test_display_callback.py @@ -166,6 +166,37 @@ def test_callback_plugin_no_log_filters(executor, cache, playbook): assert 'SENSITIVE' not in json.dumps(cache.items()) +@pytest.mark.parametrize('playbook', [ +{'no_log_on_ok.yml': ''' +- name: args should not be logged when task-level no_log is set + connection: local + hosts: all + gather_facts: no + tasks: + - shell: echo "SENSITIVE" + - shell: echo "PRIVATE" + no_log: true +'''}, # noqa +]) +def test_callback_plugin_task_args_leak(executor, cache, playbook): + executor.run() + events = cache.values() + assert events[0]['event'] == 'playbook_on_start' + assert events[1]['event'] == 'playbook_on_play_start' + + # task 1 + assert events[2]['event'] == 'playbook_on_task_start' + assert 'SENSITIVE' in events[2]['event_data']['task_args'] + assert events[3]['event'] == 'runner_on_ok' + assert 'SENSITIVE' in events[3]['event_data']['task_args'] + + # task 2 no_log=True + assert events[4]['event'] == 'playbook_on_task_start' + assert events[4]['event_data']['task_args'] == "the output has been hidden due to the fact that 'no_log: true' was specified for this result" # noqa + assert events[5]['event'] == 'runner_on_ok' + assert events[5]['event_data']['task_args'] == "the output has been hidden due to the fact that 'no_log: true' was specified for this result" # noqa + + @pytest.mark.parametrize('playbook', [ {'strip_env_vars.yml': ''' - name: sensitive environment variables should be stripped from events diff --git a/awx/lib/tower_display_callback/module.py b/awx/lib/tower_display_callback/module.py index 9d8cf24fb9..59575d7989 100644 --- a/awx/lib/tower_display_callback/module.py +++ b/awx/lib/tower_display_callback/module.py @@ -55,31 +55,10 @@ class BaseCallbackModule(CallbackBase): 'playbook_on_no_hosts_remaining', ] - CENSOR_FIELD_WHITELIST = [ - 'msg', - 'failed', - 'changed', - 'results', - 'start', - 'end', - 'delta', - 'cmd', - '_ansible_no_log', - 'rc', - 'failed_when_result', - 'skipped', - 'skip_reason', - ] - def __init__(self): super(BaseCallbackModule, self).__init__() self.task_uuids = set() - def censor_result(self, res): - if res.get('_ansible_no_log', False): - return {'censored': "the output has been hidden due to the fact that 'no_log: true' was specified for this result"} # noqa - return res - @contextlib.contextmanager def capture_event_data(self, event, **event_data): @@ -90,6 +69,9 @@ class BaseCallbackModule(CallbackBase): else: task = None + if event_data.get('res') and event_data['res'].get('_ansible_no_log', False): + event_data['res'] = {'censored': "the output has been hidden due to the fact that 'no_log: true' was specified for this result"} # noqa + with event_context.display_lock: try: event_context.add_local(event=event, **event_data) @@ -137,7 +119,9 @@ class BaseCallbackModule(CallbackBase): task_ctx['task_path'] = task.get_path() except AttributeError: pass - if not task.no_log: + if task.no_log: + task_ctx['task_args'] = "the output has been hidden due to the fact that 'no_log: true' was specified for this result" + else: task_args = ', '.join(('%s=%s' % a for a in task.args.items())) task_ctx['task_args'] = task_args if getattr(task, '_role', None): @@ -315,13 +299,11 @@ class BaseCallbackModule(CallbackBase): if result._task.get_name() == 'setup': result._result.get('ansible_facts', {}).pop('ansible_env', None) - res = self.censor_result(result._result) - event_data = dict( host=result._host.get_name(), remote_addr=result._host.address, task=result._task, - res=res, + res=result._result, event_loop=result._task.loop if hasattr(result._task, 'loop') else None, ) with self.capture_event_data('runner_on_ok', **event_data): @@ -329,13 +311,10 @@ class BaseCallbackModule(CallbackBase): def v2_runner_on_failed(self, result, ignore_errors=False): # FIXME: Add verbosity for exception/results output. - - res = self.censor_result(result._result) - event_data = dict( host=result._host.get_name(), remote_addr=result._host.address, - res=res, + res=result._result, task=result._task, ignore_errors=ignore_errors, event_loop=result._task.loop if hasattr(result._task, 'loop') else None, @@ -425,31 +404,28 @@ class BaseCallbackModule(CallbackBase): super(BaseCallbackModule, self).v2_on_file_diff(result) def v2_runner_item_on_ok(self, result): - res = self.censor_result(result._result) event_data = dict( host=result._host.get_name(), task=result._task, - res=res, + res=result._result, ) with self.capture_event_data('runner_item_on_ok', **event_data): super(BaseCallbackModule, self).v2_runner_item_on_ok(result) def v2_runner_item_on_failed(self, result): - res = self.censor_result(result._result) event_data = dict( host=result._host.get_name(), task=result._task, - res=res, + res=result._result, ) with self.capture_event_data('runner_item_on_failed', **event_data): super(BaseCallbackModule, self).v2_runner_item_on_failed(result) def v2_runner_item_on_skipped(self, result): - res = self.censor_result(result._result) event_data = dict( host=result._host.get_name(), task=result._task, - res=res, + res=result._result, ) with self.capture_event_data('runner_item_on_skipped', **event_data): super(BaseCallbackModule, self).v2_runner_item_on_skipped(result) From 2e2d88516e73c0639f6be0e326dfb82944610208 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Mon, 20 Mar 2017 12:05:35 -0400 Subject: [PATCH 56/58] Negative functional test added. --- awx/main/models/jobs.py | 2 -- .../functional/api/test_job_runtime_params.py | 27 +++++++++++++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index fd7718514b..388be47d17 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -300,8 +300,6 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour Return whether job template can be used to start a new job without requiring any user input. ''' - # It is worthwhile to find out if this function is now only used by - # provisioning callback. variables_needed = False if callback_extra_vars: extra_vars_dict = parse_yaml_or_json(callback_extra_vars) diff --git a/awx/main/tests/functional/api/test_job_runtime_params.py b/awx/main/tests/functional/api/test_job_runtime_params.py index 4412325043..e63a074965 100644 --- a/awx/main/tests/functional/api/test_job_runtime_params.py +++ b/awx/main/tests/functional/api/test_job_runtime_params.py @@ -348,7 +348,7 @@ def test_job_launch_unprompted_vars_with_survey(mocker, survey_spec_factory, job @pytest.mark.django_db @pytest.mark.job_runtime_vars -def test_jt_provisioning_callback(mocker, survey_spec_factory, job_template_prompts, post, admin_user, host): +def test_callback_accept_prompted_extra_var(mocker, survey_spec_factory, job_template_prompts, post, admin_user, host): job_template = job_template_prompts(True) job_template.host_config_key = "foo" job_template.survey_enabled = True @@ -360,7 +360,7 @@ def test_jt_provisioning_callback(mocker, survey_spec_factory, job_template_prom with mocker.patch.object(JobTemplate, 'create_unified_job', return_value=mock_job): with mocker.patch('awx.api.serializers.JobSerializer.to_representation', return_value={}): with mocker.patch('awx.api.views.JobTemplateCallback.find_matching_hosts', return_value=[host]): - response = post( + post( reverse('api:job_template_callback', args=[job_template.pk]), dict(extra_vars={"job_launch_var": 3, "survey_var": 4}, host_config_key="foo"), admin_user, expect=201, format='json') @@ -371,3 +371,26 @@ def test_jt_provisioning_callback(mocker, survey_spec_factory, job_template_prom 'limit': 'single-host'},) mock_job.signal_start.assert_called_once() + + +@pytest.mark.django_db +@pytest.mark.job_runtime_vars +def test_callback_ignore_unprompted_extra_var(mocker, survey_spec_factory, job_template_prompts, post, admin_user, host): + job_template = job_template_prompts(False) + job_template.host_config_key = "foo" + job_template.save() + + with mocker.patch('awx.main.access.BaseAccess.check_license'): + mock_job = mocker.MagicMock(spec=Job, id=968, extra_vars={"job_launch_var": 3, "survey_var": 4}) + with mocker.patch.object(JobTemplate, 'create_unified_job', return_value=mock_job): + with mocker.patch('awx.api.serializers.JobSerializer.to_representation', return_value={}): + with mocker.patch('awx.api.views.JobTemplateCallback.find_matching_hosts', return_value=[host]): + post( + reverse('api:job_template_callback', args=[job_template.pk]), + dict(extra_vars={"job_launch_var": 3, "survey_var": 4}, host_config_key="foo"), + admin_user, expect=201, format='json') + assert JobTemplate.create_unified_job.called + assert JobTemplate.create_unified_job.call_args == ({'launch_type': 'callback', + 'limit': 'single-host'},) + + mock_job.signal_start.assert_called_once() From d0ae8186851cab5f965f346c66255dcd97a47b84 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 20 Mar 2017 17:02:05 -0400 Subject: [PATCH 57/58] show extra variables (if any) in the adhoc command results UI see: #1744 --- .../adhoc/standard-out-adhoc.partial.html | 20 +++++++++++++++++++ .../standard-out/standard-out.controller.js | 6 +++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/awx/ui/client/src/standard-out/adhoc/standard-out-adhoc.partial.html b/awx/ui/client/src/standard-out/adhoc/standard-out-adhoc.partial.html index c2baf24ee9..b8ef147b28 100644 --- a/awx/ui/client/src/standard-out/adhoc/standard-out-adhoc.partial.html +++ b/awx/ui/client/src/standard-out/adhoc/standard-out-adhoc.partial.html @@ -99,6 +99,26 @@
Verbosity
{{ verbosity }}
+ +
+
+ Extra Variables + + +
+
+ +
+
+ diff --git a/awx/ui/client/src/standard-out/standard-out.controller.js b/awx/ui/client/src/standard-out/standard-out.controller.js index 64f34cb68f..45a5676af1 100644 --- a/awx/ui/client/src/standard-out/standard-out.controller.js +++ b/awx/ui/client/src/standard-out/standard-out.controller.js @@ -149,7 +149,11 @@ export function JobStdoutController ($rootScope, $scope, $state, $stateParams, } if (data.extra_vars) { - ParseTypeChange({ scope: $scope, field_id: 'pre-formatted-variables' }); + ParseTypeChange({ + scope: $scope, + field_id: 'pre-formatted-variables', + readOnly: true + }); } if ($scope.job.type === 'inventory_update' && !$scope.inv_manage_group_link) { From ba0e9ab7756728164404f7eb4c3fc82b264b01a1 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 20 Mar 2017 17:20:33 -0400 Subject: [PATCH 58/58] copy extra_vars when relaunching ad-hoc commands see: #1744 --- awx/api/views.py | 2 +- awx/main/models/ad_hoc_commands.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/api/views.py b/awx/api/views.py index 3c5d5b9dc9..33ef14826a 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -3666,7 +3666,7 @@ class AdHocCommandRelaunch(GenericAPIView): data = {} for field in ('job_type', 'inventory_id', 'limit', 'credential_id', 'module_name', 'module_args', 'forks', 'verbosity', - 'become_enabled'): + 'extra_vars', 'become_enabled'): if field.endswith('_id'): data[field[:-3]] = getattr(obj, field) else: diff --git a/awx/main/models/ad_hoc_commands.py b/awx/main/models/ad_hoc_commands.py index 3636aa8e0a..d6c97e6f86 100644 --- a/awx/main/models/ad_hoc_commands.py +++ b/awx/main/models/ad_hoc_commands.py @@ -190,7 +190,7 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin): data = {} for field in ('job_type', 'inventory_id', 'limit', 'credential_id', 'module_name', 'module_args', 'forks', 'verbosity', - 'become_enabled'): + 'extra_vars', 'become_enabled'): data[field] = getattr(self, field) return AdHocCommand.objects.create(**data)