From 3c241052716bc61c3dbc6e7ca0e30de226996a6d Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 24 Oct 2016 16:31:33 -0400 Subject: [PATCH 01/10] update JT playbook validation for HA --- awx/api/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index d6aa39f706..9d44a6cee1 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1847,7 +1847,7 @@ class JobOptionsSerializer(LabelsListMixin, BaseSerializer): job_type = attrs.get('job_type', self.instance and self.instance.job_type or None) if not project and job_type != PERM_INVENTORY_SCAN: raise serializers.ValidationError({'project': 'This field is required.'}) - if project and playbook and force_text(playbook) not in project.playbooks: + if project and playbook and force_text(playbook) not in project.playbook_files: raise serializers.ValidationError({'playbook': 'Playbook not found for project.'}) if project and not playbook: raise serializers.ValidationError({'playbook': 'Must select playbook for project.'}) From 3b4b1412fce1c1f0db95f80677114d40d5e36157 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 24 Oct 2016 21:04:01 -0400 Subject: [PATCH 02/10] fix tests for project playbook mocking --- awx/main/tests/factories/fixtures.py | 5 +++-- .../tests/functional/api/test_job_template.py | 21 ++++--------------- awx/main/tests/functional/conftest.py | 3 ++- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/awx/main/tests/factories/fixtures.py b/awx/main/tests/factories/fixtures.py index ac6c93348d..8f6e5df414 100644 --- a/awx/main/tests/factories/fixtures.py +++ b/awx/main/tests/factories/fixtures.py @@ -74,7 +74,8 @@ def mk_user(name, is_superuser=False, organization=None, team=None, persisted=Tr def mk_project(name, organization=None, description=None, persisted=True): description = description or '{}-description'.format(name) - project = Project(name=name, description=description) + project = Project(name=name, description=description, + playbook_files=['helloworld.yml', 'alt-helloworld.yml']) if organization is not None: project.organization = organization if persisted: @@ -134,7 +135,7 @@ def mk_job_template(name, job_type='run', extra_vars = json.dumps(extra_vars) jt = JobTemplate(name=name, job_type=job_type, extra_vars=extra_vars, - playbook='mocked') + playbook='helloworld.yml') jt.inventory = inventory if jt.inventory is None: diff --git a/awx/main/tests/functional/api/test_job_template.py b/awx/main/tests/functional/api/test_job_template.py index 68a7e7aecd..df577e84dc 100644 --- a/awx/main/tests/functional/api/test_job_template.py +++ b/awx/main/tests/functional/api/test_job_template.py @@ -11,12 +11,7 @@ from awx.main.migrations import _save_password_keys as save_password_keys from django.core.urlresolvers import reverse from django.apps import apps -@property -def project_playbooks(self): - return ['mocked', 'mocked.yml', 'alt-mocked.yml'] - @pytest.mark.django_db -@mock.patch.object(ProjectOptions, "playbooks", project_playbooks) @pytest.mark.parametrize( "grant_project, grant_credential, grant_inventory, expect", [ (True, True, True, 201), @@ -38,11 +33,10 @@ def test_create(post, project, machine_credential, inventory, alice, grant_proje 'project': project.id, 'credential': machine_credential.id, 'inventory': inventory.id, - 'playbook': 'mocked.yml', + 'playbook': 'helloworld.yml', }, alice, expect=expect) @pytest.mark.django_db -@mock.patch.object(ProjectOptions, "playbooks", project_playbooks) @pytest.mark.parametrize( "grant_project, grant_credential, grant_inventory, expect", [ (True, True, True, 200), @@ -67,11 +61,10 @@ def test_edit_sensitive_fields(patch, job_template_factory, alice, grant_project 'project': objs.project.id, 'credential': objs.credential.id, 'inventory': objs.inventory.id, - 'playbook': 'alt-mocked.yml', + 'playbook': 'alt-helloworld.yml', }, alice, expect=expect) @pytest.mark.django_db -@mock.patch.object(ProjectOptions, "playbooks", project_playbooks) def test_edit_playbook(patch, job_template_factory, alice): objs = job_template_factory('jt', organization='org1', project='prj', inventory='inv', credential='cred') objs.job_template.admin_role.members.add(alice) @@ -80,16 +73,15 @@ def test_edit_playbook(patch, job_template_factory, alice): objs.inventory.use_role.members.add(alice) patch(reverse('api:job_template_detail', args=(objs.job_template.id,)), { - 'playbook': 'alt-mocked.yml', + 'playbook': 'alt-helloworld.yml', }, alice, expect=200) objs.inventory.use_role.members.remove(alice) patch(reverse('api:job_template_detail', args=(objs.job_template.id,)), { - 'playbook': 'mocked.yml', + 'playbook': 'helloworld.yml', }, alice, expect=403) @pytest.mark.django_db -@mock.patch.object(ProjectOptions, "playbooks", project_playbooks) def test_edit_nonsenstive(patch, job_template_factory, alice): objs = job_template_factory('jt', organization='org1', project='prj', inventory='inv', credential='cred') jt = objs.job_template @@ -121,10 +113,6 @@ def jt_copy_edit(job_template_factory, project): project=project) return objects.job_template -@property -def project_playbooks(self): - return ['mocked', 'mocked.yml', 'alt-mocked.yml'] - @pytest.mark.django_db def test_job_template_role_user(post, organization_factory, job_template_factory): objects = organization_factory("org", @@ -143,7 +131,6 @@ def test_job_template_role_user(post, organization_factory, job_template_factory @pytest.mark.django_db -@mock.patch.object(ProjectOptions, "playbooks", project_playbooks) def test_jt_admin_copy_edit_functional(jt_copy_edit, rando, get, post): # Grant random user JT admin access only diff --git a/awx/main/tests/functional/conftest.py b/awx/main/tests/functional/conftest.py index 0c620feb7e..0fb6084edb 100644 --- a/awx/main/tests/functional/conftest.py +++ b/awx/main/tests/functional/conftest.py @@ -110,7 +110,8 @@ def team_member(user, team): def project(instance, organization): prj = Project.objects.create(name="test-proj", description="test-proj-desc", - organization=organization + organization=organization, + playbook_files=['helloworld.yml', 'alt-helloworld.yml'] ) return prj From 7da6d2978a75e08210f94ef0f7d84ef95e71eeaa Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 24 Oct 2016 21:30:03 -0400 Subject: [PATCH 03/10] remove test imports --- awx/main/tests/functional/api/test_job_template.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/awx/main/tests/functional/api/test_job_template.py b/awx/main/tests/functional/api/test_job_template.py index df577e84dc..ca6bdf3d31 100644 --- a/awx/main/tests/functional/api/test_job_template.py +++ b/awx/main/tests/functional/api/test_job_template.py @@ -1,10 +1,8 @@ import pytest -import mock # AWX from awx.api.serializers import JobTemplateSerializer, JobLaunchSerializer from awx.main.models.jobs import Job -from awx.main.models.projects import ProjectOptions from awx.main.migrations import _save_password_keys as save_password_keys # Django From 7c9e1c91db9b03c4802bc450d3a6f63d718a3d04 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Wed, 26 Oct 2016 15:52:49 -0400 Subject: [PATCH 04/10] Ensure workflow_job_template being set during node creation. --- awx/api/serializers.py | 2 ++ awx/main/models/workflow.py | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index d6aa39f706..c21fbfc2b5 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2322,6 +2322,8 @@ class WorkflowJobTemplateNodeSerializer(WorkflowNodeBaseSerializer): raise serializers.ValidationError({ "job_type": "%s is not a valid job type. The choices are %s." % ( attrs['char_prompts']['job_type'], job_types)}) + if self.instance is None and 'workflow_job_template' not in attrs: + raise serializers.ValidationError({"workflow_job_template": "Workflow job template is missing during creation"}) ujt_obj = attrs.get('unified_job_template', None) if isinstance(ujt_obj, (WorkflowJobTemplate, SystemJobTemplate)): raise serializers.ValidationError({ diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index 2848b38a4a..d109b43519 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -140,7 +140,6 @@ class WorkflowNodeBase(CreatedModifiedModel): 'inventory', 'credential', 'char_prompts'] class WorkflowJobTemplateNode(WorkflowNodeBase): - # TODO: Ensure the API forces workflow_job_template being set workflow_job_template = models.ForeignKey( 'WorkflowJobTemplate', related_name='workflow_job_template_nodes', @@ -149,7 +148,7 @@ class WorkflowJobTemplateNode(WorkflowNodeBase): default=None, on_delete=models.CASCADE, ) - + def get_absolute_url(self): return reverse('api:workflow_job_template_node_detail', args=(self.pk,)) From 1fec29dcb31b620b55397dcca568030cdbc65220 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 26 Oct 2016 17:22:03 -0400 Subject: [PATCH 05/10] remove awx.compat, since it looks like it is not necessary --- awx/lib/__init__.py | 2 -- awx/lib/compat.py | 33 --------------------------------- awx/main/models/projects.py | 2 +- awx/settings/defaults.py | 2 +- 4 files changed, 2 insertions(+), 37 deletions(-) delete mode 100644 awx/lib/__init__.py delete mode 100644 awx/lib/compat.py diff --git a/awx/lib/__init__.py b/awx/lib/__init__.py deleted file mode 100644 index e484e62be1..0000000000 --- a/awx/lib/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) 2015 Ansible, Inc. -# All Rights Reserved. diff --git a/awx/lib/compat.py b/awx/lib/compat.py deleted file mode 100644 index fb686dd11c..0000000000 --- a/awx/lib/compat.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2015 Ansible, Inc. -# All Rights Reserved. - -''' -Compability library for support of both Django 1.4.x and Django 1.5.x. -''' - -try: - from django.utils.html import format_html -except ImportError: - from django.utils.html import conditional_escape - from django.utils.safestring import mark_safe - - def format_html(format_string, *args, **kwargs): - args_safe = map(conditional_escape, args) - kwargs_safe = dict([(k, conditional_escape(v)) for (k, v) in - kwargs.items()]) - return mark_safe(format_string.format(*args_safe, **kwargs_safe)) - -try: - from django.utils.log import RequireDebugTrue -except ImportError: - import logging - from django.conf import settings - - class RequireDebugTrue(logging.Filter): - def filter(self, record): - return settings.DEBUG - -try: - from django.utils.text import slugify # noqa -except ImportError: - from django.template.defaultfilters import slugify # noqa diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index 9888afaaa7..1c693a6398 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -15,12 +15,12 @@ from django.conf import settings from django.db import models from django.utils.translation import ugettext_lazy as _ from django.utils.encoding import smart_str, smart_text +from django.utils.text import slugify from django.core.exceptions import ValidationError from django.core.urlresolvers import reverse from django.utils.timezone import now, make_aware, get_default_timezone # AWX -from awx.lib.compat import slugify from awx.main.models.base import * # noqa from awx.main.models.jobs import Job from awx.main.models.notifications import ( diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 928ac39afa..05a0fe3d1f 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -796,7 +796,7 @@ LOGGING = { '()': 'django.utils.log.RequireDebugFalse', }, 'require_debug_true': { - '()': 'awx.lib.compat.RequireDebugTrue', + '()': 'django.utils.log.RequireDebugTrue', }, 'require_debug_true_or_test': { '()': 'awx.main.utils.RequireDebugTrueOrTest', From 432242bdd22426f461c772de0c5a5c51b32df80d Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Thu, 27 Oct 2016 11:20:45 -0400 Subject: [PATCH 06/10] Null check added. --- awx/api/serializers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index c21fbfc2b5..e6b25953d1 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2322,7 +2322,8 @@ class WorkflowJobTemplateNodeSerializer(WorkflowNodeBaseSerializer): raise serializers.ValidationError({ "job_type": "%s is not a valid job type. The choices are %s." % ( attrs['char_prompts']['job_type'], job_types)}) - if self.instance is None and 'workflow_job_template' not in attrs: + if self.instance is None and ('workflow_job_template' not in attrs or + attrs['workflow_job_template'] is None): raise serializers.ValidationError({"workflow_job_template": "Workflow job template is missing during creation"}) ujt_obj = attrs.get('unified_job_template', None) if isinstance(ujt_obj, (WorkflowJobTemplate, SystemJobTemplate)): From 3ae5a4b9a87214ada8a78ae57c9469fb922e0be3 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 27 Oct 2016 13:17:20 -0400 Subject: [PATCH 07/10] add job timeouts to CTiT --- awx/main/conf.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/awx/main/conf.py b/awx/main/conf.py index e0d16e8542..862238f3fa 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -173,3 +173,21 @@ register( category=_('Jobs'), category_slug='jobs', ) + +register( + 'DEFAULT_JOB_TIMEOUTS', + field_class=fields.DictField, + default={ + 'Job': 0, + 'InventoryUpdate': 0, + 'ProjectUpdate': 0, + }, + label=_('Default Job Timeouts'), + help_text=_('Maximum time to allow jobs to run. Use sub-keys of Job, ' + 'InventoryUpdate, and ProjectUpdate to configure this value ' + 'for each job type. Use value of 0 to indicate that no ' + 'timeout should be imposed. A timeout set on an individual ' + 'job template will override this.'), + category=_('Jobs'), + category_slug='jobs', +) From d21de4c99f1bdbca983a90de746e2aee569d342d Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Tue, 4 Oct 2016 10:56:27 -0400 Subject: [PATCH 08/10] Add validate_license to job's can_add --- awx/main/access.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/awx/main/access.py b/awx/main/access.py index 70346d8b4b..07c25c237a 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1194,7 +1194,10 @@ class JobAccess(BaseAccess): return True return self.org_access(obj, role_types=['auditor_role', 'admin_role']) - def can_add(self, data): + def can_add(self, data, validate_license=True): + if validate_license: + self.check_license() + if not data: # So the browseable API will work return True if not self.user.is_superuser: From 6717d4f3fabbe731818eacc15b9d7b62ffbf6f49 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Thu, 27 Oct 2016 15:24:03 -0400 Subject: [PATCH 09/10] Prevent job can_change from erroneously firing license validation. --- awx/main/access.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/awx/main/access.py b/awx/main/access.py index 07c25c237a..442cbcc895 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1221,7 +1221,8 @@ class JobAccess(BaseAccess): return True def can_change(self, obj, data): - return obj.status == 'new' and self.can_read(obj) and self.can_add(data) + return obj.status == 'new' and self.can_read(obj) and\ + self.can_add(data, validate_license=False) @check_superuser def can_delete(self, obj): From 31d06ecdc422a1a2efa7298f68bbfa491095c570 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Thu, 27 Oct 2016 16:09:55 -0400 Subject: [PATCH 10/10] flake8 fix. --- awx/main/access.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/awx/main/access.py b/awx/main/access.py index 442cbcc895..3fd0ee0f0e 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1221,8 +1221,9 @@ class JobAccess(BaseAccess): return True def can_change(self, obj, data): - return obj.status == 'new' and self.can_read(obj) and\ - self.can_add(data, validate_license=False) + return (obj.status == 'new' and + self.can_read(obj) and + self.can_add(data, validate_license=False)) @check_superuser def can_delete(self, obj):