From 318b1aebd16ab57bc249d413f79fdbea2961e3b4 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 22 Jun 2017 14:21:22 -0400 Subject: [PATCH 1/5] remove special scan job logic --- awx/api/serializers.py | 16 ++----- awx/api/urls.py | 1 - awx/api/views.py | 21 +------- awx/main/access.py | 19 -------- awx/main/models/jobs.py | 13 +---- awx/main/tasks.py | 7 +-- .../functional/api/test_job_runtime_params.py | 20 -------- .../tests/functional/api/test_job_template.py | 48 +------------------ .../api/test_organization_counts.py | 7 +-- .../tests/unit/models/test_workflow_unit.py | 8 ---- 10 files changed, 10 insertions(+), 150 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index a234240a7e..d1c1b6c9e2 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1147,7 +1147,6 @@ class InventorySerializer(BaseSerializerWithVariables): update_inventory_sources = self.reverse('api:inventory_inventory_sources_update', kwargs={'pk': obj.pk}), activity_stream = self.reverse('api:inventory_activity_stream_list', kwargs={'pk': obj.pk}), job_templates = self.reverse('api:inventory_job_template_list', kwargs={'pk': obj.pk}), - scan_job_templates = self.reverse('api:inventory_scan_job_template_list', kwargs={'pk': obj.pk}), ad_hoc_commands = self.reverse('api:inventory_ad_hoc_commands_list', kwargs={'pk': obj.pk}), access_list = self.reverse('api:inventory_access_list', kwargs={'pk': obj.pk}), object_roles = self.reverse('api:inventory_object_roles_list', kwargs={'pk': obj.pk}), @@ -2336,8 +2335,7 @@ class JobOptionsSerializer(LabelsListMixin, BaseSerializer): if 'project' in self.fields and 'playbook' in self.fields: project = attrs.get('project', self.instance and self.instance.project or None) playbook = attrs.get('playbook', self.instance and self.instance.playbook or '') - job_type = attrs.get('job_type', self.instance and self.instance.job_type or None) - if not project and job_type != PERM_INVENTORY_SCAN: + if not project: raise serializers.ValidationError({'project': _('This field is required.')}) if project and project.scm_type and playbook and force_text(playbook) not in project.playbook_files: raise serializers.ValidationError({'playbook': _('Playbook not found for project.')}) @@ -2408,26 +2406,18 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO def get_field_from_model_or_attrs(fd): return attrs.get(fd, self.instance and getattr(self.instance, fd) or None) - survey_enabled = get_field_from_model_or_attrs('survey_enabled') - job_type = get_field_from_model_or_attrs('job_type') inventory = get_field_from_model_or_attrs('inventory') credential = get_field_from_model_or_attrs('credential') project = get_field_from_model_or_attrs('project') prompting_error_message = _("Must either set a default value or ask to prompt on launch.") - if job_type == "scan": - if inventory is None or attrs.get('ask_inventory_on_launch', False): - raise serializers.ValidationError({'inventory': _('Scan jobs must be assigned a fixed inventory.')}) - elif project is None: + if project is None: raise serializers.ValidationError({'project': _("Job types 'run' and 'check' must have assigned a project.")}) elif credential is None and not get_field_from_model_or_attrs('ask_credential_on_launch'): raise serializers.ValidationError({'credential': prompting_error_message}) elif inventory is None and not get_field_from_model_or_attrs('ask_inventory_on_launch'): raise serializers.ValidationError({'inventory': prompting_error_message}) - if survey_enabled and job_type == PERM_INVENTORY_SCAN: - raise serializers.ValidationError({'survey_enabled': _('Survey Enabled cannot be used with scan jobs.')}) - return super(JobTemplateSerializer, self).validate(attrs) def validate_extra_vars(self, value): @@ -2570,7 +2560,7 @@ class JobRelaunchSerializer(JobSerializer): obj = self.context.get('obj') if not obj.credential: raise serializers.ValidationError(dict(credential=[_("Credential not found or deleted.")])) - if obj.job_type != PERM_INVENTORY_SCAN and obj.project is None: + if obj.project is None: raise serializers.ValidationError(dict(errors=[_("Job Template Project is missing or undefined.")])) if obj.inventory is None: raise serializers.ValidationError(dict(errors=[_("Job Template Inventory is missing or undefined.")])) diff --git a/awx/api/urls.py b/awx/api/urls.py index c42d059b82..a21d1df9ab 100644 --- a/awx/api/urls.py +++ b/awx/api/urls.py @@ -97,7 +97,6 @@ inventory_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/update_inventory_sources/$', 'inventory_inventory_sources_update'), url(r'^(?P[0-9]+)/activity_stream/$', 'inventory_activity_stream_list'), url(r'^(?P[0-9]+)/job_templates/$', 'inventory_job_template_list'), - url(r'^(?P[0-9]+)/scan_job_templates/$', 'inventory_scan_job_template_list'), url(r'^(?P[0-9]+)/ad_hoc_commands/$', 'inventory_ad_hoc_commands_list'), url(r'^(?P[0-9]+)/access_list/$', 'inventory_access_list'), url(r'^(?P[0-9]+)/object_roles/$', 'inventory_object_roles_list'), diff --git a/awx/api/views.py b/awx/api/views.py index fb5e54565b..3b6d6ee08c 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -858,10 +858,8 @@ class OrganizationDetail(RetrieveUpdateDestroyAPIView): organization__id=org_id).count() org_counts['projects'] = Project.accessible_objects(**access_kwargs).filter( organization__id=org_id).count() - org_counts['job_templates'] = JobTemplate.accessible_objects(**access_kwargs).exclude( - job_type='scan').filter(project__organization__id=org_id).count() - org_counts['job_templates'] += JobTemplate.accessible_objects(**access_kwargs).filter( - job_type='scan').filter(inventory__organization__id=org_id).count() + org_counts['job_templates'] = JobTemplate.accessible_objects(**access_kwargs).filter( + project__organization__id=org_id).count() full_context['related_field_counts'] = {} full_context['related_field_counts'][org_id] = org_counts @@ -1907,21 +1905,6 @@ class InventoryJobTemplateList(SubListAPIView): return qs.filter(inventory=parent) -class InventoryScanJobTemplateList(SubListAPIView): - - model = JobTemplate - serializer_class = JobTemplateSerializer - parent_model = Inventory - relationship = 'jobtemplates' - new_in_220 = True - - def get_queryset(self): - parent = self.get_parent_object() - self.check_parent_access(parent) - qs = self.request.user.get_queryset(self.model) - return qs.filter(job_type=PERM_INVENTORY_SCAN, inventory=parent) - - class HostList(ListCreateAPIView): always_allow_superuser = False diff --git a/awx/main/access.py b/awx/main/access.py index 0582669173..417421e5f4 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1141,9 +1141,6 @@ class JobTemplateAccess(BaseAccess): # if reference_obj is provided, determine if it can be copied reference_obj = data.get('reference_obj', None) - if 'job_type' in data and data['job_type'] == PERM_INVENTORY_SCAN: - self.check_license(feature='system_tracking') - if 'survey_enabled' in data and data['survey_enabled']: self.check_license(feature='surveys') @@ -1175,11 +1172,6 @@ class JobTemplateAccess(BaseAccess): return False project = get_value(Project, 'project') - if 'job_type' in data and data['job_type'] == PERM_INVENTORY_SCAN: - if not inventory: - return False - elif not project: - return True # If the user has admin access to the project (as an org admin), should # be able to proceed without additional checks. if project: @@ -1194,8 +1186,6 @@ class JobTemplateAccess(BaseAccess): # Check license. if validate_license: self.check_license() - if obj.job_type == PERM_INVENTORY_SCAN: - self.check_license(feature='system_tracking') if obj.survey_enabled: self.check_license(feature='surveys') if Instance.objects.active_count() > 1: @@ -1205,12 +1195,6 @@ class JobTemplateAccess(BaseAccess): if self.user.is_superuser: return True - if obj.job_type == PERM_INVENTORY_SCAN: - # Scan job with default project, must have JT execute or be org admin - if obj.project is None and obj.inventory: - return (self.user in obj.execute_role or - self.user in obj.inventory.organization.admin_role) - return self.user in obj.execute_role def can_change(self, obj, data): @@ -1221,9 +1205,6 @@ class JobTemplateAccess(BaseAccess): data = dict(data) if self.changes_are_non_sensitive(obj, data): - if 'job_type' in data and obj.job_type != data['job_type'] and data['job_type'] == PERM_INVENTORY_SCAN: - self.check_license(feature='system_tracking') - if 'survey_enabled' in data and obj.survey_enabled != data['survey_enabled'] and data['survey_enabled']: self.check_license(feature='surveys') return True diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index a071cb48c3..3aaae69fac 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -308,10 +308,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour validation_errors['credential'] = [_("Job Template must provide 'credential' or allow prompting for it."),] # Job type dependent checks - if self.job_type == PERM_INVENTORY_SCAN: - if self.inventory is None or self.ask_inventory_on_launch: - validation_errors['inventory'] = [_("Scan jobs must be assigned a fixed inventory."),] - elif self.project is None: + if self.project is None: resources_needed_to_start.append('project') validation_errors['project'] = [_("Job types 'run' and 'check' must have assigned a project."),] @@ -407,12 +404,8 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour """ errors = {} if 'job_type' in data and self.ask_job_type_on_launch: - if ((self.job_type == PERM_INVENTORY_SCAN and not data['job_type'] == PERM_INVENTORY_SCAN) or - (data['job_type'] == PERM_INVENTORY_SCAN and not self.job_type == PERM_INVENTORY_SCAN)): + if data['job_type'] == PERM_INVENTORY_SCAN and not self.job_type == PERM_INVENTORY_SCAN: errors['job_type'] = _('Cannot override job_type to or from a scan job.') - if (self.job_type == PERM_INVENTORY_SCAN and ('inventory' in data) and self.ask_inventory_on_launch and - self.inventory != data['inventory']): - errors['inventory'] = _('Inventory cannot be changed at runtime for scan jobs.') return errors @property @@ -647,8 +640,6 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin): return data def _resources_sufficient_for_launch(self): - if self.job_type == PERM_INVENTORY_SCAN: - return self.inventory_id is not None return not (self.inventory_id is None or self.project_id is None) def display_artifacts(self): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 68172a3085..8e2f5d7423 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1057,18 +1057,13 @@ class RunJob(BaseTask): args.extend(['-e', json.dumps(extra_vars)]) # Add path to playbook (relative to project.local_path). - if job.project is None and job.job_type == PERM_INVENTORY_SCAN: - args.append("scan_facts.yml") - else: - args.append(job.playbook) + args.append(job.playbook) return args def build_safe_args(self, job, **kwargs): return self.build_args(job, display=True, **kwargs) def build_cwd(self, job, **kwargs): - if job.project is None and job.job_type == PERM_INVENTORY_SCAN: - return self.get_path_to('..', 'playbooks') cwd = job.project.get_project_path() if not cwd: root = settings.PROJECTS_ROOT diff --git a/awx/main/tests/functional/api/test_job_runtime_params.py b/awx/main/tests/functional/api/test_job_runtime_params.py index 69c82e5d28..3626ae89a7 100644 --- a/awx/main/tests/functional/api/test_job_runtime_params.py +++ b/awx/main/tests/functional/api/test_job_runtime_params.py @@ -76,14 +76,6 @@ def job_template_prompts_null(project): ) -@pytest.fixture -def bad_scan_JT(job_template_prompts): - job_template = job_template_prompts(True) - job_template.job_type = 'scan' - job_template.save() - return job_template - - # End of setup, tests start here @pytest.mark.django_db @pytest.mark.job_runtime_vars @@ -259,18 +251,6 @@ def test_job_block_scan_job_type_change(job_template_prompts, post, admin_user): assert 'job_type' in response.data -@pytest.mark.django_db -@pytest.mark.job_runtime_vars -def test_job_block_scan_job_inv_change(mocker, bad_scan_JT, runtime_data, post, admin_user): - # Assure that giving a new inventory for a scan job blocks the launch - with mocker.patch('awx.main.access.BaseAccess.check_license'): - response = post(reverse('api:job_template_launch', kwargs={'pk': bad_scan_JT.pk}), - dict(inventory=runtime_data['inventory']), admin_user, - expect=400) - - assert 'inventory' in response.data - - @pytest.mark.django_db def test_job_launch_JT_with_validation(machine_credential, deploy_jobtemplate): deploy_jobtemplate.extra_vars = '{"job_template_var": 3}' diff --git a/awx/main/tests/functional/api/test_job_template.py b/awx/main/tests/functional/api/test_job_template.py index 61892ffa82..3ef9775c13 100644 --- a/awx/main/tests/functional/api/test_job_template.py +++ b/awx/main/tests/functional/api/test_job_template.py @@ -1,7 +1,7 @@ import pytest # AWX -from awx.api.serializers import JobTemplateSerializer, JobLaunchSerializer +from awx.api.serializers import JobTemplateSerializer from awx.api.versioning import reverse from awx.main.models.jobs import Job from awx.main.migrations import _save_password_keys as save_password_keys @@ -387,7 +387,6 @@ def test_edit_nonsenstive(patch, job_template_factory, alice): 'ask_inventory_on_launch':True, 'ask_credential_on_launch': True, }, alice, expect=200) - print(res.data) assert res.data['name'] == 'updated' @@ -430,48 +429,6 @@ def test_jt_admin_copy_edit_functional(jt_copy_edit, rando, get, post): assert post_response.status_code == 403 -@pytest.mark.django_db -def test_scan_jt_no_inventory(job_template_factory): - # A user should be able to create a scan job without a project, but an inventory is required - objects = job_template_factory('jt', - credential='c', - job_type="scan", - project='p', - inventory='i', - organization='o') - serializer = JobTemplateSerializer(data={"name": "Test", "job_type": "scan", - "project": None, "inventory": objects.inventory.pk}) - assert serializer.is_valid() - serializer = JobTemplateSerializer(data={"name": "Test", "job_type": "scan", - "project": None, "inventory": None}) - assert not serializer.is_valid() - assert "inventory" in serializer.errors - serializer = JobTemplateSerializer(data={"name": "Test", "job_type": "scan", - "project": None, "inventory": None, - "ask_inventory_on_launch": True}) - assert not serializer.is_valid() - assert "inventory" in serializer.errors - - # A user shouldn't be able to launch a scan job template which is missing an inventory - obj_jt = objects.job_template - obj_jt.inventory = None - serializer = JobLaunchSerializer(instance=obj_jt, - context={'obj': obj_jt, - "data": {}}, - data={}) - assert not serializer.is_valid() - assert 'inventory' in serializer.errors - - -@pytest.mark.django_db -def test_scan_jt_surveys(inventory): - serializer = JobTemplateSerializer(data={"name": "Test", "job_type": "scan", - "project": None, "inventory": inventory.pk, - "survey_enabled": True}) - assert not serializer.is_valid() - assert "survey_enabled" in serializer.errors - - @pytest.mark.django_db def test_launch_with_pending_deletion_inventory(get, post, organization_factory, job_template_factory, machine_credential, @@ -641,9 +598,6 @@ def test_jt_without_project(inventory): serializer = JobTemplateSerializer(data=data) assert not serializer.is_valid() assert "project" in serializer.errors - data["job_type"] = "scan" - serializer = JobTemplateSerializer(data=data) - assert serializer.is_valid() @pytest.mark.django_db diff --git a/awx/main/tests/functional/api/test_organization_counts.py b/awx/main/tests/functional/api/test_organization_counts.py index 452db26dfd..9c4f536b09 100644 --- a/awx/main/tests/functional/api/test_organization_counts.py +++ b/awx/main/tests/functional/api/test_organization_counts.py @@ -163,12 +163,7 @@ def test_two_organizations(resourced_organization, organizations, user, get): @pytest.mark.django_db def test_scan_JT_counted(resourced_organization, user, get): admin_user = user('admin', True) - # Add a scan job template to the org - resourced_organization.projects.all()[0].jobtemplates.create( - job_type='scan', inventory=resourced_organization.inventories.all()[0], - name='scan-job-template') counts_dict = COUNTS_PRIMES - counts_dict['job_templates'] += 1 # Test list view list_response = get(reverse('api:organization_list'), admin_user) @@ -184,7 +179,7 @@ def test_scan_JT_counted(resourced_organization, user, get): @pytest.mark.django_db def test_JT_not_double_counted(resourced_organization, user, get): admin_user = user('admin', True) - # Add a scan job template to the org + # Add a run job template to the org resourced_organization.projects.all()[0].jobtemplates.create( job_type='run', inventory=resourced_organization.inventories.all()[0], diff --git a/awx/main/tests/unit/models/test_workflow_unit.py b/awx/main/tests/unit/models/test_workflow_unit.py index adee9b7ae7..129f18d78c 100644 --- a/awx/main/tests/unit/models/test_workflow_unit.py +++ b/awx/main/tests/unit/models/test_workflow_unit.py @@ -242,11 +242,3 @@ class TestWorkflowWarnings: assert 'credential' in job_node_with_prompts.get_prompts_warnings()['ignored'] assert len(job_node_with_prompts.get_prompts_warnings()['ignored']) == 2 - def test_warn_scan_errors_node_prompts(self, job_node_with_prompts): - job_node_with_prompts.unified_job_template.job_type = 'scan' - job_node_with_prompts.char_prompts['job_type'] = 'run' - job_node_with_prompts.inventory = Inventory(name='different-inventory', pk=23) - assert 'ignored' in job_node_with_prompts.get_prompts_warnings() - assert 'job_type' in job_node_with_prompts.get_prompts_warnings()['ignored'] - assert 'inventory' in job_node_with_prompts.get_prompts_warnings()['ignored'] - assert len(job_node_with_prompts.get_prompts_warnings()['ignored']) == 2 From 88b250b2f122d5fa2608076e1d8a3aebbc88d32e Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 22 Jun 2017 14:22:52 -0400 Subject: [PATCH 2/5] migrate scan jobs to use fact caching instead --- .../migrations/0039_v320_data_migrations.py | 2 + awx/main/migrations/_scan_jobs.py | 64 +++++++++++ .../functional/test_scan_jobs_migration.py | 100 ++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 awx/main/migrations/_scan_jobs.py create mode 100644 awx/main/tests/functional/test_scan_jobs_migration.py diff --git a/awx/main/migrations/0039_v320_data_migrations.py b/awx/main/migrations/0039_v320_data_migrations.py index fa3d6cc768..b3cc6fdbcb 100644 --- a/awx/main/migrations/0039_v320_data_migrations.py +++ b/awx/main/migrations/0039_v320_data_migrations.py @@ -9,6 +9,7 @@ from django.db import migrations from awx.main.migrations import _inventory_source as invsrc from awx.main.migrations import _migration_utils as migration_utils from awx.main.migrations import _reencrypt +from awx.main.migrations import _scan_jobs class Migration(migrations.Migration): @@ -24,4 +25,5 @@ class Migration(migrations.Migration): migrations.RunPython(invsrc.remove_inventory_source_with_no_inventory_link), migrations.RunPython(invsrc.rename_inventory_sources), migrations.RunPython(_reencrypt.replace_aesecb_fernet), + migrations.RunPython(_scan_jobs.migrate_scan_job_templates), ] diff --git a/awx/main/migrations/_scan_jobs.py b/awx/main/migrations/_scan_jobs.py new file mode 100644 index 0000000000..fc6c9d3a3f --- /dev/null +++ b/awx/main/migrations/_scan_jobs.py @@ -0,0 +1,64 @@ +import logging + +from awx.main.models.base import PERM_INVENTORY_SCAN, PERM_INVENTORY_DEPLOY + +logger = logging.getLogger('awx.main.migrations') + + +def _create_fact_scan_project(Project, org): + name = "Tower Fact Scan - {}".format(org.name if org else "No Organization") + return Project.objects.create(name=name, + scm_url='https://github.com/ansible/tower-fact-modules', + organization=org) + + +def _create_fact_scan_projects(Project, orgs): + return {org.id : _create_fact_scan_project(Project, org) for org in orgs} + + +def _get_tower_scan_job_templates(JobTemplate): + return JobTemplate.objects.filter(job_type=PERM_INVENTORY_SCAN, project__isnull=True) \ + .prefetch_related('inventory__organization') + + +def _get_orgs(Organization, job_template_ids): + return Organization.objects.filter(inventories__jobtemplates__in=job_template_ids).distinct() + + +def _migrate_scan_job_templates(apps): + Organization = apps.get_model('main', 'Organization') + Project = apps.get_model('main', 'Project') + JobTemplate = apps.get_model('main', 'JobTemplate') + + project_no_org = None + + # A scan job template with a custom project will retain the custom project. + JobTemplate.objects.filter(job_type=PERM_INVENTORY_SCAN, project__isnull=False).update(use_fact_cache=True, job_type=PERM_INVENTORY_DEPLOY) + + # Scan jobs templates using Tower's default scan playbook will now point at + # the same playbook but in a github repo. + jts = _get_tower_scan_job_templates(JobTemplate) + if jts.count() == 0: + return + + orgs = _get_orgs(Organization, jts.values_list('id')) + if orgs.count() == 0: + return + + org_proj_map = _create_fact_scan_projects(Project, orgs) + for jt in jts: + if jt.inventory and jt.inventory.organization: + jt.project = org_proj_map[jt.inventory.organization.id] + # Job Templates without an Organization; through related Inventory + else: + # TODO: Create a project without an org and connect + if not project_no_org: + project_no_org = _create_fact_scan_project(Project, None) + jt.project = project_no_org + jt.job_type = PERM_INVENTORY_DEPLOY + jt.use_fact_cache = True + jt.save() + + +def migrate_scan_job_templates(apps, schema_editor): + _migrate_scan_job_templates(apps) diff --git a/awx/main/tests/functional/test_scan_jobs_migration.py b/awx/main/tests/functional/test_scan_jobs_migration.py new file mode 100644 index 0000000000..f7bc08364a --- /dev/null +++ b/awx/main/tests/functional/test_scan_jobs_migration.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Ansible, Inc. +# All Rights Reserved. +import pytest + +from django.apps import apps + +from awx.main.models.base import PERM_INVENTORY_SCAN, PERM_INVENTORY_DEPLOY +from awx.main.models import ( + JobTemplate, + Project, + Inventory, + Organization, +) + +from awx.main.migrations._scan_jobs import _migrate_scan_job_templates + + +@pytest.fixture +def organizations(): + return [Organization.objects.create(name="org-{}".format(x)) for x in range(3)] + + +@pytest.fixture +def inventories(organizations): + return [Inventory.objects.create(name="inv-{}".format(x), + organization=organizations[x]) for x in range(3)] + + +@pytest.fixture +def job_templates_scan(inventories): + return [JobTemplate.objects.create(name="jt-scan-{}".format(x), + job_type=PERM_INVENTORY_SCAN, + inventory=inventories[x]) for x in range(3)] + + +@pytest.fixture +def job_templates_deploy(inventories): + return [JobTemplate.objects.create(name="jt-deploy-{}".format(x), + job_type=PERM_INVENTORY_DEPLOY, + inventory=inventories[x]) for x in range(3)] + + +@pytest.fixture +def project_custom(organizations): + return Project.objects.create(name="proj-scan_custom", + scm_url='https://giggity.com', + organization=organizations[0]) + + +@pytest.fixture +def job_templates_custom_scan_project(project_custom): + return [JobTemplate.objects.create(name="jt-scan-custom-{}".format(x), + project=project_custom, + job_type=PERM_INVENTORY_SCAN) for x in range(3)] + + +@pytest.fixture +def job_template_scan_no_org(): + return JobTemplate.objects.create(name="jt-scan-no-org", + job_type=PERM_INVENTORY_SCAN) + + +@pytest.mark.django_db +def test_scan_jobs_migration(job_templates_scan, job_templates_deploy, job_templates_custom_scan_project, project_custom, job_template_scan_no_org): + _migrate_scan_job_templates(apps) + + # Ensure there are no scan job templates after the migration + assert 0 == JobTemplate.objects.filter(job_type=PERM_INVENTORY_SCAN).count() + + # Ensure special No Organization proj created + # And No Organization project is associated with correct jt + proj = Project.objects.get(name="Tower Fact Scan - No Organization") + assert proj.id == JobTemplate.objects.get(id=job_template_scan_no_org.id).project.id + + # Ensure per-org projects were created + projs = Project.objects.filter(name__startswith="Tower Fact Scan") + assert projs.count() == 4 + + # Ensure scan job templates with Tower project are migrated + for i, jt_old in enumerate(job_templates_scan): + jt = JobTemplate.objects.get(id=jt_old.id) + assert PERM_INVENTORY_DEPLOY == jt.job_type + assert jt.use_fact_cache is True + assert projs[i] == jt.project + + # Ensure scan job templates with custom projects are migrated + for jt_old in job_templates_custom_scan_project: + jt = JobTemplate.objects.get(id=jt_old.id) + assert PERM_INVENTORY_DEPLOY == jt.job_type + assert jt.use_fact_cache is True + assert project_custom == jt.project + + # Ensure other job template aren't touched + for jt_old in job_templates_deploy: + jt = JobTemplate.objects.get(id=jt_old.id) + assert PERM_INVENTORY_DEPLOY == jt.job_type + assert jt.project is None + From 4b84cec1e813a09e0c51bf9c8aa01fbf907fd0b8 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Fri, 30 Jun 2017 10:17:05 -0400 Subject: [PATCH 3/5] do not launch proj update on migrate --- awx/main/migrations/_scan_jobs.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/awx/main/migrations/_scan_jobs.py b/awx/main/migrations/_scan_jobs.py index fc6c9d3a3f..872ef2c2d0 100644 --- a/awx/main/migrations/_scan_jobs.py +++ b/awx/main/migrations/_scan_jobs.py @@ -7,9 +7,14 @@ logger = logging.getLogger('awx.main.migrations') def _create_fact_scan_project(Project, org): name = "Tower Fact Scan - {}".format(org.name if org else "No Organization") - return Project.objects.create(name=name, - scm_url='https://github.com/ansible/tower-fact-modules', - organization=org) + proj = Project(name=name, + scm_url='https://github.com/ansible/tower-fact-modules', + scm_type='git', + scm_update_on_launch=True, + scm_update_cache_timeout=86400, + organization=org) + proj.save(skip_update=True) + return proj def _create_fact_scan_projects(Project, orgs): @@ -51,7 +56,6 @@ def _migrate_scan_job_templates(apps): jt.project = org_proj_map[jt.inventory.organization.id] # Job Templates without an Organization; through related Inventory else: - # TODO: Create a project without an org and connect if not project_no_org: project_no_org = _create_fact_scan_project(Project, None) jt.project = project_no_org From 3d4f8b00744bbceed256465535cf03ec34c2da23 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 5 Jul 2017 10:26:49 -0400 Subject: [PATCH 4/5] support clear_facts --- awx/plugins/fact_caching/tower.py | 3 +-- docs/fact_cache.md | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/awx/plugins/fact_caching/tower.py b/awx/plugins/fact_caching/tower.py index c588ac848f..a6f8852362 100755 --- a/awx/plugins/fact_caching/tower.py +++ b/awx/plugins/fact_caching/tower.py @@ -104,8 +104,7 @@ class CacheModule(BaseCacheModule): return False def delete(self, key): - self.mc.delete(self.translate_host_key(key)) - self.mc.delete(self.translate_modified_key(key)) + self.set(key, {}) def flush(self): host_names = self.mc.get(self.host_names_key) diff --git a/docs/fact_cache.md b/docs/fact_cache.md index 8bea9b6189..4026eff334 100644 --- a/docs/fact_cache.md +++ b/docs/fact_cache.md @@ -17,3 +17,7 @@ Tower will always inject the host `ansible_facts` into memcached. The Ansible To ## Tower Fact Logging New and changed facts will be logged via Tower's logging facility. Specifically, to the `system_tracking` namespace or logger. The logging payload will include the fields: `host_name`, `inventory_id`, and `ansible_facts`. Where `ansible_facts` is a dictionary of all ansible facts for `host_name` in Tower Inventory `inventory_id`. +## Integration Testing +* ensure `clear_facts` set's `hosts//ansible_facts` to `{}` +* ensure that `gather_facts: False` does NOT result in clearing existing facts +* ensure that the when a host fact timeout is reached, that the facts are not used from the cache From 1331865749b8d79f9bcd9717951655c9b8f2adab Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 5 Jul 2017 10:53:09 -0400 Subject: [PATCH 5/5] take into account memcached key restrictions * Keys can't contain spaces or control characters --- awx/main/models/jobs.py | 5 +++-- awx/main/tests/unit/models/test_jobs.py | 11 ++++++----- awx/plugins/fact_caching/tower.py | 5 +++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 3aaae69fac..f469809647 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -8,6 +8,7 @@ import hmac import logging import time import json +import base64 from urlparse import urljoin # Django @@ -705,10 +706,10 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin): return '{}'.format(self.inventory.id) def memcached_fact_host_key(self, host_name): - return '{}-{}'.format(self.inventory.id, host_name) + return '{}-{}'.format(self.inventory.id, base64.b64encode(host_name)) def memcached_fact_modified_key(self, host_name): - return '{}-{}-modified'.format(self.inventory.id, host_name) + return '{}-{}-modified'.format(self.inventory.id, base64.b64encode(host_name)) def _get_inventory_hosts(self, only=['name', 'ansible_facts', 'modified',]): return self.inventory.hosts.only(*only) diff --git a/awx/main/tests/unit/models/test_jobs.py b/awx/main/tests/unit/models/test_jobs.py index 9b2e3a60d3..0e9113cf7a 100644 --- a/awx/main/tests/unit/models/test_jobs.py +++ b/awx/main/tests/unit/models/test_jobs.py @@ -8,6 +8,7 @@ from awx.main.models import ( import datetime import json +import base64 from dateutil.tz import tzutc @@ -89,8 +90,8 @@ def test_start_job_fact_cache(hosts, job, inventory, mocker): job._get_memcache_connection().set.assert_any_call('5', [h.name for h in hosts]) for host in hosts: - job._get_memcache_connection().set.assert_any_call('{}-{}'.format(5, host.name), json.dumps(host.ansible_facts)) - job._get_memcache_connection().set.assert_any_call('{}-{}-modified'.format(5, host.name), host.ansible_facts_modified.isoformat()) + job._get_memcache_connection().set.assert_any_call('{}-{}'.format(5, base64.b64encode(host.name)), json.dumps(host.ansible_facts)) + job._get_memcache_connection().set.assert_any_call('{}-{}-modified'.format(5, base64.b64encode(host.name)), host.ansible_facts_modified.isoformat()) def test_start_job_fact_cache_existing_host(hosts, hosts2, job, job2, inventory, mocker): @@ -98,15 +99,15 @@ def test_start_job_fact_cache_existing_host(hosts, hosts2, job, job2, inventory, job.start_job_fact_cache() for host in hosts: - job._get_memcache_connection().set.assert_any_call('{}-{}'.format(5, host.name), json.dumps(host.ansible_facts)) - job._get_memcache_connection().set.assert_any_call('{}-{}-modified'.format(5, host.name), host.ansible_facts_modified.isoformat()) + job._get_memcache_connection().set.assert_any_call('{}-{}'.format(5, base64.b64encode(host.name)), json.dumps(host.ansible_facts)) + job._get_memcache_connection().set.assert_any_call('{}-{}-modified'.format(5, base64.b64encode(host.name)), host.ansible_facts_modified.isoformat()) job._get_memcache_connection().set.reset_mock() job2.start_job_fact_cache() # Ensure hosts2 ansible_facts didn't overwrite hosts ansible_facts - ansible_facts_cached = job._get_memcache_connection().get('{}-{}'.format(5, hosts2[0].name)) + ansible_facts_cached = job._get_memcache_connection().get('{}-{}'.format(5, base64.b64encode(hosts2[0].name))) assert ansible_facts_cached == json.dumps(hosts[1].ansible_facts) diff --git a/awx/plugins/fact_caching/tower.py b/awx/plugins/fact_caching/tower.py index a6f8852362..86624f75da 100755 --- a/awx/plugins/fact_caching/tower.py +++ b/awx/plugins/fact_caching/tower.py @@ -33,6 +33,7 @@ import os import memcache import json import datetime +import base64 from dateutil import parser from dateutil.tz import tzutc @@ -56,10 +57,10 @@ class CacheModule(BaseCacheModule): return '{}'.format(self._inventory_id) def translate_host_key(self, host_name): - return '{}-{}'.format(self._inventory_id, host_name) + return '{}-{}'.format(self._inventory_id, base64.b64encode(host_name)) def translate_modified_key(self, host_name): - return '{}-{}-modified'.format(self._inventory_id, host_name) + return '{}-{}-modified'.format(self._inventory_id, base64.b64encode(host_name)) def get(self, key): host_key = self.translate_host_key(key)