From 88b250b2f122d5fa2608076e1d8a3aebbc88d32e Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 22 Jun 2017 14:22:52 -0400 Subject: [PATCH] migrate scan jobs to use fact caching instead --- .../migrations/0039_v320_data_migrations.py | 2 + awx/main/migrations/_scan_jobs.py | 64 +++++++++++ .../functional/test_scan_jobs_migration.py | 100 ++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 awx/main/migrations/_scan_jobs.py create mode 100644 awx/main/tests/functional/test_scan_jobs_migration.py diff --git a/awx/main/migrations/0039_v320_data_migrations.py b/awx/main/migrations/0039_v320_data_migrations.py index fa3d6cc768..b3cc6fdbcb 100644 --- a/awx/main/migrations/0039_v320_data_migrations.py +++ b/awx/main/migrations/0039_v320_data_migrations.py @@ -9,6 +9,7 @@ from django.db import migrations from awx.main.migrations import _inventory_source as invsrc from awx.main.migrations import _migration_utils as migration_utils from awx.main.migrations import _reencrypt +from awx.main.migrations import _scan_jobs class Migration(migrations.Migration): @@ -24,4 +25,5 @@ class Migration(migrations.Migration): migrations.RunPython(invsrc.remove_inventory_source_with_no_inventory_link), migrations.RunPython(invsrc.rename_inventory_sources), migrations.RunPython(_reencrypt.replace_aesecb_fernet), + migrations.RunPython(_scan_jobs.migrate_scan_job_templates), ] diff --git a/awx/main/migrations/_scan_jobs.py b/awx/main/migrations/_scan_jobs.py new file mode 100644 index 0000000000..fc6c9d3a3f --- /dev/null +++ b/awx/main/migrations/_scan_jobs.py @@ -0,0 +1,64 @@ +import logging + +from awx.main.models.base import PERM_INVENTORY_SCAN, PERM_INVENTORY_DEPLOY + +logger = logging.getLogger('awx.main.migrations') + + +def _create_fact_scan_project(Project, org): + name = "Tower Fact Scan - {}".format(org.name if org else "No Organization") + return Project.objects.create(name=name, + scm_url='https://github.com/ansible/tower-fact-modules', + organization=org) + + +def _create_fact_scan_projects(Project, orgs): + return {org.id : _create_fact_scan_project(Project, org) for org in orgs} + + +def _get_tower_scan_job_templates(JobTemplate): + return JobTemplate.objects.filter(job_type=PERM_INVENTORY_SCAN, project__isnull=True) \ + .prefetch_related('inventory__organization') + + +def _get_orgs(Organization, job_template_ids): + return Organization.objects.filter(inventories__jobtemplates__in=job_template_ids).distinct() + + +def _migrate_scan_job_templates(apps): + Organization = apps.get_model('main', 'Organization') + Project = apps.get_model('main', 'Project') + JobTemplate = apps.get_model('main', 'JobTemplate') + + project_no_org = None + + # A scan job template with a custom project will retain the custom project. + JobTemplate.objects.filter(job_type=PERM_INVENTORY_SCAN, project__isnull=False).update(use_fact_cache=True, job_type=PERM_INVENTORY_DEPLOY) + + # Scan jobs templates using Tower's default scan playbook will now point at + # the same playbook but in a github repo. + jts = _get_tower_scan_job_templates(JobTemplate) + if jts.count() == 0: + return + + orgs = _get_orgs(Organization, jts.values_list('id')) + if orgs.count() == 0: + return + + org_proj_map = _create_fact_scan_projects(Project, orgs) + for jt in jts: + if jt.inventory and jt.inventory.organization: + jt.project = org_proj_map[jt.inventory.organization.id] + # Job Templates without an Organization; through related Inventory + else: + # TODO: Create a project without an org and connect + if not project_no_org: + project_no_org = _create_fact_scan_project(Project, None) + jt.project = project_no_org + jt.job_type = PERM_INVENTORY_DEPLOY + jt.use_fact_cache = True + jt.save() + + +def migrate_scan_job_templates(apps, schema_editor): + _migrate_scan_job_templates(apps) diff --git a/awx/main/tests/functional/test_scan_jobs_migration.py b/awx/main/tests/functional/test_scan_jobs_migration.py new file mode 100644 index 0000000000..f7bc08364a --- /dev/null +++ b/awx/main/tests/functional/test_scan_jobs_migration.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Ansible, Inc. +# All Rights Reserved. +import pytest + +from django.apps import apps + +from awx.main.models.base import PERM_INVENTORY_SCAN, PERM_INVENTORY_DEPLOY +from awx.main.models import ( + JobTemplate, + Project, + Inventory, + Organization, +) + +from awx.main.migrations._scan_jobs import _migrate_scan_job_templates + + +@pytest.fixture +def organizations(): + return [Organization.objects.create(name="org-{}".format(x)) for x in range(3)] + + +@pytest.fixture +def inventories(organizations): + return [Inventory.objects.create(name="inv-{}".format(x), + organization=organizations[x]) for x in range(3)] + + +@pytest.fixture +def job_templates_scan(inventories): + return [JobTemplate.objects.create(name="jt-scan-{}".format(x), + job_type=PERM_INVENTORY_SCAN, + inventory=inventories[x]) for x in range(3)] + + +@pytest.fixture +def job_templates_deploy(inventories): + return [JobTemplate.objects.create(name="jt-deploy-{}".format(x), + job_type=PERM_INVENTORY_DEPLOY, + inventory=inventories[x]) for x in range(3)] + + +@pytest.fixture +def project_custom(organizations): + return Project.objects.create(name="proj-scan_custom", + scm_url='https://giggity.com', + organization=organizations[0]) + + +@pytest.fixture +def job_templates_custom_scan_project(project_custom): + return [JobTemplate.objects.create(name="jt-scan-custom-{}".format(x), + project=project_custom, + job_type=PERM_INVENTORY_SCAN) for x in range(3)] + + +@pytest.fixture +def job_template_scan_no_org(): + return JobTemplate.objects.create(name="jt-scan-no-org", + job_type=PERM_INVENTORY_SCAN) + + +@pytest.mark.django_db +def test_scan_jobs_migration(job_templates_scan, job_templates_deploy, job_templates_custom_scan_project, project_custom, job_template_scan_no_org): + _migrate_scan_job_templates(apps) + + # Ensure there are no scan job templates after the migration + assert 0 == JobTemplate.objects.filter(job_type=PERM_INVENTORY_SCAN).count() + + # Ensure special No Organization proj created + # And No Organization project is associated with correct jt + proj = Project.objects.get(name="Tower Fact Scan - No Organization") + assert proj.id == JobTemplate.objects.get(id=job_template_scan_no_org.id).project.id + + # Ensure per-org projects were created + projs = Project.objects.filter(name__startswith="Tower Fact Scan") + assert projs.count() == 4 + + # Ensure scan job templates with Tower project are migrated + for i, jt_old in enumerate(job_templates_scan): + jt = JobTemplate.objects.get(id=jt_old.id) + assert PERM_INVENTORY_DEPLOY == jt.job_type + assert jt.use_fact_cache is True + assert projs[i] == jt.project + + # Ensure scan job templates with custom projects are migrated + for jt_old in job_templates_custom_scan_project: + jt = JobTemplate.objects.get(id=jt_old.id) + assert PERM_INVENTORY_DEPLOY == jt.job_type + assert jt.use_fact_cache is True + assert project_custom == jt.project + + # Ensure other job template aren't touched + for jt_old in job_templates_deploy: + jt = JobTemplate.objects.get(id=jt_old.id) + assert PERM_INVENTORY_DEPLOY == jt.job_type + assert jt.project is None +