migrate scan jobs to use fact caching instead

awx/main/migrations/0039_v320_data_migrations.py

@@ -9,6 +9,7 @@ from django.db import migrations
 from awx.main.migrations import _inventory_source as invsrc
 from awx.main.migrations import _migration_utils as migration_utils
 from awx.main.migrations import _reencrypt
+from awx.main.migrations import _scan_jobs
 
 
 class Migration(migrations.Migration):
@@ -24,4 +25,5 @@ class Migration(migrations.Migration):
         migrations.RunPython(invsrc.remove_inventory_source_with_no_inventory_link),
         migrations.RunPython(invsrc.rename_inventory_sources),
         migrations.RunPython(_reencrypt.replace_aesecb_fernet),
+        migrations.RunPython(_scan_jobs.migrate_scan_job_templates),
     ]

awx/main/migrations/_scan_jobs.py

@@ -0,0 +1,64 @@
+import logging
+
+from awx.main.models.base import PERM_INVENTORY_SCAN, PERM_INVENTORY_DEPLOY
+
+logger = logging.getLogger('awx.main.migrations')
+
+
+def _create_fact_scan_project(Project, org):
+    name = "Tower Fact Scan - {}".format(org.name if org else "No Organization")
+    return Project.objects.create(name=name, 
+                                  scm_url='https://github.com/ansible/tower-fact-modules',
+                                  organization=org)
+
+
+def _create_fact_scan_projects(Project, orgs):
+    return {org.id : _create_fact_scan_project(Project, org) for org in orgs}
+
+
+def _get_tower_scan_job_templates(JobTemplate):
+    return JobTemplate.objects.filter(job_type=PERM_INVENTORY_SCAN, project__isnull=True) \
+                              .prefetch_related('inventory__organization')
+
+
+def _get_orgs(Organization, job_template_ids):
+    return Organization.objects.filter(inventories__jobtemplates__in=job_template_ids).distinct()
+
+
+def _migrate_scan_job_templates(apps):
+    Organization = apps.get_model('main', 'Organization')
+    Project = apps.get_model('main', 'Project')
+    JobTemplate = apps.get_model('main', 'JobTemplate')
+
+    project_no_org = None
+    
+    # A scan job template with a custom project will retain the custom project.
+    JobTemplate.objects.filter(job_type=PERM_INVENTORY_SCAN, project__isnull=False).update(use_fact_cache=True, job_type=PERM_INVENTORY_DEPLOY)
+    
+    # Scan jobs templates using Tower's default scan playbook will now point at
+    # the same playbook but in a github repo.
+    jts = _get_tower_scan_job_templates(JobTemplate)
+    if jts.count() == 0:
+        return
+
+    orgs = _get_orgs(Organization, jts.values_list('id'))
+    if orgs.count() == 0:
+        return
+
+    org_proj_map = _create_fact_scan_projects(Project, orgs)
+    for jt in jts:
+        if jt.inventory and jt.inventory.organization:
+            jt.project = org_proj_map[jt.inventory.organization.id]
+        # Job Templates without an Organization; through related Inventory
+        else:
+            # TODO: Create a project without an org and connect
+            if not project_no_org:
+                project_no_org = _create_fact_scan_project(Project, None)
+            jt.project = project_no_org
+        jt.job_type = PERM_INVENTORY_DEPLOY
+        jt.use_fact_cache = True
+        jt.save()
+
+
+def migrate_scan_job_templates(apps, schema_editor):
+    _migrate_scan_job_templates(apps)

awx/main/tests/functional/test_scan_jobs_migration.py

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+import pytest
+
+from django.apps import apps
+
+from awx.main.models.base import PERM_INVENTORY_SCAN, PERM_INVENTORY_DEPLOY
+from awx.main.models import (
+    JobTemplate,
+    Project,
+    Inventory,
+    Organization,
+)
+
+from awx.main.migrations._scan_jobs import _migrate_scan_job_templates
+
+
+@pytest.fixture
+def organizations():
+    return [Organization.objects.create(name="org-{}".format(x)) for x in range(3)]
+
+
+@pytest.fixture
+def inventories(organizations):
+    return [Inventory.objects.create(name="inv-{}".format(x), 
+                                     organization=organizations[x]) for x in range(3)]
+
+
+@pytest.fixture
+def job_templates_scan(inventories):
+    return [JobTemplate.objects.create(name="jt-scan-{}".format(x), 
+                                       job_type=PERM_INVENTORY_SCAN,
+                                       inventory=inventories[x]) for x in range(3)]
+
+
+@pytest.fixture
+def job_templates_deploy(inventories):
+    return [JobTemplate.objects.create(name="jt-deploy-{}".format(x), 
+                                       job_type=PERM_INVENTORY_DEPLOY,
+                                       inventory=inventories[x]) for x in range(3)]
+
+
+@pytest.fixture
+def project_custom(organizations):
+    return Project.objects.create(name="proj-scan_custom", 
+                                  scm_url='https://giggity.com',
+                                  organization=organizations[0])
+
+
+@pytest.fixture
+def job_templates_custom_scan_project(project_custom):
+    return [JobTemplate.objects.create(name="jt-scan-custom-{}".format(x), 
+                                       project=project_custom, 
+                                       job_type=PERM_INVENTORY_SCAN) for x in range(3)]
+
+
+@pytest.fixture
+def job_template_scan_no_org():
+    return JobTemplate.objects.create(name="jt-scan-no-org",
+                                      job_type=PERM_INVENTORY_SCAN)
+
+
+@pytest.mark.django_db
+def test_scan_jobs_migration(job_templates_scan, job_templates_deploy, job_templates_custom_scan_project, project_custom, job_template_scan_no_org):
+    _migrate_scan_job_templates(apps)
+
+    # Ensure there are no scan job templates after the migration
+    assert 0 == JobTemplate.objects.filter(job_type=PERM_INVENTORY_SCAN).count()
+
+    # Ensure special No Organization proj created
+    # And No Organization project is associated with correct jt
+    proj = Project.objects.get(name="Tower Fact Scan - No Organization")
+    assert proj.id == JobTemplate.objects.get(id=job_template_scan_no_org.id).project.id
+
+    # Ensure per-org projects were created
+    projs = Project.objects.filter(name__startswith="Tower Fact Scan")
+    assert projs.count() == 4
+
+    # Ensure scan job templates with Tower project are migrated
+    for i, jt_old in enumerate(job_templates_scan):
+        jt = JobTemplate.objects.get(id=jt_old.id)
+        assert PERM_INVENTORY_DEPLOY == jt.job_type
+        assert jt.use_fact_cache is True
+        assert projs[i] == jt.project
+
+    # Ensure scan job templates with custom projects are migrated
+    for jt_old in job_templates_custom_scan_project:
+        jt = JobTemplate.objects.get(id=jt_old.id)
+        assert PERM_INVENTORY_DEPLOY == jt.job_type
+        assert jt.use_fact_cache is True
+        assert project_custom == jt.project
+
+    # Ensure other job template aren't touched
+    for jt_old in job_templates_deploy:
+        jt = JobTemplate.objects.get(id=jt_old.id)
+        assert PERM_INVENTORY_DEPLOY == jt.job_type
+        assert jt.project is None
+