Merge branch 'devel' of github.com:ansible/ansible-tower into python27_el6

Conflicts:
	packaging/rpm/ansible-tower.spec
	tools/docker-compose/start_development.sh

awx/main/management/commands/run_fact_cache_receiver.py

@@ -67,12 +67,12 @@ class FactCacheReceiver(object):
         self.timestamp = datetime.fromtimestamp(date_key, None)
 
         # Update existing Fact entry
-        fact_obj = Fact.get_host_fact(host_obj.id, module_name, self.timestamp)
-        if fact_obj:
+        try:
+            fact_obj = Fact.objects.get(host__id=host_obj.id, module=module_name, timestamp=self.timestamp)
             fact_obj.facts = facts
             fact_obj.save()
             logger.info('Updated existing fact <%s>' % (fact_obj.id))
-        else:
+        except Fact.DoesNotExist:
             # Create new Fact entry
             fact_obj = Fact.add_fact(host_obj.id, module_name, self.timestamp, facts)
             logger.info('Created new fact <fact_id, module> <%s, %s>' % (fact_obj.id, module_name))

awx/main/management/commands/run_task_system.py

@@ -108,6 +108,8 @@ class SimpleDAG(object):
             return "inventory_update"
         elif type(obj) == ProjectUpdate:
             return "project_update"
+        elif type(obj) == SystemJob:
+            return "system_job"
         return "unknown"
 
     def get_dependencies(self, obj):

awx/main/migrations/0003_v300_notification_changes.py

@@ -13,7 +13,7 @@ class Migration(migrations.Migration):
     dependencies = [
         ('taggit', '0002_auto_20150616_2121'),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-        ('main', '0002_v300_changes'),
+        ('main', '0002_v300_tower_settings_changes'),
     ]
 
     operations = [

awx/main/migrations/0004_v300_fact_changes.py

@@ -8,7 +8,7 @@ import jsonbfield.fields
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('main', '0003_v300_changes'),
+        ('main', '0003_v300_notification_changes'),
     ]
 
     operations = [

awx/main/migrations/0005_v300_migrate_facts.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from awx.main.migrations import _system_tracking as system_tracking
+from django.db import migrations
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0004_v300_fact_changes'),
+    ]
+
+    operations = [
+        migrations.RunPython(system_tracking.migrate_facts),
+    ]

awx/main/migrations/0006_v300_create_system_job_templates.py

@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+from django.utils.timezone import now
+
+from awx.api.license import feature_enabled
+
+
+def create_system_job_templates(apps, schema_editor):
+    '''
+    Create default system job templates if not present. Create default schedules
+    only if new system job templates were created (i.e. new database).
+    '''
+
+    SystemJobTemplate = apps.get_model('main', 'SystemJobTemplate')
+    ContentType = apps.get_model('contenttypes', 'ContentType')
+    sjt_ct = ContentType.objects.get_for_model(SystemJobTemplate)
+    now_dt = now()
+    now_str = now_dt.strftime('%Y%m%dT%H%M%SZ')
+
+    sjt, created = SystemJobTemplate.objects.get_or_create(
+        job_type='cleanup_jobs',
+        defaults=dict(
+            name='Cleanup Job Details',
+            description='Remove job history older than X days',
+            created=now_dt,
+            modified=now_dt,
+            polymorphic_ctype=sjt_ct,
+        ),
+    )
+    if created:
+        sjt.schedules.create(
+            name='Cleanup Job Schedule',
+            rrule='DTSTART:%s RRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=SU' % now_str,
+            description='Automatically Generated Schedule',
+            enabled=True,
+            extra_data={'days': '120'},
+            created=now_dt,
+            modified=now_dt,
+        )
+
+    sjt, created = SystemJobTemplate.objects.get_or_create(
+        job_type='cleanup_deleted',
+        defaults=dict(
+            name='Cleanup Deleted Data',
+            description='Remove deleted object history older than X days',
+            created=now_dt,
+            modified=now_dt,
+            polymorphic_ctype=sjt_ct,
+        ),
+    )
+    if created:
+        sjt.schedules.create(
+            name='Cleanup Deleted Data Schedule',
+            rrule='DTSTART:%s RRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=MO' % now_str,
+            description='Automatically Generated Schedule',
+            enabled=True,
+            extra_data={'days': '30'},
+            created=now_dt,
+            modified=now_dt,
+        )
+
+    sjt, created = SystemJobTemplate.objects.get_or_create(
+        job_type='cleanup_activitystream',
+        defaults=dict(
+            name='Cleanup Activity Stream',
+            description='Remove activity stream history older than X days',
+            created=now_dt,
+            modified=now_dt,
+            polymorphic_ctype=sjt_ct,
+        ),
+    )
+    if created:
+        sjt.schedules.create(
+            name='Cleanup Activity Schedule',
+            rrule='DTSTART:%s RRULE:FREQ=WEEKLY;INTERVAL=1;BYDAY=TU' % now_str,
+            description='Automatically Generated Schedule',
+            enabled=True,
+            extra_data={'days': '355'},
+            created=now_dt,
+            modified=now_dt,
+        )
+
+    sjt, created = SystemJobTemplate.objects.get_or_create(
+        job_type='cleanup_facts',
+        defaults=dict(
+            name='Cleanup Fact Details',
+            description='Remove system tracking history',
+            created=now_dt,
+            modified=now_dt,
+            polymorphic_ctype=sjt_ct,
+        ),
+    )
+    if created and feature_enabled('system_tracking', bypass_database=True):
+        sjt.schedules.create(
+            name='Cleanup Fact Schedule',
+            rrule='DTSTART:%s RRULE:FREQ=MONTHLY;INTERVAL=1;BYMONTHDAY=1' % now_str,
+            description='Automatically Generated Schedule',
+            enabled=True,
+            extra_data={'older_than': '120d', 'granularity': '1w'},
+            created=now_dt,
+            modified=now_dt,
+        )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0005_v300_migrate_facts'),
+    ]
+
+    operations = [
+        migrations.RunPython(create_system_job_templates, migrations.RunPython.noop),
+    ]

awx/main/migrations/_system_tracking.py

@@ -0,0 +1,47 @@
+
+from awx.fact.models import FactVersion
+from mongoengine.connection import ConnectionError
+from pymongo.errors import OperationFailure
+from django.conf import settings
+
+def drop_system_tracking_db():
+    try:
+        db = FactVersion._get_db()
+        db.connection.drop_database(settings.MONGO_DB)
+    except ConnectionError:
+        # TODO: Log this. Not a deal-breaker. Just let the user know they
+        # may need to manually drop/delete the database.
+        pass
+    except OperationFailure:
+        # TODO: This means the database was up but something happened when we tried to query it
+        pass
+
+def migrate_facts(apps, schema_editor):
+    Fact = apps.get_model('main', "Fact")
+    Host = apps.get_model('main', "Host")
+
+    try:
+        n = FactVersion.objects.all().count()
+    except ConnectionError:
+        # TODO: Let the user know about the error.  Likely this is
+        # a new install and we just don't need to do this
+        return (0, 0)
+    except OperationFailure:
+        # TODO: This means the database was up but something happened when we tried to query it
+        return (0, 0)
+
+    migrated_count = 0
+    not_migrated_count = 0
+    for factver in FactVersion.objects.all():
+        fact_obj = factver.fact 
+        try:
+            host = Host.objects.only('id').get(inventory__id=factver.host.inventory_id, name=factver.host.hostname)
+            Fact.objects.create(host_id=host.id, timestamp=fact_obj.timestamp, module=fact_obj.module, facts=fact_obj.fact).save()
+            migrated_count += 1
+        except Host.DoesNotExist:
+            # TODO: Log this. No host was found to migrate the facts to.
+            # This isn't a hard error. Just something the user would want to know.
+            not_migrated_count += 1
+
+    drop_system_tracking_db()
+    return (migrated_count, not_migrated_count)

awx/main/models/jobs.py

@@ -1065,6 +1065,13 @@ class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions):
     def cache_timeout_blocked(self):
         return False
 
+    @property
+    def notifiers(self):
+        base_notifiers = Notifier.objects.filter(active=True)
+        error_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_errors__in=[self]))
+        success_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_success__in=[self]))
+        any_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_any__in=[self]))
+        return dict(error=list(error_notifiers), success=list(success_notifiers), any=list(any_notifiers))
 
 
 class SystemJob(UnifiedJob, SystemJobOptions):

awx/main/signals.py

@@ -393,9 +393,11 @@ def activity_stream_associate(sender, instance, **kwargs):
             obj2_id = entity_acted
             obj2_actual = obj2.objects.get(id=obj2_id)
             object2 = camelcase_to_underscore(obj2.__name__)
-            # Skip recording any inventory source changes here.
+            # Skip recording any inventory source, or system job template changes here.
             if isinstance(obj1, InventorySource) or isinstance(obj2_actual, InventorySource):
                 continue
+            if isinstance(obj1, SystemJobTemplate) or isinstance(obj2_actual, SystemJobTemplate):
+                continue
             activity_entry = ActivityStream(
                 operation=action,
                 object1=object1,

awx/main/tasks.py

@@ -14,7 +14,6 @@ import pipes
 import re
 import shutil
 import stat
-import subprocess
 import tempfile
 import thread
 import time
@@ -53,7 +52,6 @@ from awx.main.task_engine import TaskSerializer, TASK_TIMEOUT_INTERVAL
 from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, update_scm_url,
                             ignore_inventory_computed_fields, emit_websocket_notification,
                             check_proot_installed, build_proot_temp_dir, wrap_args_with_proot)
-from awx.fact.utils.connection import test_mongo_connection
 
 __all__ = ['RunJob', 'RunSystemJob', 'RunProjectUpdate', 'RunInventoryUpdate',
            'RunAdHocCommand', 'handle_work_error', 'handle_work_success',
@@ -181,30 +179,6 @@ def notify_task_runner(metadata_dict):
     queue = FifoQueue('tower_task_manager')
     queue.push(metadata_dict)
 
-@task()
-def mongodb_control(cmd):
-    # Sanity check: Do not send arbitrary commands.
-    if cmd not in ('start', 'stop'):
-        raise ValueError('Only "start" and "stop" are allowed.')
-
-    # Either start or stop mongo, as requested.
-    p = subprocess.Popen('sudo service mongod %s' % cmd, shell=True,
-                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    out, err = p.communicate()
-    p.wait()
-
-    # Check to make sure the stop actually succeeded
-    p = subprocess.Popen('pidof mongod', shell=True)
-    shutdown_failed = p.wait() == 0
-
-    # If there was an error, log it.
-    if err:
-        logger.error(err)
-
-    if cmd == 'stop' and shutdown_failed:
-        p = subprocess.Popen('sudo mongod --shutdown -f /etc/mongod.conf', shell=True)
-        p.wait()
-
 @task(bind=True)
 def handle_work_success(self, result, task_actual):
     if task_actual['type'] == 'project_update':
@@ -227,6 +201,11 @@ def handle_work_success(self, result, task_actual):
         instance_name = instance.module_name
         notifiers = [] # TODO: Ad-hoc commands need to notify someone
         friendly_name = "AdHoc Command"
+    elif task_actual['type'] == 'system_job':
+        instance = SystemJob.objects.get(id=task_actual['id'])
+        instance_name = instance.system_job_template.name
+        notifiers = instance.system_job_template.notifiers
+        friendly_name = "System Job"
     else:
         return
     notification_body = instance.notification_data()
@@ -234,7 +213,8 @@ def handle_work_success(self, result, task_actual):
                                                                                task_actual['id'],
                                                                                instance_name,
                                                                                notification_body['url'])
-    send_notifications.delay([n.generate_notification(notification_subject, notification_body)
+    notification_body['friendly_name'] = friendly_name
+    send_notifications.delay([n.generate_notification(notification_subject, notification_body).id
                               for n in set(notifiers.get('success', []) + notifiers.get('any', []))],
                              job_id=task_actual['id'])
 
@@ -269,6 +249,11 @@ def handle_work_error(self, task_id, subtasks=None):
                 instance_name = instance.module_name
                 notifiers = []
                 friendly_name = "AdHoc Command"
+            elif task_actual['type'] == 'system_job':
+                instance = SystemJob.objects.get(id=task_actual['id'])
+                instance_name = instance.system_job_template.name
+                notifiers = instance.system_job_template.notifiers
+                friendly_name = "System Job"
             else:
                 # Unknown task type
                 break
@@ -957,11 +942,6 @@ class RunJob(BaseTask):
         '''
         return getattr(tower_settings, 'AWX_PROOT_ENABLED', False)
 
-    def pre_run_hook(self, job, **kwargs):
-        if job.job_type == PERM_INVENTORY_SCAN:
-            if not test_mongo_connection():
-                raise RuntimeError("Fact Scan Database is offline")
-
     def post_run_hook(self, job, **kwargs):
         '''
         Hook for actions to run after job/task has completed.

awx/main/tests/base.py

@@ -11,6 +11,7 @@ import shutil
 import sys
 import tempfile
 import time
+import urllib
 from multiprocessing import Process
 from subprocess import Popen
 import re
@@ -463,6 +464,8 @@ class BaseTestMixin(QueueTestMixin, MockCommonlySlowTestMixin):
                 response = method(url, json.dumps(data), 'application/json')
             elif data_type == 'yaml':
                 response = method(url, yaml.safe_dump(data), 'application/yaml')
+            elif data_type == 'form':
+                response = method(url, urllib.urlencode(data), 'application/x-www-form-urlencoded')
             else:
                 self.fail('Unsupported data_type %s' % data_type)
         else:

awx/main/tests/functional/api/test_fact_versions.py

@@ -16,6 +16,9 @@ from django.utils import timezone
 def mock_feature_enabled(feature, bypass_database=None):
     return True
 
+def mock_feature_disabled(feature, bypass_database=None):
+    return False
+
 def setup_common(hosts, fact_scans, get, user, epoch=timezone.now(), get_params={}, host_count=1):
     hosts = hosts(host_count=host_count)
     fact_scans(fact_scans=3, timestamp_epoch=epoch)
@@ -42,8 +45,33 @@ def check_response_facts(facts_known, response):
         assert timestamp_apiformat(fact_known.timestamp) == response.data['results'][i]['timestamp']
         check_url(response.data['results'][i]['related']['fact_view'], fact_known, fact_known.module)
 
+def check_system_tracking_feature_forbidden(response):
+    assert 402 == response.status_code
+    assert 'Your license does not permit use of system tracking.' == response.data['detail']
+
+@mock.patch('awx.api.views.feature_enabled', new=mock_feature_disabled)
+@pytest.mark.django_db
+@pytest.mark.license_feature
+def test_system_tracking_license_get(hosts, get, user):
+    hosts = hosts(host_count=1)
+    url = reverse('api:host_fact_versions_list', args=(hosts[0].pk,))
+    response = get(url, user('admin', True))
+
+    check_system_tracking_feature_forbidden(response)
+
+@mock.patch('awx.api.views.feature_enabled', new=mock_feature_disabled)
+@pytest.mark.django_db
+@pytest.mark.license_feature
+def test_system_tracking_license_options(hosts, options, user):
+    hosts = hosts(host_count=1)
+    url = reverse('api:host_fact_versions_list', args=(hosts[0].pk,))
+    response = options(url, None, user('admin', True))
+
+    check_system_tracking_feature_forbidden(response)
+
 @mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
 @pytest.mark.django_db
+@pytest.mark.license_feature
 def test_no_facts_db(hosts, get, user):
     hosts = hosts(host_count=1)
     url = reverse('api:host_fact_versions_list', args=(hosts[0].pk,))
@@ -72,6 +100,7 @@ def test_basic_fields(hosts, fact_scans, get, user):
 
 @mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
 @pytest.mark.django_db
+@pytest.mark.license_feature
 def test_basic_options_fields(hosts, fact_scans, options, user):
     hosts = hosts(host_count=1)
     fact_scans(fact_scans=1)
@@ -79,9 +108,6 @@ def test_basic_options_fields(hosts, fact_scans, options, user):
     url = reverse('api:host_fact_versions_list', args=(hosts[0].pk,))
     response = options(url, None, user('admin', True), pk=hosts[0].id)
 
-    import json
-    print(json.dumps(response.data))
-
     assert 'related' in response.data['actions']['GET']
     assert 'module' in response.data['actions']['GET']
     assert ("ansible", "Ansible") in response.data['actions']['GET']['module']['choices']

awx/main/tests/functional/api/test_fact_view.py

@@ -9,6 +9,9 @@ from django.utils import timezone
 def mock_feature_enabled(feature, bypass_database=None):
     return True
 
+def mock_feature_disabled(feature, bypass_database=None):
+    return False
+
 # TODO: Consider making the fact_scan() fixture a Class, instead of a function, and move this method into it
 def find_fact(facts, host_id, module_name, timestamp):
     for f in facts:
@@ -26,6 +29,30 @@ def setup_common(hosts, fact_scans, get, user, epoch=timezone.now(), module_name
     fact_known = find_fact(facts, hosts[0].id, module_name, epoch)
     return (fact_known, response)
 
+def check_system_tracking_feature_forbidden(response):
+    assert 402 == response.status_code
+    assert 'Your license does not permit use of system tracking.' == response.data['detail']
+
+@mock.patch('awx.api.views.feature_enabled', new=mock_feature_disabled)
+@pytest.mark.django_db
+@pytest.mark.license_feature
+def test_system_tracking_license_get(hosts, get, user):
+    hosts = hosts(host_count=1)
+    url = reverse('api:host_fact_compare_view', args=(hosts[0].pk,))
+    response = get(url, user('admin', True))
+
+    check_system_tracking_feature_forbidden(response)
+
+@mock.patch('awx.api.views.feature_enabled', new=mock_feature_disabled)
+@pytest.mark.django_db
+@pytest.mark.license_feature
+def test_system_tracking_license_options(hosts, options, user):
+    hosts = hosts(host_count=1)
+    url = reverse('api:host_fact_compare_view', args=(hosts[0].pk,))
+    response = options(url, None, user('admin', True))
+
+    check_system_tracking_feature_forbidden(response)
+
 @mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
 @pytest.mark.django_db
 def test_no_fact_found(hosts, get, user):

awx/main/tests/functional/api/test_organization_counts.py

@@ -0,0 +1,144 @@
+import pytest
+
+from django.core.urlresolvers import reverse
+
+@pytest.fixture
+def resourced_organization(organization, project, team, inventory, user):
+    admin_user = user('test-admin', True)
+    member_user = user('org-member')
+
+    # Associate one resource of every type with the organization
+    organization.users.add(member_user)
+    organization.admins.add(admin_user)
+    organization.projects.add(project)
+    # organization.teams.create(name='org-team')
+    # inventory = organization.inventories.create(name="associated-inv")
+    project.jobtemplates.create(name="test-jt",
+                                description="test-job-template-desc",
+                                inventory=inventory,
+                                playbook="test_playbook.yml")
+
+    return organization
+
+@pytest.mark.django_db
+def test_org_counts_admin(resourced_organization, user, get):
+    # Check that all types of resources are counted by a superuser
+    external_admin = user('admin', True)
+    response = get(reverse('api:organization_list', args=[]), external_admin)
+    assert response.status_code == 200
+
+    counts = response.data['results'][0]['summary_fields']['related_field_counts']
+    assert counts == {
+        'users': 1,
+        'admins': 1,
+        'job_templates': 1,
+        'projects': 1,
+        'inventories': 1,
+        'teams': 1
+    }
+
+@pytest.mark.django_db
+def test_org_counts_member(resourced_organization, get):
+    # Check that a non-admin user can only see the full project and
+    #   user count, consistent with the RBAC rules
+    member_user = resourced_organization.users.get(username='org-member')
+    response = get(reverse('api:organization_list', args=[]), member_user)
+    assert response.status_code == 200
+
+    counts = response.data['results'][0]['summary_fields']['related_field_counts']
+
+    assert counts == {
+        'users': 1,  # User can see themselves
+        'admins': 0,
+        'job_templates': 0,
+        'projects': 1,  # Projects are shared with all the organization
+        'inventories': 0,
+        'teams': 0
+    }
+
+@pytest.mark.django_db
+def test_new_org_zero_counts(user, post):
+    # Check that a POST to the organization list endpoint returns
+    #   correct counts, including the new record
+    org_list_url = reverse('api:organization_list', args=[])
+    post_response = post(url=org_list_url, data={'name': 'test organization',
+                         'description': ''}, user=user('admin', True))
+    assert post_response.status_code == 201
+
+    new_org_list = post_response.render().data
+    counts_dict = new_org_list['summary_fields']['related_field_counts']
+    assert counts_dict == {
+        'users': 0,
+        'admins': 0,
+        'job_templates': 0,
+        'projects': 0,
+        'inventories': 0,
+        'teams': 0
+    }
+
+@pytest.mark.django_db
+def test_two_organizations(resourced_organization, organizations, user, get):
+    # Check correct results for two organizations are returned
+    external_admin = user('admin', True)
+    organization_zero = organizations(1)[0]
+    response = get(reverse('api:organization_list', args=[]), external_admin)
+    assert response.status_code == 200
+
+    org_id_full = resourced_organization.id
+    org_id_zero = organization_zero.id
+    counts = {}
+    for i in range(2):
+        org_id = response.data['results'][i]['id']
+        counts[org_id] = response.data['results'][i]['summary_fields']['related_field_counts']
+
+    assert counts[org_id_full] == {
+        'users': 1,
+        'admins': 1,
+        'job_templates': 1,
+        'projects': 1,
+        'inventories': 1,
+        'teams': 1
+    }
+    assert counts[org_id_zero] == {
+        'users': 0,
+        'admins': 0,
+        'job_templates': 0,
+        'projects': 0,
+        'inventories': 0,
+        'teams': 0
+    }
+
+@pytest.mark.django_db
+def test_JT_associated_with_project(organizations, project, user, get):
+    # Check that adding a project to an organization gets the project's JT
+    #  included in the organization's JT count
+    external_admin = user('admin', True)
+    two_orgs = organizations(2)
+    organization = two_orgs[0]
+    other_org = two_orgs[1]
+
+    unrelated_inv = other_org.inventories.create(name='not-in-organization')
+    project.jobtemplates.create(name="test-jt",
+                                description="test-job-template-desc",
+                                inventory=unrelated_inv,
+                                playbook="test_playbook.yml")
+    organization.projects.add(project)
+
+    response = get(reverse('api:organization_list', args=[]), external_admin)
+    assert response.status_code == 200
+
+    org_id = organization.id
+    counts = {}
+    for i in range(2):
+        working_id = response.data['results'][i]['id']
+        counts[working_id] = response.data['results'][i]['summary_fields']['related_field_counts']
+
+    assert counts[org_id] == {
+        'users': 0,
+        'admins': 0,
+        'job_templates': 1,
+        'projects': 1,
+        'inventories': 0,
+        'teams': 0
+    }
+

awx/main/tests/functional/commands/test_cleanup_facts.py

@@ -3,6 +3,7 @@
 
 # Python
 import pytest
+import mock
 from dateutil.relativedelta import relativedelta
 from datetime import timedelta
 
@@ -15,6 +16,12 @@ from awx.main.management.commands.cleanup_facts import CleanupFacts, Command
 from awx.main.models.fact import Fact
 from awx.main.models.inventory import Host
 
+def mock_feature_enabled(feature, bypass_database=None):
+    return True
+
+def mock_feature_disabled(feature, bypass_database=None):
+    return False
+
 @pytest.mark.django_db
 def test_cleanup_granularity(fact_scans, hosts):
     epoch = timezone.now()
@@ -88,6 +95,16 @@ def test_cleanup_logic(fact_scans, hosts):
             timestamp_pivot -= granularity
             assert fact.timestamp == timestamp_pivot
 
+@mock.patch('awx.main.management.commands.cleanup_facts.feature_enabled', new=mock_feature_disabled)
+@pytest.mark.django_db
+@pytest.mark.license_feature
+def test_system_tracking_feature_disabled(mocker):
+    cmd = Command()
+    with pytest.raises(CommandError) as err:
+        cmd.handle(None)
+    assert 'The System Tracking feature is not enabled for your Tower instance' in err.value
+
+@mock.patch('awx.main.management.commands.cleanup_facts.feature_enabled', new=mock_feature_enabled)
 @pytest.mark.django_db
 def test_parameters_ok(mocker):
     run = mocker.patch('awx.main.management.commands.cleanup_facts.CleanupFacts.run')
@@ -158,6 +175,7 @@ def test_string_time_to_timestamp_invalid():
         res = cmd.string_time_to_timestamp(kv['time'])
         assert res is None
 
+@mock.patch('awx.main.management.commands.cleanup_facts.feature_enabled', new=mock_feature_enabled)
 @pytest.mark.django_db
 def test_parameters_fail(mocker):
     # Mock run() just in case, but it should never get called because an error should be thrown

awx/main/tests/functional/migrations/conftest.py

@@ -0,0 +1,84 @@
+# Python
+import pytest
+from datetime import timedelta
+
+# Django
+from django.utils import timezone
+from django.conf import settings
+
+# AWX
+from awx.fact.models.fact import Fact, FactHost
+
+# MongoEngine
+from mongoengine.connection import ConnectionError
+
+@pytest.fixture(autouse=True)
+def mongo_db(request):
+    marker = request.keywords.get('mongo_db', None)
+    if marker:
+        # Drop mongo database
+        try:
+            db = Fact._get_db()
+            db.connection.drop_database(settings.MONGO_DB)
+        except ConnectionError:
+            raise
+
+@pytest.fixture
+def inventories(organization):
+    def rf(inventory_count=1):
+        invs = []
+        for i in xrange(0, inventory_count):
+            inv = organization.inventories.create(name="test-inv-%d" % i, description="test-inv-desc")
+            invs.append(inv)
+        return invs
+    return rf
+
+'''
+hosts naming convension should align with hosts_mongo 
+'''
+@pytest.fixture
+def hosts(organization):
+    def rf(host_count=1, inventories=[]):
+        hosts = []
+        for inv in inventories:
+            for i in xrange(0, host_count):
+                name = '%s-host-%s' % (inv.name, i)
+                host = inv.hosts.create(name=name)
+                hosts.append(host)
+        return hosts
+    return rf
+
+@pytest.fixture
+def hosts_mongo(organization):
+    def rf(host_count=1, inventories=[]):
+        hosts = []
+        for inv in inventories:
+            for i in xrange(0, host_count):
+                name = '%s-host-%s' % (inv.name, i)
+                (host, created) = FactHost.objects.get_or_create(hostname=name, inventory_id=inv.id)
+                hosts.append(host)
+        return hosts
+    return rf
+
+@pytest.fixture
+def fact_scans(organization, fact_ansible_json, fact_packages_json, fact_services_json):
+    def rf(fact_scans=1, inventories=[], timestamp_epoch=timezone.now()):
+        facts_json = {}
+        facts = []
+        module_names = ['ansible', 'services', 'packages']
+
+        facts_json['ansible'] = fact_ansible_json
+        facts_json['packages'] = fact_packages_json
+        facts_json['services'] = fact_services_json
+
+        for inv in inventories:
+            for host_obj in FactHost.objects.filter(inventory_id=inv.id):
+                timestamp_current = timestamp_epoch
+                for i in xrange(0, fact_scans):
+                    for module_name in module_names:
+                        facts.append(Fact.add_fact(timestamp_current, facts_json[module_name],  host_obj, module_name))
+                    timestamp_current += timedelta(days=1)
+        return facts
+    return rf
+
+

awx/main/tests/functional/migrations/test_fact.py

@@ -0,0 +1,79 @@
+import pytest
+import datetime
+
+from django.apps import apps
+
+from awx.main.models.inventory import Host
+from awx.main.models.fact import Fact
+
+from awx.main.migrations import _system_tracking as system_tracking
+
+from awx.fact.models.fact import Fact as FactMongo
+from awx.fact.models.fact import FactVersion, FactHost
+
+def micro_to_milli(micro):
+    return micro - (((int)(micro / 1000)) * 1000)
+
+@pytest.mark.django_db
+@pytest.mark.mongo_db
+def test_migrate_facts(inventories, hosts, hosts_mongo, fact_scans):
+    inventory_objs = inventories(2)
+    hosts(2, inventory_objs)
+    hosts_mongo(2, inventory_objs)
+    facts_known = fact_scans(2, inventory_objs)
+
+    (migrated_count, not_migrated_count) = system_tracking.migrate_facts(apps, None)
+    # 4 hosts w/ 2 fact scans each, 3 modules each scan
+    assert migrated_count == 24
+    assert not_migrated_count == 0
+
+    
+    for fact_mongo, fact_version in facts_known:
+        host = Host.objects.get(inventory_id=fact_mongo.host.inventory_id, name=fact_mongo.host.hostname)
+        t = fact_mongo.timestamp - datetime.timedelta(microseconds=micro_to_milli(fact_mongo.timestamp.microsecond))
+        fact = Fact.objects.filter(host_id=host.id, timestamp=t, module=fact_mongo.module)
+
+        assert len(fact) == 1
+        assert fact[0] is not None
+
+@pytest.mark.django_db
+@pytest.mark.mongo_db
+def test_migrate_facts_hostname_does_not_exist(inventories, hosts, hosts_mongo, fact_scans):
+    inventory_objs = inventories(2)
+    host_objs = hosts(1, inventory_objs)
+    hosts_mongo(2, inventory_objs)
+    facts_known = fact_scans(2, inventory_objs)
+
+    (migrated_count, not_migrated_count) = system_tracking.migrate_facts(apps, None)
+    assert migrated_count == 12
+    assert not_migrated_count == 12
+
+
+    for fact_mongo, fact_version in facts_known:
+        # Facts that don't match the only host will not be migrated
+        if fact_mongo.host.hostname != host_objs[0].name:
+            continue
+
+        host = Host.objects.get(inventory_id=fact_mongo.host.inventory_id, name=fact_mongo.host.hostname)
+        t = fact_mongo.timestamp - datetime.timedelta(microseconds=micro_to_milli(fact_mongo.timestamp.microsecond))
+        fact = Fact.objects.filter(host_id=host.id, timestamp=t, module=fact_mongo.module)
+
+        assert len(fact) == 1
+        assert fact[0] is not None
+   
+@pytest.mark.django_db
+@pytest.mark.mongo_db
+def test_drop_system_tracking_db(inventories, hosts, hosts_mongo, fact_scans):
+    inventory_objs = inventories(1)
+    hosts_mongo(1, inventory_objs)
+    fact_scans(1, inventory_objs)
+
+    assert FactMongo.objects.all().count() > 0
+    assert FactVersion.objects.all().count() > 0
+    assert FactHost.objects.all().count() > 0
+
+    system_tracking.drop_system_tracking_db()
+  
+    assert FactMongo.objects.all().count() == 0
+    assert FactVersion.objects.all().count() == 0
+    assert FactHost.objects.all().count() == 0

awx/main/tests/old/jobs/jobs_monolithic.py

@@ -803,6 +803,21 @@ class JobTemplateCallbackTest(BaseJobTestMixin, django.test.LiveServerTestCase):
         self.assertEqual(job.hosts.count(), 1)
         self.assertEqual(job.hosts.all()[0], host)
 
+        # Create the job itself using URL-encoded form data instead of JSON.
+        result = self.post(url, data, expect=202, remote_addr=host_ip, data_type='form')
+
+        # Establish that we got back what we expect, and made the changes
+        # that we expect.
+        self.assertTrue('Location' in result.response, result.response)
+        self.assertEqual(jobs_qs.count(), 2)
+        job = jobs_qs[0]
+        self.assertEqual(urlparse.urlsplit(result.response['Location']).path,
+                         job.get_absolute_url())
+        self.assertEqual(job.launch_type, 'callback')
+        self.assertEqual(job.limit, host.name)
+        self.assertEqual(job.hosts.count(), 1)
+        self.assertEqual(job.hosts.all()[0], host)
+
         # Run the callback job again with extra vars and verify their presence
         data.update(dict(extra_vars=dict(key="value")))
         result = self.post(url, data, expect=202, remote_addr=host_ip)
@@ -853,9 +868,9 @@ class JobTemplateCallbackTest(BaseJobTestMixin, django.test.LiveServerTestCase):
             if host_ip:
                 break
         self.assertTrue(host)
-        self.assertEqual(jobs_qs.count(), 2)
-        self.post(url, data, expect=202, remote_addr=host_ip)
         self.assertEqual(jobs_qs.count(), 3)
+        self.post(url, data, expect=202, remote_addr=host_ip)
+        self.assertEqual(jobs_qs.count(), 4)
         job = jobs_qs[0]
         self.assertEqual(job.launch_type, 'callback')
         self.assertEqual(job.limit, host.name)
@@ -878,9 +893,9 @@ class JobTemplateCallbackTest(BaseJobTestMixin, django.test.LiveServerTestCase):
             if host_ip:
                 break
         self.assertTrue(host)
-        self.assertEqual(jobs_qs.count(), 3)
-        self.post(url, data, expect=202, remote_addr=host_ip)
         self.assertEqual(jobs_qs.count(), 4)
+        self.post(url, data, expect=202, remote_addr=host_ip)
+        self.assertEqual(jobs_qs.count(), 5)
         job = jobs_qs[0]
         self.assertEqual(job.launch_type, 'callback')
         self.assertEqual(job.limit, host.name)
@@ -892,9 +907,9 @@ class JobTemplateCallbackTest(BaseJobTestMixin, django.test.LiveServerTestCase):
         host_qs = host_qs.filter(variables__icontains='ansible_ssh_host')
         host = host_qs[0]
         host_ip = host.variables_dict['ansible_ssh_host']
-        self.assertEqual(jobs_qs.count(), 4)
-        self.post(url, data, expect=202, remote_addr=host_ip)
         self.assertEqual(jobs_qs.count(), 5)
+        self.post(url, data, expect=202, remote_addr=host_ip)
+        self.assertEqual(jobs_qs.count(), 6)
         job = jobs_qs[0]
         self.assertEqual(job.launch_type, 'callback')
         self.assertEqual(job.limit, host.name)
@@ -926,9 +941,9 @@ class JobTemplateCallbackTest(BaseJobTestMixin, django.test.LiveServerTestCase):
                 host_ip = list(ips)[0]
                 break
         self.assertTrue(host)
-        self.assertEqual(jobs_qs.count(), 5)
-        self.post(url, data, expect=202, remote_addr=host_ip)
         self.assertEqual(jobs_qs.count(), 6)
+        self.post(url, data, expect=202, remote_addr=host_ip)
+        self.assertEqual(jobs_qs.count(), 7)
         job = jobs_qs[0]
         self.assertEqual(job.launch_type, 'callback')
         self.assertEqual(job.limit, ':&'.join([job_template.limit, host.name]))