migrate data from mongo to postgres

awx/main/migrations/0005_v300_fact_migrations.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from awx.main.migrations import _system_tracking as system_tracking
+from django.db import migrations
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0004_v300_fact_changes'),
+    ]
+
+    operations = [
+        migrations.RunPython(system_tracking.migrate_facts),
+    ]

awx/main/migrations/0006_v300_changes.py

@@ -107,7 +107,7 @@ def create_system_job_templates(apps, schema_editor):
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('main', '0004_v300_changes'),
+        ('main', '0005_v300_fact_migrations'),
     ]
 
     operations = [

awx/main/migrations/_system_tracking.py

@@ -0,0 +1,21 @@
+
+from awx.fact.models import FactVersion
+
+def migrate_facts(apps, schema_editor):
+    Fact = apps.get_model('main', "Fact")
+    Host = apps.get_model('main', "Host")
+
+    migrated_count = 0
+    not_migrated_count = 0
+    for factver in FactVersion.objects.all():
+        fact_obj = factver.fact 
+        try:
+            host = Host.objects.only('id').get(inventory__id=factver.host.inventory_id, name=factver.host.hostname)
+            Fact.objects.create(host_id=host.id, timestamp=fact_obj.timestamp, module=fact_obj.module, facts=fact_obj.fact).save()
+            migrated_count += 1
+        except Host.DoesNotExist:
+            # TODO: Log this. No host was found to migrate the facts to.
+            # This isn't a hard error. Just something the user would want to know.
+            not_migrated_count += 1
+
+    return (migrated_count, not_migrated_count)

awx/main/tests/functional/migrations/conftest.py

@@ -0,0 +1,84 @@
+# Python
+import pytest
+from datetime import timedelta
+
+# Django
+from django.utils import timezone
+from django.conf import settings
+
+# AWX
+from awx.fact.models.fact import Fact, FactHost
+
+# MongoEngine
+from mongoengine.connection import ConnectionError
+
+@pytest.fixture(autouse=True)
+def mongo_db(request):
+    marker = request.keywords.get('mongo_db', None)
+    if marker:
+        # Drop mongo database
+        try:
+            db = Fact._get_db()
+            db.connection.drop_database(settings.MONGO_DB)
+        except ConnectionError:
+            raise
+
+@pytest.fixture
+def inventories(organization):
+    def rf(inventory_count=1):
+        invs = []
+        for i in xrange(0, inventory_count):
+            inv = organization.inventories.create(name="test-inv-%d" % i, description="test-inv-desc")
+            invs.append(inv)
+        return invs
+    return rf
+
+'''
+hosts naming convension should align with hosts_mongo 
+'''
+@pytest.fixture
+def hosts(organization):
+    def rf(host_count=1, inventories=[]):
+        hosts = []
+        for inv in inventories:
+            for i in xrange(0, host_count):
+                name = '%s-host-%s' % (inv.name, i)
+                host = inv.hosts.create(name=name)
+                hosts.append(host)
+        return hosts
+    return rf
+
+@pytest.fixture
+def hosts_mongo(organization):
+    def rf(host_count=1, inventories=[]):
+        hosts = []
+        for inv in inventories:
+            for i in xrange(0, host_count):
+                name = '%s-host-%s' % (inv.name, i)
+                (host, created) = FactHost.objects.get_or_create(hostname=name, inventory_id=inv.id)
+                hosts.append(host)
+        return hosts
+    return rf
+
+@pytest.fixture
+def fact_scans(organization, fact_ansible_json, fact_packages_json, fact_services_json):
+    def rf(fact_scans=1, inventories=[], timestamp_epoch=timezone.now()):
+        facts_json = {}
+        facts = []
+        module_names = ['ansible', 'services', 'packages']
+
+        facts_json['ansible'] = fact_ansible_json
+        facts_json['packages'] = fact_packages_json
+        facts_json['services'] = fact_services_json
+
+        for inv in inventories:
+            for host_obj in FactHost.objects.filter(inventory_id=inv.id):
+                timestamp_current = timestamp_epoch
+                for i in xrange(0, fact_scans):
+                    for module_name in module_names:
+                        facts.append(Fact.add_fact(timestamp_current, facts_json[module_name],  host_obj, module_name))
+                    timestamp_current += timedelta(days=1)
+        return facts
+    return rf
+
+

awx/main/tests/functional/migrations/test_fact.py

@@ -0,0 +1,60 @@
+import pytest
+import datetime
+
+from django.apps import apps
+
+from awx.main.models.inventory import Host
+from awx.main.models.fact import Fact
+
+from awx.main.migrations import _system_tracking as system_tracking
+
+def micro_to_milli(micro):
+    return micro - (((int)(micro / 1000)) * 1000)
+
+@pytest.mark.django_db
+@pytest.mark.mongo_db
+def test_migrate_facts(inventories, hosts, hosts_mongo, fact_scans):
+    inventory_objs = inventories(2)
+    hosts(2, inventory_objs)
+    hosts_mongo(2, inventory_objs)
+    facts_known = fact_scans(2, inventory_objs)
+
+    (migrated_count, not_migrated_count) = system_tracking.migrate_facts(apps, None)
+    # 4 hosts w/ 2 fact scans each, 3 modules each scan
+    assert migrated_count == 24
+    assert not_migrated_count == 0
+
+    
+    for fact_mongo, fact_version in facts_known:
+        host = Host.objects.get(inventory_id=fact_mongo.host.inventory_id, name=fact_mongo.host.hostname)
+        t = fact_mongo.timestamp - datetime.timedelta(microseconds=micro_to_milli(fact_mongo.timestamp.microsecond))
+        fact = Fact.objects.filter(host_id=host.id, timestamp=t, module=fact_mongo.module)
+
+        assert len(fact) == 1
+        assert fact[0] is not None
+
+@pytest.mark.django_db
+@pytest.mark.mongo_db
+def test_migrate_facts_hostname_does_not_exist(inventories, hosts, hosts_mongo, fact_scans):
+    inventory_objs = inventories(2)
+    host_objs = hosts(1, inventory_objs)
+    hosts_mongo(2, inventory_objs)
+    facts_known = fact_scans(2, inventory_objs)
+
+    (migrated_count, not_migrated_count) = system_tracking.migrate_facts(apps, None)
+    assert migrated_count == 12
+    assert not_migrated_count == 12
+
+
+    for fact_mongo, fact_version in facts_known:
+        # Facts that don't match the only host will not be migrated
+        if fact_mongo.host.hostname != host_objs[0].name:
+            continue
+
+        host = Host.objects.get(inventory_id=fact_mongo.host.inventory_id, name=fact_mongo.host.hostname)
+        t = fact_mongo.timestamp - datetime.timedelta(microseconds=micro_to_milli(fact_mongo.timestamp.microsecond))
+        fact = Fact.objects.filter(host_id=host.id, timestamp=t, module=fact_mongo.module)
+
+        assert len(fact) == 1
+        assert fact[0] is not None
+   

pytest.ini

@@ -7,3 +7,4 @@ addopts = --reuse-db
 markers =
     ac: access control test
     license_feature: ensure license features are accessible or not depending on license
+    mongo_db: drop mongodb test database before test runs