Merge pull request #3620 from AlanCoding/gce_instance_id

Add instance_id for gce imported hosts

Reviewed-by: https://github.com/softwarefactory-project-zuul[bot]

awx/main/migrations/0070_v350_gce_instance_id.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# Created manually 2019-03-05
+from __future__ import unicode_literals
+
+import logging
+
+from django.db import migrations
+
+from awx.main.migrations._inventory_source import set_new_instance_id, back_out_new_instance_id
+
+
+logger = logging.getLogger('awx.main.migrations')
+
+
+# new value introduced in awx/settings/defaults.py, frozen in time here
+GCE_INSTANCE_ID_VAR = 'gce_id'
+
+
+def gce_id_forward(apps, schema_editor):
+    set_new_instance_id(apps, 'gce', GCE_INSTANCE_ID_VAR)
+
+
+def gce_id_backward(apps, schema_editor):
+    back_out_new_instance_id(apps, 'gce', GCE_INSTANCE_ID_VAR)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0069_v350_generate_unique_install_uuid'),
+    ]
+
+    operations = [
+        migrations.RunPython(gce_id_forward, gce_id_backward)
+    ]

awx/main/migrations/_inventory_source.py

@@ -1,6 +1,9 @@
 import logging
 
 from django.db.models import Q
+from django.utils.encoding import smart_text
+
+from awx.main.utils.common import parse_yaml_or_json
 
 logger = logging.getLogger('awx.main.migrations')
 
@@ -62,3 +65,84 @@ def remove_azure_inventory_sources(apps, schema_editor):
     logger.debug("Removing all Azure InventorySource from database.")
     InventorySource.objects.filter(source='azure').delete()
 
+
+def _get_instance_id(from_dict, new_id, default=''):
+    '''logic mostly duplicated with inventory_import command Command._get_instance_id
+    frozen in time here, for purposes of migrations
+    '''
+    instance_id = default
+    for key in new_id.split('.'):
+        if not hasattr(from_dict, 'get'):
+            instance_id = default
+            break
+        instance_id = from_dict.get(key, default)
+        from_dict = instance_id
+    return smart_text(instance_id)
+
+
+def _get_instance_id_for_upgrade(host, new_id):
+    if host.instance_id:
+        # this should not have happened, but nothing to really do about it...
+        logger.debug('Unexpectedly, host {}-{} has instance_id set'.format(host.name, host.pk))
+        return None
+    host_vars = parse_yaml_or_json(host.variables)
+    new_id_value = _get_instance_id(host_vars, new_id)
+    if not new_id_value:
+        # another source type with overwrite_vars or pesky users could have done this
+        logger.info('Host {}-{} has no {} var, probably due to separate modifications'.format(
+            host.name, host.pk, new_id
+        ))
+        return None
+    if len(new_id) > 255:
+        # this should never happen
+        logger.warn('Computed instance id "{}"" for host {}-{} is too long'.format(
+            new_id_value, host.name, host.pk
+        ))
+        return None
+    return new_id_value
+
+
+def set_new_instance_id(apps, source, new_id):
+    '''This methods adds an instance_id in cases where there was not one before
+    '''
+    from django.conf import settings
+    id_from_settings = getattr(settings, '{}_INSTANCE_ID_VAR'.format(source.upper()))
+    if id_from_settings != new_id:
+        # User applied an instance ID themselves, so nope on out of there
+        logger.warn('You have an instance ID set for {}, not migrating'.format(source))
+        return
+    logger.debug('Migrating inventory instance_id for {} to {}'.format(source, new_id))
+    Host = apps.get_model('main', 'Host')
+    modified_ct = 0
+    for host in Host.objects.filter(inventory_sources__source=source).iterator():
+        new_id_value = _get_instance_id_for_upgrade(host, new_id)
+        if not new_id_value:
+            continue
+        host.instance_id = new_id_value
+        host.save(update_fields=['instance_id'])
+        modified_ct += 1
+    if modified_ct:
+        logger.info('Migrated instance ID for {} hosts imported by {} source'.format(
+            modified_ct, source
+        ))
+
+
+def back_out_new_instance_id(apps, source, new_id):
+    Host = apps.get_model('main', 'Host')
+    modified_ct = 0
+    for host in Host.objects.filter(inventory_sources__source=source).iterator():
+        host_vars = parse_yaml_or_json(host.variables)
+        predicted_id_value = _get_instance_id(host_vars, new_id)
+        if predicted_id_value != host.instance_id:
+            logger.debug('Host {}-{} did not get its instance_id from {}, skipping'.format(
+                host.name, host.pk, new_id
+            ))
+            continue
+        host.instance_id = ''
+        host.save(update_fields=['instance_id'])
+        modified_ct += 1
+    if modified_ct:
+        logger.info('Reverse migrated instance ID for {} hosts imported by {} source'.format(
+            modified_ct, source
+        ))
+

awx/main/tests/functional/test_inventory_source_migration.py

@@ -1,4 +1,5 @@
 import pytest
+from unittest import mock
 
 from awx.main.migrations import _inventory_source as invsrc
 from awx.main.models import InventorySource
@@ -45,3 +46,36 @@ def test_azure_inv_src_removal(inventory_source):
     assert InventorySource.objects.filter(pk=inventory_source.pk).exists()
     invsrc.remove_azure_inventory_sources(apps, None)
     assert not InventorySource.objects.filter(pk=inventory_source.pk).exists()
+
+
+@pytest.mark.parametrize('vars,id_var,result', [
+    ({'foo': {'bar': '1234'}}, 'foo.bar', '1234'),
+    ({'cat': 'meow'}, 'cat', 'meow'),
+    ({'dog': 'woof'}, 'cat', '')
+])
+def test_instance_id(vars, id_var, result):
+    assert invsrc._get_instance_id(vars, id_var) == result
+
+
+@pytest.mark.django_db
+def test_apply_new_instance_id(inventory_source):
+    host1 = inventory_source.hosts.create(
+        name='foo1', inventory=inventory_source.inventory,
+        variables={'foo': 'bar'}, instance_id=''
+    )
+    host2 = inventory_source.hosts.create(
+        name='foo2', inventory=inventory_source.inventory,
+        variables={'foo': 'bar'}, instance_id='bad_user'
+    )
+    with mock.patch('django.conf.settings.{}_INSTANCE_ID_VAR'.format(inventory_source.source.upper()), 'foo'):
+        invsrc.set_new_instance_id(apps, inventory_source.source, 'foo')
+    host1.refresh_from_db()
+    host2.refresh_from_db()
+    assert host1.instance_id == 'bar'
+    assert host2.instance_id == 'bad_user'
+    with mock.patch('django.conf.settings.{}_INSTANCE_ID_VAR'.format(inventory_source.source.upper()), 'foo'):
+        invsrc.back_out_new_instance_id(apps, inventory_source.source, 'foo')
+    host1.refresh_from_db()
+    host2.refresh_from_db()
+    assert host1.instance_id == ''
+    assert host2.instance_id == 'bad_user'

awx/settings/defaults.py

@@ -814,7 +814,7 @@ GCE_ENABLED_VALUE = 'running'
 GCE_GROUP_FILTER = r'^.+$'
 GCE_HOST_FILTER = r'^.+$'
 GCE_EXCLUDE_EMPTY_GROUPS = True
-GCE_INSTANCE_ID_VAR = None
+GCE_INSTANCE_ID_VAR = 'gce_id'
 
 # --------------------------------------
 # -- Microsoft Azure Resource Manager --