diff --git a/awx/main/migrations/0070_v350_gce_instance_id.py b/awx/main/migrations/0070_v350_gce_instance_id.py new file mode 100644 index 0000000000..fe32d930c0 --- /dev/null +++ b/awx/main/migrations/0070_v350_gce_instance_id.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Created manually 2019-03-05 +from __future__ import unicode_literals + +import logging + +from django.db import migrations + +from awx.main.migrations._inventory_source import set_new_instance_id, back_out_new_instance_id + + +logger = logging.getLogger('awx.main.migrations') + + +# new value introduced in awx/settings/defaults.py, frozen in time here +GCE_INSTANCE_ID_VAR = 'gce_id' + + +def gce_id_forward(apps, schema_editor): + set_new_instance_id(apps, 'gce', GCE_INSTANCE_ID_VAR) + + +def gce_id_backward(apps, schema_editor): + back_out_new_instance_id(apps, 'gce', GCE_INSTANCE_ID_VAR) + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0069_v350_generate_unique_install_uuid'), + ] + + operations = [ + migrations.RunPython(gce_id_forward, gce_id_backward) + ] diff --git a/awx/main/migrations/_inventory_source.py b/awx/main/migrations/_inventory_source.py index 53fc2d479d..6378a83463 100644 --- a/awx/main/migrations/_inventory_source.py +++ b/awx/main/migrations/_inventory_source.py @@ -1,6 +1,9 @@ import logging from django.db.models import Q +from django.utils.encoding import smart_text + +from awx.main.utils.common import parse_yaml_or_json logger = logging.getLogger('awx.main.migrations') @@ -62,3 +65,84 @@ def remove_azure_inventory_sources(apps, schema_editor): logger.debug("Removing all Azure InventorySource from database.") InventorySource.objects.filter(source='azure').delete() + +def _get_instance_id(from_dict, new_id, default=''): + '''logic mostly duplicated with inventory_import command Command._get_instance_id + frozen in time here, for purposes of migrations + ''' + instance_id = default + for key in new_id.split('.'): + if not hasattr(from_dict, 'get'): + instance_id = default + break + instance_id = from_dict.get(key, default) + from_dict = instance_id + return smart_text(instance_id) + + +def _get_instance_id_for_upgrade(host, new_id): + if host.instance_id: + # this should not have happened, but nothing to really do about it... + logger.debug('Unexpectedly, host {}-{} has instance_id set'.format(host.name, host.pk)) + return None + host_vars = parse_yaml_or_json(host.variables) + new_id_value = _get_instance_id(host_vars, new_id) + if not new_id_value: + # another source type with overwrite_vars or pesky users could have done this + logger.info('Host {}-{} has no {} var, probably due to separate modifications'.format( + host.name, host.pk, new_id + )) + return None + if len(new_id) > 255: + # this should never happen + logger.warn('Computed instance id "{}"" for host {}-{} is too long'.format( + new_id_value, host.name, host.pk + )) + return None + return new_id_value + + +def set_new_instance_id(apps, source, new_id): + '''This methods adds an instance_id in cases where there was not one before + ''' + from django.conf import settings + id_from_settings = getattr(settings, '{}_INSTANCE_ID_VAR'.format(source.upper())) + if id_from_settings != new_id: + # User applied an instance ID themselves, so nope on out of there + logger.warn('You have an instance ID set for {}, not migrating'.format(source)) + return + logger.debug('Migrating inventory instance_id for {} to {}'.format(source, new_id)) + Host = apps.get_model('main', 'Host') + modified_ct = 0 + for host in Host.objects.filter(inventory_sources__source=source).iterator(): + new_id_value = _get_instance_id_for_upgrade(host, new_id) + if not new_id_value: + continue + host.instance_id = new_id_value + host.save(update_fields=['instance_id']) + modified_ct += 1 + if modified_ct: + logger.info('Migrated instance ID for {} hosts imported by {} source'.format( + modified_ct, source + )) + + +def back_out_new_instance_id(apps, source, new_id): + Host = apps.get_model('main', 'Host') + modified_ct = 0 + for host in Host.objects.filter(inventory_sources__source=source).iterator(): + host_vars = parse_yaml_or_json(host.variables) + predicted_id_value = _get_instance_id(host_vars, new_id) + if predicted_id_value != host.instance_id: + logger.debug('Host {}-{} did not get its instance_id from {}, skipping'.format( + host.name, host.pk, new_id + )) + continue + host.instance_id = '' + host.save(update_fields=['instance_id']) + modified_ct += 1 + if modified_ct: + logger.info('Reverse migrated instance ID for {} hosts imported by {} source'.format( + modified_ct, source + )) + diff --git a/awx/main/tests/functional/test_inventory_source_migration.py b/awx/main/tests/functional/test_inventory_source_migration.py index 9752a65579..ade2329de5 100644 --- a/awx/main/tests/functional/test_inventory_source_migration.py +++ b/awx/main/tests/functional/test_inventory_source_migration.py @@ -1,4 +1,5 @@ import pytest +from unittest import mock from awx.main.migrations import _inventory_source as invsrc from awx.main.models import InventorySource @@ -45,3 +46,36 @@ def test_azure_inv_src_removal(inventory_source): assert InventorySource.objects.filter(pk=inventory_source.pk).exists() invsrc.remove_azure_inventory_sources(apps, None) assert not InventorySource.objects.filter(pk=inventory_source.pk).exists() + + +@pytest.mark.parametrize('vars,id_var,result', [ + ({'foo': {'bar': '1234'}}, 'foo.bar', '1234'), + ({'cat': 'meow'}, 'cat', 'meow'), + ({'dog': 'woof'}, 'cat', '') +]) +def test_instance_id(vars, id_var, result): + assert invsrc._get_instance_id(vars, id_var) == result + + +@pytest.mark.django_db +def test_apply_new_instance_id(inventory_source): + host1 = inventory_source.hosts.create( + name='foo1', inventory=inventory_source.inventory, + variables={'foo': 'bar'}, instance_id='' + ) + host2 = inventory_source.hosts.create( + name='foo2', inventory=inventory_source.inventory, + variables={'foo': 'bar'}, instance_id='bad_user' + ) + with mock.patch('django.conf.settings.{}_INSTANCE_ID_VAR'.format(inventory_source.source.upper()), 'foo'): + invsrc.set_new_instance_id(apps, inventory_source.source, 'foo') + host1.refresh_from_db() + host2.refresh_from_db() + assert host1.instance_id == 'bar' + assert host2.instance_id == 'bad_user' + with mock.patch('django.conf.settings.{}_INSTANCE_ID_VAR'.format(inventory_source.source.upper()), 'foo'): + invsrc.back_out_new_instance_id(apps, inventory_source.source, 'foo') + host1.refresh_from_db() + host2.refresh_from_db() + assert host1.instance_id == '' + assert host2.instance_id == 'bad_user' diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index ca9e8502bc..38ce7e0d6c 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -814,7 +814,7 @@ GCE_ENABLED_VALUE = 'running' GCE_GROUP_FILTER = r'^.+$' GCE_HOST_FILTER = r'^.+$' GCE_EXCLUDE_EMPTY_GROUPS = True -GCE_INSTANCE_ID_VAR = None +GCE_INSTANCE_ID_VAR = 'gce_id' # -------------------------------------- # -- Microsoft Azure Resource Manager --