From a08a158672baa88d15e12790fd432addf492c25f Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 14 Sep 2017 12:58:54 -0400 Subject: [PATCH 1/5] remove azure --- awx/api/metadata.py | 2 +- awx/main/constants.py | 2 +- .../management/commands/inventory_import.py | 1 - awx/main/models/base.py | 2 +- awx/main/models/credential.py | 30 -------- awx/main/models/inventory.py | 9 +-- awx/main/tasks.py | 10 +-- .../tests/functional/api/test_credential.py | 37 ---------- awx/main/tests/functional/test_credential.py | 1 - .../functional/test_credential_migration.py | 16 ---- awx/main/tests/unit/test_tasks.py | 74 +++---------------- awx/settings/defaults.py | 27 ++----- 12 files changed, 23 insertions(+), 188 deletions(-) diff --git a/awx/api/metadata.py b/awx/api/metadata.py index 87aec4b69f..7beb3bd5ad 100644 --- a/awx/api/metadata.py +++ b/awx/api/metadata.py @@ -89,7 +89,7 @@ class Metadata(metadata.SimpleMetadata): # Special handling of inventory source_region choices that vary based on # selected inventory source. if field.field_name == 'source_regions': - for cp in ('azure', 'ec2', 'gce'): + for cp in ('azure_rm', 'ec2', 'gce'): get_regions = getattr(InventorySource, 'get_%s_region_choices' % cp) field_info['%s_region_choices' % cp] = get_regions() diff --git a/awx/main/constants.py b/awx/main/constants.py index 10be060094..4ff22662b3 100644 --- a/awx/main/constants.py +++ b/awx/main/constants.py @@ -5,7 +5,7 @@ import re from django.utils.translation import ugettext_lazy as _ -CLOUD_PROVIDERS = ('azure', 'azure_rm', 'ec2', 'gce', 'rax', 'vmware', 'openstack', 'satellite6', 'cloudforms') +CLOUD_PROVIDERS = ('azure_rm', 'ec2', 'gce', 'vmware', 'openstack', 'satellite6', 'cloudforms') SCHEDULEABLE_PROVIDERS = CLOUD_PROVIDERS + ('custom', 'scm',) PRIVILEGE_ESCALATION_METHODS = [ ('sudo', _('Sudo')), ('su', _('Su')), ('pbrun', _('Pbrun')), ('pfexec', _('Pfexec')), ('dzdo', _('DZDO')), ('pmrun', _('Pmrun')), ('runas', _('Runas'))] ANSI_SGR_PATTERN = re.compile(r'\x1b\[[0-9;]*m') diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index aa64822bfe..d84d70f2ea 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -194,7 +194,6 @@ def load_inventory_source(source, group_filter_re=None, ''' # Sanity check: We sanitize these module names for our API but Ansible proper doesn't follow # good naming conventions - source = source.replace('azure.py', 'windows_azure.py') source = source.replace('satellite6.py', 'foreman.py') source = source.replace('vmware.py', 'vmware_inventory.py') if not os.path.exists(source): diff --git a/awx/main/models/base.py b/awx/main/models/base.py index 7e22acd3ad..84e90fede4 100644 --- a/awx/main/models/base.py +++ b/awx/main/models/base.py @@ -52,7 +52,7 @@ PROJECT_UPDATE_JOB_TYPE_CHOICES = [ (PERM_INVENTORY_CHECK, _('Check')), ] -CLOUD_INVENTORY_SOURCES = ['ec2', 'rax', 'vmware', 'gce', 'azure', 'azure_rm', 'openstack', 'custom', 'satellite6', 'cloudforms', 'scm',] +CLOUD_INVENTORY_SOURCES = ['ec2', 'rax', 'vmware', 'gce', 'azure_rm', 'openstack', 'custom', 'satellite6', 'cloudforms', 'scm',] VERBOSITY_CHOICES = [ (0, '0 (Normal)'), diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 9b81498bfa..5a3f0b9662 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -57,7 +57,6 @@ class V1Credential(object): ('satellite6', 'Red Hat Satellite 6'), ('cloudforms', 'Red Hat CloudForms'), ('gce', 'Google Compute Engine'), - ('azure', 'Microsoft Azure Classic (deprecated)'), ('azure_rm', 'Microsoft Azure Resource Manager'), ('openstack', 'OpenStack'), ('insights', 'Insights'), @@ -934,35 +933,6 @@ def gce(cls): ) -@CredentialType.default -def azure(cls): - return cls( - kind='cloud', - name='Microsoft Azure Classic (deprecated)', - managed_by_tower=True, - inputs={ - 'fields': [{ - 'id': 'username', - 'label': 'Subscription ID', - 'type': 'string', - 'help_text': ('Subscription ID is an Azure construct, which is ' - 'mapped to a username.') - }, { - 'id': 'ssh_key_data', - 'label': 'Management Certificate', - 'type': 'string', - 'format': 'ssh_private_key', - 'secret': True, - 'multiline': True, - 'help_text': ('Paste the contents of the PEM file that corresponds ' - 'to the certificate you uploaded in the Microsoft ' - 'Azure console.') - }], - 'required': ['username', 'ssh_key_data'], - } - ) - - @CredentialType.default def azure_rm(cls): return cls( diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index e13db5f749..7b5e3d3dfa 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -867,7 +867,6 @@ class InventorySourceOptions(BaseModel): ('scm', _('Sourced from a Project')), ('ec2', _('Amazon EC2')), ('gce', _('Google Compute Engine')), - ('azure', _('Microsoft Azure Classic (deprecated)')), ('azure_rm', _('Microsoft Azure Resource Manager')), ('vmware', _('VMware vCenter')), ('satellite6', _('Red Hat Satellite 6')), @@ -1087,7 +1086,7 @@ class InventorySourceOptions(BaseModel): return regions @classmethod - def get_azure_region_choices(self): + def get_azure_rm_region_choices(self): """Return a complete list of regions in Microsoft Azure, as a list of two-tuples. """ @@ -1095,14 +1094,10 @@ class InventorySourceOptions(BaseModel): # authenticating first (someone reading these might think there's # a pattern here!). Therefore, you guessed it, use a list from # settings. - regions = list(getattr(settings, 'AZURE_REGION_CHOICES', [])) + regions = list(getattr(settings, 'AZURE_RM_REGION_CHOICES', [])) regions.insert(0, ('all', 'All')) return regions - @classmethod - def get_azure_rm_region_choices(self): - return InventorySourceOptions.get_azure_region_choices() - @classmethod def get_vmware_region_choices(self): """Return a complete list of regions in VMware, as a list of two-tuples diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 2c884802c2..18f1ed5be3 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1054,9 +1054,6 @@ class RunJob(BaseTask): env['GCE_EMAIL'] = cloud_cred.username env['GCE_PROJECT'] = cloud_cred.project env['GCE_PEM_FILE_PATH'] = cred_files.get(cloud_cred, '') - elif cloud_cred and cloud_cred.kind == 'azure': - env['AZURE_SUBSCRIPTION_ID'] = cloud_cred.username - env['AZURE_CERT_PATH'] = cred_files.get(cloud_cred, '') elif cloud_cred and cloud_cred.kind == 'azure_rm': if len(cloud_cred.client) and len(cloud_cred.tenant): env['AZURE_CLIENT_ID'] = cloud_cred.client @@ -1616,8 +1613,8 @@ class RunInventoryUpdate(BaseTask): If no private data is needed, return None. """ private_data = {'credentials': {}} - # If this is Microsoft Azure or GCE, return the RSA key - if inventory_update.source in ('azure', 'gce'): + # If this is GCE, return the RSA key + if inventory_update.source == 'gce': credential = inventory_update.credential private_data['credentials'][credential] = decrypt_field(credential, 'ssh_key_data') return private_data @@ -1847,9 +1844,6 @@ class RunInventoryUpdate(BaseTask): env['EC2_INI_PATH'] = cloud_credential elif inventory_update.source == 'vmware': env['VMWARE_INI_PATH'] = cloud_credential - elif inventory_update.source == 'azure': - env['AZURE_SUBSCRIPTION_ID'] = passwords.get('source_username', '') - env['AZURE_CERT_PATH'] = cloud_credential elif inventory_update.source == 'azure_rm': if len(passwords.get('source_client', '')) and \ len(passwords.get('source_tenant', '')): diff --git a/awx/main/tests/functional/api/test_credential.py b/awx/main/tests/functional/api/test_credential.py index 8063ff01e7..feebfc08f4 100644 --- a/awx/main/tests/functional/api/test_credential.py +++ b/awx/main/tests/functional/api/test_credential.py @@ -1071,43 +1071,6 @@ def test_gce_create_ok(post, organization, admin, version, params): assert decrypt_field(cred, 'ssh_key_data') == EXAMPLE_PRIVATE_KEY -# -# Azure Classic -# -@pytest.mark.django_db -@pytest.mark.parametrize('version, params', [ - ['v1', { - 'kind': 'azure', - 'name': 'Best credential ever', - 'username': 'some_username', - 'ssh_key_data': EXAMPLE_PRIVATE_KEY - }], - ['v2', { - 'credential_type': 1, - 'name': 'Best credential ever', - 'inputs': { - 'username': 'some_username', - 'ssh_key_data': EXAMPLE_PRIVATE_KEY - } - }] -]) -def test_azure_create_ok(post, organization, admin, version, params): - azure = CredentialType.defaults['azure']() - azure.save() - params['organization'] = organization.id - response = post( - reverse('api:credential_list', kwargs={'version': version}), - params, - admin - ) - assert response.status_code == 201 - - assert Credential.objects.count() == 1 - cred = Credential.objects.all()[:1].get() - assert cred.inputs['username'] == 'some_username' - assert decrypt_field(cred, 'ssh_key_data') == EXAMPLE_PRIVATE_KEY - - # # Azure Resource Manager # diff --git a/awx/main/tests/functional/test_credential.py b/awx/main/tests/functional/test_credential.py index 4a51565d04..b4dd0cb0e4 100644 --- a/awx/main/tests/functional/test_credential.py +++ b/awx/main/tests/functional/test_credential.py @@ -19,7 +19,6 @@ EXAMPLE_ENCRYPTED_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nProc-Type: 4,ENCRY def test_default_cred_types(): assert sorted(CredentialType.defaults.keys()) == [ 'aws', - 'azure', 'azure_rm', 'cloudforms', 'gce', diff --git a/awx/main/tests/functional/test_credential_migration.py b/awx/main/tests/functional/test_credential_migration.py index b5453c9a0d..3494545501 100644 --- a/awx/main/tests/functional/test_credential_migration.py +++ b/awx/main/tests/functional/test_credential_migration.py @@ -269,22 +269,6 @@ def test_gce_migration(): assert Credential.objects.count() == 1 -@pytest.mark.django_db -def test_azure_classic_migration(): - cred = Credential(name='My Credential') - with migrate(cred, 'azure'): - cred.__dict__.update({ - 'username': 'bob', - 'ssh_key_data': EXAMPLE_PRIVATE_KEY - }) - - assert cred.credential_type.name == 'Microsoft Azure Classic (deprecated)' - assert cred.inputs['username'] == 'bob' - assert cred.inputs['ssh_key_data'].startswith('$encrypted$') - assert decrypt_field(cred, 'ssh_key_data') == EXAMPLE_PRIVATE_KEY - assert Credential.objects.count() == 1 - - @pytest.mark.django_db def test_azure_rm_migration(): cred = Credential(name='My Credential') diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 99ad34d766..dfaeab121b 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -533,29 +533,6 @@ class TestJobCredentials(TestJobExecution): self.run_pexpect.side_effect = run_pexpect_side_effect self.task.run(self.pk) - def test_azure_credentials(self): - azure = CredentialType.defaults['azure']() - credential = Credential( - pk=1, - credential_type=azure, - inputs = { - 'username': 'bob', - 'ssh_key_data': self.EXAMPLE_PRIVATE_KEY - } - ) - credential.inputs['ssh_key_data'] = encrypt_field(credential, 'ssh_key_data') - self.instance.extra_credentials.add(credential) - - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - assert env['AZURE_SUBSCRIPTION_ID'] == 'bob' - ssh_key_data = env['AZURE_CERT_PATH'] - assert open(ssh_key_data, 'rb').read() == self.EXAMPLE_PRIVATE_KEY - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) - def test_azure_rm_with_tenant(self): azure = CredentialType.defaults['azure_rm']() credential = Credential( @@ -1027,29 +1004,25 @@ class TestJobCredentials(TestJobExecution): gce_credential.inputs['ssh_key_data'] = encrypt_field(gce_credential, 'ssh_key_data') self.instance.extra_credentials.add(gce_credential) - azure = CredentialType.defaults['azure']() - azure_credential = Credential( + azure_rm = CredentialType.defaults['azure_rm']() + azure_rm_credential = Credential( pk=2, - credential_type=azure, + credential_type=azure_rm, inputs = { - 'username': 'joe', - 'ssh_key_data': 'AZURE: %s' % self.EXAMPLE_PRIVATE_KEY + 'subscription': 'some-subscription', + 'username': 'bob', + 'password': 'secret' } ) - azure_credential.inputs['ssh_key_data'] = encrypt_field(azure_credential, 'ssh_key_data') - self.instance.extra_credentials.add(azure_credential) + azure_rm_credential.inputs['secret'] = encrypt_field(azure_rm_credential, 'secret') + self.instance.extra_credentials.add(azure_rm_credential) def run_pexpect_side_effect(*args, **kwargs): args, cwd, env, stdout = args - assert env['GCE_EMAIL'] == 'bob' - assert env['GCE_PROJECT'] == 'some-project' - ssh_key_data = env['GCE_PEM_FILE_PATH'] - assert open(ssh_key_data, 'rb').read() == 'GCE: %s' % self.EXAMPLE_PRIVATE_KEY - - assert env['AZURE_SUBSCRIPTION_ID'] == 'joe' - ssh_key_data = env['AZURE_CERT_PATH'] - assert open(ssh_key_data, 'rb').read() == 'AZURE: %s' % self.EXAMPLE_PRIVATE_KEY + assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription' + assert env['AZURE_AD_USER'] == 'bob' + assert env['AZURE_PASSWORD'] == 'secret' return ['successful', 0] @@ -1246,31 +1219,6 @@ class TestInventoryUpdateCredentials(TestJobExecution): self.run_pexpect.side_effect = run_pexpect_side_effect self.task.run(self.pk) - def test_azure_source(self): - azure = CredentialType.defaults['azure']() - self.instance.source = 'azure' - self.instance.credential = Credential( - pk=1, - credential_type=azure, - inputs = { - 'username': 'bob', - 'ssh_key_data': self.EXAMPLE_PRIVATE_KEY - } - ) - self.instance.credential.inputs['ssh_key_data'] = encrypt_field( - self.instance.credential, 'ssh_key_data' - ) - - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - assert env['AZURE_SUBSCRIPTION_ID'] == 'bob' - ssh_key_data = env['AZURE_CERT_PATH'] - assert open(ssh_key_data, 'rb').read() == self.EXAMPLE_PRIVATE_KEY - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) - def test_gce_source(self): gce = CredentialType.defaults['gce']() self.instance.source = 'gce' diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index c4af5d8d10..29c40dc230 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -770,14 +770,12 @@ GCE_HOST_FILTER = r'^.+$' GCE_EXCLUDE_EMPTY_GROUPS = True GCE_INSTANCE_ID_VAR = None - -# ------------------- -# -- Microsoft Azure -- -# ------------------- - +# -------------------------------------- +# -- Microsoft Azure Resource Manager -- +# -------------------------------------- # It's not possible to get zones in Azure without authenticating, so we # provide a list here. -AZURE_REGION_CHOICES = [ +AZURE_RM_REGION_CHOICES = [ ('eastus', _('US East')), ('eastus2', _('US East 2')), ('centralus', _('US Central')), @@ -804,23 +802,8 @@ AZURE_REGION_CHOICES = [ ('koreacentral', _('Korea Central')), ('koreasouth', _('Korea South')), ] -AZURE_REGIONS_BLACKLIST = [] +AZURE_RM_REGIONS_BLACKLIST = [] -# Inventory variable name/value for determining whether a host is active -# in Microsoft Azure. -AZURE_ENABLED_VAR = 'instance_status' -AZURE_ENABLED_VALUE = 'ReadyRole' - -# Filter for allowed group and host names when importing inventory from -# Microsoft Azure. -AZURE_GROUP_FILTER = r'^.+$' -AZURE_HOST_FILTER = r'^.+$' -AZURE_EXCLUDE_EMPTY_GROUPS = True -AZURE_INSTANCE_ID_VAR = 'private_id' - -# -------------------------------------- -# -- Microsoft Azure Resource Manager -- -# -------------------------------------- AZURE_RM_GROUP_FILTER = r'^.+$' AZURE_RM_HOST_FILTER = r'^.+$' AZURE_RM_ENABLED_VAR = 'powerstate' From 0b403311073a8eb21ee0bbe7a8f85423c1cd729f Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Fri, 15 Sep 2017 11:19:40 -0400 Subject: [PATCH 2/5] azure migrations * delete azure credentials * delete azure inventory sources --- awx/main/migrations/0006_v320_release.py | 4 ++-- awx/main/migrations/0007_v320_data_migrations.py | 3 +++ awx/main/migrations/_azure_credentials.py | 15 +++++++++++++++ awx/main/migrations/_inventory_source.py | 9 +++++++++ .../functional/test_inventory_source_migration.py | 10 ++++++++++ 5 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 awx/main/migrations/_azure_credentials.py diff --git a/awx/main/migrations/0006_v320_release.py b/awx/main/migrations/0006_v320_release.py index e9af9962c5..45eabbebc5 100644 --- a/awx/main/migrations/0006_v320_release.py +++ b/awx/main/migrations/0006_v320_release.py @@ -145,12 +145,12 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='inventorysource', name='source', - field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'File, Directory or Script'), (b'scm', 'Sourced from a Project'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'satellite6', 'Red Hat Satellite 6'), (b'cloudforms', 'Red Hat CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]), + field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'File, Directory or Script'), (b'scm', 'Sourced from a Project'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'satellite6', 'Red Hat Satellite 6'), (b'cloudforms', 'Red Hat CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]), ), migrations.AlterField( model_name='inventoryupdate', name='source', - field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'File, Directory or Script'), (b'scm', 'Sourced from a Project'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'satellite6', 'Red Hat Satellite 6'), (b'cloudforms', 'Red Hat CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]), + field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'File, Directory or Script'), (b'scm', 'Sourced from a Project'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'satellite6', 'Red Hat Satellite 6'), (b'cloudforms', 'Red Hat CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]), ), migrations.AlterField( model_name='inventorysource', diff --git a/awx/main/migrations/0007_v320_data_migrations.py b/awx/main/migrations/0007_v320_data_migrations.py index 2971dba213..9461e81bcb 100644 --- a/awx/main/migrations/0007_v320_data_migrations.py +++ b/awx/main/migrations/0007_v320_data_migrations.py @@ -11,6 +11,7 @@ from awx.main.migrations import _migration_utils as migration_utils from awx.main.migrations import _reencrypt as reencrypt from awx.main.migrations import _scan_jobs as scan_jobs from awx.main.migrations import _credentialtypes as credentialtypes +from awx.main.migrations import _azure_credentials as azurecreds import awx.main.fields @@ -24,6 +25,8 @@ class Migration(migrations.Migration): # Inventory Refresh migrations.RunPython(migration_utils.set_current_apps_for_migrations), migrations.RunPython(invsrc.remove_rax_inventory_sources), + migrations.RunPython(azurecreds.remove_azure_credentials), + migrations.RunPython(invsrc.remove_azure_inventory_sources), migrations.RunPython(invsrc.remove_inventory_source_with_no_inventory_link), migrations.RunPython(invsrc.rename_inventory_sources), migrations.RunPython(reencrypt.replace_aesecb_fernet), diff --git a/awx/main/migrations/_azure_credentials.py b/awx/main/migrations/_azure_credentials.py new file mode 100644 index 0000000000..93981ea8f9 --- /dev/null +++ b/awx/main/migrations/_azure_credentials.py @@ -0,0 +1,15 @@ +import logging + +from django.db.models import Q + +logger = logging.getLogger('awx.main.migrations') + + +def remove_azure_credentials(apps, schema_editor): + '''Azure is not supported as of 3.2 and greater. Instead, azure_rm is + supported. + ''' + Credential = apps.get_model('main', 'Credential') + logger.debug("Removing all Azure Credentials from database.") + Credential.objects.filter(kind='azure').delete() + diff --git a/awx/main/migrations/_inventory_source.py b/awx/main/migrations/_inventory_source.py index baf5576e90..e2401ee3ae 100644 --- a/awx/main/migrations/_inventory_source.py +++ b/awx/main/migrations/_inventory_source.py @@ -51,3 +51,12 @@ def remove_inventory_source_with_no_inventory_link(apps, schema_editor): InventorySource = apps.get_model('main', 'InventorySource') logger.debug("Removing all InventorySource that have no link to an Inventory from database.") InventorySource.objects.filter(Q(inventory__organization=None) & Q(deprecated_group__inventory=None)).delete() + + +def remove_azure_inventory_sources(apps, schema_editor): + '''Azure inventory sources are not supported since 3.2, remove them. + ''' + InventorySource = apps.get_model('main', 'InventorySource') + logger.debug("Removing all Azure InventorySource from database.") + InventorySource.objects.filter(source='azure').delete() + diff --git a/awx/main/tests/functional/test_inventory_source_migration.py b/awx/main/tests/functional/test_inventory_source_migration.py index 75d7a15704..9752a65579 100644 --- a/awx/main/tests/functional/test_inventory_source_migration.py +++ b/awx/main/tests/functional/test_inventory_source_migration.py @@ -35,3 +35,13 @@ def test_inv_src_rename(inventory_source_factory): inv_src01.refresh_from_db() # inv-is-t1 is generated in the inventory_source_factory assert inv_src01.name == 't1 - inv-is-t1 - 0' + + +@pytest.mark.django_db +def test_azure_inv_src_removal(inventory_source): + inventory_source.source = 'azure' + inventory_source.save() + + assert InventorySource.objects.filter(pk=inventory_source.pk).exists() + invsrc.remove_azure_inventory_sources(apps, None) + assert not InventorySource.objects.filter(pk=inventory_source.pk).exists() From 4dd265633e2a15cb798b05fdd0fe10984067ed8c Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 18 Sep 2017 10:41:49 -0400 Subject: [PATCH 3/5] remove legacy azure inventory script see: ansible/ansible-tower#7629 --- awx/plugins/inventory/windows_azure.py | 284 ------------------------- 1 file changed, 284 deletions(-) delete mode 100755 awx/plugins/inventory/windows_azure.py diff --git a/awx/plugins/inventory/windows_azure.py b/awx/plugins/inventory/windows_azure.py deleted file mode 100755 index cceed36bcc..0000000000 --- a/awx/plugins/inventory/windows_azure.py +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/env python - -''' -Windows Azure external inventory script -======================================= - -Generates inventory that Ansible can understand by making API request to -Windows Azure using the azure python library. - -NOTE: This script assumes Ansible is being executed where azure is already -installed. - - pip install azure - -Adapted from the ansible Linode plugin by Dan Slimmon. -''' - -# (c) 2013, John Whitbeck -# -# This file is part of Ansible, -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -###################################################################### - -# Standard imports -import re -import sys -import argparse -import os -from urlparse import urlparse -from time import time -try: - import json -except ImportError: - import simplejson as json - -try: - from azure.servicemanagement import ServiceManagementService -except ImportError as e: - sys.exit("ImportError: {0}".format(str(e))) - -# Imports for ansible -import ConfigParser - -class AzureInventory(object): - def __init__(self): - """Main execution path.""" - # Inventory grouped by display group - self.inventory = {} - # Index of deployment name -> host - self.index = {} - self.host_metadata = {} - - # Cache setting defaults. - # These can be overridden in settings (see `read_settings`). - cache_dir = os.path.expanduser('~') - self.cache_path_cache = os.path.join(cache_dir, '.ansible-azure.cache') - self.cache_path_index = os.path.join(cache_dir, '.ansible-azure.index') - self.cache_max_age = 0 - - # Read settings and parse CLI arguments - self.read_settings() - self.read_environment() - self.parse_cli_args() - - # Initialize Azure ServiceManagementService - self.sms = ServiceManagementService(self.subscription_id, self.cert_path) - - # Cache - if self.args.refresh_cache: - self.do_api_calls_update_cache() - elif not self.is_cache_valid(): - self.do_api_calls_update_cache() - - if self.args.list_images: - data_to_print = self.json_format_dict(self.get_images(), True) - elif self.args.list or self.args.host: - # Display list of nodes for inventory - if len(self.inventory) == 0: - data = json.loads(self.get_inventory_from_cache()) - else: - data = self.inventory - - if self.args.host: - data_to_print = self.get_host(self.args.host) - else: - # Add the `['_meta']['hostvars']` information. - hostvars = {} - if len(data) > 0: - for host in set([h for hosts in data.values() for h in hosts if h]): - hostvars[host] = self.get_host(host, jsonify=False) - data['_meta'] = {'hostvars': hostvars} - - # JSONify the data. - data_to_print = self.json_format_dict(data, pretty=True) - print(data_to_print) - - def get_host(self, hostname, jsonify=True): - """Return information about the given hostname, based on what - the Windows Azure API provides. - """ - if hostname not in self.host_metadata: - return "No host found: %s" % json.dumps(self.host_metadata) - if jsonify: - return json.dumps(self.host_metadata[hostname]) - return self.host_metadata[hostname] - - def get_images(self): - images = [] - for image in self.sms.list_os_images(): - if str(image.label).lower().find(self.args.list_images.lower()) >= 0: - images.append(vars(image)) - return json.loads(json.dumps(images, default=lambda o: o.__dict__)) - - def is_cache_valid(self): - """Determines if the cache file has expired, or if it is still valid.""" - if os.path.isfile(self.cache_path_cache): - mod_time = os.path.getmtime(self.cache_path_cache) - current_time = time() - if (mod_time + self.cache_max_age) > current_time: - if os.path.isfile(self.cache_path_index): - return True - return False - - def read_settings(self): - """Reads the settings from the .ini file.""" - config = ConfigParser.SafeConfigParser() - config.read(os.path.dirname(os.path.realpath(__file__)) + '/windows_azure.ini') - - # Credentials related - if config.has_option('azure', 'subscription_id'): - self.subscription_id = config.get('azure', 'subscription_id') - if config.has_option('azure', 'cert_path'): - self.cert_path = config.get('azure', 'cert_path') - - # Cache related - if config.has_option('azure', 'cache_path'): - cache_path = os.path.expandvars(os.path.expanduser(config.get('azure', 'cache_path'))) - self.cache_path_cache = os.path.join(cache_path, 'ansible-azure.cache') - self.cache_path_index = os.path.join(cache_path, 'ansible-azure.index') - if config.has_option('azure', 'cache_max_age'): - self.cache_max_age = config.getint('azure', 'cache_max_age') - - def read_environment(self): - ''' Reads the settings from environment variables ''' - # Credentials - if os.getenv("AZURE_SUBSCRIPTION_ID"): - self.subscription_id = os.getenv("AZURE_SUBSCRIPTION_ID") - if os.getenv("AZURE_CERT_PATH"): - self.cert_path = os.getenv("AZURE_CERT_PATH") - - def parse_cli_args(self): - """Command line argument processing""" - parser = argparse.ArgumentParser( - description='Produce an Ansible Inventory file based on Azure', - ) - parser.add_argument('--list', action='store_true', default=True, - help='List nodes (default: True)') - parser.add_argument('--list-images', action='store', - help='Get all available images.') - parser.add_argument('--refresh-cache', - action='store_true', default=False, - help='Force refresh of thecache by making API requests to Azure ' - '(default: False - use cache files)', - ) - parser.add_argument('--host', action='store', - help='Get all information about an instance.') - self.args = parser.parse_args() - - def do_api_calls_update_cache(self): - """Do API calls, and save data in cache files.""" - self.add_cloud_services() - self.write_to_cache(self.inventory, self.cache_path_cache) - self.write_to_cache(self.index, self.cache_path_index) - - def add_cloud_services(self): - """Makes an Azure API call to get the list of cloud services.""" - try: - for cloud_service in self.sms.list_hosted_services(): - self.add_deployments(cloud_service) - except Exception as e: - sys.exit("Error: Failed to access cloud services - {0}".format(str(e))) - - def add_deployments(self, cloud_service): - """Makes an Azure API call to get the list of virtual machines - associated with a cloud service. - """ - try: - for deployment in self.sms.get_hosted_service_properties(cloud_service.service_name,embed_detail=True).deployments.deployments: - self.add_deployment(cloud_service, deployment) - except Exception as e: - sys.exit("Error: Failed to access deployments - {0}".format(str(e))) - - def add_deployment(self, cloud_service, deployment): - """Adds a deployment to the inventory and index""" - for role in deployment.role_instance_list.role_instances: - try: - # Default port 22 unless port found with name 'SSH' - port = '22' - for ie in role.instance_endpoints.instance_endpoints: - if ie.name == 'SSH': - port = ie.public_port - break - except AttributeError as e: - pass - finally: - self.add_instance(role.instance_name, deployment, port, cloud_service, role.instance_status) - - def add_instance(self, hostname, deployment, ssh_port, cloud_service, status): - """Adds an instance to the inventory and index""" - - dest = urlparse(deployment.url).hostname - - # Add to index - self.index[hostname] = deployment.name - - self.host_metadata[hostname] = dict(ansible_ssh_host=dest, - ansible_ssh_port=int(ssh_port), - instance_status=status, - private_id=deployment.private_id) - - # List of all azure deployments - self.push(self.inventory, "azure", hostname) - - # Inventory: Group by service name - self.push(self.inventory, self.to_safe(cloud_service.service_name), hostname) - - if int(ssh_port) == 22: - self.push(self.inventory, "Cloud_services", hostname) - - # Inventory: Group by region - self.push(self.inventory, self.to_safe(cloud_service.hosted_service_properties.location), hostname) - - def push(self, my_dict, key, element): - """Pushed an element onto an array that may not have been defined in the dict.""" - if key in my_dict: - my_dict[key].append(element) - else: - my_dict[key] = [element] - - def get_inventory_from_cache(self): - """Reads the inventory from the cache file and returns it as a JSON object.""" - cache = open(self.cache_path_cache, 'r') - json_inventory = cache.read() - return json_inventory - - def load_index_from_cache(self): - """Reads the index from the cache file and sets self.index.""" - cache = open(self.cache_path_index, 'r') - json_index = cache.read() - self.index = json.loads(json_index) - - def write_to_cache(self, data, filename): - """Writes data in JSON format to a file.""" - json_data = self.json_format_dict(data, True) - cache = open(filename, 'w') - cache.write(json_data) - cache.close() - - def to_safe(self, word): - """Escapes any characters that would be invalid in an ansible group name.""" - return re.sub("[^A-Za-z0-9\-]", "_", word) - - def json_format_dict(self, data, pretty=False): - """Converts a dict to a JSON object and dumps it as a formatted string.""" - if pretty: - return json.dumps(data, sort_keys=True, indent=2) - else: - return json.dumps(data) - - -AzureInventory() From 14b0f9aa24ead98b10bee97fead07cc477730363 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 18 Sep 2017 10:43:02 -0400 Subject: [PATCH 4/5] remove reference to legacy rax credentials --- awx/main/models/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/models/base.py b/awx/main/models/base.py index 84e90fede4..6c038dad74 100644 --- a/awx/main/models/base.py +++ b/awx/main/models/base.py @@ -52,7 +52,7 @@ PROJECT_UPDATE_JOB_TYPE_CHOICES = [ (PERM_INVENTORY_CHECK, _('Check')), ] -CLOUD_INVENTORY_SOURCES = ['ec2', 'rax', 'vmware', 'gce', 'azure_rm', 'openstack', 'custom', 'satellite6', 'cloudforms', 'scm',] +CLOUD_INVENTORY_SOURCES = ['ec2', 'vmware', 'gce', 'azure_rm', 'openstack', 'custom', 'satellite6', 'cloudforms', 'scm',] VERBOSITY_CHOICES = [ (0, '0 (Normal)'), From cfe1f1e8e4efcd0b0c7cf38c86df627cf22e6173 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 18 Sep 2017 10:43:16 -0400 Subject: [PATCH 5/5] more legacy azure deprecation cleanup --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1ffaacf118..986c565660 100755 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,7 @@ # W391 - Blank line at end of file # W293 - Blank line contains whitespace ignore=E201,E203,E221,E225,E231,E241,E251,E261,E265,E303,E501,W291,W391,W293 -exclude=.tox,venv,awx/lib/site-packages,awx/plugins/inventory/ec2.py,awx/plugins/inventory/gce.py,awx/plugins/inventory/vmware.py,awx/plugins/inventory/windows_azure.py,awx/plugins/inventory/openstack.py,awx/ui,awx/api/urls.py,awx/main/migrations,awx/main/south_migrations,awx/main/tests/data,installer/openshift/settings.py +exclude=.tox,venv,awx/lib/site-packages,awx/plugins/inventory/ec2.py,awx/plugins/inventory/gce.py,awx/plugins/inventory/vmware.py,awx/plugins/inventory/openstack.py,awx/ui,awx/api/urls.py,awx/main/migrations,awx/main/south_migrations,awx/main/tests/data,installer/openshift/settings.py [flake8] ignore=E201,E203,E221,E225,E231,E241,E251,E261,E265,E303,E501,W291,W391,W293,E731,F405