From 768c7ba3dc07375d7006a5375941b8c08c77df6f Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 1 Sep 2017 15:14:15 -0400 Subject: [PATCH] bump azurerm dependencies to support Ansible 2.4 see: https://github.com/ansible/ansible-tower/issues/7470 --- awx/plugins/inventory/azure_rm.py | 107 +++++++++++++++++++------- requirements/requirements_ansible.in | 15 +++- requirements/requirements_ansible.txt | 52 +++++++------ 3 files changed, 122 insertions(+), 52 deletions(-) diff --git a/awx/plugins/inventory/azure_rm.py b/awx/plugins/inventory/azure_rm.py index 73b8b959d3..b3b7e1e904 100755 --- a/awx/plugins/inventory/azure_rm.py +++ b/awx/plugins/inventory/azure_rm.py @@ -49,6 +49,7 @@ Command line arguments: - tenant - ad_user - password + - cloud_environment Environment variables: - AZURE_PROFILE @@ -58,6 +59,7 @@ Environment variables: - AZURE_TENANT - AZURE_AD_USER - AZURE_PASSWORD + - AZURE_CLOUD_ENVIRONMENT Run for Specific Host ----------------------- @@ -190,22 +192,27 @@ import json import os import re import sys +import inspect +import traceback + from packaging.version import Version from os.path import expanduser +import ansible.module_utils.six.moves.urllib.parse as urlparse HAS_AZURE = True HAS_AZURE_EXC = None try: from msrestazure.azure_exceptions import CloudError + from msrestazure import azure_cloud from azure.mgmt.compute import __version__ as azure_compute_version from azure.common import AzureMissingResourceHttpError, AzureHttpError from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials - from azure.mgmt.network.network_management_client import NetworkManagementClient - from azure.mgmt.resource.resources.resource_management_client import ResourceManagementClient - from azure.mgmt.compute.compute_management_client import ComputeManagementClient + from azure.mgmt.network import NetworkManagementClient + from azure.mgmt.resource.resources import ResourceManagementClient + from azure.mgmt.compute import ComputeManagementClient except ImportError as exc: HAS_AZURE_EXC = exc HAS_AZURE = False @@ -218,7 +225,8 @@ AZURE_CREDENTIAL_ENV_MAPPING = dict( secret='AZURE_SECRET', tenant='AZURE_TENANT', ad_user='AZURE_AD_USER', - password='AZURE_PASSWORD' + password='AZURE_PASSWORD', + cloud_environment='AZURE_CLOUD_ENVIRONMENT', ) AZURE_CONFIG_SETTINGS = dict( @@ -232,7 +240,7 @@ AZURE_CONFIG_SETTINGS = dict( group_by_tag='AZURE_GROUP_BY_TAG' ) -AZURE_MIN_VERSION = "0.30.0rc5" +AZURE_MIN_VERSION = "2.0.0" def azure_id_to_dict(id): @@ -249,6 +257,7 @@ class AzureRM(object): def __init__(self, args): self._args = args + self._cloud_environment = None self._compute_client = None self._resource_client = None self._network_client = None @@ -262,6 +271,26 @@ class AzureRM(object): self.fail("Failed to get credentials. Either pass as parameters, set environment variables, " "or define a profile in ~/.azure/credentials.") + # if cloud_environment specified, look up/build Cloud object + raw_cloud_env = self.credentials.get('cloud_environment') + if not raw_cloud_env: + self._cloud_environment = azure_cloud.AZURE_PUBLIC_CLOUD # SDK default + else: + # try to look up "well-known" values via the name attribute on azure_cloud members + all_clouds = [x[1] for x in inspect.getmembers(azure_cloud) if isinstance(x[1], azure_cloud.Cloud)] + matched_clouds = [x for x in all_clouds if x.name == raw_cloud_env] + if len(matched_clouds) == 1: + self._cloud_environment = matched_clouds[0] + elif len(matched_clouds) > 1: + self.fail("Azure SDK failure: more than one cloud matched for cloud_environment name '{0}'".format(raw_cloud_env)) + else: + if not urlparse.urlparse(raw_cloud_env).scheme: + self.fail("cloud_environment must be an endpoint discovery URL or one of {0}".format([x.name for x in all_clouds])) + try: + self._cloud_environment = azure_cloud.get_cloud_from_metadata_endpoint(raw_cloud_env) + except Exception as e: + self.fail("cloud_environment {0} could not be resolved: {1}".format(raw_cloud_env, e.message)) + if self.credentials.get('subscription_id', None) is None: self.fail("Credentials did not include a subscription_id value.") self.log("setting subscription_id") @@ -272,16 +301,23 @@ class AzureRM(object): self.credentials.get('tenant') is not None: self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'], secret=self.credentials['secret'], - tenant=self.credentials['tenant']) + tenant=self.credentials['tenant'], + cloud_environment=self._cloud_environment) elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None: - self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password']) + tenant = self.credentials.get('tenant') + if not tenant: + tenant = 'common' + self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], + self.credentials['password'], + tenant=tenant, + cloud_environment=self._cloud_environment) else: self.fail("Failed to authenticate with provided credentials. Some attributes were missing. " "Credentials must include client_id, secret and tenant or ad_user and password.") def log(self, msg): if self.debug: - print (msg + u'\n') + print(msg + u'\n') def fail(self, msg): raise Exception(msg) @@ -341,6 +377,10 @@ class AzureRM(object): self.log('Received credentials from parameters.') return arg_credentials + if arg_credentials['ad_user'] is not None: + self.log('Received credentials from parameters.') + return arg_credentials + # try environment env_credentials = self._get_env_credentials() if env_credentials: @@ -372,7 +412,12 @@ class AzureRM(object): def network_client(self): self.log('Getting network client') if not self._network_client: - self._network_client = NetworkManagementClient(self.azure_credentials, self.subscription_id) + self._network_client = NetworkManagementClient( + self.azure_credentials, + self.subscription_id, + base_url=self._cloud_environment.endpoints.resource_manager, + api_version='2017-06-01' + ) self._register('Microsoft.Network') return self._network_client @@ -380,14 +425,24 @@ class AzureRM(object): def rm_client(self): self.log('Getting resource manager client') if not self._resource_client: - self._resource_client = ResourceManagementClient(self.azure_credentials, self.subscription_id) + self._resource_client = ResourceManagementClient( + self.azure_credentials, + self.subscription_id, + base_url=self._cloud_environment.endpoints.resource_manager, + api_version='2017-05-10' + ) return self._resource_client @property def compute_client(self): self.log('Getting compute client') if not self._compute_client: - self._compute_client = ComputeManagementClient(self.azure_credentials, self.subscription_id) + self._compute_client = ComputeManagementClient( + self.azure_credentials, + self.subscription_id, + base_url=self._cloud_environment.endpoints.resource_manager, + api_version='2017-03-30' + ) self._register('Microsoft.Compute') return self._compute_client @@ -440,7 +495,7 @@ class AzureInventory(object): self.include_powerstate = False self.get_inventory() - print (self._json_format_dict(pretty=self._args.pretty)) + print(self._json_format_dict(pretty=self._args.pretty)) sys.exit(0) def _parse_cli_args(self): @@ -448,13 +503,13 @@ class AzureInventory(object): parser = argparse.ArgumentParser( description='Produce an Ansible Inventory file for an Azure subscription') parser.add_argument('--list', action='store_true', default=True, - help='List instances (default: True)') + help='List instances (default: True)') parser.add_argument('--debug', action='store_true', default=False, - help='Send debug messages to STDOUT') + help='Send debug messages to STDOUT') parser.add_argument('--host', action='store', - help='Get all information about an instance') + help='Get all information about an instance') parser.add_argument('--pretty', action='store_true', default=False, - help='Pretty print JSON output(default: False)') + help='Pretty print JSON output(default: False)') parser.add_argument('--profile', action='store', help='Azure profile contained in ~/.azure/credentials') parser.add_argument('--subscription_id', action='store', @@ -465,10 +520,12 @@ class AzureInventory(object): help='Azure Client Secret') parser.add_argument('--tenant', action='store', help='Azure Tenant Id') - parser.add_argument('--ad-user', action='store', + parser.add_argument('--ad_user', action='store', help='Active Directory User') parser.add_argument('--password', action='store', help='password') + parser.add_argument('--cloud_environment', action='store', + help='Azure Cloud Environment name or metadata discovery URL') parser.add_argument('--resource-groups', action='store', help='Return inventory for comma separated list of resource group names') parser.add_argument('--tags', action='store', @@ -486,8 +543,7 @@ class AzureInventory(object): try: virtual_machines = self._compute_client.virtual_machines.list(resource_group) except Exception as exc: - sys.exit("Error: fetching virtual machines for resource group {0} - {1}".format(resource_group, - str(exc))) + sys.exit("Error: fetching virtual machines for resource group {0} - {1}".format(resource_group, str(exc))) if self._args.host or self.tags: selected_machines = self._selected_machines(virtual_machines) self._load_machines(selected_machines) @@ -510,7 +566,7 @@ class AzureInventory(object): for machine in machines: id_dict = azure_id_to_dict(machine.id) - #TODO - The API is returning an ID value containing resource group name in ALL CAPS. If/when it gets + # TODO - The API is returning an ID value containing resource group name in ALL CAPS. If/when it gets # fixed, we should remove the .lower(). Opened Issue # #574: https://github.com/Azure/azure-sdk-for-python/issues/574 resource_group = id_dict['resourceGroups'].lower() @@ -538,7 +594,7 @@ class AzureInventory(object): mac_address=None, plan=(machine.plan.name if machine.plan else None), virtual_machine_size=machine.hardware_profile.vm_size, - computer_name=machine.os_profile.computer_name, + computer_name=(machine.os_profile.computer_name if machine.os_profile else None), provisioning_state=machine.provisioning_state, ) @@ -559,7 +615,7 @@ class AzureInventory(object): ) # Add windows details - if machine.os_profile.windows_configuration is not None: + if machine.os_profile is not None and machine.os_profile.windows_configuration is not None: host_vars['windows_auto_updates_enabled'] = \ machine.os_profile.windows_configuration.enable_automatic_updates host_vars['windows_timezone'] = machine.os_profile.windows_configuration.time_zone @@ -790,13 +846,10 @@ class AzureInventory(object): def main(): if not HAS_AZURE: - sys.exit("The Azure python sdk is not installed (try `pip install 'azure>=2.0.0rc5' --upgrade`) - {0}".format(HAS_AZURE_EXC)) - - if Version(azure_compute_version) < Version(AZURE_MIN_VERSION): - sys.exit("Expecting azure.mgmt.compute.__version__ to be {0}. Found version {1} " - "Do you have Azure >= 2.0.0rc5 installed? (try `pip install 'azure>=2.0.0rc5' --upgrade`)".format(AZURE_MIN_VERSION, azure_compute_version)) + sys.exit("The Azure python sdk is not installed (try `pip install 'azure>={0}' --upgrade`) - {1}".format(AZURE_MIN_VERSION, HAS_AZURE_EXC)) AzureInventory() + if __name__ == '__main__': main() diff --git a/requirements/requirements_ansible.in b/requirements/requirements_ansible.in index 3e17a968e4..c686e9545f 100644 --- a/requirements/requirements_ansible.in +++ b/requirements/requirements_ansible.in @@ -1,5 +1,18 @@ apache-libcloud==2.0.0 -azure==2.0.0rc6 +# azure deps from https://github.com/ansible/ansible/blob/fe1153c0afa1ffd648147af97454e900560b3532/packaging/requirements/requirements-azure.txt +azure-mgmt-compute>=2.0.0,<3 +azure-mgmt-network>=1.3.0,<2 +azure-mgmt-storage>=1.2.0,<2 +azure-mgmt-resource>=1.1.0,<2 +azure-storage>=0.35.1,<0.36 +azure-cli-core>=2.0.12,<3 +msrestazure>=0.4.11,<0.5 +azure-mgmt-dns>=1.0.1,<2 +azure-mgmt-keyvault>=0.40.0,<0.41 +azure-mgmt-batch>=4.1.0,<5 +azure-mgmt-sql>=0.7.1,<0.8 +azure-mgmt-web>=0.32.0,<0.33 +azure-mgmt-containerservice>=1.0.0 backports.ssl-match-hostname==3.5.0.1 kombu==3.0.37 boto==2.46.1 diff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt index f31c02928c..20c57308e8 100644 --- a/requirements/requirements_ansible.txt +++ b/requirements/requirements_ansible.txt @@ -4,30 +4,30 @@ # # pip-compile --output-file requirements/requirements_ansible.txt requirements/requirements_ansible.in # -adal==0.4.5 # via msrestazure +adal==0.4.7 # via azure-cli-core, msrestazure amqp==1.4.9 # via kombu anyjson==0.3.3 # via kombu apache-libcloud==2.0.0 appdirs==1.4.3 # via os-client-config, python-ironicclient, setuptools +applicationinsights==0.11.0 # via azure-cli-core +argcomplete==1.9.2 # via azure-cli-core asn1crypto==0.22.0 # via cryptography -azure-batch==1.0.0 # via azure -azure-common[autorest]==1.1.4 # via azure-batch, azure-mgmt-batch, azure-mgmt-compute, azure-mgmt-keyvault, azure-mgmt-logic, azure-mgmt-network, azure-mgmt-redis, azure-mgmt-resource, azure-mgmt-scheduler, azure-mgmt-storage, azure-servicebus, azure-servicemanagement-legacy, azure-storage -azure-mgmt-batch==1.0.0 # via azure-mgmt -azure-mgmt-compute==0.30.0rc6 # via azure-mgmt -azure-mgmt-keyvault==0.30.0rc6 # via azure-mgmt -azure-mgmt-logic==1.0.0 # via azure-mgmt -azure-mgmt-network==0.30.0rc6 # via azure-mgmt -azure-mgmt-nspkg==2.0.0 # via azure-batch, azure-mgmt-batch, azure-mgmt-compute, azure-mgmt-keyvault, azure-mgmt-logic, azure-mgmt-network, azure-mgmt-redis, azure-mgmt-resource, azure-mgmt-scheduler, azure-mgmt-storage -azure-mgmt-redis==1.0.0 # via azure-mgmt -azure-mgmt-resource==0.30.0rc6 # via azure-mgmt -azure-mgmt-scheduler==1.0.0 # via azure-mgmt -azure-mgmt-storage==0.30.0rc6 # via azure-mgmt -azure-mgmt==0.30.0rc6 # via azure -azure-nspkg==2.0.0 # via azure-common, azure-mgmt-nspkg, azure-storage -azure-servicebus==0.20.3 # via azure -azure-servicemanagement-legacy==0.20.4 # via azure -azure-storage==0.33.0 # via azure -azure==2.0.0rc6 +azure-cli-core==2.0.15 +azure-cli-nspkg==3.0.1 # via azure-cli-core +azure-common==1.1.8 # via azure-mgmt-batch, azure-mgmt-compute, azure-mgmt-containerservice, azure-mgmt-dns, azure-mgmt-keyvault, azu +azure-mgmt-batch==4.1.0 +azure-mgmt-compute==2.1.0 +azure-mgmt-containerservice==1.0.0 +azure-mgmt-dns==1.0.1 +azure-mgmt-keyvault==0.40.0 +azure-mgmt-network==1.4.0 +azure-mgmt-nspkg==2.0.0 # via azure-mgmt-batch, azure-mgmt-compute, azure-mgmt-containerservice, azure-mgmt-dns, azure-mgmt-keyvault, azu +azure-mgmt-resource==1.1.0 +azure-mgmt-sql==0.7.1 +azure-mgmt-storage==1.2.1 +azure-mgmt-web==0.32.0 +azure-nspkg==2.0.0 # via azure-cli-nspkg, azure-common, azure-mgmt-nspkg, azure-storage +azure-storage==0.35.1 babel==2.3.4 # via osc-lib, oslo.i18n, python-cinderclient, python-glanceclient, python-neutronclient, python-novaclient, python-openstackclient backports.ssl-match-hostname==3.5.0.1 boto3==1.4.4 @@ -37,7 +37,8 @@ certifi==2017.4.17 # via msrest cffi==1.10.0 # via cryptography cliff==2.7.0 # via osc-lib, python-designateclient, python-neutronclient, python-openstackclient cmd2==0.7.2 # via cliff -cryptography==1.9 # via adal, azure-storage, pyopenssl, secretstorage +colorama==0.3.9 # via azure-cli-core +cryptography==2.0.3 # via adal, azure-storage, paramiko, pyopenssl, secretstorage debtcollector==1.15.0 # via oslo.config, oslo.utils, python-designateclient, python-keystoneclient, python-neutronclient decorator==4.0.11 # via shade deprecation==1.0.1 # via openstacksdk @@ -47,6 +48,7 @@ enum34==1.1.6 # via cryptography, msrest funcsigs==1.0.2 # via debtcollector, oslo.utils functools32==3.2.3.post2 # via jsonschema futures==3.1.1 # via azure-storage, s3transfer, shade +humanfriendly==4.4.1 # via azure-cli-core idna==2.5 # via cryptography ipaddress==1.0.18 # via cryptography, shade iso8601==0.1.11 # via keystoneauth1, oslo.utils, python-neutronclient, python-novaclient @@ -61,8 +63,8 @@ kombu==3.0.37 lxml==3.8.0 # via pyvmomi monotonic==1.3 # via oslo.utils msgpack-python==0.4.8 # via oslo.serialization -msrest==0.4.10 # via azure-common, msrestazure -msrestazure==0.4.9 # via azure-common +msrest==0.4.14 # via azure-cli-core, msrestazure +msrestazure==0.4.13 munch==2.1.1 # via shade netaddr==0.7.19 # via oslo.config, oslo.utils, python-neutronclient netifaces==0.10.6 # via oslo.utils, shade @@ -83,9 +85,10 @@ prettytable==0.7.2 # via cliff, python-cinderclient, python-glanceclient, psphere==0.5.2 psutil==5.2.2 pycparser==2.17 # via cffi -pyjwt==1.5.0 # via adal +pygments==2.2.0 # via azure-cli-core +pyjwt==1.5.2 # via adal, azure-cli-core pykerberos==1.1.14 # via requests-kerberos -pyopenssl==17.0.0 # via pyvmomi +pyopenssl==17.2.0 # via azure-cli-core, python-glanceclient, pyvmomi pyparsing==2.2.0 # via cliff, cmd2, oslo.utils, packaging python-cinderclient==2.2.0 # via python-openstackclient, shade python-dateutil==2.6.0 # via adal, azure-storage, botocore @@ -114,6 +117,7 @@ simplejson==3.11.1 # via osc-lib, python-cinderclient, python-neutronclie six==1.10.0 # via cliff, cmd2, cryptography, debtcollector, keystoneauth1, munch, ntlm-auth, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, packaging, pyopenssl, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-ironicclient, python-keystoneclient, python-memcached, python-neutronclient, python-novaclient, python-openstackclient, pyvmomi, pywinrm, setuptools, shade, stevedore, warlock stevedore==1.23.0 # via cliff, keystoneauth1, openstacksdk, osc-lib, oslo.config, python-designateclient, python-keystoneclient suds==0.4 # via psphere +tabulate==0.7.7 # via azure-cli-core unicodecsv==0.14.1 # via cliff warlock==1.2.0 # via python-glanceclient wrapt==1.10.10 # via debtcollector, positional, python-glanceclient