Implement Azure RM creds and inventory

* Vendor ansible's azure_rm inventory script
* Add new inventory type
* Add new credential type
* Expand host instance_id column from varchar 100 to 1024 to accept the
  long instance ids returned by Azure
* Make the inventory_import azure match rename more explicit.

awx/api/serializers.py

@@ -1566,7 +1566,7 @@ class CredentialSerializer(BaseSerializer):
                   'password', 'security_token', 'project', 'domain',
                   'ssh_key_data', 'ssh_key_unlock',
                   'become_method', 'become_username', 'become_password',
-                  'vault_password')
+                  'vault_password', 'subscription', 'tenant', 'secret', 'client')
 
     def build_standard_field(self, field_name, model_field):
         field_class, field_kwargs = super(CredentialSerializer, self).build_standard_field(field_name, model_field)

awx/main/constants.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved.
 
-CLOUD_PROVIDERS = ('azure', 'ec2', 'gce', 'rax', 'vmware', 'openstack', 'foreman', 'cloudforms')
+CLOUD_PROVIDERS = ('azure', 'azure_rm', 'ec2', 'gce', 'rax', 'vmware', 'openstack', 'foreman', 'cloudforms')
 SCHEDULEABLE_PROVIDERS = CLOUD_PROVIDERS + ('custom',)

awx/main/management/commands/inventory_import.py

@@ -466,10 +466,10 @@ def load_inventory_source(source, all_group=None, group_filter_re=None,
     '''
     Load inventory from given source directory or file.
     '''
-    # Sanity check: We need the "azure" module to be titled "windows_azure.py",
+    # Sanity check: We sanitize these module names for our API but Ansible proper doesn't follow
-    # because it depends on the "azure" package from PyPI, and naming the
+    # good naming conventions
-    # module the same way makes the importer sad.
+    if source == 'azure':
-    source = source.replace('azure', 'windows_azure')
+        source = 'windows_azure'
 
     logger.debug('Analyzing type of source: %s', source)
     original_all_group = all_group

awx/main/migrations/0019_v300_new_azure_credential.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0018_v300_host_ordering'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='credential',
+            name='client',
+            field=models.CharField(default=b'', help_text='Client Id or Application Id for the credential', max_length=128, blank=True),
+        ),
+        migrations.AddField(
+            model_name='credential',
+            name='secret',
+            field=models.CharField(default=b'', help_text='Secret Token for this credential', max_length=1024, blank=True),
+        ),
+        migrations.AddField(
+            model_name='credential',
+            name='subscription',
+            field=models.CharField(default=b'', help_text='Subscription identifier for this credential', max_length=1024, blank=True),
+        ),
+        migrations.AddField(
+            model_name='credential',
+            name='tenant',
+            field=models.CharField(default=b'', help_text='Tenant identifier for this credential', max_length=1024, blank=True),
+        ),
+        migrations.AlterField(
+            model_name='credential',
+            name='kind',
+            field=models.CharField(default=b'ssh', max_length=32, choices=[(b'ssh', 'Machine'), (b'net', 'Network'), (b'scm', 'Source Control'), (b'aws', 'Amazon Web Services'), (b'rax', 'Rackspace'), (b'vmware', 'VMware vCenter'), (b'foreman', 'Satellite 6'), (b'cloudforms', 'CloudForms'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'openstack', 'OpenStack')]),
+        ),
+        migrations.AlterField(
+            model_name='host',
+            name='instance_id',
+            field=models.CharField(default=b'', max_length=1024, blank=True),
+        ),
+        migrations.AlterField(
+            model_name='inventorysource',
+            name='source',
+            field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'Local File, Directory or Script'), (b'rax', 'Rackspace Cloud Servers'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'foreman', 'Satellite 6'), (b'cloudforms', 'CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]),
+        ),
+        migrations.AlterField(
+            model_name='inventoryupdate',
+            name='source',
+            field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'Local File, Directory or Script'), (b'rax', 'Rackspace Cloud Servers'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'foreman', 'Satellite 6'), (b'cloudforms', 'CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]),
+        ),
+
+    ]

awx/main/models/base.py

@@ -61,7 +61,7 @@ PERMISSION_TYPE_CHOICES = [
     (PERM_JOBTEMPLATE_CREATE, _('Create a Job Template')),
 ]
 
-CLOUD_INVENTORY_SOURCES = ['ec2', 'rax', 'vmware', 'gce', 'azure', 'openstack', 'custom', 'foreman', 'cloudforms']
+CLOUD_INVENTORY_SOURCES = ['ec2', 'rax', 'vmware', 'gce', 'azure', 'azure_rm', 'openstack', 'custom', 'foreman', 'cloudforms']
 
 VERBOSITY_CHOICES = [
     (0, '0 (Normal)'),

awx/main/models/credential.py

@@ -41,7 +41,8 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
         ('foreman', _('Satellite 6')),
         ('cloudforms', _('CloudForms')),
         ('gce', _('Google Compute Engine')),
-        ('azure', _('Microsoft Azure')),
+        ('azure', _('Microsoft Azure Classic (deprecated)')),
+        ('azure_rm', _('Microsoft Azure Resource Manager')),
         ('openstack', _('OpenStack')),
     ]
 
@@ -55,7 +56,7 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
     ]
 
     PASSWORD_FIELDS = ('password', 'security_token', 'ssh_key_data', 'ssh_key_unlock',
-                       'become_password', 'vault_password')
+                       'become_password', 'vault_password', 'secret')
 
     class Meta:
         app_label = 'main'
@@ -168,6 +169,30 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
         default='',
         help_text=_('Vault password (or "ASK" to prompt the user).'),
     )
+    client = models.CharField(
+        max_length=128,
+        blank=True,
+        default='',
+        help_text=_('Client Id or Application Id for the credential'),
+    )
+    secret = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        help_text=_('Secret Token for this credential'),
+    )
+    subscription = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        help_text=_('Subscription identifier for this credential'),
+    )
+    tenant = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        help_text=_('Tenant identifier for this credential'),
+    )
     owner_role = ImplicitRoleField(
         role_name='Credential Owner',
         role_description='Owner of the credential',

awx/main/models/inventory.py

@@ -355,7 +355,7 @@ class Host(CommonModelNameNotUnique, ResourceMixin):
         help_text=_('Is this host online and available for running jobs?'),
     )
     instance_id = models.CharField(
-        max_length=100,
+        max_length=1024,
         blank=True,
         default='',
     )
@@ -748,7 +748,8 @@ class InventorySourceOptions(BaseModel):
         ('rax', _('Rackspace Cloud Servers')),
         ('ec2', _('Amazon EC2')),
         ('gce', _('Google Compute Engine')),
-        ('azure', _('Microsoft Azure')),
+        ('azure', _('Microsoft Azure Classic (deprecated)')),
+        ('azure_rm', _('Microsoft Azure Resource Manager')),
         ('vmware', _('VMware vCenter')),
         ('foreman', _('Satellite 6')),
         ('cloudforms', _('CloudForms')),
@@ -969,6 +970,10 @@ class InventorySourceOptions(BaseModel):
         regions.insert(0, ('all', 'All'))
         return regions
 
+    @classmethod
+    def get_azure_rm_region_choices(self):
+        return InventorySourceOptions.get_azure_region_choices()
+
     @classmethod
     def get_vmware_region_choices(self):
         """Return a complete list of regions in VMware, as a list of two-tuples

awx/main/tasks.py

@@ -801,6 +801,16 @@ class RunJob(BaseTask):
         elif cloud_cred and cloud_cred.kind == 'azure':
             env['AZURE_SUBSCRIPTION_ID'] = cloud_cred.username
             env['AZURE_CERT_PATH'] = kwargs.get('private_data_files', {}).get('cloud_credential', '')
+        elif cloud_cred and cloud_cred.kind == 'azure_rm':
+            if len(cloud_cred.client) and len(cloud_cred.tenant):
+                env['AZURE_CLIENT_ID'] = cloud_cred.client
+                env['AZURE_SECRET'] = decrypt_field(cloud_cred, 'secret')
+                env['AZURE_TENANT'] = cloud_cred.tenant
+                env['AZURE_SUBSCRIPTION_ID'] = cloud_cred.subscription
+            else:
+                env['AZURE_SUBSCRIPTION_ID'] = cloud_cred.subscription
+                env['AZURE_AD_USER'] = cloud_cred.username
+                env['AZURE_PASSWORD'] = decrypt_field(cloud_cred, 'password')
         elif cloud_cred and cloud_cred.kind == 'vmware':
             env['VMWARE_USER'] = cloud_cred.username
             env['VMWARE_PASSWORD'] = decrypt_field(cloud_cred, 'password')
@@ -1278,6 +1288,14 @@ class RunInventoryUpdate(BaseTask):
                 cp.set(section, 'username', credential.username)
                 cp.set(section, 'password', decrypt_field(credential, 'password'))
 
+        elif inventory_update.source == 'azure_rm':
+            section = 'azure'
+            cp.add_section(section)
+            cp.set(section, 'include_powerstate', 'yes')
+            cp.set(section, 'group_by_resource_group', 'yes')
+            cp.set(section, 'group_by_location', 'yes')
+            cp.set(section, 'group_by_tag', 'yes')
+
         # Return INI content.
         if cp.sections():
             f = cStringIO.StringIO()
@@ -1298,9 +1316,9 @@ class RunInventoryUpdate(BaseTask):
         # passwords dictionary.
         credential = inventory_update.credential
         if credential:
-            for subkey in ('username', 'host', 'project'):
+            for subkey in ('username', 'host', 'project', 'client', 'tenant', 'subscription'):
                 passwords['source_%s' % subkey] = getattr(credential, subkey)
-            for passkey in ('password', 'ssh_key_data', 'security_token'):
+            for passkey in ('password', 'ssh_key_data', 'security_token', 'secret'):
                 k = 'source_%s' % passkey
                 passwords[k] = decrypt_field(credential, passkey)
         return passwords
@@ -1353,6 +1371,17 @@ class RunInventoryUpdate(BaseTask):
         elif inventory_update.source == 'azure':
             env['AZURE_SUBSCRIPTION_ID'] = passwords.get('source_username', '')
             env['AZURE_CERT_PATH'] = cloud_credential
+        elif inventory_update.source == 'azure_rm':
+            if len(passwords.get('source_client', '')) and \
+               len(passwords.get('source_tenant', '')):
+                   env['AZURE_CLIENT_ID'] = passwords.get('source_client', '')
+                   env['AZURE_SECRET'] = passwords.get('source_secret', '')
+                   env['AZURE_TENANT'] = passwords.get('source_tenant', '')
+                   env['AZURE_SUBSCRIPTION_ID'] = passwords.get('source_subscription', '')
+            else:
+                env['AZURE_SUBSCRIPTION_ID'] = passwords.get('source_subscription', '')
+                env['AZURE_AD_USER'] = passwords.get('source_username', '')
+                env['AZURE_PASSWORD'] = passwords.get('source_password', '')
         elif inventory_update.source == 'gce':
             env['GCE_EMAIL'] = passwords.get('source_username', '')
             env['GCE_PROJECT'] = passwords.get('source_project', '')

awx/plugins/inventory/azure_rm.ini.example

@@ -0,0 +1,19 @@
+#
+# Configuration file for azure_rm_invetory.py
+# 
+[azure]
+# Control which resource groups are included. By default all resources groups are included.
+# Set resource_groups to a comma separated list of resource groups names.
+#resource_groups=
+
+# Control which tags are included. Set tags to a comma separated list of keys or key:value pairs
+#tags=
+
+# Include powerstate. If you don't need powerstate information, turning it off improves runtime performance.
+include_powerstate=yes
+
+# Control grouping with the following boolean flags. Valid values: yes, no, true, false, True, False, 0, 1.
+group_by_resource_group=yes
+group_by_location=yes
+group_by_security_group=no
+group_by_tag=yes

awx/plugins/inventory/azure_rm.py

@@ -0,0 +1,786 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2016 Matt Davis, <mdavis@ansible.com>
+#                    Chris Houseknecht, <house@redhat.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+'''
+Azure External Inventory Script
+===============================
+Generates dynamic inventory by making API requests to the Azure Resource
+Manager using the AAzure Python SDK. For instruction on installing the
+Azure Python SDK see http://azure-sdk-for-python.readthedocs.org/
+
+Authentication
+--------------
+The order of precedence is command line arguments, environment variables,
+and finally the [default] profile found in ~/.azure/credentials.
+
+If using a credentials file, it should be an ini formatted file with one or
+more sections, which we refer to as profiles. The script looks for a 
+[default] section, if a profile is not specified either on the command line
+or with an environment variable. The keys in a profile will match the
+list of command line arguments below.
+
+For command line arguments and environment variables specify a profile found
+in your ~/.azure/credentials file, or a service principal or Active Directory
+user.
+
+Command line arguments:
+ - profile 
+ - client_id
+ - secret
+ - subscription_id
+ - tenant
+ - ad_user
+ - password
+
+Environment variables:
+ - AZURE_PROFILE
+ - AZURE_CLIENT_ID
+ - AZURE_SECRET
+ - AZURE_SUBSCRIPTION_ID
+ - AZURE_TENANT
+ - AZURE_AD_USER
+ - AZURE_PASSWORD
+
+Run for Specific Host
+-----------------------
+When run for a specific host using the --host option, a resource group is 
+required. For a specific host, this script returns the following variables:
+
+{
+  "ansible_host": "XXX.XXX.XXX.XXX",
+  "computer_name": "computer_name2",
+  "fqdn": null,
+  "id": "/subscriptions/subscription-id/resourceGroups/galaxy-production/providers/Microsoft.Compute/virtualMachines/object-name",
+  "image": {
+    "offer": "CentOS",
+    "publisher": "OpenLogic",
+    "sku": "7.1",
+    "version": "latest"
+  },
+  "location": "westus",
+  "mac_address": "00-0D-3A-31-2C-EC",
+  "name": "object-name",
+  "network_interface": "interface-name",
+  "network_interface_id": "/subscriptions/subscription-id/resourceGroups/galaxy-production/providers/Microsoft.Network/networkInterfaces/object-name1",
+  "network_security_group": null,
+  "network_security_group_id": null,
+  "os_disk": {
+    "name": "object-name",
+    "operating_system_type": "Linux"
+  },
+  "plan": null,
+  "powerstate": "running",
+  "private_ip": "172.26.3.6",
+  "private_ip_alloc_method": "Static",
+  "provisioning_state": "Succeeded",
+  "public_ip": "XXX.XXX.XXX.XXX",
+  "public_ip_alloc_method": "Static",
+  "public_ip_id": "/subscriptions/subscription-id/resourceGroups/galaxy-production/providers/Microsoft.Network/publicIPAddresses/object-name",
+  "public_ip_name": "object-name",
+  "resource_group": "galaxy-production",
+  "security_group": "object-name",
+  "security_group_id": "/subscriptions/subscription-id/resourceGroups/galaxy-production/providers/Microsoft.Network/networkSecurityGroups/object-name",
+  "tags": {
+      "db": "database"
+  },
+  "type": "Microsoft.Compute/virtualMachines",
+  "virtual_machine_size": "Standard_DS4"
+}
+
+Groups
+------
+When run in --list mode, instances are grouped by the following categories:
+ - azure
+ - location
+ - resource_group
+ - security_group
+ - tag key
+ - tag key_value
+
+Control groups using azure_rm_inventory.ini or set environment variables:
+
+AZURE_GROUP_BY_RESOURCE_GROUP=yes
+AZURE_GROUP_BY_LOCATION=yes
+AZURE_GROUP_BY_SECURITY_GROUP=yes
+AZURE_GROUP_BY_TAG=yes
+
+Select hosts within specific resource groups by assigning a comma separated list to:
+
+AZURE_RESOURCE_GROUPS=resource_group_a,resource_group_b
+
+Select hosts for specific tag key by assigning a comma separated list of tag keys to:
+
+AZURE_TAGS=key1,key2,key3
+
+Or, select hosts for specific tag key:value pairs by assigning a comma separated list key:value pairs to:
+
+AZURE_TAGS=key1:value1,key2:value2
+
+If you don't need the powerstate, you can improve performance by turning off powerstate fetching:
+AZURE_INCLUDE_POWERSTATE=no
+
+azure_rm_inventory.ini
+----------------------
+As mentioned above you can control execution using environment variables or an .ini file. A sample
+azure_rm_inventory.ini is included. The name of the .ini file is the basename of the inventory script (in this case
+'azure_rm_inventory') with a .ini extension. This provides you with the flexibility of copying and customizing this
+script and having matching .ini files. Go forth and customize your Azure inventory!
+
+Powerstate:
+-----------
+The powerstate attribute indicates whether or not a host is running. If the value is 'running', the machine is
+up. If the value is anything other than 'running', the machine is down, and will be unreachable.
+
+Examples:
+---------
+  Execute /bin/uname on all instances in the galaxy-qa resource group
+  $ ansible -i azure_rm_inventory.py galaxy-qa -m shell -a "/bin/uname -a"
+
+  Use the inventory script to print instance specific information
+  $ contrib/inventory/azure_rm_inventory.py --host my_instance_host_name --pretty
+
+  Use with a playbook
+  $ ansible-playbook -i contrib/inventory/azure_rm_inventory.py my_playbook.yml --limit galaxy-qa
+
+
+Insecure Platform Warning
+-------------------------
+If you receive InsecurePlatformWarning from urllib3, install the
+requests security packages:
+
+    pip install requests[security]
+
+
+author:
+    - Chris Houseknecht (@chouseknecht)
+    - Matt Davis (@nitzmahone)
+
+Company: Ansible by Red Hat
+
+Version: 1.0.0
+'''
+
+import argparse
+import ConfigParser
+import json 
+import os
+import re
+import sys
+
+from os.path import expanduser
+
+HAS_AZURE = True
+HAS_AZURE_EXC = None
+
+try:
+    from msrestazure.azure_exceptions import CloudError
+    from azure.mgmt.compute import __version__ as azure_compute_version
+    from azure.common import AzureMissingResourceHttpError, AzureHttpError
+    from azure.common.credentials import ServicePrincipalCredentials, UserPassCredentials
+    from azure.mgmt.network.network_management_client import NetworkManagementClient,\
+                                                             NetworkManagementClientConfiguration
+    from azure.mgmt.resource.resources.resource_management_client import ResourceManagementClient,\
+                                                                         ResourceManagementClientConfiguration
+    from azure.mgmt.compute.compute_management_client import ComputeManagementClient,\
+                                                             ComputeManagementClientConfiguration
+except ImportError as exc:
+    HAS_AZURE_EXC = exc
+    HAS_AZURE = False
+
+
+AZURE_CREDENTIAL_ENV_MAPPING = dict(
+    profile='AZURE_PROFILE',
+    subscription_id='AZURE_SUBSCRIPTION_ID',
+    client_id='AZURE_CLIENT_ID',
+    secret='AZURE_SECRET',
+    tenant='AZURE_TENANT',
+    ad_user='AZURE_AD_USER',
+    password='AZURE_PASSWORD'
+)
+
+AZURE_CONFIG_SETTINGS = dict(
+    resource_groups='AZURE_RESOURCE_GROUPS',
+    tags='AZURE_TAGS',
+    include_powerstate='AZURE_INCLUDE_POWERSTATE',
+    group_by_resource_group='AZURE_GROUP_BY_RESOURCE_GROUP',
+    group_by_location='AZURE_GROUP_BY_LOCATION',
+    group_by_security_group='AZURE_GROUP_BY_SECURITY_GROUP',
+    group_by_tag='AZURE_GROUP_BY_TAG'
+)
+
+AZURE_MIN_VERSION = "2016-03-30"
+
+
+def azure_id_to_dict(id):
+    pieces = re.sub(r'^\/', '', id).split('/')
+    result = {}
+    index = 0
+    while index < len(pieces) - 1:
+        result[pieces[index]] = pieces[index + 1]
+        index += 1
+    return result
+
+
+class AzureRM(object):
+
+    def __init__(self, args):
+        self._args = args
+        self._compute_client = None
+        self._resource_client = None
+        self._network_client = None
+
+        self.debug = False
+        if args.debug:
+            self.debug = True
+
+        self.credentials = self._get_credentials(args)
+        if not self.credentials:
+            self.fail("Failed to get credentials. Either pass as parameters, set environment variables, "
+                      "or define a profile in ~/.azure/credentials.")
+
+        if self.credentials.get('subscription_id', None) is None:
+            self.fail("Credentials did not include a subscription_id value.")
+        self.log("setting subscription_id")
+        self.subscription_id = self.credentials['subscription_id']
+
+        if self.credentials.get('client_id') is not None and \
+           self.credentials.get('secret') is not None and \
+           self.credentials.get('tenant') is not None:
+            self.azure_credentials = ServicePrincipalCredentials(client_id=self.credentials['client_id'],
+                                                                 secret=self.credentials['secret'],
+                                                                 tenant=self.credentials['tenant'])
+        elif self.credentials.get('ad_user') is not None and self.credentials.get('password') is not None:
+            self.azure_credentials = UserPassCredentials(self.credentials['ad_user'], self.credentials['password'])
+        else:
+            self.fail("Failed to authenticate with provided credentials. Some attributes were missing. "
+                      "Credentials must include client_id, secret and tenant or ad_user and password.")
+
+    def log(self, msg):
+        if self.debug:
+            print (msg + u'\n')
+
+    def fail(self, msg):
+        raise Exception(msg)
+
+    def _get_profile(self, profile="default"):
+        path = expanduser("~")
+        path += "/.azure/credentials"
+        try:
+            config = ConfigParser.ConfigParser()
+            config.read(path)
+        except Exception as exc:
+            self.fail("Failed to access {0}. Check that the file exists and you have read "
+                      "access. {1}".format(path, str(exc)))
+        credentials = dict()
+        for key in AZURE_CREDENTIAL_ENV_MAPPING:
+            try:
+                credentials[key] = config.get(profile, key, raw=True)
+            except:
+                pass
+
+        if credentials.get('client_id') is not None or credentials.get('ad_user') is not None:
+            return credentials
+
+        return None
+
+    def _get_env_credentials(self):
+        env_credentials = dict()
+        for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.iteritems():
+            env_credentials[attribute] = os.environ.get(env_variable, None)
+
+        if env_credentials['profile'] is not None:
+            credentials = self._get_profile(env_credentials['profile'])
+            return credentials
+
+        if env_credentials['client_id'] is not None or env_credentials['ad_user'] is not None:
+            return env_credentials
+
+        return None
+
+    def _get_credentials(self, params):
+        # Get authentication credentials.
+        # Precedence: cmd line parameters-> environment variables-> default profile in ~/.azure/credentials.
+
+        self.log('Getting credentials')
+
+        arg_credentials = dict()
+        for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.iteritems():
+            arg_credentials[attribute] = getattr(params, attribute)
+
+        # try module params
+        if arg_credentials['profile'] is not None:
+            self.log('Retrieving credentials with profile parameter.')
+            credentials = self._get_profile(arg_credentials['profile'])
+            return credentials
+
+        if arg_credentials['client_id'] is not None:
+            self.log('Received credentials from parameters.')
+            return arg_credentials
+
+        # try environment
+        env_credentials = self._get_env_credentials()
+        if env_credentials:
+            self.log('Received credentials from env.')
+            return env_credentials
+
+        # try default profile from ~./azure/credentials
+        default_credentials = self._get_profile()
+        if default_credentials:
+            self.log('Retrieved default profile credentials from ~/.azure/credentials.')
+            return default_credentials
+
+        return None
+
+    def _register(self, key):
+        try:
+            # We have to perform the one-time registration here. Otherwise, we receive an error the first
+            # time we attempt to use the requested client.
+            resource_client = self.rm_client
+            resource_client.providers.register(key)
+        except Exception as exc:
+            self.fail("One-time registration of {0} failed - {1}".format(key, str(exc)))
+
+    @property
+    def network_client(self):
+        self.log('Getting network client')
+        if not self._network_client:
+            self._network_client = NetworkManagementClient(
+                NetworkManagementClientConfiguration(self.azure_credentials, self.subscription_id))
+            self._register('Microsoft.Network')
+        return self._network_client
+
+    @property
+    def rm_client(self):
+        self.log('Getting resource manager client')
+        if not self._resource_client:
+            self._resource_client = ResourceManagementClient(
+                ResourceManagementClientConfiguration(self.azure_credentials, self.subscription_id))
+        return self._resource_client
+
+    @property
+    def compute_client(self):
+        self.log('Getting compute client')
+        if not self._compute_client:
+            self._compute_client = ComputeManagementClient(
+                ComputeManagementClientConfiguration(self.azure_credentials, self.subscription_id))
+            self._register('Microsoft.Compute')
+        return self._compute_client
+
+
+class AzureInventory(object):
+
+    def __init__(self):
+
+        self._args = self._parse_cli_args()
+
+        try:
+            rm = AzureRM(self._args)
+        except Exception as e:
+            sys.exit("{0}".format(str(e)))
+
+        self._compute_client = rm.compute_client
+        self._network_client = rm.network_client
+        self._resource_client = rm.rm_client
+        self._security_groups = None
+
+        self.resource_groups = []
+        self.tags = None
+        self.replace_dash_in_groups = False
+        self.group_by_resource_group = True
+        self.group_by_location = True
+        self.group_by_security_group = True
+        self.group_by_tag = True
+        self.include_powerstate = True
+
+        self._inventory = dict(
+            _meta=dict(
+                hostvars=dict()
+            ),
+            azure=[]
+        )
+
+        self._get_settings()
+
+        if self._args.resource_groups:
+            self.resource_groups = self._args.resource_groups.split(',')
+
+        if self._args.tags:
+            self.tags = self._args.tags.split(',')
+
+        if self._args.no_powerstate:
+            self.include_powerstate = False
+
+        self.get_inventory()
+        print (self._json_format_dict(pretty=self._args.pretty))
+        sys.exit(0)
+
+    def _parse_cli_args(self):
+        # Parse command line arguments
+        parser = argparse.ArgumentParser(
+                description='Produce an Ansible Inventory file for an Azure subscription')
+        parser.add_argument('--list', action='store_true', default=True,
+                           help='List instances (default: True)')
+        parser.add_argument('--debug', action='store_true', default=False,
+                           help='Send debug messages to STDOUT')
+        parser.add_argument('--host', action='store',
+                           help='Get all information about an instance')
+        parser.add_argument('--pretty', action='store_true', default=False,
+                           help='Pretty print JSON output(default: False)')
+        parser.add_argument('--profile', action='store',
+                            help='Azure profile contained in ~/.azure/credentials')
+        parser.add_argument('--subscription_id', action='store',
+                            help='Azure Subscription Id')
+        parser.add_argument('--client_id', action='store',
+                            help='Azure Client Id ')
+        parser.add_argument('--secret', action='store',
+                            help='Azure Client Secret')
+        parser.add_argument('--tenant', action='store',
+                            help='Azure Tenant Id')
+        parser.add_argument('--ad-user', action='store',
+                            help='Active Directory User')
+        parser.add_argument('--password', action='store',
+                            help='password')
+        parser.add_argument('--resource-groups', action='store',
+                            help='Return inventory for comma separated list of resource group names')
+        parser.add_argument('--tags', action='store',
+                            help='Return inventory for comma separated list of tag key:value pairs')
+        parser.add_argument('--no-powerstate', action='store_true', default=False,
+                            help='Do not include the power state of each virtual host')
+        return parser.parse_args()
+
+    def get_inventory(self):
+        if len(self.resource_groups) > 0:
+            # get VMs for requested resource groups
+            for resource_group in self.resource_groups:
+                try:
+                    virtual_machines = self._compute_client.virtual_machines.list(resource_group)
+                except Exception as exc:
+                    sys.exit("Error: fetching virtual machines for resource group {0} - {1}".format(resource_group,
+                                                                                                   str(exc)))
+                if self._args.host or self.tags:
+                    selected_machines = self._selected_machines(virtual_machines)
+                    self._load_machines(selected_machines)
+                else:
+                    self._load_machines(virtual_machines)
+        else:
+            # get all VMs within the subscription
+            try:
+                virtual_machines = self._compute_client.virtual_machines.list_all()
+            except Exception as exc:
+                sys.exit("Error: fetching virtual machines - {0}".format(str(exc)))
+
+            if self._args.host or self.tags > 0:
+                selected_machines = self._selected_machines(virtual_machines)
+                self._load_machines(selected_machines)
+            else:
+                self._load_machines(virtual_machines)
+
+    def _load_machines(self, machines):
+        for machine in machines:
+            id_dict = azure_id_to_dict(machine.id)
+
+            #TODO - The API is returning an ID value containing resource group name in ALL CAPS. If/when it gets
+            #       fixed, we should remove the .lower(). Opened Issue
+            #       #574: https://github.com/Azure/azure-sdk-for-python/issues/574
+            resource_group = id_dict['resourceGroups'].lower()
+
+            if self.group_by_security_group:
+                self._get_security_groups(resource_group)
+
+            host_vars = dict(
+                ansible_host=None,
+                private_ip=None,
+                private_ip_alloc_method=None,
+                public_ip=None,
+                public_ip_name=None,
+                public_ip_id=None,
+                public_ip_alloc_method=None,
+                fqdn=None,
+                location=machine.location,
+                name=machine.name,
+                type=machine.type,
+                id=machine.id,
+                tags=machine.tags,
+                network_interface_id=None,
+                network_interface=None,
+                resource_group=resource_group,
+                mac_address=None,
+                plan=(machine.plan.name if machine.plan else None),
+                virtual_machine_size=machine.hardware_profile.vm_size.value,
+                computer_name=machine.os_profile.computer_name,
+                provisioning_state=machine.provisioning_state,
+            )
+
+            host_vars['os_disk'] = dict(
+                name=machine.storage_profile.os_disk.name,
+                operating_system_type=machine.storage_profile.os_disk.os_type.value
+            )
+
+            if self.include_powerstate:
+                host_vars['powerstate'] = self._get_powerstate(resource_group, machine.name)
+
+            if machine.storage_profile.image_reference:
+                host_vars['image'] = dict(
+                    offer=machine.storage_profile.image_reference.offer,
+                    publisher=machine.storage_profile.image_reference.publisher,
+                    sku=machine.storage_profile.image_reference.sku,
+                    version=machine.storage_profile.image_reference.version
+                )
+
+            # Add windows details
+            if machine.os_profile.windows_configuration is not None:
+                host_vars['windows_auto_updates_enabled'] = \
+                    machine.os_profile.windows_configuration.enable_automatic_updates
+                host_vars['windows_timezone'] = machine.os_profile.windows_configuration.time_zone
+                host_vars['windows_rm'] = None
+                if machine.os_profile.windows_configuration.win_rm is not None:
+                    host_vars['windows_rm'] = dict(listeners=None)
+                    if machine.os_profile.windows_configuration.win_rm.listeners is not None:
+                        host_vars['windows_rm']['listeners'] = []
+                        for listener in machine.os_profile.windows_configuration.win_rm.listeners:
+                            host_vars['windows_rm']['listeners'].append(dict(protocol=listener.protocol,
+                                                                             certificate_url=listener.certificate_url))
+
+            for interface in machine.network_profile.network_interfaces:
+                interface_reference = self._parse_ref_id(interface.id)
+                network_interface = self._network_client.network_interfaces.get(
+                    interface_reference['resourceGroups'],
+                    interface_reference['networkInterfaces'])
+                if network_interface.primary:
+                    if self.group_by_security_group and \
+                       self._security_groups[resource_group].get(network_interface.id, None):
+                        host_vars['security_group'] = \
+                            self._security_groups[resource_group][network_interface.id]['name']
+                        host_vars['security_group_id'] = \
+                            self._security_groups[resource_group][network_interface.id]['id']
+                    host_vars['network_interface'] = network_interface.name
+                    host_vars['network_interface_id'] = network_interface.id
+                    host_vars['mac_address'] = network_interface.mac_address
+                    for ip_config in network_interface.ip_configurations:
+                        host_vars['private_ip'] = ip_config.private_ip_address
+                        host_vars['private_ip_alloc_method'] = ip_config.private_ip_allocation_method.value
+                        if ip_config.public_ip_address:
+                            public_ip_reference = self._parse_ref_id(ip_config.public_ip_address.id)
+                            public_ip_address = self._network_client.public_ip_addresses.get(
+                                public_ip_reference['resourceGroups'],
+                                public_ip_reference['publicIPAddresses'])
+                            host_vars['ansible_host'] = public_ip_address.ip_address
+                            host_vars['public_ip'] = public_ip_address.ip_address
+                            host_vars['public_ip_name'] = public_ip_address.name
+                            host_vars['public_ip_alloc_method'] = public_ip_address.public_ip_allocation_method.value
+                            host_vars['public_ip_id'] = public_ip_address.id
+                            if public_ip_address.dns_settings:
+                                host_vars['fqdn'] = public_ip_address.dns_settings.fqdn
+
+            self._add_host(host_vars)
+
+    def _selected_machines(self, virtual_machines):
+        selected_machines = []
+        for machine in virtual_machines:
+            if self._args.host and self._args.host == machine.name:
+                selected_machines.append(machine)
+            if self.tags and self._tags_match(machine.tags, self.tags):
+                selected_machines.append(machine)
+        return selected_machines
+
+    def _get_security_groups(self, resource_group):
+        ''' For a given resource_group build a mapping of network_interface.id to security_group name '''
+        if not self._security_groups:
+            self._security_groups = dict()
+        if not self._security_groups.get(resource_group):
+            self._security_groups[resource_group] = dict()
+            for group in self._network_client.network_security_groups.list(resource_group):
+                if group.network_interfaces:
+                    for interface in group.network_interfaces:
+                        self._security_groups[resource_group][interface.id] = dict(
+                            name=group.name,
+                            id=group.id
+                        )
+
+    def _get_powerstate(self, resource_group, name):
+        try:
+            vm = self._compute_client.virtual_machines.get(resource_group,
+                                                           name,
+                                                           expand='instanceview')
+        except Exception as exc:
+            sys.exit("Error: fetching instanceview for host {0} - {1}".format(name, str(exc)))
+
+        return next((s.code.replace('PowerState/', '')
+                    for s in vm.instance_view.statuses if s.code.startswith('PowerState')), None)
+
+    def _add_host(self, vars):
+
+        host_name = self._to_safe(vars['name'])
+        resource_group = self._to_safe(vars['resource_group'])
+        security_group = None
+        if vars.get('security_group'):
+            security_group = self._to_safe(vars['security_group'])
+
+        if self.group_by_resource_group:
+            if not self._inventory.get(resource_group):
+                self._inventory[resource_group] = []
+            self._inventory[resource_group].append(host_name)
+
+        if self.group_by_location:
+            if not self._inventory.get(vars['location']):
+                self._inventory[vars['location']] = []
+            self._inventory[vars['location']].append(host_name)
+
+        if self.group_by_security_group and security_group:
+            if not self._inventory.get(security_group):
+                self._inventory[security_group] = []
+            self._inventory[security_group].append(host_name)
+
+        self._inventory['_meta']['hostvars'][host_name] = vars
+        self._inventory['azure'].append(host_name)
+
+        if self.group_by_tag and vars.get('tags'):
+            for key, value in vars['tags'].iteritems():
+                safe_key = self._to_safe(key)
+                safe_value = safe_key + '_' + self._to_safe(value)
+                if not self._inventory.get(safe_key):
+                    self._inventory[safe_key] = []
+                if not self._inventory.get(safe_value):
+                    self._inventory[safe_value] = []
+                self._inventory[safe_key].append(host_name)
+                self._inventory[safe_value].append(host_name)
+
+    def _json_format_dict(self, pretty=False):
+        # convert inventory to json
+        if pretty:
+            return json.dumps(self._inventory, sort_keys=True, indent=2)
+        else:
+            return json.dumps(self._inventory)
+
+    def _get_settings(self):
+        # Load settings from the .ini, if it exists. Otherwise,
+        # look for environment values.
+        file_settings = self._load_settings()
+        if file_settings:
+            for key in AZURE_CONFIG_SETTINGS:
+                if key in ('resource_groups', 'tags') and file_settings.get(key, None) is not None:
+                    values = file_settings.get(key).split(',')
+                    if len(values) > 0:
+                        setattr(self, key, values)
+                elif file_settings.get(key, None) is not None:
+                    val = self._to_boolean(file_settings[key])
+                    setattr(self, key, val)
+        else:
+            env_settings = self._get_env_settings()
+            for key in AZURE_CONFIG_SETTINGS:
+                if key in('resource_groups', 'tags') and env_settings.get(key, None) is not None:
+                    values = env_settings.get(key).split(',')
+                    if len(values) > 0:
+                        setattr(self, key, values)
+                elif env_settings.get(key, None) is not None:
+                    val = self._to_boolean(env_settings[key])
+                    setattr(self, key, val)
+
+    def _parse_ref_id(self, reference):
+        response = {}
+        keys = reference.strip('/').split('/')
+        for index in range(len(keys)):
+            if index < len(keys) - 1 and index % 2 == 0:
+                response[keys[index]] = keys[index + 1]
+        return response
+
+    def _to_boolean(self, value):
+        if value in ['Yes', 'yes', 1, 'True', 'true', True]:
+            result = True
+        elif value in ['No', 'no', 0, 'False', 'false', False]:
+            result = False
+        else:
+            result = True
+        return result
+
+    def _get_env_settings(self):
+        env_settings = dict()
+        for attribute, env_variable in AZURE_CONFIG_SETTINGS.iteritems():
+            env_settings[attribute] = os.environ.get(env_variable, None)
+        return env_settings
+
+    def _load_settings(self):
+        basename = os.path.splitext(os.path.basename(__file__))[0]
+        path = basename + '.ini'
+        config = None
+        settings = None
+        try:
+            config = ConfigParser.ConfigParser()
+            config.read(path)
+        except:
+            pass
+
+        if config is not None:
+            settings = dict()
+            for key in AZURE_CONFIG_SETTINGS:
+                try:
+                    settings[key] = config.get('azure', key, raw=True)
+                except:
+                    pass
+
+        return settings
+
+    def _tags_match(self, tag_obj, tag_args):
+        '''
+        Return True if the tags object from a VM contains the requested tag values.
+
+        :param tag_obj:  Dictionary of string:string pairs
+        :param tag_args: List of strings in the form key=value
+        :return: boolean
+        '''
+
+        if not tag_obj:
+            return False
+
+        matches = 0
+        for arg in tag_args:
+            arg_key = arg
+            arg_value = None
+            if re.search(r':', arg):
+                arg_key, arg_value = arg.split(':')
+            if arg_value and tag_obj.get(arg_key, None) == arg_value:
+                matches += 1
+            elif not arg_value and tag_obj.get(arg_key, None) is not None:
+                matches += 1
+        if matches == len(tag_args):
+            return True
+        return False
+
+    def _to_safe(self, word):
+        ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups '''
+        regex = "[^A-Za-z0-9\_"
+        if not self.replace_dash_in_groups:
+            regex += "\-"
+        return re.sub(regex + "]", "_", word)
+
+
+def main():
+    if not HAS_AZURE:
+        sys.exit("The Azure python sdk is not installed (try 'pip install azure') - {0}".format(HAS_AZURE_EXC))
+
+    if azure_compute_version < AZURE_MIN_VERSION:
+        sys.exit("Expecting azure.mgmt.compute.__version__ to be >= {0}. Found version {1} "
+                 "Do you have Azure >= 2.0.0rc2 installed?".format(AZURE_MIN_VERSION, azure_compute_version))
+
+    AzureInventory()
+
+if __name__ == '__main__':
+    main()

awx/settings/defaults.py

@@ -651,6 +651,16 @@ AZURE_HOST_FILTER = r'^.+$'
 AZURE_EXCLUDE_EMPTY_GROUPS = True
 AZURE_INSTANCE_ID_VAR = 'private_id'
 
+# --------------------------------------
+# -- Microsoft Azure Resource Manager --
+# --------------------------------------
+AZURE_RM_GROUP_FILTER = r'^.+$'
+AZURE_RM_HOST_FILTER = r'^.+$'
+AZURE_RM_ENABLED_VAR = 'powerstate'
+AZURE_RM_ENABLED_VALUE = 'running'
+AZURE_RM_INSTANCE_ID_VAR = 'id'
+AZURE_RM_EXCLUDE_EMPTY_GROUPS = True
+
 # ---------------------
 # ----- OpenStack -----
 # ---------------------

requirements/requirements_ansible.txt

@@ -1,7 +1,7 @@
 anyjson==0.3.3
 apache-libcloud==0.15.1
 appdirs==1.4.0
-azure==0.9.0
+azure==2.0.0rc2
 Babel==2.2.0
 boto==2.34.0
 cliff==1.15.0