From b72a148cbcce0085992e0a7713f0e1f01526cad5 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 19 Dec 2016 16:39:10 -0500 Subject: [PATCH 1/5] vmware_inventory dyn inv script support --- .../management/commands/inventory_import.py | 1 + awx/main/tasks.py | 21 +- awx/plugins/inventory/vmware_inventory.py | 690 ++++++++++++++++++ 3 files changed, 706 insertions(+), 6 deletions(-) create mode 100755 awx/plugins/inventory/vmware_inventory.py diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 955bcae559..90b15faf02 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -482,6 +482,7 @@ def load_inventory_source(source, all_group=None, group_filter_re=None, # good naming conventions source = source.replace('azure.py', 'windows_azure.py') source = source.replace('satellite6.py', 'foreman.py') + source = source.replace('vmware.py', 'vmware_inventory.py') logger.debug('Analyzing type of source: %s', source) original_all_group = all_group if not os.path.exists(source): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index ef636010d7..f1d28938f5 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1349,10 +1349,22 @@ class RunInventoryUpdate(BaseTask): 'password')) # Allow custom options to vmware inventory script. elif inventory_update.source == 'vmware': - section = 'defaults' + credential = inventory_update.credential + + section = 'vmware' cp.add_section(section) + cp.set('vmware', 'cache_max_age', 0) + + cp.set('vmware', 'username', credential.username) + cp.set('vmware', 'password', decrypt_field(credential, 'password')) + cp.set('vmware', 'server', credential.host) + vmware_opts = dict(inventory_update.source_vars_dict.items()) - vmware_opts.setdefault('guests_only', 'True') + if inventory_update.instance_filters: + inventory_update.setdefault('host_filters', inventory_update.instance_filters) + if inventory_update.group_by: + inventory_update.setdefault('groupby_patterns', inventory_update.groupby_patterns) + for k,v in vmware_opts.items(): cp.set(section, k, unicode(v)) @@ -1472,10 +1484,7 @@ class RunInventoryUpdate(BaseTask): # complain about not being able to determine its version number. env['PBR_VERSION'] = '0.5.21' elif inventory_update.source == 'vmware': - env['VMWARE_INI'] = cloud_credential - env['VMWARE_HOST'] = passwords.get('source_host', '') - env['VMWARE_USER'] = passwords.get('source_username', '') - env['VMWARE_PASSWORD'] = passwords.get('source_password', '') + env['VMWARE_INI_PATH'] = cloud_credential elif inventory_update.source == 'azure': env['AZURE_SUBSCRIPTION_ID'] = passwords.get('source_username', '') env['AZURE_CERT_PATH'] = cloud_credential diff --git a/awx/plugins/inventory/vmware_inventory.py b/awx/plugins/inventory/vmware_inventory.py new file mode 100755 index 0000000000..a221293edc --- /dev/null +++ b/awx/plugins/inventory/vmware_inventory.py @@ -0,0 +1,690 @@ +#!/usr/bin/env python + +# Requirements +# - pyvmomi >= 6.0.0.2016.4 + +# TODO: +# * more jq examples +# * optional folder heriarchy + +""" +$ jq '._meta.hostvars[].config' data.json | head +{ + "alternateguestname": "", + "instanceuuid": "5035a5cd-b8e8-d717-e133-2d383eb0d675", + "memoryhotaddenabled": false, + "guestfullname": "Red Hat Enterprise Linux 7 (64-bit)", + "changeversion": "2016-05-16T18:43:14.977925Z", + "uuid": "4235fc97-5ddb-7a17-193b-9a3ac97dc7b4", + "cpuhotremoveenabled": false, + "vpmcenabled": false, + "firmware": "bios", +""" + +from __future__ import print_function + +import argparse +import atexit +import datetime +import getpass +import jinja2 +import os +import six +import ssl +import sys +import uuid + +from collections import defaultdict +from six.moves import configparser +from time import time + +HAS_PYVMOMI = False +try: + from pyVmomi import vim + from pyVim.connect import SmartConnect, Disconnect + HAS_PYVMOMI = True +except ImportError: + pass + +try: + import json +except ImportError: + import simplejson as json + +hasvcr = False +try: + import vcr + hasvcr = True +except ImportError: + pass + +class VMwareMissingHostException(Exception): + pass + +class VMWareInventory(object): + + __name__ = 'VMWareInventory' + + guest_props = False + instances = [] + debug = False + load_dumpfile = None + write_dumpfile = None + maxlevel = 1 + lowerkeys = True + config = None + cache_max_age = None + cache_path_cache = None + cache_path_index = None + server = None + port = None + username = None + password = None + host_filters = [] + groupby_patterns = [] + + if (sys.version_info > (3, 0)): + safe_types = [int, bool, str, float, None] + else: + safe_types = [int, long, bool, str, float, None] + iter_types = [dict, list] + + bad_types = ['Array', 'disabledMethod', 'declaredAlarmState'] + skip_keys = ['declaredalarmstate', + 'disabledmethod', + 'dynamicproperty', + 'dynamictype', + 'environmentbrowser', + 'managedby', + 'parent', + 'childtype'] + + # translation table for attributes to fetch for known vim types + if not HAS_PYVMOMI: + vimTable = {} + else: + vimTable = { + vim.Datastore: ['_moId', 'name'], + vim.ResourcePool: ['_moId', 'name'], + } + + def _empty_inventory(self): + return {"_meta" : {"hostvars" : {}}} + + + def __init__(self, load=True): + self.inventory = self._empty_inventory() + + if load: + # Read settings and parse CLI arguments + self.parse_cli_args() + self.read_settings() + + # Check the cache + cache_valid = self.is_cache_valid() + + # Handle Cache + if self.args.refresh_cache or not cache_valid: + self.do_api_calls_update_cache() + else: + self.debugl('loading inventory from cache') + self.inventory = self.get_inventory_from_cache() + + def debugl(self, text): + if self.args.debug: + try: + text = str(text) + except UnicodeEncodeError: + text = text.encode('ascii','ignore') + print('%s %s' % (datetime.datetime.now(), text)) + + def show(self): + # Data to print + self.debugl('dumping results') + data_to_print = None + if self.args.host: + data_to_print = self.get_host_info(self.args.host) + elif self.args.list: + # Display list of instances for inventory + data_to_print = self.inventory + return json.dumps(data_to_print, indent=2) + + + def is_cache_valid(self): + + ''' Determines if the cache files have expired, or if it is still valid ''' + + valid = False + + if os.path.isfile(self.cache_path_cache): + mod_time = os.path.getmtime(self.cache_path_cache) + current_time = time() + if (mod_time + self.cache_max_age) > current_time: + valid = True + + return valid + + + def do_api_calls_update_cache(self): + + ''' Get instances and cache the data ''' + + instances = self.get_instances() + self.instances = instances + self.inventory = self.instances_to_inventory(instances) + self.write_to_cache(self.inventory, self.cache_path_cache) + + + def write_to_cache(self, data, cache_path): + + ''' Dump inventory to json file ''' + + with open(self.cache_path_cache, 'wb') as f: + f.write(json.dumps(data)) + + + def get_inventory_from_cache(self): + + ''' Read in jsonified inventory ''' + + jdata = None + with open(self.cache_path_cache, 'rb') as f: + jdata = f.read() + return json.loads(jdata) + + + def read_settings(self): + + ''' Reads the settings from the vmware_inventory.ini file ''' + + scriptbasename = __file__ + scriptbasename = os.path.basename(scriptbasename) + scriptbasename = scriptbasename.replace('.py', '') + + defaults = {'vmware': { + 'server': '', + 'port': 443, + 'username': '', + 'password': '', + 'validate_certs': True, + 'ini_path': os.path.join(os.path.dirname(__file__), '%s.ini' % scriptbasename), + 'cache_name': 'ansible-vmware', + 'cache_path': '~/.ansible/tmp', + 'cache_max_age': 3600, + 'max_object_level': 1, + 'alias_pattern': '{{ config.name + "_" + config.uuid }}', + 'host_pattern': '{{ guest.ipaddress }}', + 'host_filters': '{{ guest.gueststate == "running" }}', + 'groupby_patterns': '{{ guest.guestid }},{{ "templates" if config.template else "guests"}}', + 'lower_var_keys': True } + } + + if six.PY3: + config = configparser.ConfigParser() + else: + config = configparser.SafeConfigParser() + + # where is the config? + vmware_ini_path = os.environ.get('VMWARE_INI_PATH', defaults['vmware']['ini_path']) + vmware_ini_path = os.path.expanduser(os.path.expandvars(vmware_ini_path)) + config.read(vmware_ini_path) + + # apply defaults + for k,v in defaults['vmware'].items(): + if not config.has_option('vmware', k): + config.set('vmware', k, str(v)) + + # where is the cache? + self.cache_dir = os.path.expanduser(config.get('vmware', 'cache_path')) + if self.cache_dir and not os.path.exists(self.cache_dir): + os.makedirs(self.cache_dir) + + # set the cache filename and max age + cache_name = config.get('vmware', 'cache_name') + self.cache_path_cache = self.cache_dir + "/%s.cache" % cache_name + self.debugl('cache path is %s' % self.cache_path_cache) + self.cache_max_age = int(config.getint('vmware', 'cache_max_age')) + + # mark the connection info + self.server = os.environ.get('VMWARE_SERVER', config.get('vmware', 'server')) + self.debugl('server is %s' % self.server) + self.port = int(os.environ.get('VMWARE_PORT', config.get('vmware', 'port'))) + self.username = os.environ.get('VMWARE_USERNAME', config.get('vmware', 'username')) + self.debugl('username is %s' % self.username) + self.password = os.environ.get('VMWARE_PASSWORD', config.get('vmware', 'password')) + self.validate_certs = os.environ.get('VMWARE_VALIDATE_CERTS', config.get('vmware', 'validate_certs')) + if self.validate_certs in ['no', 'false', 'False', False]: + self.validate_certs = False + else: + self.validate_certs = True + self.debugl('cert validation is %s' % self.validate_certs) + + # behavior control + self.maxlevel = int(config.get('vmware', 'max_object_level')) + self.debugl('max object level is %s' % self.maxlevel) + self.lowerkeys = config.get('vmware', 'lower_var_keys') + if type(self.lowerkeys) != bool: + if str(self.lowerkeys).lower() in ['yes', 'true', '1']: + self.lowerkeys = True + else: + self.lowerkeys = False + self.debugl('lower keys is %s' % self.lowerkeys) + + self.host_filters = list(config.get('vmware', 'host_filters').split(',')) + self.debugl('host filters are %s' % self.host_filters) + self.groupby_patterns = list(config.get('vmware', 'groupby_patterns').split(',')) + self.debugl('groupby patterns are %s' % self.groupby_patterns) + + # Special feature to disable the brute force serialization of the + # virtulmachine objects. The key name for these properties does not + # matter because the values are just items for a larger list. + if config.has_section('properties'): + self.guest_props = [] + for prop in config.items('properties'): + self.guest_props.append(prop[1]) + + # save the config + self.config = config + + + def parse_cli_args(self): + + ''' Command line argument processing ''' + + parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on PyVmomi') + parser.add_argument('--debug', action='store_true', default=False, + help='show debug info') + parser.add_argument('--list', action='store_true', default=True, + help='List instances (default: True)') + parser.add_argument('--host', action='store', + help='Get all the variables about a specific instance') + parser.add_argument('--refresh-cache', action='store_true', default=False, + help='Force refresh of cache by making API requests to VSphere (default: False - use cache files)') + parser.add_argument('--max-instances', default=None, type=int, + help='maximum number of instances to retrieve') + self.args = parser.parse_args() + + + def get_instances(self): + + ''' Get a list of vm instances with pyvmomi ''' + + instances = [] + + kwargs = {'host': self.server, + 'user': self.username, + 'pwd': self.password, + 'port': int(self.port) } + + if hasattr(ssl, 'SSLContext') and not self.validate_certs: + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_NONE + kwargs['sslContext'] = context + + instances = self._get_instances(kwargs) + return instances + + + def _get_instances(self, inkwargs): + + ''' Make API calls ''' + + instances = [] + si = SmartConnect(**inkwargs) + + self.debugl('retrieving all instances') + if not si: + print("Could not connect to the specified host using specified " + "username and password") + return -1 + atexit.register(Disconnect, si) + content = si.RetrieveContent() + + # Create a search container for virtualmachines + self.debugl('creating containerview for virtualmachines') + container = content.rootFolder + viewType = [vim.VirtualMachine] + recursive = True + containerView = content.viewManager.CreateContainerView(container, viewType, recursive) + children = containerView.view + for child in children: + # If requested, limit the total number of instances + if self.args.max_instances: + if len(instances) >= (self.args.max_instances): + break + instances.append(child) + self.debugl("%s total instances in container view" % len(instances)) + + if self.args.host: + instances = [x for x in instances if x.name == self.args.host] + + instance_tuples = [] + for instance in sorted(instances): + if self.guest_props != False: + ifacts = self.facts_from_proplist(instance) + else: + ifacts = self.facts_from_vobj(instance) + instance_tuples.append((instance, ifacts)) + self.debugl('facts collected for all instances') + return instance_tuples + + + def instances_to_inventory(self, instances): + + ''' Convert a list of vm objects into a json compliant inventory ''' + + self.debugl('re-indexing instances based on ini settings') + inventory = self._empty_inventory() + inventory['all'] = {} + inventory['all']['hosts'] = [] + last_idata = None + total = len(instances) + for idx,instance in enumerate(instances): + + # make a unique id for this object to avoid vmware's + # numerous uuid's which aren't all unique. + thisid = str(uuid.uuid4()) + idata = instance[1] + + # Put it in the inventory + inventory['all']['hosts'].append(thisid) + inventory['_meta']['hostvars'][thisid] = idata.copy() + inventory['_meta']['hostvars'][thisid]['ansible_uuid'] = thisid + + # Make a map of the uuid to the alias the user wants + name_mapping = self.create_template_mapping(inventory, + self.config.get('vmware', 'alias_pattern')) + + # Make a map of the uuid to the ssh hostname the user wants + host_mapping = self.create_template_mapping(inventory, + self.config.get('vmware', 'host_pattern')) + + + # Reset the inventory keys + for k,v in name_mapping.items(): + + if not host_mapping or not k in host_mapping: + continue + + # set ansible_host (2.x) + try: + inventory['_meta']['hostvars'][k]['ansible_host'] = host_mapping[k] + # 1.9.x backwards compliance + inventory['_meta']['hostvars'][k]['ansible_ssh_host'] = host_mapping[k] + except Exception as e: + continue + + if k == v: + continue + + # add new key + inventory['all']['hosts'].append(v) + inventory['_meta']['hostvars'][v] = inventory['_meta']['hostvars'][k] + + # cleanup old key + inventory['all']['hosts'].remove(k) + inventory['_meta']['hostvars'].pop(k, None) + + self.debugl('pre-filtered hosts:') + for i in inventory['all']['hosts']: + self.debugl(' * %s' % i) + # Apply host filters + for hf in self.host_filters: + if not hf: + continue + self.debugl('filter: %s' % hf) + filter_map = self.create_template_mapping(inventory, hf, dtype='boolean') + for k,v in filter_map.items(): + if not v: + # delete this host + inventory['all']['hosts'].remove(k) + inventory['_meta']['hostvars'].pop(k, None) + + self.debugl('post-filter hosts:') + for i in inventory['all']['hosts']: + self.debugl(' * %s' % i) + + # Create groups + for gbp in self.groupby_patterns: + groupby_map = self.create_template_mapping(inventory, gbp) + for k,v in groupby_map.items(): + if v not in inventory: + inventory[v] = {} + inventory[v]['hosts'] = [] + if k not in inventory[v]['hosts']: + inventory[v]['hosts'].append(k) + + return inventory + + + def create_template_mapping(self, inventory, pattern, dtype='string'): + + ''' Return a hash of uuid to templated string from pattern ''' + + mapping = {} + for k,v in inventory['_meta']['hostvars'].items(): + t = jinja2.Template(pattern) + newkey = None + try: + newkey = t.render(v) + newkey = newkey.strip() + except Exception as e: + self.debugl(e) + if not newkey: + continue + elif dtype == 'integer': + newkey = int(newkey) + elif dtype == 'boolean': + if newkey.lower() == 'false': + newkey = False + elif newkey.lower() == 'true': + newkey = True + elif dtype == 'string': + pass + mapping[k] = newkey + return mapping + + def facts_from_proplist(self, vm): + '''Get specific properties instead of serializing everything''' + + rdata = {} + for prop in self.guest_props: + self.debugl('getting %s property for %s' % (prop, vm.name)) + key = prop + if self.lowerkeys: + key = key.lower() + + if not '.' in prop: + # props without periods are direct attributes of the parent + rdata[key] = getattr(vm, prop) + else: + # props with periods are subkeys of parent attributes + parts = prop.split('.') + total = len(parts) - 1 + + # pointer to the current object + val = None + # pointer to the current result key + lastref = rdata + + for idx,x in enumerate(parts): + + # if the val wasn't set yet, get it from the parent + if not val: + val = getattr(vm, x) + else: + # in a subkey, get the subprop from the previous attrib + try: + val = getattr(val, x) + except AttributeError as e: + self.debugl(e) + + # lowercase keys if requested + if self.lowerkeys: + x = x.lower() + + # change the pointer or set the final value + if idx != total: + if x not in lastref: + lastref[x] = {} + lastref = lastref[x] + else: + lastref[x] = val + + return rdata + + + def facts_from_vobj(self, vobj, level=0): + + ''' Traverse a VM object and return a json compliant data structure ''' + + # pyvmomi objects are not yet serializable, but may be one day ... + # https://github.com/vmware/pyvmomi/issues/21 + + # WARNING: + # Accessing an object attribute will trigger a SOAP call to the remote. + # Increasing the attributes collected or the depth of recursion greatly + # increases runtime duration and potentially memory+network utilization. + + if level == 0: + try: + self.debugl("get facts for %s" % vobj.name) + except Exception as e: + self.debugl(e) + + rdata = {} + + methods = dir(vobj) + methods = [str(x) for x in methods if not x.startswith('_')] + methods = [x for x in methods if not x in self.bad_types] + methods = [x for x in methods if not x.lower() in self.skip_keys] + methods = sorted(methods) + + for method in methods: + # Attempt to get the method, skip on fail + try: + methodToCall = getattr(vobj, method) + except Exception as e: + continue + + # Skip callable methods + if callable(methodToCall): + continue + + if self.lowerkeys: + method = method.lower() + + rdata[method] = self._process_object_types( + methodToCall, + thisvm=vobj, + inkey=method + ) + + return rdata + + + def _process_object_types(self, vobj, thisvm=None, inkey=None, level=0): + ''' Serialize an object ''' + rdata = {} + + if vobj is None: + rdata = None + + elif type(vobj) in self.vimTable: + rdata = {} + for key in self.vimTable[type(vobj)]: + rdata[key] = getattr(vobj, key) + + elif issubclass(type(vobj), str) or isinstance(vobj, str): + if vobj.isalnum(): + rdata = vobj + else: + rdata = vobj.decode('ascii', 'ignore') + elif issubclass(type(vobj), bool) or isinstance(vobj, bool): + rdata = vobj + elif issubclass(type(vobj), int) or isinstance(vobj, int): + rdata = vobj + elif issubclass(type(vobj), float) or isinstance(vobj, float): + rdata = vobj + elif issubclass(type(vobj), long) or isinstance(vobj, long): + rdata = vobj + elif issubclass(type(vobj), list) or issubclass(type(vobj), tuple): + rdata = [] + try: + vobj = sorted(vobj) + except Exception as e: + pass + + for idv, vii in enumerate(vobj): + + if (level+1 <= self.maxlevel): + + vid = self._process_object_types( + vii, + thisvm=thisvm, + inkey=inkey+'['+str(idv)+']', + level=(level+1) + ) + + if vid: + rdata.append(vid) + + elif issubclass(type(vobj), dict): + pass + + elif issubclass(type(vobj), object): + methods = dir(vobj) + methods = [str(x) for x in methods if not x.startswith('_')] + methods = [x for x in methods if not x in self.bad_types] + methods = [x for x in methods if not x.lower() in self.skip_keys] + methods = sorted(methods) + + for method in methods: + # Attempt to get the method, skip on fail + try: + methodToCall = getattr(vobj, method) + except Exception as e: + continue + if callable(methodToCall): + continue + if self.lowerkeys: + method = method.lower() + if (level+1 <= self.maxlevel): + rdata[method] = self._process_object_types( + methodToCall, + thisvm=thisvm, + inkey=inkey+'.'+method, + level=(level+1) + ) + else: + pass + + return rdata + + def get_host_info(self, host): + + ''' Return hostvars for a single host ''' + + if host in self.inventory['_meta']['hostvars']: + return self.inventory['_meta']['hostvars'][host] + elif self.args.host and self.inventory['_meta']['hostvars']: + # check if the machine has the name requested + keys = self.inventory['_meta']['hostvars'].keys() + match = None + for k,v in self.inventory['_meta']['hostvars'].items(): + if self.inventory['_meta']['hostvars'][k]['name'] == self.args.host: + match = k + break + if match: + return self.inventory['_meta']['hostvars'][match] + else: + raise VMwareMissingHostException('%s not found' % host) + else: + raise VMwareMissingHostException('%s not found' % host) + + +if __name__ == "__main__": + # Run the script + print(VMWareInventory().show()) + + From 139d80bcbba98b60288fbaccb1cc6c4146a20e5a Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 19 Dec 2016 16:39:38 -0500 Subject: [PATCH 2/5] deps for new vmware inv script --- requirements/requirements.in | 1 + requirements/requirements.txt | 23 +++++++++++---------- requirements/requirements_ansible.in | 1 + requirements/requirements_ansible.txt | 29 ++++++++++++++------------- 4 files changed, 29 insertions(+), 25 deletions(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index 2182c5aada..2a30d786b0 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -38,6 +38,7 @@ python-memcached==1.58 python-radius==1.0 python-saml==2.2.0 python-social-auth==0.2.21 +pyvmomi==6.5 redbaron==0.6.2 requests-futures==0.9.7 service-identity==16.0.0 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 4d3d27b1de..bfff181962 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -7,7 +7,7 @@ git+https://github.com/ansible/ansiconv.git@tower_1.0.0#egg=ansiconv git+https://github.com/ansible/django-qsstats-magic.git@tower_0.7.2#egg=django-qsstats-magic git+https://github.com/ansible/dm.xmlsec.binding.git@master#egg=dm.xmlsec.binding -git+https://github.com/ansible/django-jsonbfield@master#egg=jsonbfield +git+https://github.com/ansible/django-jsonbfield@fix-sqlite_serialization#egg=jsonbfield git+https://github.com/chrismeyersfsu/pyrax@tower#egg=pyrax adal==0.4.3 # via msrestazure amqp==1.4.9 # via kombu @@ -19,7 +19,7 @@ asgiref==1.0.0 # via asgi-amqp, channels, daphne attrs==16.3.0 # via service-identity autobahn==0.17.0 # via daphne azure-batch==1.0.0 # via azure -azure-common[autorest]==1.1.4 # via azure-batch, azure-mgmt-batch, azure-mgmt-compute, azure-mgmt-keyvault, azure-mgmt-logic, azure-mgmt-network, azure-mgmt-redis, azure-mgmt-resource, azure-mgmt-scheduler, azure-mgmt-storage, azure-servicebus, azure-servicemanagement-legacy, azure-storage +azure-common==1.1.4 # via azure-batch, azure-mgmt-batch, azure-mgmt-compute, azure-mgmt-keyvault, azure-mgmt-logic, azure-mgmt-network, azure-mgmt-redis, azure-mgmt-resource, azure-mgmt-scheduler, azure-mgmt-storage, azure-servicebus, azure-servicemanagement-legacy, azure-storage azure-mgmt-batch==1.0.0 # via azure-mgmt azure-mgmt-compute==0.30.0rc6 # via azure-mgmt azure-mgmt-keyvault==0.30.0rc6 # via azure-mgmt @@ -36,14 +36,14 @@ azure-servicebus==0.20.3 # via azure azure-servicemanagement-legacy==0.20.4 # via azure azure-storage==0.33.0 # via azure azure==2.0.0rc6 -Babel==2.3.4 # via osc-lib, oslo.i18n, python-cinderclient, python-glanceclient, python-heatclient, python-magnumclient, python-neutronclient, python-novaclient, python-openstackclient, python-troveclient +babel==2.3.4 # via osc-lib, oslo.i18n, python-cinderclient, python-glanceclient, python-heatclient, python-magnumclient, python-neutronclient, python-novaclient, python-openstackclient, python-troveclient backports.functools-lru-cache==1.3 # via jaraco.functools backports.ssl-match-hostname==3.5.0.1 # via websocket-client baron==0.6.2 # via redbaron billiard==3.3.0.23 # via celery boto==2.43.0 celery==3.1.17 -#certifi==2016.9.26 # via msrest --- chris meyers removed this because it fails gce inv sync and azure "classic". By removing this, we coerce libcloud into using the cert file '/etc/pki/tls/certs/ca-bundle.crt ' +#certifi==2016.9.26 # via msrest cffi==1.9.1 # via cryptography channels==0.17.3 chardet==2.3.0 # via msrest @@ -66,7 +66,7 @@ django-solo==1.1.2 django-split-settings==0.2.2 django-taggit==0.21.3 django-transaction-hooks==0.2 -Django==1.8.16 # via channels, django-auth-ldap, django-crum, django-split-settings, django-transaction-hooks +django==1.8.16 djangorestframework-yaml==1.0.3 djangorestframework==3.3.3 dogpile.cache==0.6.2 # via python-ironicclient, shade @@ -128,7 +128,7 @@ oslo.utils==3.20.0 # via osc-lib, oslo.serialization, python-cinderclient pbr==1.10.0 # via cliff, debtcollector, keystoneauth1, mock, openstacksdk, osc-lib, oslo.i18n, oslo.serialization, oslo.utils, positional, python-cinderclient, python-designateclient, python-glanceclient, python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-mistralclient, python-neutronclient, python-novaclient, python-openstackclient, python-troveclient, requestsexceptions, shade, stevedore pexpect==3.1 positional==1.1.1 # via keystoneauth1, python-keystoneclient -PrettyTable==0.7.2 # via cliff, python-cinderclient, python-glanceclient, python-heatclient, python-ironicclient, python-magnumclient, python-novaclient, python-troveclient +prettytable==0.7.2 # via cliff, python-cinderclient, python-glanceclient, python-heatclient, python-ironicclient, python-magnumclient, python-novaclient, python-troveclient psphere==0.5.2 psutil==5.0.0 psycopg2==2.6.2 @@ -136,8 +136,8 @@ pyasn1-modules==0.0.8 # via service-identity pyasn1==0.1.9 # via cryptography, pyasn1-modules, service-identity pycparser==2.17 # via cffi pygerduty==0.35.1 -PyJWT==1.4.2 # via adal, python-social-auth -pyOpenSSL==16.2.0 # via service-identity +pyjwt==1.4.2 # via adal, python-social-auth +pyopenssl==16.2.0 pyparsing==2.1.10 # via cliff, cmd2, oslo.utils pyrad==2.0 # via django-radius python-cinderclient==1.9.0 # via python-openstackclient, shade @@ -162,7 +162,8 @@ python-social-auth==0.2.21 python-swiftclient==3.2.0 # via python-heatclient, python-troveclient, shade python-troveclient==2.7.0 # via shade pytz==2016.10 # via babel, celery, irc, oslo.serialization, oslo.utils, tempora, twilio -PyYAML==3.12 # via cliff, djangorestframework-yaml, os-client-config, psphere, python-heatclient, python-ironicclient, python-mistralclient +pyvmomi==6.5 +pyyaml==3.12 # via cliff, djangorestframework-yaml, os-client-config, psphere, python-heatclient, python-ironicclient, python-mistralclient rackspace-auth-openstack==1.3 # via rackspace-novaclient rackspace-novaclient==2.1 rax-default-network-flags-python-novaclient-ext==0.4.0 # via rackspace-novaclient @@ -170,7 +171,7 @@ rax-scheduled-images-python-novaclient-ext==0.3.1 # via rackspace-novaclient redbaron==0.6.2 requests-futures==0.9.7 requests-oauthlib==0.7.0 # via msrest, python-social-auth -requests==2.11.1 # via adal, azure-servicebus, azure-servicemanagement-legacy, azure-storage, keystoneauth1, msrest, python-cinderclient, python-designateclient, python-glanceclient, python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-mistralclient, python-neutronclient, python-novaclient, python-social-auth, python-swiftclient, python-troveclient, requests-futures, requests-oauthlib, slackclient +requests==2.11.1 # via adal, azure-servicebus, azure-servicemanagement-legacy, azure-storage, keystoneauth1, msrest, python-cinderclient, python-designateclient, python-glanceclient, python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-mistralclient, python-neutronclient, python-novaclient, python-social-auth, python-swiftclient, python-troveclient, pyvmomi, requests-futures, requests-oauthlib, slackclient requestsexceptions==1.1.3 # via os-client-config, shade rfc3986==0.4.1 # via oslo.config rply==0.7.4 # via baron @@ -178,7 +179,7 @@ secretstorage==2.3.1 # via keyring service-identity==16.0.0 shade==1.13.1 simplejson==3.10.0 # via osc-lib, python-cinderclient, python-neutronclient, python-novaclient, python-troveclient -six==1.10.0 # via asgi-amqp, asgiref, autobahn, cliff, cryptography, debtcollector, django-extensions, irc, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.stream, keystoneauth1, mock, more-itertools, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, pygerduty, pyopenssl, pyrad, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-memcached, python-mistralclient, python-neutronclient, python-novaclient, python-openstackclient, python-social-auth, python-swiftclient, python-troveclient, shade, slackclient, stevedore, tempora, twilio, txaio, warlock, websocket-client +six==1.10.0 # via asgi-amqp, asgiref, autobahn, cliff, cryptography, debtcollector, django-extensions, irc, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.stream, keystoneauth1, mock, more-itertools, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, pygerduty, pyopenssl, pyrad, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-memcached, python-mistralclient, python-neutronclient, python-novaclient, python-openstackclient, python-social-auth, python-swiftclient, python-troveclient, pyvmomi, shade, slackclient, stevedore, tempora, twilio, txaio, warlock, websocket-client slackclient==1.0.2 stevedore==1.19.1 # via cliff, keystoneauth1, openstacksdk, osc-lib, oslo.config, python-designateclient, python-keystoneclient, python-magnumclient suds==0.4 # via psphere diff --git a/requirements/requirements_ansible.in b/requirements/requirements_ansible.in index 8027e6a92f..3ac5ad0d21 100644 --- a/requirements/requirements_ansible.in +++ b/requirements/requirements_ansible.in @@ -5,5 +5,6 @@ kombu==3.0.35 boto==2.43.0 psphere==0.5.2 psutil==5.0.0 +pyvmomi==6.5 secretstorage==2.3.1 shade==1.13.1 diff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt index b1a669bc91..02e324b400 100644 --- a/requirements/requirements_ansible.txt +++ b/requirements/requirements_ansible.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile # To update, run: # -# pip-compile --output-file requirements_ansible.txt requirements_ansible.in +# pip-compile --output-file requirements/requirements_ansible.txt requirements/requirements_ansible.in # git+https://github.com/chrismeyersfsu/pyrax@tower#egg=pyrax adal==0.4.3 # via msrestazure @@ -30,12 +30,12 @@ azure-storage==0.33.0 # via azure azure==2.0.0rc6 Babel==2.3.4 # via osc-lib, oslo.i18n, python-cinderclient, python-glanceclient, python-heatclient, python-magnumclient, python-neutronclient, python-novaclient, python-openstackclient, python-troveclient boto==2.43.0 -#certifi==2016.9.26 # via msrest --- chris meyers removed this because it fails gce inv sync and azure "classic". By removing this, we coerce libcloud into using the cert file '/etc/pki/tls/certs/ca-bundle.crt ' +#certifi==2016.9.26 # via msrest cffi==1.9.1 # via cryptography chardet==2.3.0 # via msrest cliff==2.3.0 # via osc-lib, python-designateclient, python-heatclient, python-mistralclient, python-neutronclient, python-openstackclient cmd2==0.6.9 # via cliff -cryptography==1.6 # via adal, azure-storage, python-magnumclient, secretstorage +cryptography==1.7.1 # via adal, azure-storage, python-magnumclient, secretstorage debtcollector==1.10.0 # via oslo.config, oslo.utils, python-designateclient, python-keystoneclient, python-neutronclient decorator==4.0.10 # via python-magnumclient, shade dogpile.cache==0.6.2 # via python-ironicclient, shade @@ -59,21 +59,21 @@ mock==2.0.0 monotonic==1.2 # via oslo.utils msgpack-python==0.4.8 # via oslo.serialization msrest==0.4.4 # via azure-common, msrestazure -msrestazure==0.4.5 # via azure-common +msrestazure==0.4.6 # via azure-common munch==2.0.4 # via shade netaddr==0.7.18 # via oslo.config, oslo.utils, python-neutronclient netifaces==0.10.5 # via oslo.utils, shade oauthlib==2.0.1 # via requests-oauthlib -openstacksdk==0.9.10 # via python-openstackclient +openstacksdk==0.9.11 # via python-openstackclient os-client-config==1.24.0 # via openstacksdk, osc-lib, python-magnumclient, python-neutronclient, shade os-diskconfig-python-novaclient-ext==0.1.3 # via rackspace-novaclient os-networksv2-python-novaclient-ext==0.26 # via rackspace-novaclient os-virtual-interfacesv2-python-novaclient-ext==0.20 # via rackspace-novaclient osc-lib==1.2.0 # via python-designateclient, python-heatclient, python-ironicclient, python-mistralclient, python-neutronclient, python-openstackclient -oslo.config==3.20.0 # via python-keystoneclient +oslo.config==3.21.0 # via python-keystoneclient oslo.i18n==3.11.0 # via osc-lib, oslo.config, oslo.utils, python-cinderclient, python-glanceclient, python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-neutronclient, python-novaclient, python-openstackclient, python-troveclient oslo.serialization==2.15.0 # via python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-neutronclient, python-novaclient -oslo.utils==3.19.0 # via osc-lib, oslo.serialization, python-cinderclient, python-designateclient, python-glanceclient, python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-mistralclient, python-neutronclient, python-novaclient, python-openstackclient, python-troveclient +oslo.utils==3.20.0 # via osc-lib, oslo.serialization, python-cinderclient, python-designateclient, python-glanceclient, python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-mistralclient, python-neutronclient, python-novaclient, python-openstackclient, python-troveclient pbr==1.10.0 # via cliff, debtcollector, keystoneauth1, mock, openstacksdk, osc-lib, oslo.i18n, oslo.serialization, oslo.utils, positional, python-cinderclient, python-designateclient, python-glanceclient, python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-mistralclient, python-neutronclient, python-novaclient, python-openstackclient, python-troveclient, requestsexceptions, shade, stevedore positional==1.1.1 # via keystoneauth1, python-keystoneclient PrettyTable==0.7.2 # via cliff, python-cinderclient, python-glanceclient, python-heatclient, python-ironicclient, python-magnumclient, python-novaclient, python-troveclient @@ -85,33 +85,34 @@ PyJWT==1.4.2 # via adal pyparsing==2.1.10 # via cliff, cmd2, oslo.utils python-cinderclient==1.9.0 # via python-openstackclient, shade python-dateutil==2.6.0 # via adal, azure-storage -python-designateclient==2.3.0 # via shade +python-designateclient==2.4.0 # via shade python-glanceclient==2.5.0 # via python-openstackclient, shade -python-heatclient==1.6.1 # via shade +python-heatclient==1.7.0 # via shade python-ironicclient==1.8.0 # via shade python-keystoneclient==3.8.0 # via python-glanceclient, python-mistralclient, python-openstackclient, shade python-magnumclient==2.3.1 # via shade python-mistralclient==2.1.2 # via python-troveclient python-neutronclient==6.0.0 # via shade python-novaclient==6.0.0 # via ip-associations-python-novaclient-ext, os-diskconfig-python-novaclient-ext, os-networksv2-python-novaclient-ext, os-virtual-interfacesv2-python-novaclient-ext, python-openstackclient, rackspace-auth-openstack, rackspace-novaclient, rax-default-network-flags-python-novaclient-ext, rax-scheduled-images-python-novaclient-ext, shade -python-openstackclient==3.4.1 # via python-ironicclient +python-openstackclient==3.5.0 # via python-ironicclient python-swiftclient==3.2.0 # via python-heatclient, python-troveclient, shade -python-troveclient==2.6.0 # via shade +python-troveclient==2.7.0 # via shade pytz==2016.10 # via babel, oslo.serialization, oslo.utils +pyvmomi==6.5 PyYAML==3.12 # via cliff, os-client-config, psphere, python-heatclient, python-ironicclient, python-mistralclient rackspace-auth-openstack==1.3 # via rackspace-novaclient rackspace-novaclient==2.1 rax-default-network-flags-python-novaclient-ext==0.4.0 # via rackspace-novaclient rax-scheduled-images-python-novaclient-ext==0.3.1 # via rackspace-novaclient requests-oauthlib==0.7.0 # via msrest -requests==2.11.1 # via adal, azure-servicebus, azure-servicemanagement-legacy, azure-storage, keystoneauth1, msrest, python-cinderclient, python-designateclient, python-glanceclient, python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-mistralclient, python-neutronclient, python-novaclient, python-swiftclient, python-troveclient, requests-oauthlib +requests==2.11.1 # via adal, azure-servicebus, azure-servicemanagement-legacy, azure-storage, keystoneauth1, msrest, python-cinderclient, python-designateclient, python-glanceclient, python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-mistralclient, python-neutronclient, python-novaclient, python-swiftclient, python-troveclient, pyvmomi, requests-oauthlib requestsexceptions==1.1.3 # via os-client-config, shade rfc3986==0.4.1 # via oslo.config secretstorage==2.3.1 shade==1.13.1 simplejson==3.10.0 # via osc-lib, python-cinderclient, python-neutronclient, python-novaclient, python-troveclient -six==1.10.0 # via cliff, cryptography, debtcollector, keystoneauth1, mock, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-mistralclient, python-neutronclient, python-novaclient, python-openstackclient, python-swiftclient, python-troveclient, shade, stevedore, warlock -stevedore==1.19.0 # via cliff, keystoneauth1, openstacksdk, osc-lib, oslo.config, python-designateclient, python-keystoneclient, python-magnumclient +six==1.10.0 # via cliff, cryptography, debtcollector, keystoneauth1, mock, openstacksdk, osc-lib, oslo.config, oslo.i18n, oslo.serialization, oslo.utils, python-cinderclient, python-dateutil, python-designateclient, python-glanceclient, python-heatclient, python-ironicclient, python-keystoneclient, python-magnumclient, python-mistralclient, python-neutronclient, python-novaclient, python-openstackclient, python-swiftclient, python-troveclient, pyvmomi, shade, stevedore, warlock +stevedore==1.19.1 # via cliff, keystoneauth1, openstacksdk, osc-lib, oslo.config, python-designateclient, python-keystoneclient, python-magnumclient suds==0.4 # via psphere unicodecsv==0.14.1 # via cliff warlock==1.2.0 # via python-glanceclient From 2f9a5409c3478762787469dde3cdb0e4e4fdc0fe Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 19 Dec 2016 17:08:27 -0500 Subject: [PATCH 3/5] code logic fix and vmware active var updated --- awx/main/tasks.py | 4 ++-- awx/settings/defaults.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index f1d28938f5..06a5288d6e 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1361,9 +1361,9 @@ class RunInventoryUpdate(BaseTask): vmware_opts = dict(inventory_update.source_vars_dict.items()) if inventory_update.instance_filters: - inventory_update.setdefault('host_filters', inventory_update.instance_filters) + vmware_opts.setdefault('host_filters', inventory_update.instance_filters) if inventory_update.group_by: - inventory_update.setdefault('groupby_patterns', inventory_update.groupby_patterns) + vmware_opts.setdefault('groupby_patterns', inventory_update.groupby_patterns) for k,v in vmware_opts.items(): cp.set(section, k, unicode(v)) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 35911f4c71..b283387350 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -663,8 +663,8 @@ VMWARE_REGIONS_BLACKLIST = [] # Inventory variable name/values for determining whether a host is # active in vSphere. -VMWARE_ENABLED_VAR = 'vmware_powerState' -VMWARE_ENABLED_VALUE = 'poweredOn' +VMWARE_ENABLED_VAR = 'guest.gueststate' +VMWARE_ENABLED_VALUE = 'running' # Inventory variable name containing the unique instance ID. VMWARE_INSTANCE_ID_VAR = 'vmware_uuid' From dbef3b7571e906bae6113cbf55f216b6727bc26a Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 19 Dec 2016 17:10:56 -0500 Subject: [PATCH 4/5] update vmware inv sync unique instance id --- awx/settings/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index b283387350..83f3e2d90a 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -667,7 +667,7 @@ VMWARE_ENABLED_VAR = 'guest.gueststate' VMWARE_ENABLED_VALUE = 'running' # Inventory variable name containing the unique instance ID. -VMWARE_INSTANCE_ID_VAR = 'vmware_uuid' +VMWARE_INSTANCE_ID_VAR = 'config.uuid' # Filter for allowed group and host names when importing inventory # from VMware. From d985f0783863b965777689d907aba4b02b8db8e0 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 19 Dec 2016 17:12:52 -0500 Subject: [PATCH 5/5] use vmware uuid that is most unique --- awx/settings/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 83f3e2d90a..f1e51a698e 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -667,7 +667,7 @@ VMWARE_ENABLED_VAR = 'guest.gueststate' VMWARE_ENABLED_VALUE = 'running' # Inventory variable name containing the unique instance ID. -VMWARE_INSTANCE_ID_VAR = 'config.uuid' +VMWARE_INSTANCE_ID_VAR = 'config.instanceuuid' # Filter for allowed group and host names when importing inventory # from VMware.