diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 955bcae559..90b15faf02 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -482,6 +482,7 @@ def load_inventory_source(source, all_group=None, group_filter_re=None, # good naming conventions source = source.replace('azure.py', 'windows_azure.py') source = source.replace('satellite6.py', 'foreman.py') + source = source.replace('vmware.py', 'vmware_inventory.py') logger.debug('Analyzing type of source: %s', source) original_all_group = all_group if not os.path.exists(source): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index ef636010d7..f1d28938f5 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1349,10 +1349,22 @@ class RunInventoryUpdate(BaseTask): 'password')) # Allow custom options to vmware inventory script. elif inventory_update.source == 'vmware': - section = 'defaults' + credential = inventory_update.credential + + section = 'vmware' cp.add_section(section) + cp.set('vmware', 'cache_max_age', 0) + + cp.set('vmware', 'username', credential.username) + cp.set('vmware', 'password', decrypt_field(credential, 'password')) + cp.set('vmware', 'server', credential.host) + vmware_opts = dict(inventory_update.source_vars_dict.items()) - vmware_opts.setdefault('guests_only', 'True') + if inventory_update.instance_filters: + inventory_update.setdefault('host_filters', inventory_update.instance_filters) + if inventory_update.group_by: + inventory_update.setdefault('groupby_patterns', inventory_update.groupby_patterns) + for k,v in vmware_opts.items(): cp.set(section, k, unicode(v)) @@ -1472,10 +1484,7 @@ class RunInventoryUpdate(BaseTask): # complain about not being able to determine its version number. env['PBR_VERSION'] = '0.5.21' elif inventory_update.source == 'vmware': - env['VMWARE_INI'] = cloud_credential - env['VMWARE_HOST'] = passwords.get('source_host', '') - env['VMWARE_USER'] = passwords.get('source_username', '') - env['VMWARE_PASSWORD'] = passwords.get('source_password', '') + env['VMWARE_INI_PATH'] = cloud_credential elif inventory_update.source == 'azure': env['AZURE_SUBSCRIPTION_ID'] = passwords.get('source_username', '') env['AZURE_CERT_PATH'] = cloud_credential diff --git a/awx/plugins/inventory/vmware_inventory.py b/awx/plugins/inventory/vmware_inventory.py new file mode 100755 index 0000000000..a221293edc --- /dev/null +++ b/awx/plugins/inventory/vmware_inventory.py @@ -0,0 +1,690 @@ +#!/usr/bin/env python + +# Requirements +# - pyvmomi >= 6.0.0.2016.4 + +# TODO: +# * more jq examples +# * optional folder heriarchy + +""" +$ jq '._meta.hostvars[].config' data.json | head +{ + "alternateguestname": "", + "instanceuuid": "5035a5cd-b8e8-d717-e133-2d383eb0d675", + "memoryhotaddenabled": false, + "guestfullname": "Red Hat Enterprise Linux 7 (64-bit)", + "changeversion": "2016-05-16T18:43:14.977925Z", + "uuid": "4235fc97-5ddb-7a17-193b-9a3ac97dc7b4", + "cpuhotremoveenabled": false, + "vpmcenabled": false, + "firmware": "bios", +""" + +from __future__ import print_function + +import argparse +import atexit +import datetime +import getpass +import jinja2 +import os +import six +import ssl +import sys +import uuid + +from collections import defaultdict +from six.moves import configparser +from time import time + +HAS_PYVMOMI = False +try: + from pyVmomi import vim + from pyVim.connect import SmartConnect, Disconnect + HAS_PYVMOMI = True +except ImportError: + pass + +try: + import json +except ImportError: + import simplejson as json + +hasvcr = False +try: + import vcr + hasvcr = True +except ImportError: + pass + +class VMwareMissingHostException(Exception): + pass + +class VMWareInventory(object): + + __name__ = 'VMWareInventory' + + guest_props = False + instances = [] + debug = False + load_dumpfile = None + write_dumpfile = None + maxlevel = 1 + lowerkeys = True + config = None + cache_max_age = None + cache_path_cache = None + cache_path_index = None + server = None + port = None + username = None + password = None + host_filters = [] + groupby_patterns = [] + + if (sys.version_info > (3, 0)): + safe_types = [int, bool, str, float, None] + else: + safe_types = [int, long, bool, str, float, None] + iter_types = [dict, list] + + bad_types = ['Array', 'disabledMethod', 'declaredAlarmState'] + skip_keys = ['declaredalarmstate', + 'disabledmethod', + 'dynamicproperty', + 'dynamictype', + 'environmentbrowser', + 'managedby', + 'parent', + 'childtype'] + + # translation table for attributes to fetch for known vim types + if not HAS_PYVMOMI: + vimTable = {} + else: + vimTable = { + vim.Datastore: ['_moId', 'name'], + vim.ResourcePool: ['_moId', 'name'], + } + + def _empty_inventory(self): + return {"_meta" : {"hostvars" : {}}} + + + def __init__(self, load=True): + self.inventory = self._empty_inventory() + + if load: + # Read settings and parse CLI arguments + self.parse_cli_args() + self.read_settings() + + # Check the cache + cache_valid = self.is_cache_valid() + + # Handle Cache + if self.args.refresh_cache or not cache_valid: + self.do_api_calls_update_cache() + else: + self.debugl('loading inventory from cache') + self.inventory = self.get_inventory_from_cache() + + def debugl(self, text): + if self.args.debug: + try: + text = str(text) + except UnicodeEncodeError: + text = text.encode('ascii','ignore') + print('%s %s' % (datetime.datetime.now(), text)) + + def show(self): + # Data to print + self.debugl('dumping results') + data_to_print = None + if self.args.host: + data_to_print = self.get_host_info(self.args.host) + elif self.args.list: + # Display list of instances for inventory + data_to_print = self.inventory + return json.dumps(data_to_print, indent=2) + + + def is_cache_valid(self): + + ''' Determines if the cache files have expired, or if it is still valid ''' + + valid = False + + if os.path.isfile(self.cache_path_cache): + mod_time = os.path.getmtime(self.cache_path_cache) + current_time = time() + if (mod_time + self.cache_max_age) > current_time: + valid = True + + return valid + + + def do_api_calls_update_cache(self): + + ''' Get instances and cache the data ''' + + instances = self.get_instances() + self.instances = instances + self.inventory = self.instances_to_inventory(instances) + self.write_to_cache(self.inventory, self.cache_path_cache) + + + def write_to_cache(self, data, cache_path): + + ''' Dump inventory to json file ''' + + with open(self.cache_path_cache, 'wb') as f: + f.write(json.dumps(data)) + + + def get_inventory_from_cache(self): + + ''' Read in jsonified inventory ''' + + jdata = None + with open(self.cache_path_cache, 'rb') as f: + jdata = f.read() + return json.loads(jdata) + + + def read_settings(self): + + ''' Reads the settings from the vmware_inventory.ini file ''' + + scriptbasename = __file__ + scriptbasename = os.path.basename(scriptbasename) + scriptbasename = scriptbasename.replace('.py', '') + + defaults = {'vmware': { + 'server': '', + 'port': 443, + 'username': '', + 'password': '', + 'validate_certs': True, + 'ini_path': os.path.join(os.path.dirname(__file__), '%s.ini' % scriptbasename), + 'cache_name': 'ansible-vmware', + 'cache_path': '~/.ansible/tmp', + 'cache_max_age': 3600, + 'max_object_level': 1, + 'alias_pattern': '{{ config.name + "_" + config.uuid }}', + 'host_pattern': '{{ guest.ipaddress }}', + 'host_filters': '{{ guest.gueststate == "running" }}', + 'groupby_patterns': '{{ guest.guestid }},{{ "templates" if config.template else "guests"}}', + 'lower_var_keys': True } + } + + if six.PY3: + config = configparser.ConfigParser() + else: + config = configparser.SafeConfigParser() + + # where is the config? + vmware_ini_path = os.environ.get('VMWARE_INI_PATH', defaults['vmware']['ini_path']) + vmware_ini_path = os.path.expanduser(os.path.expandvars(vmware_ini_path)) + config.read(vmware_ini_path) + + # apply defaults + for k,v in defaults['vmware'].items(): + if not config.has_option('vmware', k): + config.set('vmware', k, str(v)) + + # where is the cache? + self.cache_dir = os.path.expanduser(config.get('vmware', 'cache_path')) + if self.cache_dir and not os.path.exists(self.cache_dir): + os.makedirs(self.cache_dir) + + # set the cache filename and max age + cache_name = config.get('vmware', 'cache_name') + self.cache_path_cache = self.cache_dir + "/%s.cache" % cache_name + self.debugl('cache path is %s' % self.cache_path_cache) + self.cache_max_age = int(config.getint('vmware', 'cache_max_age')) + + # mark the connection info + self.server = os.environ.get('VMWARE_SERVER', config.get('vmware', 'server')) + self.debugl('server is %s' % self.server) + self.port = int(os.environ.get('VMWARE_PORT', config.get('vmware', 'port'))) + self.username = os.environ.get('VMWARE_USERNAME', config.get('vmware', 'username')) + self.debugl('username is %s' % self.username) + self.password = os.environ.get('VMWARE_PASSWORD', config.get('vmware', 'password')) + self.validate_certs = os.environ.get('VMWARE_VALIDATE_CERTS', config.get('vmware', 'validate_certs')) + if self.validate_certs in ['no', 'false', 'False', False]: + self.validate_certs = False + else: + self.validate_certs = True + self.debugl('cert validation is %s' % self.validate_certs) + + # behavior control + self.maxlevel = int(config.get('vmware', 'max_object_level')) + self.debugl('max object level is %s' % self.maxlevel) + self.lowerkeys = config.get('vmware', 'lower_var_keys') + if type(self.lowerkeys) != bool: + if str(self.lowerkeys).lower() in ['yes', 'true', '1']: + self.lowerkeys = True + else: + self.lowerkeys = False + self.debugl('lower keys is %s' % self.lowerkeys) + + self.host_filters = list(config.get('vmware', 'host_filters').split(',')) + self.debugl('host filters are %s' % self.host_filters) + self.groupby_patterns = list(config.get('vmware', 'groupby_patterns').split(',')) + self.debugl('groupby patterns are %s' % self.groupby_patterns) + + # Special feature to disable the brute force serialization of the + # virtulmachine objects. The key name for these properties does not + # matter because the values are just items for a larger list. + if config.has_section('properties'): + self.guest_props = [] + for prop in config.items('properties'): + self.guest_props.append(prop[1]) + + # save the config + self.config = config + + + def parse_cli_args(self): + + ''' Command line argument processing ''' + + parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on PyVmomi') + parser.add_argument('--debug', action='store_true', default=False, + help='show debug info') + parser.add_argument('--list', action='store_true', default=True, + help='List instances (default: True)') + parser.add_argument('--host', action='store', + help='Get all the variables about a specific instance') + parser.add_argument('--refresh-cache', action='store_true', default=False, + help='Force refresh of cache by making API requests to VSphere (default: False - use cache files)') + parser.add_argument('--max-instances', default=None, type=int, + help='maximum number of instances to retrieve') + self.args = parser.parse_args() + + + def get_instances(self): + + ''' Get a list of vm instances with pyvmomi ''' + + instances = [] + + kwargs = {'host': self.server, + 'user': self.username, + 'pwd': self.password, + 'port': int(self.port) } + + if hasattr(ssl, 'SSLContext') and not self.validate_certs: + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.verify_mode = ssl.CERT_NONE + kwargs['sslContext'] = context + + instances = self._get_instances(kwargs) + return instances + + + def _get_instances(self, inkwargs): + + ''' Make API calls ''' + + instances = [] + si = SmartConnect(**inkwargs) + + self.debugl('retrieving all instances') + if not si: + print("Could not connect to the specified host using specified " + "username and password") + return -1 + atexit.register(Disconnect, si) + content = si.RetrieveContent() + + # Create a search container for virtualmachines + self.debugl('creating containerview for virtualmachines') + container = content.rootFolder + viewType = [vim.VirtualMachine] + recursive = True + containerView = content.viewManager.CreateContainerView(container, viewType, recursive) + children = containerView.view + for child in children: + # If requested, limit the total number of instances + if self.args.max_instances: + if len(instances) >= (self.args.max_instances): + break + instances.append(child) + self.debugl("%s total instances in container view" % len(instances)) + + if self.args.host: + instances = [x for x in instances if x.name == self.args.host] + + instance_tuples = [] + for instance in sorted(instances): + if self.guest_props != False: + ifacts = self.facts_from_proplist(instance) + else: + ifacts = self.facts_from_vobj(instance) + instance_tuples.append((instance, ifacts)) + self.debugl('facts collected for all instances') + return instance_tuples + + + def instances_to_inventory(self, instances): + + ''' Convert a list of vm objects into a json compliant inventory ''' + + self.debugl('re-indexing instances based on ini settings') + inventory = self._empty_inventory() + inventory['all'] = {} + inventory['all']['hosts'] = [] + last_idata = None + total = len(instances) + for idx,instance in enumerate(instances): + + # make a unique id for this object to avoid vmware's + # numerous uuid's which aren't all unique. + thisid = str(uuid.uuid4()) + idata = instance[1] + + # Put it in the inventory + inventory['all']['hosts'].append(thisid) + inventory['_meta']['hostvars'][thisid] = idata.copy() + inventory['_meta']['hostvars'][thisid]['ansible_uuid'] = thisid + + # Make a map of the uuid to the alias the user wants + name_mapping = self.create_template_mapping(inventory, + self.config.get('vmware', 'alias_pattern')) + + # Make a map of the uuid to the ssh hostname the user wants + host_mapping = self.create_template_mapping(inventory, + self.config.get('vmware', 'host_pattern')) + + + # Reset the inventory keys + for k,v in name_mapping.items(): + + if not host_mapping or not k in host_mapping: + continue + + # set ansible_host (2.x) + try: + inventory['_meta']['hostvars'][k]['ansible_host'] = host_mapping[k] + # 1.9.x backwards compliance + inventory['_meta']['hostvars'][k]['ansible_ssh_host'] = host_mapping[k] + except Exception as e: + continue + + if k == v: + continue + + # add new key + inventory['all']['hosts'].append(v) + inventory['_meta']['hostvars'][v] = inventory['_meta']['hostvars'][k] + + # cleanup old key + inventory['all']['hosts'].remove(k) + inventory['_meta']['hostvars'].pop(k, None) + + self.debugl('pre-filtered hosts:') + for i in inventory['all']['hosts']: + self.debugl(' * %s' % i) + # Apply host filters + for hf in self.host_filters: + if not hf: + continue + self.debugl('filter: %s' % hf) + filter_map = self.create_template_mapping(inventory, hf, dtype='boolean') + for k,v in filter_map.items(): + if not v: + # delete this host + inventory['all']['hosts'].remove(k) + inventory['_meta']['hostvars'].pop(k, None) + + self.debugl('post-filter hosts:') + for i in inventory['all']['hosts']: + self.debugl(' * %s' % i) + + # Create groups + for gbp in self.groupby_patterns: + groupby_map = self.create_template_mapping(inventory, gbp) + for k,v in groupby_map.items(): + if v not in inventory: + inventory[v] = {} + inventory[v]['hosts'] = [] + if k not in inventory[v]['hosts']: + inventory[v]['hosts'].append(k) + + return inventory + + + def create_template_mapping(self, inventory, pattern, dtype='string'): + + ''' Return a hash of uuid to templated string from pattern ''' + + mapping = {} + for k,v in inventory['_meta']['hostvars'].items(): + t = jinja2.Template(pattern) + newkey = None + try: + newkey = t.render(v) + newkey = newkey.strip() + except Exception as e: + self.debugl(e) + if not newkey: + continue + elif dtype == 'integer': + newkey = int(newkey) + elif dtype == 'boolean': + if newkey.lower() == 'false': + newkey = False + elif newkey.lower() == 'true': + newkey = True + elif dtype == 'string': + pass + mapping[k] = newkey + return mapping + + def facts_from_proplist(self, vm): + '''Get specific properties instead of serializing everything''' + + rdata = {} + for prop in self.guest_props: + self.debugl('getting %s property for %s' % (prop, vm.name)) + key = prop + if self.lowerkeys: + key = key.lower() + + if not '.' in prop: + # props without periods are direct attributes of the parent + rdata[key] = getattr(vm, prop) + else: + # props with periods are subkeys of parent attributes + parts = prop.split('.') + total = len(parts) - 1 + + # pointer to the current object + val = None + # pointer to the current result key + lastref = rdata + + for idx,x in enumerate(parts): + + # if the val wasn't set yet, get it from the parent + if not val: + val = getattr(vm, x) + else: + # in a subkey, get the subprop from the previous attrib + try: + val = getattr(val, x) + except AttributeError as e: + self.debugl(e) + + # lowercase keys if requested + if self.lowerkeys: + x = x.lower() + + # change the pointer or set the final value + if idx != total: + if x not in lastref: + lastref[x] = {} + lastref = lastref[x] + else: + lastref[x] = val + + return rdata + + + def facts_from_vobj(self, vobj, level=0): + + ''' Traverse a VM object and return a json compliant data structure ''' + + # pyvmomi objects are not yet serializable, but may be one day ... + # https://github.com/vmware/pyvmomi/issues/21 + + # WARNING: + # Accessing an object attribute will trigger a SOAP call to the remote. + # Increasing the attributes collected or the depth of recursion greatly + # increases runtime duration and potentially memory+network utilization. + + if level == 0: + try: + self.debugl("get facts for %s" % vobj.name) + except Exception as e: + self.debugl(e) + + rdata = {} + + methods = dir(vobj) + methods = [str(x) for x in methods if not x.startswith('_')] + methods = [x for x in methods if not x in self.bad_types] + methods = [x for x in methods if not x.lower() in self.skip_keys] + methods = sorted(methods) + + for method in methods: + # Attempt to get the method, skip on fail + try: + methodToCall = getattr(vobj, method) + except Exception as e: + continue + + # Skip callable methods + if callable(methodToCall): + continue + + if self.lowerkeys: + method = method.lower() + + rdata[method] = self._process_object_types( + methodToCall, + thisvm=vobj, + inkey=method + ) + + return rdata + + + def _process_object_types(self, vobj, thisvm=None, inkey=None, level=0): + ''' Serialize an object ''' + rdata = {} + + if vobj is None: + rdata = None + + elif type(vobj) in self.vimTable: + rdata = {} + for key in self.vimTable[type(vobj)]: + rdata[key] = getattr(vobj, key) + + elif issubclass(type(vobj), str) or isinstance(vobj, str): + if vobj.isalnum(): + rdata = vobj + else: + rdata = vobj.decode('ascii', 'ignore') + elif issubclass(type(vobj), bool) or isinstance(vobj, bool): + rdata = vobj + elif issubclass(type(vobj), int) or isinstance(vobj, int): + rdata = vobj + elif issubclass(type(vobj), float) or isinstance(vobj, float): + rdata = vobj + elif issubclass(type(vobj), long) or isinstance(vobj, long): + rdata = vobj + elif issubclass(type(vobj), list) or issubclass(type(vobj), tuple): + rdata = [] + try: + vobj = sorted(vobj) + except Exception as e: + pass + + for idv, vii in enumerate(vobj): + + if (level+1 <= self.maxlevel): + + vid = self._process_object_types( + vii, + thisvm=thisvm, + inkey=inkey+'['+str(idv)+']', + level=(level+1) + ) + + if vid: + rdata.append(vid) + + elif issubclass(type(vobj), dict): + pass + + elif issubclass(type(vobj), object): + methods = dir(vobj) + methods = [str(x) for x in methods if not x.startswith('_')] + methods = [x for x in methods if not x in self.bad_types] + methods = [x for x in methods if not x.lower() in self.skip_keys] + methods = sorted(methods) + + for method in methods: + # Attempt to get the method, skip on fail + try: + methodToCall = getattr(vobj, method) + except Exception as e: + continue + if callable(methodToCall): + continue + if self.lowerkeys: + method = method.lower() + if (level+1 <= self.maxlevel): + rdata[method] = self._process_object_types( + methodToCall, + thisvm=thisvm, + inkey=inkey+'.'+method, + level=(level+1) + ) + else: + pass + + return rdata + + def get_host_info(self, host): + + ''' Return hostvars for a single host ''' + + if host in self.inventory['_meta']['hostvars']: + return self.inventory['_meta']['hostvars'][host] + elif self.args.host and self.inventory['_meta']['hostvars']: + # check if the machine has the name requested + keys = self.inventory['_meta']['hostvars'].keys() + match = None + for k,v in self.inventory['_meta']['hostvars'].items(): + if self.inventory['_meta']['hostvars'][k]['name'] == self.args.host: + match = k + break + if match: + return self.inventory['_meta']['hostvars'][match] + else: + raise VMwareMissingHostException('%s not found' % host) + else: + raise VMwareMissingHostException('%s not found' % host) + + +if __name__ == "__main__": + # Run the script + print(VMWareInventory().show()) + +