Update cloudforms dynamic inventory

Pulling the latest from
https://github.com/ansible/ansible/blob/devel/contrib/inventory/cloudforms.py

Related #3168
This commit is contained in:
James Laska
2016-08-19 09:30:15 -04:00
parent a847495bea
commit 188f81ef3d
2 changed files with 417 additions and 94 deletions

View File

@@ -1319,9 +1319,14 @@ class RunInventoryUpdate(BaseTask):
credential = inventory_update.credential credential = inventory_update.credential
if credential: if credential:
cp.set(section, 'hostname', credential.host) cp.set(section, 'url', credential.host)
cp.set(section, 'username', credential.username) cp.set(section, 'username', credential.username)
cp.set(section, 'password', decrypt_field(credential, 'password')) cp.set(section, 'password', decrypt_field(credential, 'password'))
cp.set(section, 'ssl_verify', "false")
section = 'cache'
cp.add_section(section)
cp.set(section, 'max_age', "0")
elif inventory_update.source == 'azure_rm': elif inventory_update.source == 'azure_rm':
section = 'azure' section = 'azure'

View File

@@ -1,144 +1,462 @@
#!/usr/bin/python #!/usr/bin/python
# vim: set fileencoding=utf-8 :
#
# Copyright (C) 2016 Guido Günther <agx@sigxcpu.org>
#
# This script is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with it. If not, see <http://www.gnu.org/licenses/>.
#
# This is loosely based on the foreman inventory script
# -- Josh Preston <jpreston@redhat.com>
#
''' from __future__ import print_function
CloudForms external inventory script
==================================================
Generates inventory that Ansible can understand by making API request to CloudForms.
Modeled after https://raw.githubusercontent.com/ansible/ansible/stable-1.9/plugins/inventory/ec2.py
jlabocki <at> redhat.com or @jameslabocki on twitter
'''
import os
import argparse import argparse
import ConfigParser import ConfigParser
import os
import re
from time import time
import requests import requests
import json from requests.auth import HTTPBasicAuth
import warnings
# This disables warnings and is not a good idea, but hey, this is a demo try:
# http://urllib3.readthedocs.org/en/latest/security.html#disabling-warnings import json
requests.packages.urllib3.disable_warnings() except ImportError:
import simplejson as json
class CloudFormsInventory(object): class CloudFormsInventory(object):
def _empty_inventory(self):
return {"_meta": {"hostvars": {}}}
def __init__(self): def __init__(self):
''' Main execution path ''' """
Main execution path
"""
self.inventory = dict() # A list of groups and the hosts in that group
self.hosts = dict() # Details about hosts in the inventory
# Inventory grouped by instance IDs, tags, security groups, regions, # Parse CLI arguments
# and availability zones
self.inventory = self._empty_inventory()
# Index of hostname (address) to instance ID
self.index = {}
# Read CLI arguments
self.read_settings()
self.parse_cli_args() self.parse_cli_args()
# Get Hosts # Read settings
if self.args.list: self.read_settings()
self.get_hosts()
# This doesn't exist yet and needs to be added # Cache
if self.args.refresh_cache or not self.is_cache_valid():
self.update_cache()
else:
self.load_inventory_from_cache()
self.load_hosts_from_cache()
data_to_print = ""
# Data to print
if self.args.host: if self.args.host:
data2 = {} if self.args.debug:
print json.dumps(data2, indent=2) print("Fetching host [%s]" % self.args.host)
data_to_print += self.get_host_info(self.args.host)
else:
self.inventory['_meta'] = {'hostvars': {}}
for hostname in self.hosts:
self.inventory['_meta']['hostvars'][hostname] = {
'cloudforms': self.hosts[hostname],
}
# include the ansible_ssh_host in the top level
if 'ansible_ssh_host' in self.hosts[hostname]:
self.inventory['_meta']['hostvars'][hostname]['ansible_ssh_host'] = self.hosts[hostname]['ansible_ssh_host']
def parse_cli_args(self): data_to_print += self.json_format_dict(self.inventory, self.args.pretty)
''' Command line argument processing '''
parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on CloudForms') print(data_to_print)
parser.add_argument('--list', action='store_true', default=False,
help='List instances (default: False)') def is_cache_valid(self):
parser.add_argument('--host', action='store', """
help='Get all the variables about a specific instance') Determines if the cache files have expired, or if it is still valid
self.args = parser.parse_args() """
if self.args.debug:
print("Determining if cache [%s] is still valid (< %s seconds old)" % (self.cache_path_hosts, self.cache_max_age))
if os.path.isfile(self.cache_path_hosts):
mod_time = os.path.getmtime(self.cache_path_hosts)
current_time = time()
if (mod_time + self.cache_max_age) > current_time:
if os.path.isfile(self.cache_path_inventory):
if self.args.debug:
print("Cache is still valid!")
return True
if self.args.debug:
print("Cache is stale or does not exist.")
return False
def read_settings(self): def read_settings(self):
''' Reads the settings from the cloudforms.ini file ''' """
Reads the settings from the cloudforms.ini file
"""
config = ConfigParser.SafeConfigParser() config = ConfigParser.SafeConfigParser()
config_paths = [ config_paths = [
os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cloudforms.ini'), os.path.dirname(os.path.realpath(__file__)) + '/cloudforms.ini',
"/opt/rh/cloudforms.ini", "/etc/ansible/cloudforms.ini",
] ]
env_value = os.environ.get('CLOUDFORMS_INI_PATH') env_value = os.environ.get('CLOUDFORMS_INI_PATH')
if env_value is not None: if env_value is not None:
config_paths.append(os.path.expanduser(os.path.expandvars(env_value))) config_paths.append(os.path.expanduser(os.path.expandvars(env_value)))
if self.args.debug:
for config_path in config_paths:
print("Reading from configuration file [%s]" % config_path)
config.read(config_paths) config.read(config_paths)
# Version # CloudForms API related
if config.has_option('cloudforms', 'version'): if config.has_option('cloudforms', 'url'):
self.cloudforms_version = config.get('cloudforms', 'version') self.cloudforms_url = config.get('cloudforms', 'url')
else: else:
self.cloudforms_version = "none" self.cloudforms_url = None
# CloudForms Endpoint if not self.cloudforms_url:
if config.has_option('cloudforms', 'hostname'): warnings.warn("No url specified, expected something like 'https://cfme.example.com'")
self.cloudforms_hostname = config.get('cloudforms', 'hostname')
else:
self.cloudforms_hostname = None
# CloudForms Username
if config.has_option('cloudforms', 'username'): if config.has_option('cloudforms', 'username'):
self.cloudforms_username = config.get('cloudforms', 'username') self.cloudforms_username = config.get('cloudforms', 'username')
else: else:
self.cloudforms_username = "none" self.cloudforms_username = None
if not self.cloudforms_username:
warnings.warn("No username specified, you need to specify a CloudForms username.")
# CloudForms Password
if config.has_option('cloudforms', 'password'): if config.has_option('cloudforms', 'password'):
self.cloudforms_password = config.get('cloudforms', 'password') self.cloudforms_pw = config.get('cloudforms', 'password')
else: else:
self.cloudforms_password = "none" self.cloudforms_pw = None
def get_hosts(self): if not self.cloudforms_pw:
''' Gets host from CloudForms ''' warnings.warn("No password specified, you need to specify a password for the CloudForms user.")
r = requests.get("https://{0}/api/vms?expand=resources&attributes=all".format(self.cloudforms_hostname),
auth=(self.cloudforms_username, self.cloudforms_password), verify=False)
obj = r.json()
# Create groups+hosts based on host data if config.has_option('cloudforms', 'ssl_verify'):
for resource in obj.get('resources', []): self.cloudforms_ssl_verify = config.getboolean('cloudforms', 'ssl_verify')
else:
self.cloudforms_ssl_verify = True
# Maintain backwards compat by creating `Dynamic_CloudForms` group if config.has_option('cloudforms', 'version'):
if 'Dynamic_CloudForms' not in self.inventory: self.cloudforms_version = config.get('cloudforms', 'version')
self.inventory['Dynamic_CloudForms'] = [] else:
self.inventory['Dynamic_CloudForms'].append(resource['name']) self.cloudforms_version = None
# Add host to desired groups if config.has_option('cloudforms', 'limit'):
for key in ('vendor', 'type', 'location'): self.cloudforms_limit = config.getint('cloudforms', 'limit')
if key in resource: else:
# Create top-level group self.cloudforms_limit = 100
if key not in self.inventory:
self.inventory[key] = dict(children=[], vars={}, hosts=[])
# if resource['name'] not in self.inventory[key]['hosts']:
# self.inventory[key]['hosts'].append(resource['name'])
# Create sub-group if config.has_option('cloudforms', 'purge_actions'):
if resource[key] not in self.inventory: self.cloudforms_purge_actions = config.getboolean('cloudforms', 'purge_actions')
self.inventory[resource[key]] = dict(children=[], vars={}, hosts=[]) else:
# self.inventory[resource[key]]['hosts'].append(resource['name']) self.cloudforms_purge_actions = True
# Add sub-group, as a child of top-level if config.has_option('cloudforms', 'clean_group_keys'):
if resource[key] not in self.inventory[key]['children']: self.cloudforms_clean_group_keys = config.getboolean('cloudforms', 'clean_group_keys')
self.inventory[key]['children'].append(resource[key]) else:
self.cloudforms_clean_group_keys = True
if config.has_option('cloudforms', 'nest_tags'):
self.cloudforms_nest_tags = config.getboolean('cloudforms', 'nest_tags')
else:
self.cloudforms_nest_tags = False
# Ansible related
try:
group_patterns = config.get('ansible', 'group_patterns')
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
group_patterns = "[]"
self.group_patterns = eval(group_patterns)
# Cache related
try:
cache_path = os.path.expanduser(config.get('cache', 'path'))
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
cache_path = '.'
(script, ext) = os.path.splitext(os.path.basename(__file__))
self.cache_path_hosts = cache_path + "/%s.hosts" % script
self.cache_path_inventory = cache_path + "/%s.inventory" % script
self.cache_max_age = config.getint('cache', 'max_age')
if self.args.debug:
print("CloudForms settings:")
print("cloudforms_url = %s" % self.cloudforms_url)
print("cloudforms_username = %s" % self.cloudforms_username)
print("cloudforms_pw = %s" % self.cloudforms_pw)
print("cloudforms_ssl_verify = %s" % self.cloudforms_ssl_verify)
print("cloudforms_version = %s" % self.cloudforms_version)
print("cloudforms_limit = %s" % self.cloudforms_limit)
print("cloudforms_purge_actions = %s" % self.cloudforms_purge_actions)
print("Cache settings:")
print("cache_max_age = %s" % self.cache_max_age)
print("cache_path_hosts = %s" % self.cache_path_hosts)
print("cache_path_inventory = %s" % self.cache_path_inventory)
def parse_cli_args(self):
"""
Command line argument processing
"""
parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on CloudForms managed VMs')
parser.add_argument('--list', action='store_true', default=True, help='List instances (default: True)')
parser.add_argument('--host', action='store', help='Get all the variables about a specific instance')
parser.add_argument('--pretty', action='store_true', default=False, help='Pretty print JSON output (default: False)')
parser.add_argument('--refresh-cache', action='store_true', default=False,
help='Force refresh of cache by making API requests to CloudForms (default: False - use cache files)')
parser.add_argument('--debug', action='store_true', default=False, help='Show debug output while running (default: False)')
self.args = parser.parse_args()
def _get_json(self, url):
"""
Make a request and return the JSON
"""
results = []
ret = requests.get(url,
auth=HTTPBasicAuth(self.cloudforms_username, self.cloudforms_pw),
verify=self.cloudforms_ssl_verify)
ret.raise_for_status()
try:
results = json.loads(ret.text)
except ValueError:
warnings.warn("Unexpected response from {0} ({1}): {2}".format(self.cloudforms_url, ret.status_code, ret.reason))
results = {}
if self.args.debug:
print("=======================================================================")
print("=======================================================================")
print("=======================================================================")
print(ret.text)
print("=======================================================================")
print("=======================================================================")
print("=======================================================================")
return results
def _get_hosts(self):
"""
Get all hosts by paging through the results
"""
limit = self.cloudforms_limit
page = 0
last_page = False
results = []
while not last_page:
offset = page * limit
ret = self._get_json("%s/api/vms?offset=%s&limit=%s&expand=resources,tags,hosts,&attributes=ipaddresses" % (self.cloudforms_url, offset, limit))
results += ret['resources']
if ret['subcount'] < limit:
last_page = True
page += 1
return results
def update_cache(self):
"""
Make calls to cloudforms and save the output in a cache
"""
self.groups = dict()
self.hosts = dict()
if self.args.debug:
print("Updating cache...")
for host in self._get_hosts():
# Ignore VMs that are not powered on
if host['power_state'] != 'on':
if self.args.debug:
print("Skipping %s because power_state = %s" % (host['name'], host['power_state']))
continue
# purge actions
if self.cloudforms_purge_actions and 'actions' in host:
del host['actions']
# Create ansible groups for tags
if 'tags' in host:
# Create top-level group
if 'tags' not in self.inventory:
self.inventory['tags'] = dict(children=[], vars={}, hosts=[])
if not self.cloudforms_nest_tags:
# don't expand tags, just use them in a safe way
for group in host['tags']:
# Add sub-group, as a child of top-level
safe_key = self.to_safe(group['name'])
if safe_key:
if self.args.debug:
print("Adding sub-group '%s' to parent 'tags'" % safe_key)
if safe_key not in self.inventory['tags']['children']:
self.push(self.inventory['tags'], 'children', safe_key)
self.push(self.inventory, safe_key, host['name'])
if self.args.debug:
print("Found tag [%s] for host which will be mapped to [%s]" % (group['name'], safe_key))
else:
# expand the tags into nested groups / sub-groups
# Create nested groups for tags
safe_parent_tag_name = 'tags'
for tag in host['tags']:
tag_hierarchy = tag['name'][1:].split('/')
if self.args.debug:
print("Working on list %s" % tag_hierarchy)
for tag_name in tag_hierarchy:
if self.args.debug:
print("Working on tag_name = %s" % tag_name)
safe_tag_name = self.to_safe(tag_name)
if self.args.debug:
print("Using sanitized name %s" % safe_tag_name)
# Create sub-group
if safe_tag_name not in self.inventory:
self.inventory[safe_tag_name] = dict(children=[], vars={}, hosts=[])
# Add sub-group, as a child of top-level
if safe_parent_tag_name:
if self.args.debug:
print("Adding sub-group '%s' to parent '%s'" % (safe_tag_name, safe_parent_tag_name))
if safe_tag_name not in self.inventory[safe_parent_tag_name]['children']:
self.push(self.inventory[safe_parent_tag_name], 'children', safe_tag_name)
# Make sure the next one uses this one as it's parent
safe_parent_tag_name = safe_tag_name
# Add the host to the last tag
self.push(self.inventory[safe_parent_tag_name], 'hosts', host['name'])
# Set ansible_ssh_host to the first available ip address
if 'ipaddresses' in host and host['ipaddresses'] and isinstance(host['ipaddresses'], list):
host['ansible_ssh_host'] = host['ipaddresses'][0]
# Create additional groups
for key in ('location', 'type', 'vendor'):
safe_key = self.to_safe(host[key])
# Create top-level group
if key not in self.inventory:
self.inventory[key] = dict(children=[], vars={}, hosts=[])
# Create sub-group
if safe_key not in self.inventory:
self.inventory[safe_key] = dict(children=[], vars={}, hosts=[])
# Add sub-group, as a child of top-level
if safe_key not in self.inventory[key]['children']:
self.push(self.inventory[key], 'children', safe_key)
if key in host:
# Add host to sub-group # Add host to sub-group
if resource['name'] not in self.inventory[resource[key]]: self.push(self.inventory[safe_key], 'hosts', host['name'])
self.inventory[resource[key]]['hosts'].append(resource['name'])
# Delete 'actions' key self.hosts[host['name']] = host
del resource['actions'] self.push(self.inventory, 'all', host['name'])
# Add _meta hostvars if self.args.debug:
self.inventory['_meta']['hostvars'][resource['name']] = resource print("Saving cached data")
print json.dumps(self.inventory, indent=2) self.write_to_cache(self.hosts, self.cache_path_hosts)
self.write_to_cache(self.inventory, self.cache_path_inventory)
def get_host_info(self, host):
"""
Get variables about a specific host
"""
if not self.hosts or len(self.hosts) == 0:
# Need to load cache from cache
self.load_hosts_from_cache()
if host not in self.hosts:
if self.args.debug:
print("[%s] not found in cache." % host)
# try updating the cache
self.update_cache()
if host not in self.hosts:
if self.args.debug:
print("[%s] does not exist after cache update." % host)
# host might not exist anymore
return self.json_format_dict({}, self.args.pretty)
return self.json_format_dict(self.hosts[host], self.args.pretty)
def push(self, d, k, v):
"""
Safely puts a new entry onto an array.
"""
if k in d:
d[k].append(v)
else:
d[k] = [v]
def load_inventory_from_cache(self):
"""
Reads the inventory from the cache file sets self.inventory
"""
cache = open(self.cache_path_inventory, 'r')
json_inventory = cache.read()
self.inventory = json.loads(json_inventory)
def load_hosts_from_cache(self):
"""
Reads the cache from the cache file sets self.hosts
"""
cache = open(self.cache_path_hosts, 'r')
json_cache = cache.read()
self.hosts = json.loads(json_cache)
def write_to_cache(self, data, filename):
"""
Writes data in JSON format to a file
"""
json_data = self.json_format_dict(data, True)
cache = open(filename, 'w')
cache.write(json_data)
cache.close()
def to_safe(self, word):
"""
Converts 'bad' characters in a string to underscores so they can be used as Ansible groups
"""
if self.cloudforms_clean_group_keys:
regex = "[^A-Za-z0-9\_]"
return re.sub(regex, "_", word.replace(" ", ""))
else:
return word
def json_format_dict(self, data, pretty=False):
"""
Converts a dict to a JSON object and dumps it as a formatted string
"""
if pretty:
return json.dumps(data, sort_keys=True, indent=2)
else:
return json.dumps(data)
# Run the script
CloudFormsInventory() CloudFormsInventory()