From 1c4e6c55f594b44b29d4677aa7d668530ba5774f Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 19 Dec 2016 11:26:43 -0500 Subject: [PATCH] bump foreman and expose new grouping features related to #3467 related to #5226 related to #4373 --- awx/main/tasks.py | 4 +- awx/plugins/inventory/foreman.ini.example | 3 + awx/plugins/inventory/foreman.py | 151 ++++++++++++++-------- awx/settings/defaults.py | 2 + 4 files changed, 105 insertions(+), 55 deletions(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 505172070c..f0cbf9cc4e 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1373,7 +1373,9 @@ class RunInventoryUpdate(BaseTask): section = 'ansible' cp.add_section(section) - cp.set(section, 'group_patterns', '["{app}-{tier}-{color}", "{app}-{color}", "{app}", "{tier}"]') + cp.set(section, 'group_patterns', os.environ.get('SATELLITE6_GROUP_PATTERNS', [])) + cp.set(section, 'want_facts', True) + cp.set(section, 'group_prefix', os.environ.get('SATELLITE6_GROUP_PREFIX', 'foreman_')) section = 'cache' cp.add_section(section) diff --git a/awx/plugins/inventory/foreman.ini.example b/awx/plugins/inventory/foreman.ini.example index d5cd56e441..42312dac6c 100644 --- a/awx/plugins/inventory/foreman.ini.example +++ b/awx/plugins/inventory/foreman.ini.example @@ -9,6 +9,9 @@ group_patterns = ["{app}-{tier}-{color}", "{app}-{color}", "{app}", "{tier}"] +group_prefix = foreman_ +# Whether to fetch facts from Foreman and store them on the host +want_facts = True [cache] path = . diff --git a/awx/plugins/inventory/foreman.py b/awx/plugins/inventory/foreman.py index ddcb912fd5..f911f66230 100755 --- a/awx/plugins/inventory/foreman.py +++ b/awx/plugins/inventory/foreman.py @@ -18,14 +18,22 @@ # # This is somewhat based on cobbler inventory +from __future__ import print_function + import argparse -import ConfigParser import copy import os import re -from time import time import requests from requests.auth import HTTPBasicAuth +import sys +from time import time + +try: + import ConfigParser +except ImportError: + import configparser as ConfigParser + try: import json @@ -34,19 +42,33 @@ except ImportError: class ForemanInventory(object): + config_paths = [ + "/etc/ansible/foreman.ini", + os.path.dirname(os.path.realpath(__file__)) + '/foreman.ini', + ] + def __init__(self): - """ Main execution path """ self.inventory = dict() # A list of groups and the hosts in that group - self.cache = dict() # Details about hosts in the inventory - self.params = dict() # Params of each host - self.facts = dict() # Facts of each host + self.cache = dict() # Details about hosts in the inventory + self.params = dict() # Params of each host + self.facts = dict() # Facts of each host self.hostgroups = dict() # host groups - # Read settings and parse CLI arguments - self.read_settings() - self.parse_cli_args() + def run(self): + if not self._read_settings(): + return False + self._get_inventory() + self._print_data() + return True - # Cache + def _read_settings(self): + # Read settings and parse CLI arguments + if not self.read_settings(): + return False + self.parse_cli_args() + return True + + def _get_inventory(self): if self.args.refresh_cache: self.update_cache() elif not self.is_cache_valid(): @@ -57,9 +79,8 @@ class ForemanInventory(object): self.load_facts_from_cache() self.load_cache_from_cache() + def _print_data(self): data_to_print = "" - - # Data to print if self.args.host: data_to_print += self.get_host_info() else: @@ -77,38 +98,36 @@ class ForemanInventory(object): print(data_to_print) def is_cache_valid(self): - """ Determines if the cache files have expired, or if it is still valid """ - + """Determines if the cache is still valid""" if os.path.isfile(self.cache_path_cache): mod_time = os.path.getmtime(self.cache_path_cache) current_time = time() if (mod_time + self.cache_max_age) > current_time: if (os.path.isfile(self.cache_path_inventory) and os.path.isfile(self.cache_path_params) and - os.path.isfile(self.cache_path_facts)): + os.path.isfile(self.cache_path_facts)): return True return False def read_settings(self): - """ Reads the settings from the foreman.ini file """ + """Reads the settings from the foreman.ini file""" config = ConfigParser.SafeConfigParser() - config_paths = [ - "/etc/ansible/foreman.ini", - os.path.dirname(os.path.realpath(__file__)) + '/foreman.ini', - ] - env_value = os.environ.get('FOREMAN_INI_PATH') if env_value is not None: - config_paths.append(os.path.expanduser(os.path.expandvars(env_value))) + self.config_paths.append(os.path.expanduser(os.path.expandvars(env_value))) - config.read(config_paths) + config.read(self.config_paths) # Foreman API related - self.foreman_url = config.get('foreman', 'url') - self.foreman_user = config.get('foreman', 'user') - self.foreman_pw = config.get('foreman', 'password') - self.foreman_ssl_verify = config.getboolean('foreman', 'ssl_verify') + try: + self.foreman_url = config.get('foreman', 'url') + self.foreman_user = config.get('foreman', 'user') + self.foreman_pw = config.get('foreman', 'password') + self.foreman_ssl_verify = config.getboolean('foreman', 'ssl_verify') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e: + print("Error parsing configuration: %s" % e, file=sys.stderr) + return False # Ansible related try: @@ -138,10 +157,14 @@ class ForemanInventory(object): self.cache_path_inventory = cache_path + "/%s.index" % script self.cache_path_params = cache_path + "/%s.params" % script self.cache_path_facts = cache_path + "/%s.facts" % script - self.cache_max_age = config.getint('cache', 'max_age') + try: + self.cache_max_age = config.getint('cache', 'max_age') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.cache_max_age = 60 + return True def parse_cli_args(self): - """ Command line argument processing """ + """Command line argument processing""" parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on foreman') parser.add_argument('--list', action='store_true', default=True, help='List instances (default: True)') @@ -162,14 +185,22 @@ class ForemanInventory(object): break ret.raise_for_status() json = ret.json() - if not json.has_key('results'): + # /hosts/:id has not results key + if 'results' not in json: return json - if type(json['results']) == type({}): + # Facts are returned as dict in results not list + if isinstance(json['results'], dict): return json['results'] + # List of all hosts is returned paginaged results = results + json['results'] if len(results) >= json['total']: break page += 1 + if len(json['results']) == 0: + print("Did not make any progress during loop. " + "expected %d got %d" % (json['total'], len(results)), + file=sys.stderr) + break return results def _get_hosts(self): @@ -184,7 +215,8 @@ class ForemanInventory(object): def _get_all_params_by_id(self, hid): url = "%s/api/v2/hosts/%s" % (self.foreman_url, hid) ret = self._get_json(url, [404]) - if ret == []: ret = {} + if ret == []: + ret = {} return ret.get('all_parameters', {}) def _get_facts_by_id(self, hid): @@ -192,9 +224,7 @@ class ForemanInventory(object): return self._get_json(url) def _resolve_params(self, host): - """ - Fetch host params and convert to dict - """ + """Fetch host params and convert to dict""" params = {} for param in self._get_all_params_by_id(host['id']): @@ -204,9 +234,7 @@ class ForemanInventory(object): return params def _get_facts(self, host): - """ - Fetch all host facts of the host - """ + """Fetch all host facts of the host""" if not self.want_facts: return {} @@ -214,7 +242,7 @@ class ForemanInventory(object): if len(ret.values()) == 0: facts = {} elif len(ret.values()) == 1: - facts = ret.values()[0] + facts = list(ret.values())[0] else: raise ValueError("More than one set of facts returned for '%s'" % host) return facts @@ -228,8 +256,15 @@ class ForemanInventory(object): for host in self._get_hosts(): dns_name = host['name'] - # Create ansible groups for hostgroup, environment, location and organization - for group in ['hostgroup', 'environment', 'location', 'organization']: + # Create ansible groups for hostgroup + group = 'hostgroup' + val = host.get('%s_title' % group) or host.get('%s_name' % group) + if val: + safe_key = self.to_safe('%s%s_%s' % (self.group_prefix, group, val.lower())) + self.push(self.inventory, safe_key, dns_name) + + # Create ansible groups for environment, location and organization + for group in ['environment', 'location', 'organization']: val = host.get('%s_name' % group) if val: safe_key = self.to_safe('%s%s_%s' % (self.group_prefix, group, val.lower())) @@ -247,7 +282,7 @@ class ForemanInventory(object): # attributes. groupby = copy.copy(params) for k, v in host.items(): - if isinstance(v, basestring): + if isinstance(v, str): groupby[k] = self.to_safe(v) elif isinstance(v, int): groupby[k] = v @@ -264,14 +299,16 @@ class ForemanInventory(object): self.params[dns_name] = params self.facts[dns_name] = self._get_facts(host) self.push(self.inventory, 'all', dns_name) + self._write_cache() + def _write_cache(self): self.write_to_cache(self.cache, self.cache_path_cache) self.write_to_cache(self.inventory, self.cache_path_inventory) self.write_to_cache(self.params, self.cache_path_params) self.write_to_cache(self.facts, self.cache_path_facts) def get_host_info(self): - """ Get variables about a specific host """ + """Get variables about a specific host""" if not self.cache or len(self.cache) == 0: # Need to load index from cache @@ -294,21 +331,21 @@ class ForemanInventory(object): d[k] = [v] def load_inventory_from_cache(self): - """ Reads the index from the cache file sets self.index """ + """Read the index from the cache file sets self.index""" cache = open(self.cache_path_inventory, 'r') json_inventory = cache.read() self.inventory = json.loads(json_inventory) def load_params_from_cache(self): - """ Reads the index from the cache file sets self.index """ + """Read the index from the cache file sets self.index""" cache = open(self.cache_path_params, 'r') json_params = cache.read() self.params = json.loads(json_params) def load_facts_from_cache(self): - """ Reads the index from the cache file sets self.index """ + """Read the index from the cache file sets self.facts""" if not self.want_facts: return cache = open(self.cache_path_facts, 'r') @@ -316,26 +353,33 @@ class ForemanInventory(object): self.facts = json.loads(json_facts) def load_cache_from_cache(self): - """ Reads the cache from the cache file sets self.cache """ + """Read the cache from the cache file sets self.cache""" cache = open(self.cache_path_cache, 'r') json_cache = cache.read() self.cache = json.loads(json_cache) def write_to_cache(self, data, filename): - """ Writes data in JSON format to a file """ + """Write data in JSON format to a file""" json_data = self.json_format_dict(data, True) cache = open(filename, 'w') cache.write(json_data) cache.close() - def to_safe(self, word): - ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups ''' + @staticmethod + def to_safe(word): + '''Converts 'bad' characters in a string to underscores + + so they can be used as Ansible groups + + >>> ForemanInventory.to_safe("foo-bar baz") + 'foo_barbaz' + ''' regex = "[^A-Za-z0-9\_]" return re.sub(regex, "_", word.replace(" ", "")) def json_format_dict(self, data, pretty=False): - """ Converts a dict to a JSON object and dumps it as a formatted string """ + """Converts a dict to a JSON object and dumps it as a formatted string""" if pretty: return json.dumps(data, sort_keys=True, indent=2) @@ -343,6 +387,5 @@ class ForemanInventory(object): return json.dumps(data) if __name__ == '__main__': - ForemanInventory() - - + inv = ForemanInventory() + sys.exit(not inv.run()) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 876ba56c85..35911f4c71 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -777,6 +777,8 @@ SATELLITE6_GROUP_FILTER = r'^.+$' SATELLITE6_HOST_FILTER = r'^.+$' SATELLITE6_EXCLUDE_EMPTY_GROUPS = True SATELLITE6_INSTANCE_ID_VAR = 'foreman.id' +SATELLITE6_GROUP_PREFIX = 'foreman_' +SATELLITE6_GROUP_PATTERNS = ["{app}-{tier}-{color}", "{app}-{color}", "{app}", "{tier}"] # --------------------- # ----- CloudForms -----