bump foreman and expose new grouping features

related to #3467
related to #5226
related to #4373

awx/main/tasks.py

@@ -1373,7 +1373,9 @@ class RunInventoryUpdate(BaseTask):
 
             section = 'ansible'
             cp.add_section(section)
-            cp.set(section, 'group_patterns', '["{app}-{tier}-{color}", "{app}-{color}", "{app}", "{tier}"]')
+            cp.set(section, 'group_patterns', os.environ.get('SATELLITE6_GROUP_PATTERNS', []))
+            cp.set(section, 'want_facts', True)
+            cp.set(section, 'group_prefix', os.environ.get('SATELLITE6_GROUP_PREFIX', 'foreman_'))
 
             section = 'cache'
             cp.add_section(section)

awx/plugins/inventory/foreman.ini.example

@@ -9,6 +9,9 @@ group_patterns = ["{app}-{tier}-{color}",
 	          "{app}-{color}",
 	          "{app}",
 		  "{tier}"]
+group_prefix = foreman_
+# Whether to fetch facts from Foreman and store them on the host
+want_facts = True
 
 [cache]
 path = .

awx/plugins/inventory/foreman.py

@@ -18,14 +18,22 @@
 #
 # This is somewhat based on cobbler inventory
 
+from __future__ import print_function
+
 import argparse
-import ConfigParser
 import copy
 import os
 import re
-from time import time
 import requests
 from requests.auth import HTTPBasicAuth
+import sys
+from time import time
+
+try:
+    import ConfigParser
+except ImportError:
+    import configparser as ConfigParser
+
 
 try:
     import json
@@ -34,19 +42,33 @@ except ImportError:
 
 
 class ForemanInventory(object):
+    config_paths = [
+        "/etc/ansible/foreman.ini",
+        os.path.dirname(os.path.realpath(__file__)) + '/foreman.ini',
+    ]
+
     def __init__(self):
-        """ Main execution path """
         self.inventory = dict()  # A list of groups and the hosts in that group
-        self.cache = dict()  # Details about hosts in the inventory
+        self.cache = dict()   # Details about hosts in the inventory
-        self.params = dict() # Params of each host
+        self.params = dict()  # Params of each host
-        self.facts = dict() # Facts of each host
+        self.facts = dict()   # Facts of each host
         self.hostgroups = dict()  # host groups
 
-        # Read settings and parse CLI arguments
+    def run(self):
-        self.read_settings()
+        if not self._read_settings():
-        self.parse_cli_args()
+            return False
+        self._get_inventory()
+        self._print_data()
+        return True
 
-        # Cache
+    def _read_settings(self):
+        # Read settings and parse CLI arguments
+        if not self.read_settings():
+            return False
+        self.parse_cli_args()
+        return True
+
+    def _get_inventory(self):
         if self.args.refresh_cache:
             self.update_cache()
         elif not self.is_cache_valid():
@@ -57,9 +79,8 @@ class ForemanInventory(object):
             self.load_facts_from_cache()
             self.load_cache_from_cache()
 
+    def _print_data(self):
         data_to_print = ""
-
-        # Data to print
         if self.args.host:
             data_to_print += self.get_host_info()
         else:
@@ -77,38 +98,36 @@ class ForemanInventory(object):
         print(data_to_print)
 
     def is_cache_valid(self):
-        """ Determines if the cache files have expired, or if it is still valid """
+        """Determines if the cache is still valid"""
-
         if os.path.isfile(self.cache_path_cache):
             mod_time = os.path.getmtime(self.cache_path_cache)
             current_time = time()
             if (mod_time + self.cache_max_age) > current_time:
                 if (os.path.isfile(self.cache_path_inventory) and
                     os.path.isfile(self.cache_path_params) and
-                    os.path.isfile(self.cache_path_facts)):
+                        os.path.isfile(self.cache_path_facts)):
                     return True
         return False
 
     def read_settings(self):
-        """ Reads the settings from the foreman.ini file """
+        """Reads the settings from the foreman.ini file"""
 
         config = ConfigParser.SafeConfigParser()
-        config_paths = [
-            "/etc/ansible/foreman.ini",
-            os.path.dirname(os.path.realpath(__file__)) + '/foreman.ini',
-        ]
-
         env_value = os.environ.get('FOREMAN_INI_PATH')
         if env_value is not None:
-            config_paths.append(os.path.expanduser(os.path.expandvars(env_value)))
+            self.config_paths.append(os.path.expanduser(os.path.expandvars(env_value)))
 
-        config.read(config_paths)
+        config.read(self.config_paths)
 
         # Foreman API related
-        self.foreman_url = config.get('foreman', 'url')
+        try:
-        self.foreman_user = config.get('foreman', 'user')
+            self.foreman_url = config.get('foreman', 'url')
-        self.foreman_pw = config.get('foreman', 'password')
+            self.foreman_user = config.get('foreman', 'user')
-        self.foreman_ssl_verify = config.getboolean('foreman', 'ssl_verify')
+            self.foreman_pw = config.get('foreman', 'password')
+            self.foreman_ssl_verify = config.getboolean('foreman', 'ssl_verify')
+        except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e:
+            print("Error parsing configuration: %s" % e, file=sys.stderr)
+            return False
 
         # Ansible related
         try:
@@ -138,10 +157,14 @@ class ForemanInventory(object):
         self.cache_path_inventory = cache_path + "/%s.index" % script
         self.cache_path_params = cache_path + "/%s.params" % script
         self.cache_path_facts = cache_path + "/%s.facts" % script
-        self.cache_max_age = config.getint('cache', 'max_age')
+        try:
+            self.cache_max_age = config.getint('cache', 'max_age')
+        except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
+            self.cache_max_age = 60
+        return True
 
     def parse_cli_args(self):
-        """ Command line argument processing """
+        """Command line argument processing"""
 
         parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on foreman')
         parser.add_argument('--list', action='store_true', default=True, help='List instances (default: True)')
@@ -162,14 +185,22 @@ class ForemanInventory(object):
                 break
             ret.raise_for_status()
             json = ret.json()
-            if not json.has_key('results'):
+            # /hosts/:id has not results key
+            if 'results' not in json:
                 return json
-            if type(json['results']) == type({}):
+            # Facts are returned as dict in results not list
+            if isinstance(json['results'], dict):
                 return json['results']
+            # List of all hosts is returned paginaged
             results = results + json['results']
             if len(results) >= json['total']:
                 break
             page += 1
+            if len(json['results']) == 0:
+                print("Did not make any progress during loop. "
+                      "expected %d got %d" % (json['total'], len(results)),
+                      file=sys.stderr)
+                break
         return results
 
     def _get_hosts(self):
@@ -184,7 +215,8 @@ class ForemanInventory(object):
     def _get_all_params_by_id(self, hid):
         url = "%s/api/v2/hosts/%s" % (self.foreman_url, hid)
         ret = self._get_json(url, [404])
-        if ret == []: ret = {}
+        if ret == []:
+            ret = {}
         return ret.get('all_parameters', {})
 
     def _get_facts_by_id(self, hid):
@@ -192,9 +224,7 @@ class ForemanInventory(object):
         return self._get_json(url)
 
     def _resolve_params(self, host):
-        """
+        """Fetch host params and convert to dict"""
-        Fetch host params and convert to dict
-        """
         params = {}
 
         for param in self._get_all_params_by_id(host['id']):
@@ -204,9 +234,7 @@ class ForemanInventory(object):
         return params
 
     def _get_facts(self, host):
-        """
+        """Fetch all host facts of the host"""
-        Fetch all host facts of the host
-        """
         if not self.want_facts:
             return {}
 
@@ -214,7 +242,7 @@ class ForemanInventory(object):
         if len(ret.values()) == 0:
             facts = {}
         elif len(ret.values()) == 1:
-            facts = ret.values()[0]
+            facts = list(ret.values())[0]
         else:
             raise ValueError("More than one set of facts returned for '%s'" % host)
         return facts
@@ -228,8 +256,15 @@ class ForemanInventory(object):
         for host in self._get_hosts():
             dns_name = host['name']
 
-            # Create ansible groups for hostgroup, environment, location and organization
+            # Create ansible groups for hostgroup
-            for group in ['hostgroup', 'environment', 'location', 'organization']:
+            group = 'hostgroup'
+            val = host.get('%s_title' % group) or host.get('%s_name' % group)
+            if val:
+                safe_key = self.to_safe('%s%s_%s' % (self.group_prefix, group, val.lower()))
+                self.push(self.inventory, safe_key, dns_name)
+
+            # Create ansible groups for environment, location and organization
+            for group in ['environment', 'location', 'organization']:
                 val = host.get('%s_name' % group)
                 if val:
                     safe_key = self.to_safe('%s%s_%s' % (self.group_prefix, group, val.lower()))
@@ -247,7 +282,7 @@ class ForemanInventory(object):
             # attributes.
             groupby = copy.copy(params)
             for k, v in host.items():
-                if isinstance(v, basestring):
+                if isinstance(v, str):
                     groupby[k] = self.to_safe(v)
                 elif isinstance(v, int):
                     groupby[k] = v
@@ -264,14 +299,16 @@ class ForemanInventory(object):
             self.params[dns_name] = params
             self.facts[dns_name] = self._get_facts(host)
             self.push(self.inventory, 'all', dns_name)
+            self._write_cache()
 
+    def _write_cache(self):
         self.write_to_cache(self.cache, self.cache_path_cache)
         self.write_to_cache(self.inventory, self.cache_path_inventory)
         self.write_to_cache(self.params, self.cache_path_params)
         self.write_to_cache(self.facts, self.cache_path_facts)
 
     def get_host_info(self):
-        """ Get variables about a specific host """
+        """Get variables about a specific host"""
 
         if not self.cache or len(self.cache) == 0:
             # Need to load index from cache
@@ -294,21 +331,21 @@ class ForemanInventory(object):
             d[k] = [v]
 
     def load_inventory_from_cache(self):
-        """ Reads the index from the cache file sets self.index """
+        """Read the index from the cache file sets self.index"""
 
         cache = open(self.cache_path_inventory, 'r')
         json_inventory = cache.read()
         self.inventory = json.loads(json_inventory)
 
     def load_params_from_cache(self):
-        """ Reads the index from the cache file sets self.index """
+        """Read the index from the cache file sets self.index"""
 
         cache = open(self.cache_path_params, 'r')
         json_params = cache.read()
         self.params = json.loads(json_params)
 
     def load_facts_from_cache(self):
-        """ Reads the index from the cache file sets self.index """
+        """Read the index from the cache file sets self.facts"""
         if not self.want_facts:
             return
         cache = open(self.cache_path_facts, 'r')
@@ -316,26 +353,33 @@ class ForemanInventory(object):
         self.facts = json.loads(json_facts)
 
     def load_cache_from_cache(self):
-        """ Reads the cache from the cache file sets self.cache """
+        """Read the cache from the cache file sets self.cache"""
 
         cache = open(self.cache_path_cache, 'r')
         json_cache = cache.read()
         self.cache = json.loads(json_cache)
 
     def write_to_cache(self, data, filename):
-        """ Writes data in JSON format to a file """
+        """Write data in JSON format to a file"""
         json_data = self.json_format_dict(data, True)
         cache = open(filename, 'w')
         cache.write(json_data)
         cache.close()
 
-    def to_safe(self, word):
+    @staticmethod
-        ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups '''
+    def to_safe(word):
+        '''Converts 'bad' characters in a string to underscores
+
+        so they can be used as Ansible groups
+
+        >>> ForemanInventory.to_safe("foo-bar baz")
+        'foo_barbaz'
+        '''
         regex = "[^A-Za-z0-9\_]"
         return re.sub(regex, "_", word.replace(" ", ""))
 
     def json_format_dict(self, data, pretty=False):
-        """ Converts a dict to a JSON object and dumps it as a formatted string """
+        """Converts a dict to a JSON object and dumps it as a formatted string"""
 
         if pretty:
             return json.dumps(data, sort_keys=True, indent=2)
@@ -343,6 +387,5 @@ class ForemanInventory(object):
             return json.dumps(data)
 
 if __name__ == '__main__':
-  ForemanInventory()
+    inv = ForemanInventory()
-
+    sys.exit(not inv.run())
-

awx/settings/defaults.py

@@ -777,6 +777,8 @@ SATELLITE6_GROUP_FILTER = r'^.+$'
 SATELLITE6_HOST_FILTER = r'^.+$'
 SATELLITE6_EXCLUDE_EMPTY_GROUPS = True
 SATELLITE6_INSTANCE_ID_VAR = 'foreman.id'
+SATELLITE6_GROUP_PREFIX = 'foreman_'
+SATELLITE6_GROUP_PATTERNS = ["{app}-{tier}-{color}", "{app}-{color}", "{app}", "{tier}"]
 
 # ---------------------
 # ----- CloudForms -----