External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-02-23 22:16:00 -03:30
Code Issues Packages Projects Releases Wiki Activity

Dependency Updates

Browse Source 
* Dynamic Inventory Source
Template against ansible 2.3 dynamic inventory sources.
The major change is removal of `rax.py`. Most upstream scripts except
`foreman.py` has quite trivial coding style changes, or minor functional
extensions  that does not affect Tower inventory update runs.
`foreman.py`, on the other hand, went through quite a major refactoring,
but functionalities stay the same.

Major python dependency updates include apache-libcloud (1.3.0 -->
2.0.0), boto (2.45.0 --> 2.46.1) and shade (1.19.0 --> 1.20.0). Minor
python dependency updates include indirect updates via `pip-compile`,
which are determined by base dependencies.

Some minor `task.py` extensions:
 - `.ini` file for ec2 has one more field `stack_filter=False`, which
   reveals changes in `ec2.py`.
 - `.ini` file for cloudforms will catch these four options from
   `source_vars_dict` of inventory update: `'version', 'purge_actions',
   'clean_group_keys', 'nest_tags'`. These four options have always been
   available in `cloudforms.py` but `cloudforms.ini.example` has not
   mentioned them until the latest version. For consistency with upstream
   docs, we should make these fields available for tower user to customize.
 - YAML file of openstack will catch ansible options `use_hostnames`,
   `expand_hostvars` and `fail_on_errors` from `source_vars_dict` of
   inventory update as a response to issue #6075.

* Remove Rackspace support
Supports of Rackspace as both a dynamic inventory source and a cloud
credential are fully removed. Data migrations have been added to support
arbitrary credential types feature and delete rackspace inventory
sources.

Note also requirement `jsonschema` has been moved from
`requirements.txt` to `requirements.in` as a primary dependency to
reflect it's usage in `/main/fields.py`.

Connected issue: #6080.

* `pexpect` major update
`pexpect` stands at the very core of our task system and underwent a
major update from 3.1 to 4.2.1. Although verified during devel, please
still be mindful of any suspicious issues on celery side even after this
PR gets merged.

* Miscellaneous
 - requests now explicitly declared in `requirements.in` at version 2.11.1
   in response to upstream issue
 - celery: 3.1.17 -> 3.1.25
 - django-extensions: 1.7.4 -> 1.7.8
 - django-polymorphic: 0.7.2 -> 1.2
 - django-split-settings: 0.2.2 -> 0.2.5
 - django-taggit: 0.21.3 -> 0.22.1
 - irc: 15.0.4 -> 15.1.1
 - pygerduty: 0.35.1 -> 0.35.2
 - pyOpenSSL: 16.2.0 -> 17.0.0
 - python-saml: 2.2.0 -> 2.2.1
 - redbaron: 0.6.2 -> 0.6.3
 - slackclient: 1.0.2 -> 1.0.5
 - tacacs_plus: 0.1 -> 0.2
 - xmltodict: 0.10.2 -> 0.11.0
 - pip: 8.1.2 -> 9.0.1
 - setuptools: 23.0.0 -> 35.0.2
 - (requirements_ansible.in only)kombu: 3.0.35 -> 3.0.37
This commit is contained in:
Aaron Tan
2017-04-20 16:47:53 -04:00
parent 3ed9ebed89
commit cfb633e8a6
35 changed files with 666 additions and 1022 deletions
Download Patch File Download Diff File Expand all files Collapse all files

276
awx/plugins/inventory/foreman.py
View File

@@ -1,7 +1,8 @@
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
#
# Copyright (C) 2016 Guido Günther <agx@sigxcpu.org>
# Copyright (C) 2016 Guido Günther <agx@sigxcpu.org>,
# Daniel Lobato Garcia <dlobatog@redhat.com>
#
# This script is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -18,106 +19,62 @@
#
# This is somewhat based on cobbler inventory
# Stdlib imports
# __future__ imports must occur at the beginning of file
from __future__ import print_function
try:
# Python 2 version
import ConfigParser
except ImportError:
# Python 3 version
import configparser as ConfigParser
import json
import argparse
import copy
import os
import re
import requests
from requests.auth import HTTPBasicAuth
import sys
from time import time
from collections import defaultdict
from distutils.version import LooseVersion, StrictVersion
try:
import ConfigParser
except ImportError:
import configparser as ConfigParser
# 3rd party imports
import requests
if LooseVersion(requests.__version__) < LooseVersion('1.1.0'):
print('This script requires python-requests 1.1 as a minimum version')
sys.exit(1)
from requests.auth import HTTPBasicAuth
try:
import json
except ImportError:
import simplejson as json
def json_format_dict(data, pretty=False):
"""Converts a dict to a JSON object and dumps it as a formatted string"""
if pretty:
return json.dumps(data, sort_keys=True, indent=2)
else:
return json.dumps(data)
class ForemanInventory(object):
config_paths = [
"/etc/ansible/foreman.ini",
os.path.dirname(os.path.realpath(__file__)) + '/foreman.ini',
]
def __init__(self):
self.inventory = dict() # A list of groups and the hosts in that group
self.inventory = defaultdict(list) # A list of groups and the hosts in that group
self.cache = dict() # Details about hosts in the inventory
self.params = dict() # Params of each host
self.facts = dict() # Facts of each host
self.hostgroups = dict() # host groups
self.session = None # Requests session
def run(self):
if not self._read_settings():
return False
self._get_inventory()
self._print_data()
return True
def _read_settings(self):
# Read settings and parse CLI arguments
if not self.read_settings():
return False
self.parse_cli_args()
return True
def _get_inventory(self):
if self.args.refresh_cache:
self.update_cache()
elif not self.is_cache_valid():
self.update_cache()
else:
self.load_inventory_from_cache()
self.load_params_from_cache()
self.load_facts_from_cache()
self.load_cache_from_cache()
def _print_data(self):
data_to_print = ""
if self.args.host:
data_to_print += self.get_host_info()
else:
self.inventory['_meta'] = {'hostvars': {}}
for hostname in self.cache:
self.inventory['_meta']['hostvars'][hostname] = {
'foreman': self.cache[hostname],
'foreman_params': self.params[hostname],
}
if self.want_facts:
self.inventory['_meta']['hostvars'][hostname]['foreman_facts'] = self.facts[hostname]
data_to_print += self.json_format_dict(self.inventory, True)
print(data_to_print)
def is_cache_valid(self):
"""Determines if the cache is still valid"""
if os.path.isfile(self.cache_path_cache):
mod_time = os.path.getmtime(self.cache_path_cache)
current_time = time()
if (mod_time + self.cache_max_age) > current_time:
if (os.path.isfile(self.cache_path_inventory) and
os.path.isfile(self.cache_path_params) and
os.path.isfile(self.cache_path_facts)):
return True
return False
self.config_paths = [
"/etc/ansible/foreman.ini",
os.path.dirname(os.path.realpath(__file__)) + '/foreman.ini',
]
env_value = os.environ.get('FOREMAN_INI_PATH')
if env_value is not None:
self.config_paths.append(os.path.expanduser(os.path.expandvars(env_value)))
def read_settings(self):
"""Reads the settings from the foreman.ini file"""
config = ConfigParser.SafeConfigParser()
env_value = os.environ.get('FOREMAN_INI_PATH')
if env_value is not None:
self.config_paths.append(os.path.expanduser(os.path.expandvars(env_value)))
config.read(self.config_paths)
# Foreman API related
@@ -136,7 +93,7 @@ class ForemanInventory(object):
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
group_patterns = "[]"
self.group_patterns = eval(group_patterns)
self.group_patterns = json.loads(group_patterns)
try:
self.group_prefix = config.get('ansible', 'group_prefix')
@@ -212,12 +169,6 @@ class ForemanInventory(object):
def _get_hosts(self):
return self._get_json("%s/api/v2/hosts" % self.foreman_url)
def _get_hostgroup_by_id(self, hid):
if hid not in self.hostgroups:
url = "%s/api/v2/hostgroups/%s" % (self.foreman_url, hid)
self.hostgroups[hid] = self._get_json(url)
return self.hostgroups[hid]
def _get_all_params_by_id(self, hid):
url = "%s/api/v2/hosts/%s" % (self.foreman_url, hid)
ret = self._get_json(url, [404])
@@ -225,10 +176,6 @@ class ForemanInventory(object):
ret = {}
return ret.get('all_parameters', {})
def _get_facts_by_id(self, hid):
url = "%s/api/v2/hosts/%s/facts" % (self.foreman_url, hid)
return self._get_json(url)
def _resolve_params(self, host):
"""Fetch host params and convert to dict"""
params = {}
@@ -239,6 +186,10 @@ class ForemanInventory(object):
return params
def _get_facts_by_id(self, hid):
url = "%s/api/v2/hosts/%s/facts" % (self.foreman_url, hid)
return self._get_json(url)
def _get_facts(self, host):
"""Fetch all host facts of the host"""
if not self.want_facts:
@@ -253,6 +204,29 @@ class ForemanInventory(object):
raise ValueError("More than one set of facts returned for '%s'" % host)
return facts
def write_to_cache(self, data, filename):
"""Write data in JSON format to a file"""
json_data = json_format_dict(data, True)
cache = open(filename, 'w')
cache.write(json_data)
cache.close()
def _write_cache(self):
self.write_to_cache(self.cache, self.cache_path_cache)
self.write_to_cache(self.inventory, self.cache_path_inventory)
self.write_to_cache(self.params, self.cache_path_params)
self.write_to_cache(self.facts, self.cache_path_facts)
def to_safe(self, word):
'''Converts 'bad' characters in a string to underscores
so they can be used as Ansible groups
>>> ForemanInventory.to_safe("foo-bar baz")
'foo_barbaz'
'''
regex = "[^A-Za-z0-9\_]"
return re.sub(regex, "_", word.replace(" ", ""))
def update_cache(self):
"""Make calls to foreman and save the output in a cache"""
@@ -267,20 +241,20 @@ class ForemanInventory(object):
val = host.get('%s_title' % group) or host.get('%s_name' % group)
if val:
safe_key = self.to_safe('%s%s_%s' % (self.group_prefix, group, val.lower()))
self.push(self.inventory, safe_key, dns_name)
self.inventory[safe_key].append(dns_name)
# Create ansible groups for environment, location and organization
for group in ['environment', 'location', 'organization']:
val = host.get('%s_name' % group)
if val:
safe_key = self.to_safe('%s%s_%s' % (self.group_prefix, group, val.lower()))
self.push(self.inventory, safe_key, dns_name)
self.inventory[safe_key].append(dns_name)
for group in ['lifecycle_environment', 'content_view']:
val = host.get('content_facet_attributes', {}).get('%s_name' % group)
if val:
safe_key = self.to_safe('%s%s_%s' % (self.group_prefix, group, val.lower()))
self.push(self.inventory, safe_key, dns_name)
self.inventory[safe_key].append(dns_name)
params = self._resolve_params(host)
@@ -297,44 +271,27 @@ class ForemanInventory(object):
for pattern in self.group_patterns:
try:
key = pattern.format(**groupby)
self.push(self.inventory, key, dns_name)
self.inventory[key].append(dns_name)
except KeyError:
pass # Host not part of this group
self.cache[dns_name] = host
self.params[dns_name] = params
self.facts[dns_name] = self._get_facts(host)
self.push(self.inventory, 'all', dns_name)
self.inventory['all'].append(dns_name)
self._write_cache()
def _write_cache(self):
self.write_to_cache(self.cache, self.cache_path_cache)
self.write_to_cache(self.inventory, self.cache_path_inventory)
self.write_to_cache(self.params, self.cache_path_params)
self.write_to_cache(self.facts, self.cache_path_facts)
def get_host_info(self):
"""Get variables about a specific host"""
if not self.cache or len(self.cache) == 0:
# Need to load index from cache
self.load_cache_from_cache()
if self.args.host not in self.cache:
# try updating the cache
self.update_cache()
if self.args.host not in self.cache:
# host might not exist anymore
return self.json_format_dict({}, True)
return self.json_format_dict(self.cache[self.args.host], True)
def push(self, d, k, v):
if k in d:
d[k].append(v)
else:
d[k] = [v]
def is_cache_valid(self):
"""Determines if the cache is still valid"""
if os.path.isfile(self.cache_path_cache):
mod_time = os.path.getmtime(self.cache_path_cache)
current_time = time()
if (mod_time + self.cache_max_age) > current_time:
if (os.path.isfile(self.cache_path_inventory) and
os.path.isfile(self.cache_path_params) and
os.path.isfile(self.cache_path_facts)):
return True
return False
def load_inventory_from_cache(self):
"""Read the index from the cache file sets self.index"""
@@ -365,33 +322,58 @@ class ForemanInventory(object):
json_cache = cache.read()
self.cache = json.loads(json_cache)
def write_to_cache(self, data, filename):
"""Write data in JSON format to a file"""
json_data = self.json_format_dict(data, True)
cache = open(filename, 'w')
cache.write(json_data)
cache.close()
@staticmethod
def to_safe(word):
'''Converts 'bad' characters in a string to underscores
so they can be used as Ansible groups
>>> ForemanInventory.to_safe("foo-bar baz")
'foo_barbaz'
'''
regex = "[^A-Za-z0-9\_]"
return re.sub(regex, "_", word.replace(" ", ""))
def json_format_dict(self, data, pretty=False):
"""Converts a dict to a JSON object and dumps it as a formatted string"""
if pretty:
return json.dumps(data, sort_keys=True, indent=2)
def get_inventory(self):
if self.args.refresh_cache or not self.is_cache_valid():
self.update_cache()
else:
return json.dumps(data)
self.load_inventory_from_cache()
self.load_params_from_cache()
self.load_facts_from_cache()
self.load_cache_from_cache()
def get_host_info(self):
"""Get variables about a specific host"""
if not self.cache or len(self.cache) == 0:
# Need to load index from cache
self.load_cache_from_cache()
if self.args.host not in self.cache:
# try updating the cache
self.update_cache()
if self.args.host not in self.cache:
# host might not exist anymore
return json_format_dict({}, True)
return json_format_dict(self.cache[self.args.host], True)
def _print_data(self):
data_to_print = ""
if self.args.host:
data_to_print += self.get_host_info()
else:
self.inventory['_meta'] = {'hostvars': {}}
for hostname in self.cache:
self.inventory['_meta']['hostvars'][hostname] = {
'foreman': self.cache[hostname],
'foreman_params': self.params[hostname],
}
if self.want_facts:
self.inventory['_meta']['hostvars'][hostname]['foreman_facts'] = self.facts[hostname]
data_to_print += json_format_dict(self.inventory, True)
print(data_to_print)
def run(self):
# Read settings and parse CLI arguments
if not self.read_settings():
return False
self.parse_cli_args()
self.get_inventory()
self._print_data()
return True
if __name__ == '__main__':
inv = ForemanInventory()
sys.exit(not inv.run())
sys.exit(not ForemanInventory().run())