diff --git a/awx/main/conf.py b/awx/main/conf.py index 3c3f53cadc..b0284a70ea 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -2,6 +2,7 @@ import json import logging import os +from distutils.version import LooseVersion as Version # Django from django.utils.translation import ugettext_lazy as _ @@ -410,6 +411,15 @@ register( category_slug='system', ) +register( + 'PROJECT_UPDATE_VVV', + field_class=fields.BooleanField, + label=_('Run Project Updates With Higher Verbosity'), + help_text=_('Adds the CLI -vvv flag to ansible-playbook runs of project_update.yml used for project updates.'), + category=_('Jobs'), + category_slug='jobs', +) + register( 'AWX_ROLES_ENABLED', field_class=fields.BooleanField, @@ -430,6 +440,75 @@ register( category_slug='jobs', ) +register( + 'PRIMARY_GALAXY_URL', + field_class=fields.URLField, + required=False, + allow_blank=True, + label=_('Primary Galaxy Server URL'), + help_text=_( + 'For organizations that run their own Galaxy service, this gives the option to specify a ' + 'host as the primary galaxy server. Requirements will be downloaded from the primary if the ' + 'specific role or collection is available there. If the content is not avilable in the primary, ' + 'or if this field is left blank, it will default to galaxy.ansible.com.' + ), + category=_('Jobs'), + category_slug='jobs' +) + +register( + 'PRIMARY_GALAXY_USERNAME', + field_class=fields.CharField, + required=False, + allow_blank=True, + label=_('Primary Galaxy Server Username'), + help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. ' + 'The username to use for basic authentication against the Galaxy instance, ' + 'this is mutually exclusive with PRIMARY_GALAXY_TOKEN.'), + category=_('Jobs'), + category_slug='jobs' +) + +register( + 'PRIMARY_GALAXY_PASSWORD', + field_class=fields.CharField, + encrypted=True, + required=False, + allow_blank=True, + label=_('Primary Galaxy Server Password'), + help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. ' + 'The password to use for basic authentication against the Galaxy instance, ' + 'this is mutually exclusive with PRIMARY_GALAXY_TOKEN.'), + category=_('Jobs'), + category_slug='jobs' +) + +register( + 'PRIMARY_GALAXY_TOKEN', + field_class=fields.CharField, + encrypted=True, + required=False, + allow_blank=True, + label=_('Primary Galaxy Server Token'), + help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. ' + 'The token to use for connecting with the Galaxy instance, ' + 'this is mutually exclusive with corresponding username and password settings.'), + category=_('Jobs'), + category_slug='jobs' +) + +register( + 'PRIMARY_GALAXY_AUTH_URL', + field_class=fields.CharField, + required=False, + allow_blank=True, + label=_('Primary Galaxy Authentication URL'), + help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. ' + 'The token_endpoint of a Keycloak server.'), + category=_('Jobs'), + category_slug='jobs' +) + register( 'STDOUT_MAX_BYTES_DISPLAY', field_class=fields.IntegerField, @@ -700,4 +779,75 @@ def logging_validate(serializer, attrs): return attrs +def galaxy_validate(serializer, attrs): + """Ansible Galaxy config options have mutual exclusivity rules, these rules + are enforced here on serializer validation so that users will not be able + to save settings which obviously break all project updates. + """ + prefix = 'PRIMARY_GALAXY_' + + from awx.main.constants import GALAXY_SERVER_FIELDS + if not any('{}{}'.format(prefix, subfield.upper()) in attrs for subfield in GALAXY_SERVER_FIELDS): + return attrs + + def _new_value(setting_name): + if setting_name in attrs: + return attrs[setting_name] + elif not serializer.instance: + return '' + return getattr(serializer.instance, setting_name, '') + + galaxy_data = {} + for subfield in GALAXY_SERVER_FIELDS: + galaxy_data[subfield] = _new_value('{}{}'.format(prefix, subfield.upper())) + errors = {} + if not galaxy_data['url']: + for k, v in galaxy_data.items(): + if v: + setting_name = '{}{}'.format(prefix, k.upper()) + errors.setdefault(setting_name, []) + errors[setting_name].append(_( + 'Cannot provide field if PRIMARY_GALAXY_URL is not set.' + )) + for k in GALAXY_SERVER_FIELDS: + if galaxy_data[k]: + setting_name = '{}{}'.format(prefix, k.upper()) + if (not serializer.instance) or (not getattr(serializer.instance, setting_name, '')): + # new auth is applied, so check if compatible with version + from awx.main.utils import get_ansible_version + current_version = get_ansible_version() + min_version = '2.9' + if Version(current_version) < Version(min_version): + errors.setdefault(setting_name, []) + errors[setting_name].append(_( + 'Galaxy server settings are not available until Ansible {min_version}, ' + 'you are running {current_version}.' + ).format(min_version=min_version, current_version=current_version)) + if (galaxy_data['password'] or galaxy_data['username']) and (galaxy_data['token'] or galaxy_data['auth_url']): + for k in ('password', 'username', 'token', 'auth_url'): + setting_name = '{}{}'.format(prefix, k.upper()) + if setting_name in attrs: + errors.setdefault(setting_name, []) + errors[setting_name].append(_( + 'Setting Galaxy token and authentication URL is mutually exclusive with username and password.' + )) + if bool(galaxy_data['username']) != bool(galaxy_data['password']): + msg = _('If authenticating via username and password, both must be provided.') + for k in ('username', 'password'): + setting_name = '{}{}'.format(prefix, k.upper()) + errors.setdefault(setting_name, []) + errors[setting_name].append(msg) + if bool(galaxy_data['token']) != bool(galaxy_data['auth_url']): + msg = _('If authenticating via token, both token and authentication URL must be provided.') + for k in ('token', 'auth_url'): + setting_name = '{}{}'.format(prefix, k.upper()) + errors.setdefault(setting_name, []) + errors[setting_name].append(msg) + + if errors: + raise serializers.ValidationError(errors) + return attrs + + register_validate('logging', logging_validate) +register_validate('jobs', galaxy_validate) diff --git a/awx/main/constants.py b/awx/main/constants.py index eea1a55820..4c98d264dd 100644 --- a/awx/main/constants.py +++ b/awx/main/constants.py @@ -51,3 +51,7 @@ LOGGER_BLACKLIST = ( # loggers that may be called getting logging settings 'awx.conf' ) + +# these correspond to both AWX and Ansible settings to keep naming consistent +# for instance, settings.PRIMARY_GALAXY_AUTH_URL vs env var ANSIBLE_GALAXY_SERVER_FOO_AUTH_URL +GALAXY_SERVER_FIELDS = ('url', 'username', 'password', 'token', 'auth_url') diff --git a/awx/main/models/credential/__init__.py b/awx/main/models/credential/__init__.py index 7c84226d7d..ce3295cc69 100644 --- a/awx/main/models/credential/__init__.py +++ b/awx/main/models/credential/__init__.py @@ -64,7 +64,7 @@ def build_safe_env(env): for k, v in safe_env.items(): if k == 'AWS_ACCESS_KEY_ID': continue - elif k.startswith('ANSIBLE_') and not k.startswith('ANSIBLE_NET'): + elif k.startswith('ANSIBLE_') and not k.startswith('ANSIBLE_NET') and not k.startswith('ANSIBLE_GALAXY_SERVER'): continue elif hidden_re.search(k): safe_env[k] = HIDDEN_PASSWORD diff --git a/awx/main/redact.py b/awx/main/redact.py index ec6211910b..ae60684377 100644 --- a/awx/main/redact.py +++ b/awx/main/redact.py @@ -1,6 +1,8 @@ import re import urllib.parse as urlparse +from django.conf import settings + REPLACE_STR = '$encrypted$' @@ -10,14 +12,22 @@ class UriCleaner(object): @staticmethod def remove_sensitive(cleartext): + if settings.PRIMARY_GALAXY_URL: + exclude_list = [settings.PRIMARY_GALAXY_URL] + [server['url'] for server in settings.FALLBACK_GALAXY_SERVERS] + else: + exclude_list = [server['url'] for server in settings.FALLBACK_GALAXY_SERVERS] redactedtext = cleartext text_index = 0 while True: match = UriCleaner.SENSITIVE_URI_PATTERN.search(redactedtext, text_index) if not match: break + uri_str = match.group(1) + # Do not redact items from the exclude list + if any(uri_str.startswith(exclude_uri) for exclude_uri in exclude_list): + text_index = match.start() + len(uri_str) + continue try: - uri_str = match.group(1) # May raise a ValueError if invalid URI for one reason or another o = urlparse.urlsplit(uri_str) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index ad396a45cf..eb2d48546d 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -52,7 +52,7 @@ import ansible_runner # AWX from awx import __version__ as awx_application_version -from awx.main.constants import CLOUD_PROVIDERS, PRIVILEGE_ESCALATION_METHODS, STANDARD_INVENTORY_UPDATE_ENV +from awx.main.constants import CLOUD_PROVIDERS, PRIVILEGE_ESCALATION_METHODS, STANDARD_INVENTORY_UPDATE_ENV, GALAXY_SERVER_FIELDS from awx.main.access import access_registry from awx.main.models import ( Schedule, TowerScheduleState, Instance, InstanceGroup, @@ -1883,6 +1883,24 @@ class RunProjectUpdate(BaseTask): env['TMP'] = settings.AWX_PROOT_BASE_PATH env['PROJECT_UPDATE_ID'] = str(project_update.pk) env['ANSIBLE_CALLBACK_PLUGINS'] = self.get_path_to('..', 'plugins', 'callback') + env['ANSIBLE_GALAXY_IGNORE'] = True + # Set up the fallback server, which is the normal Ansible Galaxy by default + galaxy_servers = list(settings.FALLBACK_GALAXY_SERVERS) + # If private galaxy URL is non-blank, that means this feature is enabled + if settings.PRIMARY_GALAXY_URL: + galaxy_servers = [{'id': 'primary_galaxy'}] + galaxy_servers + for key in GALAXY_SERVER_FIELDS: + value = getattr(settings, 'PRIMARY_GALAXY_{}'.format(key.upper())) + if value: + galaxy_servers[0][key] = value + for server in galaxy_servers: + for key in GALAXY_SERVER_FIELDS: + if not server.get(key): + continue + env_key = ('ANSIBLE_GALAXY_SERVER_{}_{}'.format(server.get('id', 'unnamed'), key)).upper() + env[env_key] = server[key] + # now set the precedence of galaxy servers + env['ANSIBLE_GALAXY_SERVER_LIST'] = ','.join([server.get('id', 'unnamed') for server in galaxy_servers]) return env def _build_scm_url_extra_vars(self, project_update): diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 4c49af8b5d..77c13fffd2 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -130,6 +130,8 @@ def test_send_notifications_list(mock_notifications_filter, mock_job_get, mocker ('VMWARE_PASSWORD', 'SECRET'), ('API_SECRET', 'SECRET'), ('CALLBACK_CONNECTION', 'amqp://tower:password@localhost:5672/tower'), + ('ANSIBLE_GALAXY_SERVER_PRIMARY_GALAXY_PASSWORD', 'SECRET'), + ('ANSIBLE_GALAXY_SERVER_PRIMARY_GALAXY_TOKEN', 'SECRET'), ]) def test_safe_env_filtering(key, value): assert build_safe_env({key: value})[key] == tasks.HIDDEN_PASSWORD diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index fb24b7014f..8d8ba9f008 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -609,6 +609,9 @@ AWX_REBUILD_SMART_MEMBERSHIP = False # By default, allow arbitrary Jinja templating in extra_vars defined on a Job Template ALLOW_JINJA_IN_EXTRA_VARS = 'template' +# Run project updates with extra verbosity +PROJECT_UPDATE_VVV = False + # Enable dynamically pulling roles from a requirement.yml file # when updating SCM projects # Note: This setting may be overridden by database settings. @@ -619,6 +622,23 @@ AWX_ROLES_ENABLED = True # Note: This setting may be overridden by database settings. AWX_COLLECTIONS_ENABLED = True +# Settings for primary galaxy server, should be set in the UI +PRIMARY_GALAXY_URL = '' +PRIMARY_GALAXY_USERNAME = '' +PRIMARY_GALAXY_TOKEN = '' +PRIMARY_GALAXY_PASSWORD = '' +PRIMARY_GALAXY_AUTH_URL = '' +# Settings for the fallback galaxy server(s), normally this is the +# actual Ansible Galaxy site. +# server options: 'id', 'url', 'username', 'password', 'token', 'auth_url' +# To not use any fallback servers set this to [] +FALLBACK_GALAXY_SERVERS = [ + { + 'id': 'galaxy', + 'url': 'https://galaxy.ansible.com' + } +] + # Enable bubblewrap support for running jobs (playbook runs only). # Note: This setting may be overridden by database settings. AWX_PROOT_ENABLED = True diff --git a/awx/ui/client/src/configuration/forms/jobs-form/configuration-jobs.form.js b/awx/ui/client/src/configuration/forms/jobs-form/configuration-jobs.form.js index 4c844a7b7d..445b0864a2 100644 --- a/awx/ui/client/src/configuration/forms/jobs-form/configuration-jobs.form.js +++ b/awx/ui/client/src/configuration/forms/jobs-form/configuration-jobs.form.js @@ -58,12 +58,37 @@ export default ['i18n', function(i18n) { type: 'text', reset: 'ANSIBLE_FACT_CACHE_TIMEOUT', }, + PROJECT_UPDATE_VVV: { + type: 'toggleSwitch', + }, AWX_ROLES_ENABLED: { type: 'toggleSwitch', }, AWX_COLLECTIONS_ENABLED: { type: 'toggleSwitch', }, + PRIMARY_GALAXY_URL: { + type: 'text', + reset: 'PRIMARY_GALAXY_URL', + }, + PRIMARY_GALAXY_USERNAME: { + type: 'text', + reset: 'PRIMARY_GALAXY_USERNAME', + }, + PRIMARY_GALAXY_PASSWORD: { + type: 'sensitive', + hasShowInputButton: true, + reset: 'PRIMARY_GALAXY_PASSWORD', + }, + PRIMARY_GALAXY_TOKEN: { + type: 'sensitive', + hasShowInputButton: true, + reset: 'PRIMARY_GALAXY_TOKEN', + }, + PRIMARY_GALAXY_AUTH_URL: { + type: 'text', + reset: 'PRIMARY_GALAXY_AUTH_URL', + }, AWX_TASK_ENV: { type: 'textarea', reset: 'AWX_TASK_ENV', diff --git a/docs/collections.md b/docs/collections.md index 23d805b74c..5cbf3eab7a 100644 --- a/docs/collections.md +++ b/docs/collections.md @@ -5,14 +5,14 @@ AWX supports the use of Ansible Collections. This section will give ways to use ### Project Collections Requirements If you specify a Collections requirements file in SCM at `collections/requirements.yml`, -then AWX will install Collections in that file in the implicit project sync -before a job run. The invocation is: +then AWX will install Collections from that file in the implicit project sync +before a job run. The invocation looks like: ``` -ansible-galaxy collection install -r requirements.yml -p +ansible-galaxy collection install -r requirements.yml -p /requirements_collections ``` -Example of `tmp` directory where job is running: +Example of the resultant `tmp` directory where job is running: ``` ├── project @@ -52,3 +52,31 @@ Example of `tmp` directory where job is running: └── tmp_6wod58k ``` + +### Galaxy Server Selection + +Ansible core default settings will download collections from the public +Galaxy server at `https://galaxy.ansible.com`. For details on +how Galaxy servers are configured in Ansible in general see: + +https://docs.ansible.com/ansible/devel/user_guide/collections_using.html +(if "devel" link goes stale in the future, it is for Ansible 2.9) + +You can set a different server to be the primary Galaxy server to download +roles and collections from in AWX project updates. +This is done via the setting `PRIMARY_GALAXY_URL` and similar +`PRIMARY_GALAXY_xxxx` settings for authentication. + +If the `PRIMARY_GALAXY_URL` setting is not blank, then the server list is defined +to be `primary_galaxy,galaxy`. The `primary_galaxy` server definition uses the URL +from those settings, as well as username, password, and/or token and auth_url if applicable. +the `galaxy` server definition uses public Galaxy (`https://galaxy.ansible.com`) +with no authentication. + +This configuration causes requirements to be downloaded from the user-specified +primary galaxy server if they are available there. If a requirement is +not available from the primary galaxy server, then it will fallback to +downloading it from the public Galaxy server. + +Even when these settings are enabled, this can still be bypassed for a specific +requirement by using the `source:` option, as described in Ansible documentation.