Merge pull request #4589 from AlanCoding/mah_galaxy

Allow use of user-specified private galaxy server with fallback

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

awx/main/conf.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import os
+from distutils.version import LooseVersion as Version
 
 # Django
 from django.utils.translation import ugettext_lazy as _
@@ -410,6 +411,15 @@ register(
     category_slug='system',
 )
 
+register(
+    'PROJECT_UPDATE_VVV',
+    field_class=fields.BooleanField,
+    label=_('Run Project Updates With Higher Verbosity'),
+    help_text=_('Adds the CLI -vvv flag to ansible-playbook runs of project_update.yml used for project updates.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+)
+
 register(
     'AWX_ROLES_ENABLED',
     field_class=fields.BooleanField,
@@ -430,6 +440,75 @@ register(
     category_slug='jobs',
 )
 
+register(
+    'PRIMARY_GALAXY_URL',
+    field_class=fields.URLField,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Server URL'),
+    help_text=_(
+        'For organizations that run their own Galaxy service, this gives the option to specify a '
+        'host as the primary galaxy server. Requirements will be downloaded from the primary if the '
+        'specific role or collection is available there. If the content is not avilable in the primary, '
+        'or if this field is left blank, it will default to galaxy.ansible.com.'
+    ),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'PRIMARY_GALAXY_USERNAME',
+    field_class=fields.CharField,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Server Username'),
+    help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
+                'The username to use for basic authentication against the Galaxy instance, '
+                'this is mutually exclusive with PRIMARY_GALAXY_TOKEN.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'PRIMARY_GALAXY_PASSWORD',
+    field_class=fields.CharField,
+    encrypted=True,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Server Password'),
+    help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
+                'The password to use for basic authentication against the Galaxy instance, '
+                'this is mutually exclusive with PRIMARY_GALAXY_TOKEN.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'PRIMARY_GALAXY_TOKEN',
+    field_class=fields.CharField,
+    encrypted=True,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Server Token'),
+    help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
+                'The token to use for connecting with the Galaxy instance, '
+                'this is mutually exclusive with corresponding username and password settings.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
+register(
+    'PRIMARY_GALAXY_AUTH_URL',
+    field_class=fields.CharField,
+    required=False,
+    allow_blank=True,
+    label=_('Primary Galaxy Authentication URL'),
+    help_text=_('For using a galaxy server at higher precedence than the public Ansible Galaxy. '
+                'The token_endpoint of a Keycloak server.'),
+    category=_('Jobs'),
+    category_slug='jobs'
+)
+
 register(
     'STDOUT_MAX_BYTES_DISPLAY',
     field_class=fields.IntegerField,
@@ -700,4 +779,75 @@ def logging_validate(serializer, attrs):
     return attrs
 
 
+def galaxy_validate(serializer, attrs):
+    """Ansible Galaxy config options have mutual exclusivity rules, these rules
+    are enforced here on serializer validation so that users will not be able
+    to save settings which obviously break all project updates.
+    """
+    prefix = 'PRIMARY_GALAXY_'
+
+    from awx.main.constants import GALAXY_SERVER_FIELDS
+    if not any('{}{}'.format(prefix, subfield.upper()) in attrs for subfield in GALAXY_SERVER_FIELDS):
+        return attrs
+
+    def _new_value(setting_name):
+        if setting_name in attrs:
+            return attrs[setting_name]
+        elif not serializer.instance:
+            return ''
+        return getattr(serializer.instance, setting_name, '')
+
+    galaxy_data = {}
+    for subfield in GALAXY_SERVER_FIELDS:
+        galaxy_data[subfield] = _new_value('{}{}'.format(prefix, subfield.upper()))
+    errors = {}
+    if not galaxy_data['url']:
+        for k, v in galaxy_data.items():
+            if v:
+                setting_name = '{}{}'.format(prefix, k.upper())
+                errors.setdefault(setting_name, [])
+                errors[setting_name].append(_(
+                    'Cannot provide field if PRIMARY_GALAXY_URL is not set.'
+                ))
+    for k in GALAXY_SERVER_FIELDS:
+        if galaxy_data[k]:
+            setting_name = '{}{}'.format(prefix, k.upper())
+            if (not serializer.instance) or (not getattr(serializer.instance, setting_name, '')):
+                # new auth is applied, so check if compatible with version
+                from awx.main.utils import get_ansible_version
+                current_version = get_ansible_version()
+                min_version = '2.9'
+                if Version(current_version) < Version(min_version):
+                    errors.setdefault(setting_name, [])
+                    errors[setting_name].append(_(
+                        'Galaxy server settings are not available until Ansible {min_version}, '
+                        'you are running {current_version}.'
+                    ).format(min_version=min_version, current_version=current_version))
+    if (galaxy_data['password'] or galaxy_data['username']) and (galaxy_data['token'] or galaxy_data['auth_url']):
+        for k in ('password', 'username', 'token', 'auth_url'):
+            setting_name = '{}{}'.format(prefix, k.upper())
+            if setting_name in attrs:
+                errors.setdefault(setting_name, [])
+                errors[setting_name].append(_(
+                    'Setting Galaxy token and authentication URL is mutually exclusive with username and password.'
+                ))
+    if bool(galaxy_data['username']) != bool(galaxy_data['password']):
+        msg = _('If authenticating via username and password, both must be provided.')
+        for k in ('username', 'password'):
+            setting_name = '{}{}'.format(prefix, k.upper())
+            errors.setdefault(setting_name, [])
+            errors[setting_name].append(msg)
+    if bool(galaxy_data['token']) != bool(galaxy_data['auth_url']):
+        msg = _('If authenticating via token, both token and authentication URL must be provided.')
+        for k in ('token', 'auth_url'):
+            setting_name = '{}{}'.format(prefix, k.upper())
+            errors.setdefault(setting_name, [])
+            errors[setting_name].append(msg)
+
+    if errors:
+        raise serializers.ValidationError(errors)
+    return attrs
+
+
 register_validate('logging', logging_validate)
+register_validate('jobs', galaxy_validate)

awx/main/constants.py

@@ -51,3 +51,7 @@ LOGGER_BLACKLIST = (
     # loggers that may be called getting logging settings
     'awx.conf'
 )
+
+# these correspond to both AWX and Ansible settings to keep naming consistent
+# for instance, settings.PRIMARY_GALAXY_AUTH_URL vs env var ANSIBLE_GALAXY_SERVER_FOO_AUTH_URL
+GALAXY_SERVER_FIELDS = ('url', 'username', 'password', 'token', 'auth_url')

awx/main/models/credential/__init__.py

@@ -64,7 +64,7 @@ def build_safe_env(env):
     for k, v in safe_env.items():
         if k == 'AWS_ACCESS_KEY_ID':
             continue
-        elif k.startswith('ANSIBLE_') and not k.startswith('ANSIBLE_NET'):
+        elif k.startswith('ANSIBLE_') and not k.startswith('ANSIBLE_NET') and not k.startswith('ANSIBLE_GALAXY_SERVER'):
             continue
         elif hidden_re.search(k):
             safe_env[k] = HIDDEN_PASSWORD

awx/main/redact.py

@@ -1,6 +1,8 @@
 import re
 import urllib.parse as urlparse
 
+from django.conf import settings
+
 REPLACE_STR = '$encrypted$'
 
 
@@ -10,14 +12,22 @@ class UriCleaner(object):
 
     @staticmethod
     def remove_sensitive(cleartext):
+        if settings.PRIMARY_GALAXY_URL:
+            exclude_list = [settings.PRIMARY_GALAXY_URL] + [server['url'] for server in settings.FALLBACK_GALAXY_SERVERS]
+        else:
+            exclude_list = [server['url'] for server in settings.FALLBACK_GALAXY_SERVERS]
         redactedtext = cleartext
         text_index = 0
         while True:
             match = UriCleaner.SENSITIVE_URI_PATTERN.search(redactedtext, text_index)
             if not match:
                 break
+            uri_str = match.group(1)
+            # Do not redact items from the exclude list
+            if any(uri_str.startswith(exclude_uri) for exclude_uri in exclude_list):
+                text_index = match.start() + len(uri_str)
+                continue
             try:
-                uri_str = match.group(1)
                 # May raise a ValueError if invalid URI for one reason or another
                 o = urlparse.urlsplit(uri_str)
 

awx/main/tasks.py

@@ -52,7 +52,7 @@ import ansible_runner
 
 # AWX
 from awx import __version__ as awx_application_version
-from awx.main.constants import CLOUD_PROVIDERS, PRIVILEGE_ESCALATION_METHODS, STANDARD_INVENTORY_UPDATE_ENV
+from awx.main.constants import CLOUD_PROVIDERS, PRIVILEGE_ESCALATION_METHODS, STANDARD_INVENTORY_UPDATE_ENV, GALAXY_SERVER_FIELDS
 from awx.main.access import access_registry
 from awx.main.models import (
     Schedule, TowerScheduleState, Instance, InstanceGroup,
@@ -1883,6 +1883,24 @@ class RunProjectUpdate(BaseTask):
         env['TMP'] = settings.AWX_PROOT_BASE_PATH
         env['PROJECT_UPDATE_ID'] = str(project_update.pk)
         env['ANSIBLE_CALLBACK_PLUGINS'] = self.get_path_to('..', 'plugins', 'callback')
+        env['ANSIBLE_GALAXY_IGNORE'] = True
+        # Set up the fallback server, which is the normal Ansible Galaxy by default
+        galaxy_servers = list(settings.FALLBACK_GALAXY_SERVERS)
+        # If private galaxy URL is non-blank, that means this feature is enabled
+        if settings.PRIMARY_GALAXY_URL:
+            galaxy_servers = [{'id': 'primary_galaxy'}] + galaxy_servers
+            for key in GALAXY_SERVER_FIELDS:
+                value = getattr(settings, 'PRIMARY_GALAXY_{}'.format(key.upper()))
+                if value:
+                    galaxy_servers[0][key] = value
+        for server in galaxy_servers:
+            for key in GALAXY_SERVER_FIELDS:
+                if not server.get(key):
+                    continue
+                env_key = ('ANSIBLE_GALAXY_SERVER_{}_{}'.format(server.get('id', 'unnamed'), key)).upper()
+                env[env_key] = server[key]
+        # now set the precedence of galaxy servers
+        env['ANSIBLE_GALAXY_SERVER_LIST'] = ','.join([server.get('id', 'unnamed') for server in galaxy_servers])
         return env
 
     def _build_scm_url_extra_vars(self, project_update):

awx/main/tests/unit/test_tasks.py

@@ -130,6 +130,8 @@ def test_send_notifications_list(mock_notifications_filter, mock_job_get, mocker
     ('VMWARE_PASSWORD', 'SECRET'),
     ('API_SECRET', 'SECRET'),
     ('CALLBACK_CONNECTION', 'amqp://tower:password@localhost:5672/tower'),
+    ('ANSIBLE_GALAXY_SERVER_PRIMARY_GALAXY_PASSWORD', 'SECRET'),
+    ('ANSIBLE_GALAXY_SERVER_PRIMARY_GALAXY_TOKEN', 'SECRET'),
 ])
 def test_safe_env_filtering(key, value):
     assert build_safe_env({key: value})[key] == tasks.HIDDEN_PASSWORD

awx/settings/defaults.py

@@ -609,6 +609,9 @@ AWX_REBUILD_SMART_MEMBERSHIP = False
 # By default, allow arbitrary Jinja templating in extra_vars defined on a Job Template
 ALLOW_JINJA_IN_EXTRA_VARS = 'template'
 
+# Run project updates with extra verbosity
+PROJECT_UPDATE_VVV = False
+
 # Enable dynamically pulling roles from a requirement.yml file
 # when updating SCM projects
 # Note: This setting may be overridden by database settings.
@@ -619,6 +622,23 @@ AWX_ROLES_ENABLED = True
 # Note: This setting may be overridden by database settings.
 AWX_COLLECTIONS_ENABLED = True
 
+# Settings for primary galaxy server, should be set in the UI
+PRIMARY_GALAXY_URL = ''
+PRIMARY_GALAXY_USERNAME = ''
+PRIMARY_GALAXY_TOKEN = ''
+PRIMARY_GALAXY_PASSWORD = ''
+PRIMARY_GALAXY_AUTH_URL = ''
+# Settings for the fallback galaxy server(s), normally this is the
+# actual Ansible Galaxy site.
+# server options: 'id', 'url', 'username', 'password', 'token', 'auth_url'
+# To not use any fallback servers set this to []
+FALLBACK_GALAXY_SERVERS = [
+    {
+        'id': 'galaxy',
+        'url': 'https://galaxy.ansible.com'
+    }
+]
+
 # Enable bubblewrap support for running jobs (playbook runs only).
 # Note: This setting may be overridden by database settings.
 AWX_PROOT_ENABLED = True

awx/ui/client/src/configuration/forms/jobs-form/configuration-jobs.form.js

@@ -58,12 +58,37 @@ export default ['i18n', function(i18n) {
                 type: 'text',
                 reset: 'ANSIBLE_FACT_CACHE_TIMEOUT',
             },
+            PROJECT_UPDATE_VVV: {
+                type: 'toggleSwitch',
+            },
             AWX_ROLES_ENABLED: {
                 type: 'toggleSwitch',
             },
             AWX_COLLECTIONS_ENABLED: {
                 type: 'toggleSwitch',
             },
+            PRIMARY_GALAXY_URL: {
+                type: 'text',
+                reset: 'PRIMARY_GALAXY_URL',
+            },
+            PRIMARY_GALAXY_USERNAME: {
+                type: 'text',
+                reset: 'PRIMARY_GALAXY_USERNAME',
+            },
+            PRIMARY_GALAXY_PASSWORD: {
+                type: 'sensitive',
+                hasShowInputButton: true,
+                reset: 'PRIMARY_GALAXY_PASSWORD',
+            },
+            PRIMARY_GALAXY_TOKEN: {
+                type: 'sensitive',
+                hasShowInputButton: true,
+                reset: 'PRIMARY_GALAXY_TOKEN',
+            },
+            PRIMARY_GALAXY_AUTH_URL: {
+                type: 'text',
+                reset: 'PRIMARY_GALAXY_AUTH_URL',
+            },
             AWX_TASK_ENV: {
                 type: 'textarea',
                 reset: 'AWX_TASK_ENV',

docs/collections.md

@@ -5,14 +5,14 @@ AWX supports the use of Ansible Collections. This section will give ways to use
 ### Project Collections Requirements
 
 If you specify a Collections requirements file in SCM at `collections/requirements.yml`,
-then AWX will install Collections in that file in the implicit project sync
+then AWX will install Collections from that file in the implicit project sync
-before a job run. The invocation is:
+before a job run. The invocation looks like:
 
 ```
-ansible-galaxy collection install -r requirements.yml -p <job tmp location>
+ansible-galaxy collection install -r requirements.yml -p <job tmp location>/requirements_collections
 ```
 
-Example of `tmp` directory where job is running:
+Example of the resultant `tmp` directory where job is running:
 
 ```
 ├── project
@@ -52,3 +52,31 @@ Example of `tmp` directory where job is running:
 └── tmp_6wod58k
 
 ```
+
+### Galaxy Server Selection
+
+Ansible core default settings will download collections from the public
+Galaxy server at `https://galaxy.ansible.com`. For details on
+how Galaxy servers are configured in Ansible in general see:
+
+https://docs.ansible.com/ansible/devel/user_guide/collections_using.html
+(if "devel" link goes stale in the future, it is for Ansible 2.9)
+
+You can set a different server to be the primary Galaxy server to download
+roles and collections from in AWX project updates.
+This is done via the setting `PRIMARY_GALAXY_URL` and similar
+`PRIMARY_GALAXY_xxxx` settings for authentication.
+
+If the `PRIMARY_GALAXY_URL` setting is not blank, then the server list is defined
+to be `primary_galaxy,galaxy`. The `primary_galaxy` server definition uses the URL
+from those settings, as well as username, password, and/or token and auth_url if applicable.
+the `galaxy` server definition uses public Galaxy (`https://galaxy.ansible.com`)
+with no authentication.
+
+This configuration causes requirements to be downloaded from the user-specified
+primary galaxy server if they are available there. If a requirement is
+not available from the primary galaxy server, then it will fallback to
+downloading it from the public Galaxy server.
+
+Even when these settings are enabled, this can still be bypassed for a specific
+requirement by using the `source:` option, as described in Ansible documentation.