Merge remote-tracking branch 'origin/release_3.1.2' into devel

2026-06-28 18:08:03 -02:30 · 2017-03-10 10:57:03 -05:00
parent 5060ed4033 bb116b0266
commit bad9670a0b
100 changed files with 576 additions and 341 deletions
--- a/awx/main/migrations/0036_v311_insights.py
+++ b/awx/main/migrations/0036_v311_insights.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0035_v310_remove_tower_settings'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='project',
+            name='scm_type',
+            field=models.CharField(default=b'', choices=[(b'', 'Manual'), (b'git', 'Git'), (b'hg', 'Mercurial'), (b'svn', 'Subversion'), (b'insights', 'Red Hat Insights')], max_length=8, blank=True, help_text='Specifies the source control system used to store the project.', verbose_name='SCM Type'),
+        ),
+        migrations.AlterField(
+            model_name='projectupdate',
+            name='scm_type',
+            field=models.CharField(default=b'', choices=[(b'', 'Manual'), (b'git', 'Git'), (b'hg', 'Mercurial'), (b'svn', 'Subversion'), (b'insights', 'Red Hat Insights')], max_length=8, blank=True, help_text='Specifies the source control system used to store the project.', verbose_name='SCM Type'),
+        ),
+    ]
--- a/awx/main/models/projects.py
+++ b/awx/main/models/projects.py
@@ -43,6 +43,7 @@ class ProjectOptions(models.Model):
        ('git', _('Git')),
        ('hg', _('Mercurial')),
        ('svn', _('Subversion')),
+        ('insights', _('Red Hat Insights')),
    ]

    class Meta:
@@ -120,6 +121,8 @@ class ProjectOptions(models.Model):
        return self.scm_type or ''

    def clean_scm_url(self):
+        if self.scm_type == 'insights':
+            self.scm_url = settings.INSIGHTS_URL_BASE
        scm_url = unicode(self.scm_url or '')
        if not self.scm_type:
            return ''
@@ -141,6 +144,8 @@ class ProjectOptions(models.Model):
            if cred.kind != 'scm':
                raise ValidationError(_("Credential kind must be 'scm'."))
            try:
+                if self.scm_type == 'insights':
+                    self.scm_url = settings.INSIGHTS_URL_BASE
                scm_url = update_scm_url(self.scm_type, self.scm_url,
                                         check_special_cases=False)
                scm_url_parts = urlparse.urlsplit(scm_url)
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -471,24 +471,24 @@ class BaseTask(Task):
            env['PROOT_TMP_DIR'] = settings.AWX_PROOT_BASE_PATH
        return env

-    def build_safe_env(self, instance, **kwargs):
+    def build_safe_env(self, env, **kwargs):
        '''
        Build environment dictionary, hiding potentially sensitive information
        such as passwords or keys.
        '''
        hidden_re = re.compile(r'API|TOKEN|KEY|SECRET|PASS', re.I)
-        urlpass_re = re.compile(r'^.*?://.?:(.*?)@.*?$')
-        env = self.build_env(instance, **kwargs)
-        for k,v in env.items():
+        urlpass_re = re.compile(r'^.*?://[^:]+:(.*?)@.*?$')
+        safe_env = dict(env)
+        for k,v in safe_env.items():
            if k in ('REST_API_URL', 'AWS_ACCESS_KEY', 'AWS_ACCESS_KEY_ID'):
                continue
            elif k.startswith('ANSIBLE_') and not k.startswith('ANSIBLE_NET'):
                continue
            elif hidden_re.search(k):
-                env[k] = HIDDEN_PASSWORD
+                safe_env[k] = HIDDEN_PASSWORD
            elif type(v) == str and urlpass_re.match(v):
-                env[k] = urlpass_re.sub(HIDDEN_PASSWORD, v)
-        return env
+                safe_env[k] = urlpass_re.sub(HIDDEN_PASSWORD, v)
+        return safe_env

    def args2cmdline(self, *args):
        return ' '.join([pipes.quote(a) for a in args])
@@ -699,7 +699,7 @@ class BaseTask(Task):
            output_replacements = self.build_output_replacements(instance, **kwargs)
            cwd = self.build_cwd(instance, **kwargs)
            env = self.build_env(instance, **kwargs)
-            safe_env = self.build_safe_env(instance, **kwargs)
+            safe_env = self.build_safe_env(env, **kwargs)
            stdout_handle = self.get_stdout_handle(instance)
            if self.should_use_proot(instance, **kwargs):
                if not check_proot_installed():
@@ -1189,6 +1189,9 @@ class RunProjectUpdate(BaseTask):
                    scm_username = False
            elif scm_url_parts.scheme.endswith('ssh'):
                scm_password = False
+            elif scm_type == 'insights':
+                extra_vars['scm_username'] = scm_username
+                extra_vars['scm_password'] = scm_password
            scm_url = update_scm_url(scm_type, scm_url, scm_username,
                                     scm_password, scp_format=True)
        else:
@@ -1218,6 +1221,7 @@ class RunProjectUpdate(BaseTask):
            scm_branch = project_update.scm_branch or {'hg': 'tip'}.get(project_update.scm_type, 'HEAD')
        extra_vars.update({
            'project_path': project_update.get_project_path(check_if_exists=False),
+            'insights_url': settings.INSIGHTS_URL_BASE,
            'scm_type': project_update.scm_type,
            'scm_url': scm_url,
            'scm_branch': scm_branch,
@@ -1314,10 +1318,10 @@ class RunProjectUpdate(BaseTask):
            lines = fd.readlines()
            if lines:
                p.scm_revision = lines[0].strip()
-                p.playbook_files = p.playbooks
-                p.save()
            else:
-                logger.error("Could not find scm revision in check")
+                logger.info("Could not find scm revision in check")
+            p.playbook_files = p.playbooks
+            p.save()
        try:
            os.remove(self.revision_path)
        except Exception, e:
--- a/awx/main/tests/functional/test_python_requirements.py
+++ b/awx/main/tests/functional/test_python_requirements.py
@@ -1,11 +1,13 @@

 import os
 import re
+import pytest
 from pip.operations import freeze

 from django.conf import settings


+@pytest.mark.skip(reason="This test needs some love")
 def test_env_matches_requirements_txt():
    def check_is_in(src, dests):
        if src not in dests:
--- a/awx/main/tests/unit/test_tasks.py
+++ b/awx/main/tests/unit/test_tasks.py
@@ -71,6 +71,25 @@ def test_run_admin_checks_usage(mocker, current_instances, call_count):
            assert 'expire' in mock_sm.call_args_list[0][0][0]


+@pytest.mark.parametrize("key,value", [
+    ('REST_API_TOKEN', 'SECRET'),
+    ('SECRET_KEY', 'SECRET'),
+    ('RABBITMQ_PASS', 'SECRET'),
+    ('VMWARE_PASSWORD', 'SECRET'),
+    ('API_SECRET', 'SECRET'),
+    ('CALLBACK_CONNECTION', 'amqp://tower:password@localhost:5672/tower'),
+])
+def test_safe_env_filtering(key, value):
+    task = tasks.RunJob()
+    assert task.build_safe_env({key: value})[key] == tasks.HIDDEN_PASSWORD
+
+
+def test_safe_env_returns_new_copy():
+    task = tasks.RunJob()
+    env = {'foo': 'bar'}
+    assert task.build_safe_env(env) is not env
+
+
 def test_openstack_client_config_generation(mocker):
    update = tasks.RunInventoryUpdate()
    inventory_update = mocker.Mock(**{
--- a/awx/main/utils/common.py
+++ b/awx/main/utils/common.py
@@ -261,7 +261,7 @@ def update_scm_url(scm_type, url, username=True, password=True,
    # git: https://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS
    # hg: http://www.selenic.com/mercurial/hg.1.html#url-paths
    # svn: http://svnbook.red-bean.com/en/1.7/svn-book.html#svn.advanced.reposurls
-    if scm_type not in ('git', 'hg', 'svn'):
+    if scm_type not in ('git', 'hg', 'svn', 'insights'):
        raise ValueError(_('Unsupported SCM type "%s"') % str(scm_type))
    if not url.strip():
        return ''
@@ -307,6 +307,7 @@ def update_scm_url(scm_type, url, username=True, password=True,
        'git': ('ssh', 'git', 'git+ssh', 'http', 'https', 'ftp', 'ftps', 'file'),
        'hg': ('http', 'https', 'ssh', 'file'),
        'svn': ('http', 'https', 'svn', 'svn+ssh', 'file'),
+        'insights': ('http', 'https')
    }
    if parts.scheme not in scm_type_schemes.get(scm_type, ()):
        raise ValueError(_('Unsupported %s URL') % scm_type)
@@ -342,7 +343,7 @@ def update_scm_url(scm_type, url, username=True, password=True,
            #raise ValueError('Password not supported for SSH with Mercurial.')
            netloc_password = ''

-    if netloc_username and parts.scheme != 'file':
+    if netloc_username and parts.scheme != 'file' and scm_type != "insights":
        netloc = u':'.join([urllib.quote(x) for x in (netloc_username, netloc_password) if x])
    else:
        netloc = u''
--- a/awx/main/utils/formatters.py
+++ b/awx/main/utils/formatters.py
@@ -13,7 +13,9 @@ class LogstashFormatter(LogstashFormatterVersion1):
        ret = super(LogstashFormatter, self).__init__(**kwargs)
        if settings_module:
            self.host_id = settings_module.CLUSTER_HOST_ID
-            self.tower_uuid = settings_module.LOG_AGGREGATOR_TOWER_UUID
+            if hasattr(settings_module, 'LOG_AGGREGATOR_TOWER_UUID'):
+                self.tower_uuid = settings_module.LOG_AGGREGATOR_TOWER_UUID
+            self.message_type = settings_module.LOG_AGGREGATOR_TYPE
        return ret

    def reformat_data_for_log(self, raw_data, kind=None):