Merge remote-tracking branch 'upstream/release_2.3' into devel

* upstream/release_2.3: (91 commits)
  Include python-{paramiko,ecdsa} dependencies
  Remove extra epel testing stanzas
  Unit test for ec2 credentialless inventory
  Fix issue with ec2 iam sync with no credential.
  Use the htpasswd command instead the ansible module
  Pip is no longer needed
  check local user root or not in ./configure
  Remove unneeded when check for super user addition
  Improve distro detection in setup.sh
  Fix superuser check on upgrade
  Minor improvements to setup.sh
  Remove ansible prerequisite check from configure
  Attempt to install ansible within setup.sh
  Allow munin processes to access postgres
  Move up base package dependency install
  fixes jenkins failures
  Proper flake8 fix
  fixes executing processes with correct PYTHONPATH will pickup .pth files
  Show the repo for bundled package file dump
  Proper flake8 fix
  ...

awx/__init__.py

@@ -4,6 +4,7 @@
 import os
 import sys
 import warnings
+import site
 
 __version__ = '2.4.0'
 
@@ -39,7 +40,10 @@ def prepare_env():
     # Add local site-packages directory to path.
     local_site_packages = os.path.join(os.path.dirname(__file__), 'lib',
                                        'site-packages')
-    sys.path.insert(0, local_site_packages)
+    site.addsitedir(local_site_packages)
+    # Work around https://bugs.python.org/issue7744
+    # by moving  local_site_packages to the front of sys.path
+    sys.path.insert(0, sys.path.pop())
     # Hide DeprecationWarnings when running in production.  Need to first load
     # settings to apply our filter after Django's own warnings filter.
     from django.conf import settings

awx/api/views.py

@@ -2839,6 +2839,8 @@ class UnifiedJobStdout(RetrieveAPIView):
                 return response
             except Exception, e:
                 return Response({"error": "Error generating stdout download file: %s" % str(e)}, status=status.HTTP_400_BAD_REQUEST)
+        elif request.accepted_renderer.format == 'txt':
+            return Response(unified_job.result_stdout)
         else:
             return super(UnifiedJobStdout, self).retrieve(request, *args, **kwargs)
 

awx/main/management/commands/run_fact_cache_receiver.py

@@ -3,6 +3,7 @@
 
 # Python
 import logging
+from threading import Thread
 from datetime import datetime
 
 # Django
@@ -76,14 +77,18 @@ class FactCacheReceiver(object):
             (fact_obj, version_obj) = Fact.add_fact(self.timestamp, facts, host, module)
             logger.info('Created new fact <fact, fact_version> <%s, %s>' % (fact_obj.id, version_obj.id))
 
-    def run_receiver(self):
+    def run_receiver(self, use_processing_threads=True):
         with Socket('fact_cache', 'r') as facts:
             for message in facts.listen():
                 if 'host' not in message or 'facts' not in message or 'date_key' not in message:
                     logger.warn('Received invalid message %s' % message)
                     continue
                 logger.info('Received message %s' % message)
-                self.process_fact_message(message)
+                if use_processing_threads:
+                    wt = Thread(target=self.process_fact_message, args=(message,))
+                    wt.start()
+                else:
+                    self.process_fact_message(message)
 
 class Command(NoArgsCommand):
     '''

awx/main/migrations/0070_v221_changes.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import uuid
 import os
+import subprocess
 
 from south.utils import datetime_utils as datetime
 from south.db import db
@@ -20,6 +21,8 @@ class Migration(DataMigration):
             j.result_stdout_file = stdout_filename
             j.result_stdout_text = ""
             j.save()
+            sed_command = subprocess.Popen(["sed", "-i", "-e", "s/\\\\r\\\\n/\\n/g", stdout_filename])
+            sed_command.wait()
 
     def backwards(self, orm):
         pass

awx/main/models/inventory.py

@@ -1259,7 +1259,7 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions):
         if not super(InventoryUpdate, self).can_start:
             return False
 
-        if (self.source != 'custom' and
+        if (self.source not in ('custom', 'ec2') and
                 not (self.credential and self.credential.active)):
             return False
         return True

awx/main/models/projects.py

@@ -357,6 +357,16 @@ class ProjectUpdate(UnifiedJob, ProjectOptions):
     def result_stdout(self):
         return self._result_stdout_raw(redact_sensitive=True, escape_ascii=True)
 
+    @property
+    def result_stdout_raw(self):
+        return self._result_stdout_raw(redact_sensitive=True)
+
+    def result_stdout_raw_limited(self, start_line=0, end_line=None, redact_sensitive=True):
+        return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive=redact_sensitive)
+
+    def result_stdout_limited(self, start_line=0, end_line=None, redact_sensitive=True):
+        return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive=redact_sensitive, escape_ascii=True)
+
     def get_absolute_url(self):
         return reverse('api:project_update_detail', args=(self.pk,))
 

awx/main/models/unified_jobs.py

@@ -671,11 +671,11 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
 
         return return_buffer, start_actual, end_actual, absolute_end
 
-    def result_stdout_raw_limited(self, start_line=0, end_line=None):
-        return self._result_stdout_raw_limited(start_line, end_line)
+    def result_stdout_raw_limited(self, start_line=0, end_line=None, redact_sensitive=False):
+        return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive)
 
-    def result_stdout_limited(self, start_line=0, end_line=None):
-        return self._result_stdout_raw_limited(start_line, end_line, escape_ascii=True)
+    def result_stdout_limited(self, start_line=0, end_line=None, redact_sensitive=False):
+        return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive, escape_ascii=True)
 
     @property
     def celery_task(self):

awx/main/tests/base.py

@@ -627,7 +627,7 @@ class BaseTestMixin(QueueTestMixin, MockCommonlySlowTestMixin):
         msg += '"%s" found in: "%s"' % (substr, string)
         self.assertEqual(count, 0, msg)
 
-    def check_found(self, string, substr, count, description=None, word_boundary=False):
+    def check_found(self, string, substr, count=-1, description=None, word_boundary=False):
         if word_boundary:
             count_actual = len(re.findall(r'\b%s\b' % re.escape(substr), string))
         else:
@@ -636,8 +636,11 @@ class BaseTestMixin(QueueTestMixin, MockCommonlySlowTestMixin):
         msg = ''
         if description:
             msg = 'Test "%s".\n' % description
-        msg += 'Found %d occurances of "%s" instead of %d in: "%s"' % (count_actual, substr, count, string)
-        self.assertEqual(count_actual, count, msg)
+        if count == -1:
+            self.assertTrue(count_actual > 0)
+        else:
+            msg += 'Found %d occurances of "%s" instead of %d in: "%s"' % (count_actual, substr, count, string)
+            self.assertEqual(count_actual, count, msg)
         
     def check_job_result(self, job, expected='successful', expect_stdout=True,
                          expect_traceback=False):

awx/main/tests/commands/run_fact_cache_receiver.py

@@ -142,7 +142,7 @@ class RunFactCacheReceiverUnitTest(BaseTest, MongoDBRequired):
 
         receiver = FactCacheReceiver()
         receiver.process_fact_message = MagicMock(name='process_fact_message')
-        receiver.run_receiver()
+        receiver.run_receiver(use_processing_threads=False)
 
         receiver.process_fact_message.assert_called_once_with(TEST_MSG)
 

awx/main/tests/inventory.py

@@ -1665,6 +1665,17 @@ class InventoryUpdatesTest(BaseTransactionTest):
         inventory_source.save()
         self.check_inventory_source(inventory_source, initial=False)
 
+    def test_update_from_ec2_without_credential(self):
+        self.create_test_license_file()
+        group = self.group
+        group.name = 'ec2'
+        group.save()
+        self.group = group
+        cache_path = tempfile.mkdtemp(prefix='awx_ec2_')
+        self._temp_paths.append(cache_path)
+        inventory_source = self.update_inventory_source(self.group, source='ec2')
+        self.check_inventory_update(inventory_source, should_fail=True)
+
     def test_update_from_ec2_with_nested_groups(self):
         source_username = getattr(settings, 'TEST_AWS_ACCESS_KEY_ID', '')
         source_password = getattr(settings, 'TEST_AWS_SECRET_ACCESS_KEY', '')

awx/main/tests/views.py

@@ -4,6 +4,7 @@ from django.core.urlresolvers import reverse
 # Reuse Test code
 from awx.main.tests.base import BaseLiveServerTest, QueueStartStopTestMixin
 from awx.main.tests.base import URI
+from awx.main.models.projects import * # noqa
 
 __all__ = ['UnifiedJobStdoutRedactedTests']
 
@@ -32,12 +33,20 @@ class UnifiedJobStdoutRedactedTests(BaseLiveServerTest, QueueStartStopTestMixin)
         self.setup_instances()
         self.setup_users()        
         self.test_cases = []
+        self.negative_test_cases = []
+
+        proj = self.make_project()
 
         for e in TEST_STDOUTS:
-            e['job'] = self.make_job()
-            e['job'].result_stdout_text = e['text']
-            e['job'].save()
+            e['project'] = ProjectUpdate(project=proj)
+            e['project'].result_stdout_text = e['text']
+            e['project'].save()
             self.test_cases.append(e)
+        for d in TEST_STDOUTS:
+            d['job'] = self.make_job()
+            d['job'].result_stdout_text = d['text']
+            d['job'].save()
+            self.negative_test_cases.append(d)
 
     # This is more of a functional test than a unit test.
     # should filter out username and password
@@ -49,7 +58,13 @@ class UnifiedJobStdoutRedactedTests(BaseLiveServerTest, QueueStartStopTestMixin)
         # Ensure the host didn't get redacted
         self.check_found(response['content'], uri.host, test_data['occurrences'], test_data['description'])
 
-    def _get_url_job_stdout(self, job, format='json'):
+    def check_sensitive_not_redacted(self, test_data, response):
+        uri = test_data['uri']
+        self.assertIsNotNone(response['content'])
+        self.check_found(response['content'], uri.username, description=test_data['description'])
+        self.check_found(response['content'], uri.password, description=test_data['description'])
+
+    def _get_url_job_stdout(self, job, url_base, format='json'):
         formats = {
             'json': 'application/json',
             'ansi': 'text/plain',
@@ -57,22 +72,39 @@ class UnifiedJobStdoutRedactedTests(BaseLiveServerTest, QueueStartStopTestMixin)
             'html': 'text/html',
         }
         content_type = formats[format]
-        job_stdout_url = reverse('api:job_stdout', args=(job.pk,)) + "?format=" + format
-        return self.get(job_stdout_url, expect=200, auth=self.get_super_credentials(), accept=content_type)
+        project_update_stdout_url = reverse(url_base, args=(job.pk,)) + "?format=" + format
+        return self.get(project_update_stdout_url, expect=200, auth=self.get_super_credentials(), accept=content_type)
 
     def _test_redaction_enabled(self, format):
         for test_data in self.test_cases:
-            response = self._get_url_job_stdout(test_data['job'], format=format)
+            response = self._get_url_job_stdout(test_data['project'], "api:project_update_stdout", format=format)
             self.check_sensitive_redacted(test_data, response)
 
-    def test_redaction_enabled_json(self):
+    def _test_redaction_disabled(self, format):
+        for test_data in self.negative_test_cases:
+            response = self._get_url_job_stdout(test_data['job'], "api:job_stdout", format=format)
+            self.check_sensitive_not_redacted(test_data, response)
+
+    def test_project_update_redaction_enabled_json(self):
         self._test_redaction_enabled('json')
 
-    def test_redaction_enabled_ansi(self):
+    def test_project_update_redaction_enabled_ansi(self):
         self._test_redaction_enabled('ansi')
 
-    def test_redaction_enabled_html(self):
+    def test_project_update_redaction_enabled_html(self):
         self._test_redaction_enabled('html')
 
-    def test_redaction_enabled_txt(self):
+    def test_project_update_redaction_enabled_txt(self):
         self._test_redaction_enabled('txt')
+
+    def test_job_redaction_disabled_json(self):
+        self._test_redaction_disabled('json')
+
+    def test_job_redaction_disabled_ansi(self):
+        self._test_redaction_disabled('ansi')
+
+    def test_job_redaction_disabled_html(self):
+        self._test_redaction_disabled('html')
+
+    def test_job_redaction_disabled_txt(self):
+        self._test_redaction_disabled('txt')

awx/main/utils.py

@@ -446,7 +446,8 @@ def build_proot_temp_dir():
     '''
     Create a temporary directory for proot to use.
     '''
-    path = tempfile.mkdtemp(prefix='ansible_tower_proot_')
+    from django.conf import settings
+    path = tempfile.mkdtemp(prefix='ansible_tower_proot_', dir=settings.AWX_PROOT_BASE_PATH)
     os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
     return path
 

awx/settings/defaults.py

@@ -352,6 +352,9 @@ AWX_PROOT_SHOW_PATHS = []
 # Number of jobs to show as part of the job template history
 AWX_JOB_TEMPLATE_HISTORY = 10
 
+# The directory in which proot will create new temporary directories for its root
+AWX_PROOT_BASE_PATH = "/tmp"
+
 # Default list of modules allowed for ad hoc commands.
 AD_HOC_COMMANDS = [
     'command',