Merge remote-tracking branch 'upstream/release_2.3' into devel

* upstream/release_2.3: (91 commits)
  Include python-{paramiko,ecdsa} dependencies
  Remove extra epel testing stanzas
  Unit test for ec2 credentialless inventory
  Fix issue with ec2 iam sync with no credential.
  Use the htpasswd command instead the ansible module
  Pip is no longer needed
  check local user root or not in ./configure
  Remove unneeded when check for super user addition
  Improve distro detection in setup.sh
  Fix superuser check on upgrade
  Minor improvements to setup.sh
  Remove ansible prerequisite check from configure
  Attempt to install ansible within setup.sh
  Allow munin processes to access postgres
  Move up base package dependency install
  fixes jenkins failures
  Proper flake8 fix
  fixes executing processes with correct PYTHONPATH will pickup .pth files
  Show the repo for bundled package file dump
  Proper flake8 fix
  ...

.gitignore

@@ -27,6 +27,7 @@ __pycache__
 /deb-build
 /rpm-build
 /tar-build
+/offline_tar-build
 /dist
 *.egg-info
 *.py[c,o]

MANIFEST.in

@@ -6,9 +6,8 @@ recursive-include awx/ui/templates *.html
 recursive-include awx/ui/static *
 recursive-include awx/playbooks *.yml
 recursive-include awx/lib/site-packages *
+recursive-include requirements *.txt
 recursive-include config *
-recursive-include config/deb *
-recursive-include config/rpm *
 recursive-exclude awx devonly.py*
 recursive-exclude awx/api/tests *
 recursive-exclude awx/main/tests *
@@ -19,5 +18,6 @@ include tools/scripts/ansible-tower-service
 include tools/munin_monitors/*
 include tools/sosreport/*
 include COPYING
+include Makefile
 prune awx/public
 prune awx/projects

Makefile

@@ -7,6 +7,8 @@ TESTEM ?= ./node_modules/.bin/testem
 BROCCOLI_BIN ?= ./node_modules/.bin/broccoli
 MOCHA_BIN ?= ./node_modules/.bin/mocha
 NODE ?= node
+DEPS_SCRIPT ?= packaging/offline/deps.py
+AW_REPO_URL ?= "http://releases.ansible.com/ansible-tower"
 
 CLIENT_TEST_DIR ?= build_test
 
@@ -73,15 +75,34 @@ else
 endif
 DEBUILD = $(DEBUILD_BIN) $(DEBUILD_OPTS)
 DEB_PPA ?= reprepro
+DEB_ARCH ?= amd64
 
 # RPM build parameters
 RPM_SPECDIR= packaging/rpm
 RPM_SPEC = $(RPM_SPECDIR)/$(NAME).spec
 RPM_DIST ?= $(shell rpm --eval '%{?dist}' 2>/dev/null)
+RPM_ARCH ?= $(shell rpm --eval '%{_arch}' 2>/dev/null)
 RPM_NVR = $(NAME)-$(VERSION)-$(RELEASE)$(RPM_DIST)
 MOCK_BIN ?= mock
 MOCK_CFG ?=
 
+# Offline TAR build parameters
+DIST = $(shell echo $(RPM_DIST) | sed -e 's|^\.\(el\)\([0-9]\).*|\1|')
+DIST_MAJOR = $(shell echo $(RPM_DIST) | sed -e 's|^\.\(el\)\([0-9]\).*|\2|')
+DIST_FULL = $(DIST)$(DIST_MAJOR)
+OFFLINE_TAR_NAME = $(NAME)-offline-$(DIST_FULL)-$(VERSION)-$(RELEASE)
+OFFLINE_TAR_FILE = $(OFFLINE_TAR_NAME).tar.gz
+OFFLINE_TAR_LINK = $(NAME)-offline-$(DIST_FULL)-latest.tar.gz
+
+DISTRO := $(shell . /etc/os-release 2>/dev/null && echo $${ID} || echo redhat)
+ifeq ($(DISTRO),ubuntu)
+    SETUP_INSTALL_ARGS = --skip-build --no-compile --root=$(DESTDIR) -v --install-layout=deb
+else
+    SETUP_INSTALL_ARGS = --skip-build --no-compile --root=$(DESTDIR) -v
+endif
+
+.DEFAULT_GOAL := build
+
 .PHONY: clean rebase push requirements requirements_dev requirements_jenkins \
 	real-requirements real-requirements_dev real-requirements_jenkins \
 	develop refresh adduser syncdb migrate dbchange dbshell runserver celeryd \
@@ -89,7 +110,8 @@ MOCK_CFG ?=
 	release_build release_clean sdist rpmtar mock-rpm mock-srpm rpm-sign \
 	devjs minjs testjs testjs_ci node-tests browser-tests jshint ngdocs sync_ui \
 	deb deb-src debian reprepro setup_tarball \
-	virtualbox-ovf virtualbox-centos-7 virtualbox-centos-6
+	virtualbox-ovf virtualbox-centos-7 virtualbox-centos-6 \
+	clean-offline setup_offline_tarball
 
 # Remove setup build files
 clean-tar:
@@ -124,6 +146,9 @@ clean-packer:
 	rm -rf packaging/packer/ansible-tower*-ova
 	rm -f Vagrantfile
 
+clean-offline:
+	rm -rf offline-tar-build
+
 # Remove temporary build files, compiled Python files.
 clean: clean-rpm clean-deb clean-grunt clean-ui clean-tar clean-packer
 	rm -rf awx/lib/site-packages
@@ -140,21 +165,22 @@ push:
 	git push origin master
 
 # Install runtime, development and jenkins requirements
-requirements requirements_dev requirements_jenkins: %: real-% awx/lib/site-packages/oslo/__init__.py awx/lib/site-packages/dogpile/__init__.py
+requirements requirements_dev requirements_jenkins: %: real-%
-
-# Create missing __init__.py files
-awx/lib/site-packages/%/__init__.py:
-	touch $@
 
 # Install third-party requirements needed for development environment.
+# NOTE:
+#  * --target is only supported on newer versions of pip
+#  * https://github.com/pypa/pip/issues/3056 - the workaround is to override the `install-platlib`
+#  * --user (in conjunction with PYTHONUSERBASE="awx" may be a better option
+#  * --target implies --ignore-installed
 real-requirements:
-	pip install -r requirements/requirements.txt --target awx/lib/site-packages/ --ignore-installed
+	pip install -r requirements/requirements.txt --target awx/lib/site-packages/ --install-option="--install-platlib=\$$base/lib/python"
 
 real-requirements_dev:
-	pip install -r requirements/requirements_dev.txt --target awx/lib/site-packages/ --ignore-installed
+	pip install -r requirements/requirements_dev.txt --target awx/lib/site-packages/ --install-option="--install-platlib=\$$base/lib/python"
 
 # Install third-party requirements needed for running unittests in jenkins
-real-requirements_jenkins: real-requirements
+real-requirements_jenkins:
 	pip install -r requirements/requirements_jenkins.txt
 	npm install csslint jshint
 
@@ -373,14 +399,14 @@ tar-build/$(SETUP_TAR_FILE):
 	@cd tar-build/$(SETUP_TAR_NAME) && sed -e 's#%NAME%#$(NAME)#;s#%VERSION%#$(VERSION)#;s#%RELEASE%#$(RELEASE)#;' group_vars/all.in > group_vars/all
 	@cd tar-build && tar -czf $(SETUP_TAR_FILE) --exclude "*/all.in" $(SETUP_TAR_NAME)/
 	@ln -sf $(SETUP_TAR_FILE) tar-build/$(SETUP_TAR_LINK)
+
+setup_tarball: tar-build/$(SETUP_TAR_FILE)
 	@echo "#############################################"
 	@echo "Setup artifacts:"
 	@echo tar-build/$(SETUP_TAR_FILE)
 	@echo tar-build/$(SETUP_TAR_LINK)
 	@echo "#############################################"
 
-setup_tarball: tar-build/$(SETUP_TAR_FILE)
-
 release_clean:
 	-(rm *.tar)
 	-(rm -rf ($RELEASE))
@@ -390,8 +416,30 @@ dist/$(SDIST_TAR_FILE):
 
 sdist: minjs requirements dist/$(SDIST_TAR_FILE)
 
+# Build setup offline tarball
+offline-tar-build:
+	mkdir -p $@
+
+offline-tar-build/$(DIST_FULL):
+	mkdir -p $@
+
+# TODO - Somehow share implementation with setup_tarball
+offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_FILE):
+	cp -a setup offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_NAME)
+	cd offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_NAME) && sed -e 's#%NAME%#$(NAME)#;s#%VERSION%#$(VERSION)#;s#%RELEASE%#$(RELEASE)#;' group_vars/all.in > group_vars/all
+	$(PYTHON) $(DEPS_SCRIPT) -d $(DIST) -r $(DIST_MAJOR) -u $(AW_REPO_URL) -s offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_NAME) -v -v -v
+	cd offline-tar-build/$(DIST_FULL) && tar -czf $(OFFLINE_TAR_FILE) --exclude "*/all.in" $(OFFLINE_TAR_NAME)/
+	ln -sf $(OFFLINE_TAR_FILE) offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_LINK)
+
+setup_offline_tarball: offline-tar-build offline-tar-build/$(DIST_FULL) offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_FILE)
+	@echo "#############################################"
+	@echo "Offline artifacts:"
+	@echo offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_FILE)
+	@echo offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_LINK)
+	@echo "#############################################"
+
 rpm-build:
-	mkdir -p rpm-build
+	mkdir -p $@
 
 rpm-build/$(SDIST_TAR_FILE): rpm-build dist/$(SDIST_TAR_FILE)
 	cp packaging/rpm/$(NAME).spec rpm-build/
@@ -419,22 +467,22 @@ rpm-build/$(RPM_NVR).src.rpm: /etc/mock/$(MOCK_CFG).cfg
 
 mock-srpm: rpmtar rpm-build/$(RPM_NVR).src.rpm
 
-rpm-build/$(RPM_NVR).noarch.rpm: rpm-build/$(RPM_NVR).src.rpm
+rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm: rpm-build/$(RPM_NVR).src.rpm
 	$(MOCK_BIN) -r $(MOCK_CFG) --resultdir rpm-build --rebuild rpm-build/$(RPM_NVR).src.rpm \
 	   --define "tower_version $(VERSION)" --define "tower_release $(RELEASE)"
 	@echo "#############################################"
 	@echo "RPM artifacts:"
-	@echo rpm-build/$(RPM_NVR).noarch.rpm
+	@echo rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm
 	@echo "#############################################"
 
-mock-rpm: rpmtar rpm-build/$(RPM_NVR).noarch.rpm
+mock-rpm: rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm
 
 ifeq ($(OFFICIAL),yes)
 rpm-build/$(GPG_FILE): rpm-build
 	gpg --export -a "${GPG_KEY}" > "$@"
 
-rpm-sign: rpm-build/$(GPG_FILE) rpmtar rpm-build/$(RPM_NVR).noarch.rpm
+rpm-sign: rpm-build/$(GPG_FILE) rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm
-	rpm --define "_signature gpg" --define "_gpg_name $(GPG_KEY)" --addsign rpm-build/$(RPM_NVR).noarch.rpm
+	rpm --define "_signature gpg" --define "_gpg_name $(GPG_KEY)" --addsign rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm
 endif
 
 deb-build/$(SDIST_TAR_NAME):
@@ -446,14 +494,14 @@ deb-build/$(SDIST_TAR_NAME):
 
 debian: sdist deb-build/$(SDIST_TAR_NAME)
 
-deb-build/$(NAME)_$(VERSION)-$(RELEASE)_all.deb:
+deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb:
 	cd deb-build/$(SDIST_TAR_NAME) && $(DEBUILD) -b
 	@echo "#############################################"
 	@echo "DEB artifacts:"
-	@echo deb-build/$(NAME)_$(VERSION)-$(RELEASE)_all.deb
+	@echo deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb
 	@echo "#############################################"
 
-deb: debian deb-build/$(NAME)_$(VERSION)-$(RELEASE)_all.deb
+deb: debian deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb
 
 deb-build/$(NAME)_$(VERSION)-$(RELEASE)_source.changes:
 	cd deb-build/$(SDIST_TAR_NAME) && $(DEBUILD) -S
@@ -465,7 +513,7 @@ deb-build/$(NAME)_$(VERSION)-$(RELEASE)_source.changes:
 deb-src: debian deb-build/$(NAME)_$(VERSION)-$(RELEASE)_source.changes
 
 deb-upload: deb
-	$(DPUT_BIN) $(DPUT_OPTS) $(DEB_PPA) deb-build/$(NAME)_$(VERSION)-$(RELEASE)_amd64.changes ; \
+	$(DPUT_BIN) $(DPUT_OPTS) $(DEB_PPA) deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).changes ; \
 
 deb-src-upload: deb-src
 	$(DPUT_BIN) $(DPUT_OPTS) $(DEB_PPA) deb-build/$(NAME)_$(VERSION)-$(RELEASE)_source.changes ; \
@@ -473,7 +521,7 @@ deb-src-upload: deb-src
 reprepro: deb
 	mkdir -p reprepro/conf
 	cp -a packaging/reprepro/* reprepro/conf/
-	@DEB=deb-build/$(NAME)_$(VERSION)-$(RELEASE)_all.deb ; \
+	@DEB=deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb ; \
 	for DIST in trusty precise ; do \
 	    echo "Removing '$(NAME)' from the $${DIST} apt repo" ; \
 	    echo reprepro --export=force -b reprepro remove $${DIST} $(NAME) ; \
@@ -505,5 +553,11 @@ virtualbox-centos-7: packaging/packer/output-virtualbox-iso/centos-7.ovf
 docker-dev:
 	docker build --no-cache=true --rm=true -t ansible/tower_devel:latest tools/docker
 
+# TODO - figure out how to build the front-end and python requirements with
+# 'build'
+build:
+	$(PYTHON) setup.py build
+
+# TODO - only use --install-layout=deb on Debian
 install:
-	$(PYTHON) setup.py install egg_info -b ""
+	$(PYTHON) setup.py install $(SETUP_INSTALL_ARGS)

awx/__init__.py

@@ -4,6 +4,7 @@
 import os
 import sys
 import warnings
+import site
 
 __version__ = '2.4.0'
 
@@ -39,7 +40,10 @@ def prepare_env():
     # Add local site-packages directory to path.
     local_site_packages = os.path.join(os.path.dirname(__file__), 'lib',
                                        'site-packages')
-    sys.path.insert(0, local_site_packages)
+    site.addsitedir(local_site_packages)
+    # Work around https://bugs.python.org/issue7744
+    # by moving  local_site_packages to the front of sys.path
+    sys.path.insert(0, sys.path.pop())
     # Hide DeprecationWarnings when running in production.  Need to first load
     # settings to apply our filter after Django's own warnings filter.
     from django.conf import settings

awx/api/views.py

@@ -2839,6 +2839,8 @@ class UnifiedJobStdout(RetrieveAPIView):
                 return response
             except Exception, e:
                 return Response({"error": "Error generating stdout download file: %s" % str(e)}, status=status.HTTP_400_BAD_REQUEST)
+        elif request.accepted_renderer.format == 'txt':
+            return Response(unified_job.result_stdout)
         else:
             return super(UnifiedJobStdout, self).retrieve(request, *args, **kwargs)
 

awx/main/management/commands/run_fact_cache_receiver.py

@@ -3,6 +3,7 @@
 
 # Python
 import logging
+from threading import Thread
 from datetime import datetime
 
 # Django
@@ -76,14 +77,18 @@ class FactCacheReceiver(object):
             (fact_obj, version_obj) = Fact.add_fact(self.timestamp, facts, host, module)
             logger.info('Created new fact <fact, fact_version> <%s, %s>' % (fact_obj.id, version_obj.id))
 
-    def run_receiver(self):
+    def run_receiver(self, use_processing_threads=True):
         with Socket('fact_cache', 'r') as facts:
             for message in facts.listen():
                 if 'host' not in message or 'facts' not in message or 'date_key' not in message:
                     logger.warn('Received invalid message %s' % message)
                     continue
                 logger.info('Received message %s' % message)
-                self.process_fact_message(message)
+                if use_processing_threads:
+                    wt = Thread(target=self.process_fact_message, args=(message,))
+                    wt.start()
+                else:
+                    self.process_fact_message(message)
 
 class Command(NoArgsCommand):
     '''

awx/main/migrations/0070_v221_changes.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import uuid
 import os
+import subprocess
 
 from south.utils import datetime_utils as datetime
 from south.db import db
@@ -20,6 +21,8 @@ class Migration(DataMigration):
             j.result_stdout_file = stdout_filename
             j.result_stdout_text = ""
             j.save()
+            sed_command = subprocess.Popen(["sed", "-i", "-e", "s/\\\\r\\\\n/\\n/g", stdout_filename])
+            sed_command.wait()
 
     def backwards(self, orm):
         pass

awx/main/models/inventory.py

@@ -1259,7 +1259,7 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions):
         if not super(InventoryUpdate, self).can_start:
             return False
 
-        if (self.source != 'custom' and
+        if (self.source not in ('custom', 'ec2') and
                 not (self.credential and self.credential.active)):
             return False
         return True

awx/main/models/projects.py

@@ -357,6 +357,16 @@ class ProjectUpdate(UnifiedJob, ProjectOptions):
     def result_stdout(self):
         return self._result_stdout_raw(redact_sensitive=True, escape_ascii=True)
 
+    @property
+    def result_stdout_raw(self):
+        return self._result_stdout_raw(redact_sensitive=True)
+
+    def result_stdout_raw_limited(self, start_line=0, end_line=None, redact_sensitive=True):
+        return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive=redact_sensitive)
+
+    def result_stdout_limited(self, start_line=0, end_line=None, redact_sensitive=True):
+        return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive=redact_sensitive, escape_ascii=True)
+
     def get_absolute_url(self):
         return reverse('api:project_update_detail', args=(self.pk,))
 

awx/main/models/unified_jobs.py

@@ -671,11 +671,11 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
 
         return return_buffer, start_actual, end_actual, absolute_end
 
-    def result_stdout_raw_limited(self, start_line=0, end_line=None):
+    def result_stdout_raw_limited(self, start_line=0, end_line=None, redact_sensitive=False):
-        return self._result_stdout_raw_limited(start_line, end_line)
+        return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive)
 
-    def result_stdout_limited(self, start_line=0, end_line=None):
+    def result_stdout_limited(self, start_line=0, end_line=None, redact_sensitive=False):
-        return self._result_stdout_raw_limited(start_line, end_line, escape_ascii=True)
+        return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive, escape_ascii=True)
 
     @property
     def celery_task(self):

awx/main/tests/base.py

@@ -627,7 +627,7 @@ class BaseTestMixin(QueueTestMixin, MockCommonlySlowTestMixin):
         msg += '"%s" found in: "%s"' % (substr, string)
         self.assertEqual(count, 0, msg)
 
-    def check_found(self, string, substr, count, description=None, word_boundary=False):
+    def check_found(self, string, substr, count=-1, description=None, word_boundary=False):
         if word_boundary:
             count_actual = len(re.findall(r'\b%s\b' % re.escape(substr), string))
         else:
@@ -636,8 +636,11 @@ class BaseTestMixin(QueueTestMixin, MockCommonlySlowTestMixin):
         msg = ''
         if description:
             msg = 'Test "%s".\n' % description
-        msg += 'Found %d occurances of "%s" instead of %d in: "%s"' % (count_actual, substr, count, string)
+        if count == -1:
-        self.assertEqual(count_actual, count, msg)
+            self.assertTrue(count_actual > 0)
+        else:
+            msg += 'Found %d occurances of "%s" instead of %d in: "%s"' % (count_actual, substr, count, string)
+            self.assertEqual(count_actual, count, msg)
         
     def check_job_result(self, job, expected='successful', expect_stdout=True,
                          expect_traceback=False):

awx/main/tests/commands/run_fact_cache_receiver.py

@@ -142,7 +142,7 @@ class RunFactCacheReceiverUnitTest(BaseTest, MongoDBRequired):
 
         receiver = FactCacheReceiver()
         receiver.process_fact_message = MagicMock(name='process_fact_message')
-        receiver.run_receiver()
+        receiver.run_receiver(use_processing_threads=False)
 
         receiver.process_fact_message.assert_called_once_with(TEST_MSG)
 

awx/main/tests/inventory.py

@@ -1665,6 +1665,17 @@ class InventoryUpdatesTest(BaseTransactionTest):
         inventory_source.save()
         self.check_inventory_source(inventory_source, initial=False)
 
+    def test_update_from_ec2_without_credential(self):
+        self.create_test_license_file()
+        group = self.group
+        group.name = 'ec2'
+        group.save()
+        self.group = group
+        cache_path = tempfile.mkdtemp(prefix='awx_ec2_')
+        self._temp_paths.append(cache_path)
+        inventory_source = self.update_inventory_source(self.group, source='ec2')
+        self.check_inventory_update(inventory_source, should_fail=True)
+
     def test_update_from_ec2_with_nested_groups(self):
         source_username = getattr(settings, 'TEST_AWS_ACCESS_KEY_ID', '')
         source_password = getattr(settings, 'TEST_AWS_SECRET_ACCESS_KEY', '')

awx/main/tests/views.py

@@ -4,6 +4,7 @@ from django.core.urlresolvers import reverse
 # Reuse Test code
 from awx.main.tests.base import BaseLiveServerTest, QueueStartStopTestMixin
 from awx.main.tests.base import URI
+from awx.main.models.projects import * # noqa
 
 __all__ = ['UnifiedJobStdoutRedactedTests']
 
@@ -32,12 +33,20 @@ class UnifiedJobStdoutRedactedTests(BaseLiveServerTest, QueueStartStopTestMixin)
         self.setup_instances()
         self.setup_users()        
         self.test_cases = []
+        self.negative_test_cases = []
+
+        proj = self.make_project()
 
         for e in TEST_STDOUTS:
-            e['job'] = self.make_job()
+            e['project'] = ProjectUpdate(project=proj)
-            e['job'].result_stdout_text = e['text']
+            e['project'].result_stdout_text = e['text']
-            e['job'].save()
+            e['project'].save()
             self.test_cases.append(e)
+        for d in TEST_STDOUTS:
+            d['job'] = self.make_job()
+            d['job'].result_stdout_text = d['text']
+            d['job'].save()
+            self.negative_test_cases.append(d)
 
     # This is more of a functional test than a unit test.
     # should filter out username and password
@@ -49,7 +58,13 @@ class UnifiedJobStdoutRedactedTests(BaseLiveServerTest, QueueStartStopTestMixin)
         # Ensure the host didn't get redacted
         self.check_found(response['content'], uri.host, test_data['occurrences'], test_data['description'])
 
-    def _get_url_job_stdout(self, job, format='json'):
+    def check_sensitive_not_redacted(self, test_data, response):
+        uri = test_data['uri']
+        self.assertIsNotNone(response['content'])
+        self.check_found(response['content'], uri.username, description=test_data['description'])
+        self.check_found(response['content'], uri.password, description=test_data['description'])
+
+    def _get_url_job_stdout(self, job, url_base, format='json'):
         formats = {
             'json': 'application/json',
             'ansi': 'text/plain',
@@ -57,22 +72,39 @@ class UnifiedJobStdoutRedactedTests(BaseLiveServerTest, QueueStartStopTestMixin)
             'html': 'text/html',
         }
         content_type = formats[format]
-        job_stdout_url = reverse('api:job_stdout', args=(job.pk,)) + "?format=" + format
+        project_update_stdout_url = reverse(url_base, args=(job.pk,)) + "?format=" + format
-        return self.get(job_stdout_url, expect=200, auth=self.get_super_credentials(), accept=content_type)
+        return self.get(project_update_stdout_url, expect=200, auth=self.get_super_credentials(), accept=content_type)
 
     def _test_redaction_enabled(self, format):
         for test_data in self.test_cases:
-            response = self._get_url_job_stdout(test_data['job'], format=format)
+            response = self._get_url_job_stdout(test_data['project'], "api:project_update_stdout", format=format)
             self.check_sensitive_redacted(test_data, response)
 
-    def test_redaction_enabled_json(self):
+    def _test_redaction_disabled(self, format):
+        for test_data in self.negative_test_cases:
+            response = self._get_url_job_stdout(test_data['job'], "api:job_stdout", format=format)
+            self.check_sensitive_not_redacted(test_data, response)
+
+    def test_project_update_redaction_enabled_json(self):
         self._test_redaction_enabled('json')
 
-    def test_redaction_enabled_ansi(self):
+    def test_project_update_redaction_enabled_ansi(self):
         self._test_redaction_enabled('ansi')
 
-    def test_redaction_enabled_html(self):
+    def test_project_update_redaction_enabled_html(self):
         self._test_redaction_enabled('html')
 
-    def test_redaction_enabled_txt(self):
+    def test_project_update_redaction_enabled_txt(self):
         self._test_redaction_enabled('txt')
+
+    def test_job_redaction_disabled_json(self):
+        self._test_redaction_disabled('json')
+
+    def test_job_redaction_disabled_ansi(self):
+        self._test_redaction_disabled('ansi')
+
+    def test_job_redaction_disabled_html(self):
+        self._test_redaction_disabled('html')
+
+    def test_job_redaction_disabled_txt(self):
+        self._test_redaction_disabled('txt')

awx/main/utils.py

@@ -446,7 +446,8 @@ def build_proot_temp_dir():
     '''
     Create a temporary directory for proot to use.
     '''
-    path = tempfile.mkdtemp(prefix='ansible_tower_proot_')
+    from django.conf import settings
+    path = tempfile.mkdtemp(prefix='ansible_tower_proot_', dir=settings.AWX_PROOT_BASE_PATH)
     os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
     return path
 

awx/settings/defaults.py

@@ -352,6 +352,9 @@ AWX_PROOT_SHOW_PATHS = []
 # Number of jobs to show as part of the job template history
 AWX_JOB_TEMPLATE_HISTORY = 10
 
+# The directory in which proot will create new temporary directories for its root
+AWX_PROOT_BASE_PATH = "/tmp"
+
 # Default list of modules allowed for ad hoc commands.
 AD_HOC_COMMANDS = [
     'command',

requirements/requirements.txt

@@ -35,6 +35,7 @@ git+https://github.com/chrismeyersfsu/gevent-socketio.git@tower_0.3.6#egg=socket
 git+https://github.com/chrismeyersfsu/python-ipy.git@master#egg=ipy
 git+https://github.com/chrismeyersfsu/python-keystoneclient.git@1.3.0#egg=keystoneclient-1.3.0
 git+https://github.com/chrismeyersfsu/shade.git@tower_0.5.0#egg=shade-tower_0.5.0
+git+https://github.com/chrismeyersfsu/sitecustomize.git#egg=sitecustomize
 greenlet==0.4.7
 httplib2==0.9
 idna==2.0

requirements/requirements_dev.txt

@@ -2,3 +2,5 @@
 django-devserver
 django-debug-toolbar
 unittest2
+pep8
+flake8

requirements/requirements_jenkins.txt

@@ -1,3 +1,4 @@
+-r requirements.txt
 ansible
 django-jenkins
 coverage

tools/docker/Dockerfile

@@ -6,7 +6,7 @@ ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8  
 RUN apt-get update
 RUN apt-get install -y software-properties-common python-software-properties curl
-RUN add-apt-repository -y ppa:chris-lea/redis-server; add-apt-repository -y ppa:chris-lea/zeromq; add-apt-repository ppa:ansible/ansible
+RUN add-apt-repository -y ppa:chris-lea/redis-server; add-apt-repository -y ppa:chris-lea/zeromq; add-apt-repository -y ppa:chris-lea/node.js; add-apt-repository ppa:ansible/ansible
 RUN curl -sL https://deb.nodesource.com/setup_0.12 | bash -
 RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10 && apt-key adv --fetch-keys http://www.postgresql.org/media/keys/ACCC4CF8.asc
 RUN echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.0.list && echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" | tee /etc/apt/sources.list.d/postgres-9.4.list

tools/license-audit/TowerLicenses.csv

@@ -1,7 +1,8 @@
 Babel,BSD,http://babel.pocoo.org/,pip
-IPy,BSD,https://github.com/autocracy/python-ipy,pip
+Django,BSD,http://www.djangoproject.com/,pip
 Markdown,BSD,http://packages.python.org/Markdown/,pip
 PrettyTable,BSD,http://code.google.com/p/prettytable/,pip
+PyYAML,MIT,http://pyyaml.org/wiki/PyYAML,pip
 South,Apache 2.0,http://south.aeracode.org/,pip
 amqp,LGPL 2.1,http://github.com/celery/py-amqp,pip
 angular,MIT,https://github.com/angular/angular.js.git,js
@@ -27,8 +28,12 @@ bootstrap,MIT,https://github.com/twbs/bootstrap.git,js
 bootstrap-datepicker,Apache 2.0,https://github.com/eternicode/bootstrap-datepicker,js
 boto,MIT,https://github.com/boto/boto/,pip
 celery,BSD,http://celeryproject.org,pip
+cffi,MIT,http://cffi.readthedocs.org,pip
+cliff,Apache 2.0,https://launchpad.net/python-cliff,pip
+cmd2,MIT,http://packages.python.org/cmd2/,pip
 codemirror,MIT,https://github.com/codemirror/CodeMirror.git,js
 components-font-awesome,SIL Open Font License and MIT,http://fortawesome.github.io/Font-Awesome/,js
+cryptography,BSD or Apache 2.0,https://github.com/pyca/cryptography,pip
 d2to1,BSD,http://pypi.python.org/pypi/d2to1,pip
 d3,BSD,https://github.com/mbostock/d3.git,js
 distribute,PSF or ZPL,http://packages.python.org/distribute,pip
@@ -37,6 +42,7 @@ django-celery,BSD,http://celeryproject.org,pip
 django-crum,BSD,https://projects.ninemoreminutes.com/projects/django-crum/,pip
 django-extensions,MIT,http://github.com/django-extensions/django-extensions,pip
 django-jsonfield,BSD,http://bitbucket.org/schinckel/django-jsonfield/,pip
+django-polymorphic,BSD,https://github.com/chrisglass/django_polymorphic,pip
 django-qsstats-magic,MIT,http://bitbucket.org/kmike/django-qsstats-magic/,pip
 django-rest-framework-mongoengine,MIT,https://github.com/umutbozkurt/django-rest-framework-mongoengine,pip
 django-split-settings,BSD,http://github.com/2general/django-split-settings,pip
@@ -46,10 +52,16 @@ djangorestframework,BSD,http://www.django-rest-framework.org,pip
 dogpile.cache,BSD,http://bitbucket.org/zzzeek/dogpile.cache,pip
 dogpile.core,BSD,http://bitbucket.org/zzzeek/dogpile.core,pip
 ember-cli-test-loader,MIT,https://github.com/rjackson/ember-cli-test-loader,js
+enum34,BSD,https://pypi.python.org/pypi/enum34,pip
+gevent,MIT,http://www.gevent.org/,pip
 gevent-socketio,BSD,https://github.com/abourget/gevent-socketio,pip
 gevent-websocket,Apache 2.0,https://bitbucket.org/Jeffrey/gevent-websocket,pip
+greenlet,MIT,https://github.com/python-greenlet/greenlet,pip
 httplib2,MIT,https://github.com/jcgregorio/httplib2,pip
+idna,BSD,https://github.com/kjd/idna,pip
 importlib,PSF,https://pypi.python.org/pypi/importlib,pip
+ipaddress,PSF,https://github.com/phihag/ipaddress,pip
+ipy,BSD,https://github.com/autocracy/python-ipy,pip
 iso8601,MIT,https://bitbucket.org/micktwomey/pyiso8601,pip
 isodate,BSD,http://cheeseshop.python.org/pypi/isodate,pip
 jQuery.dotdotdot,MIT and GPL (*Ansible licenses via MIT),https://github.com/BeSite/jQuery.dotdotdot,js
@@ -59,19 +71,28 @@ jquery-ui,MIT,http://jqueryui.com/,js
 jqueryui,MIT,http://jqueryui.com/,js
 js-yaml,MIT,https://github.com/nodeca/js-yaml,js
 jsonlint,MIT,https://github.com/zaach/jsonlint.git,js
+jsonpatch,BSD,https://github.com/stefankoegl/python-json-patch,pip
+jsonpointer,BSD,https://github.com/stefankoegl/python-json-pointer,pip
+jsonschema,MIT,http://github.com/Julian/jsonschema,pip
 kapusta-jquery.sparkline,BSD,http://omnipotent.net/jquery.sparkline/,js
 keyring,PSF and MIT,http://bitbucket.org/kang/python-keyring-lib,pip
 kombu,BSD,http://kombu.readthedocs.org,pip
 loader.js,MIT,https://github.com/stefanpenner/loader.js,js
 lodash,MIT,https://github.com/lodash/lodash,js
 lrInfiniteScroll,MIT,https://github.com/lorenzofox3/lrInfiniteScroll,js
+lxml,BSD,http://lxml.de/,pip
 mock,BSD,http://www.voidspace.org.uk/python/mock/,pip
 moment,MIT,http://momentjs.com/,js
 mongoengine,MIT,http://mongoengine.org/,pip
+msgpack-python,Apache 2.0,http://msgpack.org/,pip
 netaddr,BSD,https://github.com/drkjam/netaddr/,pip
+netifaces,MIT,https://bitbucket.org/al45tair/netifaces,pip
 nvd3,Apache 2.0,http://www.nvd3.org,js
 ordereddict,MIT,https://pypi.python.org/pypi/ordereddict,pip
 os-client-config,Apache 2.0,http://www.openstack.org/,pip
+os-diskconfig-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_diskconfig_python_novaclient_ext,pip
+os-networksv2-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_networksv2_python_novaclient_ext,pip
+os-virtual-interfacesv2-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_virtual_interfacesv2_ext,pip
 os_diskconfig_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_diskconfig_python_novaclient_ext,pip
 os_networksv2_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_networksv2_python_novaclient_ext,pip
 os_virtual_interfacesv2_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_virtual_interfacesv2_ext,pip
@@ -82,19 +103,29 @@ oslo.utils,Apache 2.0,http://launchpad.net/oslo,pip
 pbr,Apache 2.0,http://pypi.python.org/pypi/pbr,pip
 pexpect,ISC,http://pexpect.readthedocs.org/,pip
 pip,MIT,http://www.pip-installer.org,pip
+prettytable,BSD,http://code.google.com/p/prettytable/,pip
 psphere,Apache 2.0,https://github.com/jkinred/psphere,pip
+psycopg2,LGPL with exceptions or ZPL,http://initd.org/psycopg/,pip
+pyOpenSSL,Apache 2.0,https://github.com/pyca/pyopenssl,pip
+pyasn1,BSD,http://sourceforge.net/projects/pyasn1/,pip
+pycparser,BSD,https://github.com/eliben/pycparser,pip
+pycrypto,Public domain,http://www.pycrypto.org/,pip
+pymongo,Apache 2.0,http://github.com/mongodb/mongo-python-driver,pip
+pyparsing,MIT,http://pyparsing.wikispaces.com/,pip
 pyrax,Apache 2.0,https://github.com/rackspace/pyrax,pip
 python-cinderclient,Apache 2.0,http://www.openstack.org/,pip
 python-dateutil,BSD,https://dateutil.readthedocs.org,pip
 python-glanceclient,Apache 2.0,http://www.openstack.org/,pip
 python-ironicclient,Apache 2.0,http://www.openstack.org/,pip
 python-keystoneclient,Apache 2.0,http://www.openstack.org/,pip
+python-ldap,Python style,http://www.python-ldap.org/,pip
 python-neutronclient,Apache 2.0,http://www.openstack.org/,pip
 python-novaclient,Apache 2.0,https://git.openstack.org/cgit/openstack/python-novaclient,pip
 python-swiftclient,Apache 2.0,http://www.openstack.org/,pip
 python-troveclient,Apache 2.0,http://www.openstack.org/,pip
 pytz,MIT,http://pythonhosted.org/pytz,pip
 pywinrm,MIT,http://github.com/diyan/pywinrm/,pip
+pyzmq,LGPL+BSD,http://github.com/zeromq/pyzmq,pip
 rackspace-auth-openstack,Apache 2.0,https://github.com/rackerlabs/rackspace-auth-openstack,pip
 rackspace-novaclient,Apache 2.0,https://github.com/rackerlabs/rackspace-novaclient,pip
 rax_default_network_flags_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/rax_default_network_flags_python_novaclient_ext,pip
@@ -112,7 +143,11 @@ sizzle,MIT,https://github.com/jquery/sizzle.git,js
 socket.io-client,MIT,https://github.com/Automattic/socket.io-client.git,js
 stevedore,Apache 2.0,https://github.com/dreamhost/stevedore,pip
 suds,LGPL 3,https://fedorahosted.org/suds,pip
+superlance,BSD,http://supervisord.org,pip
 timezone-js,Apache 2.0,https://github.com/mde/timezone-js.git,js
 twitter,MIT,http://getbootstrap.com,js
 underscore,MIT,https://github.com/jashkenas/underscore,js
+warlock,Apache 2.0,http://github.com/bcwaldon/warlock,pip
+wheel,MIT,http://bitbucket.org/pypa/wheel/,pip
+wsgiref,PSF or ZPL,http://cheeseshop.python.org/pypi/wsgiref,pip
 xmltodict,MIT,https://github.com/martinblech/xmltodict,pip

tools/license-audit/license-audit.py

@@ -17,52 +17,55 @@ def usage():
     sys.exit(1)
 
 def read_requirements(towerpath):
-    filename = '%s/awx/lib/site-packages/README' % (towerpath,)
+    filename = '%s/requirements/requirements.txt' % (towerpath,)
     ret = {}
-    f = open(filename)
+    try:
-    if not f:
+        f = open(filename)
+    except:
         print "failed to open %s" %(filename,)
         return None
     data = f.readlines()
     f.close()
     for line in data:
+        if line[0] == '#':
+            continue
         if '==' in line:
-            m = re.match(r"(\S+)==(\S+) \((\S+)",line)
+            m = re.match(r"(\S+)==(\S+)",line)
             if m:
                 name = m.group(1)
                 version = m.group(2)
-                pathname = m.group(3)
-                if pathname.endswith(',') or pathname.endswith(')'):
-                    pathname = pathname[:-1]
-                if pathname.endswith('/*'):
-                    pathname = pathname[:-2]
                 item = {}
                 item['name'] = name
                 item['version'] = version
-                item['path'] = pathname
                 ret[name] = item
-    return ret
+                continue
-
+        elif line.startswith("git+https"):
-def get_python(towerpath):
+            l = line.rsplit('/',1)
-    excludes = [ 
+            m = re.match(r"(\S+).git@(\S+)#",l[1])
-        'README*', 
+            if m:
-        '*.dist-info', 
+                name = m.group(1)
-        'funtests', 
+                version = m.group(2)
-        'easy_install.py', 
+                if version.startswith('tower_'):
-        'oslo', 
+                    version = version[6:]
-        'pkg_resources', 
+                item = {}
-        '_markerlib'
+                if name == 'python-ipy':
-    ]
+                    name='ipy'
-    directory = '%s/awx/lib/site-packages' % (towerpath,)
+                item['name'] = name
-    dirlist = os.listdir(directory)
+                item['version'] = version
-    ret = []
+                if len(version) > 20:
-    for item in dirlist:
+                    # it's a sha1sum, read it off the egg spec
-        use = True
+                    lver = l[1].rsplit('-',1)
-        for exclude in excludes:
+                    if lver[1] == l[1]:
-            if fnmatch.fnmatch(item, exclude):
+                        lver = l[1].rsplit('_',1)
-                use = False
+                    item['version'] = lver[1][:-1]
-        if use:
+                ret[name] = item
-            ret.append(item)
+                continue
+        else:
+            item = {}
+            item['name'] = line[:-1]
+            item['version'] = ''
+            ret[name] = item
+            continue
     return ret
 
 def get_js(towerpath):
@@ -116,13 +119,13 @@ def normalize_license(license):
     license = license.replace('"','')
     if license == 'None':
         return 'UNKNOWN'
-    if license in ['Apache License, Version 2.0', 'Apache License (2.0)', 'Apache License 2.0', 'Apache-2.0', 'Apache License, v2.0']:
+    if license in ['Apache License, Version 2.0', 'Apache License (2.0)', 'Apache License 2.0', 'Apache-2.0', 'Apache License, v2.0', 'APL2']:
         return 'Apache 2.0'
     if license == 'ISC license':
         return 'ISC'
     if license == 'MIT License' or license == 'MIT license':
         return 'MIT'
-    if license == 'BSD License' or license == 'Simplified BSD':
+    if license in ['BSD License', 'Simplified BSD', 'BSD-derived (http://www.repoze.org/LICENSE.txt)', 'BSD-like', 'Modified BSD License']:
         return 'BSD'
     if license == 'LGPL':
         return 'LGPL 2.1'
@@ -131,6 +134,10 @@ def normalize_license(license):
         return 'Apache 2.0'
     if license.find('https://github.com/umutbozkurt/django-rest-framework-mongoengine/blob/master/LICENSE') != -1:
         return 'MIT'
+    if license == '"BSD or Apache License, Version 2.0"':
+        return 'BSD or Apache 2.0'
+    if license == 'Modified BSD License':
+        return 'BSD'
     if license == 'Python Software Foundation License':
         return 'PSF'
     return license
@@ -183,24 +190,11 @@ if not olddata or not requirements:
     print "No starting data"
     sys.exit(1)
 
-# Get directory of vendored things from site-packages...
-python_packages = get_python(tower_path)
-
-# ... and ensure they're noted in the requirements file
-ok = True
-for package in python_packages:
-    if not search_requirements(requirements, package):
-        print "%s not in requirements!" % (package,)
-        ok = False
-if not ok:
-    sys.exit(1)
-
-
 # See if there's pip things in our current license list that we don't have now
 reqs = requirements.keys()
 for item in olddata.values():
     if item['source'] == 'pip' and item['name'] not in reqs:
-        print "No longer vendoring %s" %(item['name'],)
+        print "Potentially no longer vendoring %s" %(item['name'],)
 
 # Get directory of vendored JS things from the js dir
 js_packages = get_js(tower_path)
@@ -214,10 +208,12 @@ for item in olddata.values():
 # Take the requirements file, and get license information where necessary
 cs = yolk.pypi.CheeseShop()
 for req in requirements.values():
-    cs_info = cs.release_data(req['name'],req['version'])
+    # name sanitization
+    (pname, pvers) = cs.query_versions_pypi(req['name'])
+    cs_info = cs.release_data(pname,req['version'])
     if not cs_info:
-        print "Couldn't find '%s-%s'" %(req['name'],req['version'])
+        print "Couldn't find '%s==%s'" %(req['name'],req['version'])
-        if 'name' not in olddata:
+        if req['name'] not in olddata:
             print "... and it's not in the current data. This needs fixed!"
             sys.exit(1)
         continue
@@ -241,7 +237,7 @@ for req in requirements.values():
 
 # Update JS package info
 for pkg in js:
-    if 'pkg' in olddata:
+    if pkg in olddata:
         data = olddata[pkg]
         new = js_packages[pkg]
         if new['license'] != 'UNKNOWN' and new['license'] != data['license']: