diff --git a/.gitignore b/.gitignore index b52de3762d..b7fcfcd15a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ __pycache__ /deb-build /rpm-build /tar-build +/offline_tar-build /dist *.egg-info *.py[c,o] diff --git a/MANIFEST.in b/MANIFEST.in index 211971e20f..64382e32a8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,9 +6,8 @@ recursive-include awx/ui/templates *.html recursive-include awx/ui/static * recursive-include awx/playbooks *.yml recursive-include awx/lib/site-packages * +recursive-include requirements *.txt recursive-include config * -recursive-include config/deb * -recursive-include config/rpm * recursive-exclude awx devonly.py* recursive-exclude awx/api/tests * recursive-exclude awx/main/tests * @@ -19,5 +18,6 @@ include tools/scripts/ansible-tower-service include tools/munin_monitors/* include tools/sosreport/* include COPYING +include Makefile prune awx/public prune awx/projects diff --git a/Makefile b/Makefile index a60b0cdeed..bc5f658a64 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ TESTEM ?= ./node_modules/.bin/testem BROCCOLI_BIN ?= ./node_modules/.bin/broccoli MOCHA_BIN ?= ./node_modules/.bin/mocha NODE ?= node +DEPS_SCRIPT ?= packaging/offline/deps.py +AW_REPO_URL ?= "http://releases.ansible.com/ansible-tower" CLIENT_TEST_DIR ?= build_test @@ -73,15 +75,34 @@ else endif DEBUILD = $(DEBUILD_BIN) $(DEBUILD_OPTS) DEB_PPA ?= reprepro +DEB_ARCH ?= amd64 # RPM build parameters RPM_SPECDIR= packaging/rpm RPM_SPEC = $(RPM_SPECDIR)/$(NAME).spec RPM_DIST ?= $(shell rpm --eval '%{?dist}' 2>/dev/null) +RPM_ARCH ?= $(shell rpm --eval '%{_arch}' 2>/dev/null) RPM_NVR = $(NAME)-$(VERSION)-$(RELEASE)$(RPM_DIST) MOCK_BIN ?= mock MOCK_CFG ?= +# Offline TAR build parameters +DIST = $(shell echo $(RPM_DIST) | sed -e 's|^\.\(el\)\([0-9]\).*|\1|') +DIST_MAJOR = $(shell echo $(RPM_DIST) | sed -e 's|^\.\(el\)\([0-9]\).*|\2|') +DIST_FULL = $(DIST)$(DIST_MAJOR) +OFFLINE_TAR_NAME = $(NAME)-offline-$(DIST_FULL)-$(VERSION)-$(RELEASE) +OFFLINE_TAR_FILE = $(OFFLINE_TAR_NAME).tar.gz +OFFLINE_TAR_LINK = $(NAME)-offline-$(DIST_FULL)-latest.tar.gz + +DISTRO := $(shell . /etc/os-release 2>/dev/null && echo $${ID} || echo redhat) +ifeq ($(DISTRO),ubuntu) + SETUP_INSTALL_ARGS = --skip-build --no-compile --root=$(DESTDIR) -v --install-layout=deb +else + SETUP_INSTALL_ARGS = --skip-build --no-compile --root=$(DESTDIR) -v +endif + +.DEFAULT_GOAL := build + .PHONY: clean rebase push requirements requirements_dev requirements_jenkins \ real-requirements real-requirements_dev real-requirements_jenkins \ develop refresh adduser syncdb migrate dbchange dbshell runserver celeryd \ @@ -89,7 +110,8 @@ MOCK_CFG ?= release_build release_clean sdist rpmtar mock-rpm mock-srpm rpm-sign \ devjs minjs testjs testjs_ci node-tests browser-tests jshint ngdocs sync_ui \ deb deb-src debian reprepro setup_tarball \ - virtualbox-ovf virtualbox-centos-7 virtualbox-centos-6 + virtualbox-ovf virtualbox-centos-7 virtualbox-centos-6 \ + clean-offline setup_offline_tarball # Remove setup build files clean-tar: @@ -124,6 +146,9 @@ clean-packer: rm -rf packaging/packer/ansible-tower*-ova rm -f Vagrantfile +clean-offline: + rm -rf offline-tar-build + # Remove temporary build files, compiled Python files. clean: clean-rpm clean-deb clean-grunt clean-ui clean-tar clean-packer rm -rf awx/lib/site-packages @@ -140,21 +165,22 @@ push: git push origin master # Install runtime, development and jenkins requirements -requirements requirements_dev requirements_jenkins: %: real-% awx/lib/site-packages/oslo/__init__.py awx/lib/site-packages/dogpile/__init__.py - -# Create missing __init__.py files -awx/lib/site-packages/%/__init__.py: - touch $@ +requirements requirements_dev requirements_jenkins: %: real-% # Install third-party requirements needed for development environment. +# NOTE: +# * --target is only supported on newer versions of pip +# * https://github.com/pypa/pip/issues/3056 - the workaround is to override the `install-platlib` +# * --user (in conjunction with PYTHONUSERBASE="awx" may be a better option +# * --target implies --ignore-installed real-requirements: - pip install -r requirements/requirements.txt --target awx/lib/site-packages/ --ignore-installed + pip install -r requirements/requirements.txt --target awx/lib/site-packages/ --install-option="--install-platlib=\$$base/lib/python" real-requirements_dev: - pip install -r requirements/requirements_dev.txt --target awx/lib/site-packages/ --ignore-installed + pip install -r requirements/requirements_dev.txt --target awx/lib/site-packages/ --install-option="--install-platlib=\$$base/lib/python" # Install third-party requirements needed for running unittests in jenkins -real-requirements_jenkins: real-requirements +real-requirements_jenkins: pip install -r requirements/requirements_jenkins.txt npm install csslint jshint @@ -373,14 +399,14 @@ tar-build/$(SETUP_TAR_FILE): @cd tar-build/$(SETUP_TAR_NAME) && sed -e 's#%NAME%#$(NAME)#;s#%VERSION%#$(VERSION)#;s#%RELEASE%#$(RELEASE)#;' group_vars/all.in > group_vars/all @cd tar-build && tar -czf $(SETUP_TAR_FILE) --exclude "*/all.in" $(SETUP_TAR_NAME)/ @ln -sf $(SETUP_TAR_FILE) tar-build/$(SETUP_TAR_LINK) + +setup_tarball: tar-build/$(SETUP_TAR_FILE) @echo "#############################################" @echo "Setup artifacts:" @echo tar-build/$(SETUP_TAR_FILE) @echo tar-build/$(SETUP_TAR_LINK) @echo "#############################################" -setup_tarball: tar-build/$(SETUP_TAR_FILE) - release_clean: -(rm *.tar) -(rm -rf ($RELEASE)) @@ -390,8 +416,30 @@ dist/$(SDIST_TAR_FILE): sdist: minjs requirements dist/$(SDIST_TAR_FILE) +# Build setup offline tarball +offline-tar-build: + mkdir -p $@ + +offline-tar-build/$(DIST_FULL): + mkdir -p $@ + +# TODO - Somehow share implementation with setup_tarball +offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_FILE): + cp -a setup offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_NAME) + cd offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_NAME) && sed -e 's#%NAME%#$(NAME)#;s#%VERSION%#$(VERSION)#;s#%RELEASE%#$(RELEASE)#;' group_vars/all.in > group_vars/all + $(PYTHON) $(DEPS_SCRIPT) -d $(DIST) -r $(DIST_MAJOR) -u $(AW_REPO_URL) -s offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_NAME) -v -v -v + cd offline-tar-build/$(DIST_FULL) && tar -czf $(OFFLINE_TAR_FILE) --exclude "*/all.in" $(OFFLINE_TAR_NAME)/ + ln -sf $(OFFLINE_TAR_FILE) offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_LINK) + +setup_offline_tarball: offline-tar-build offline-tar-build/$(DIST_FULL) offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_FILE) + @echo "#############################################" + @echo "Offline artifacts:" + @echo offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_FILE) + @echo offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_LINK) + @echo "#############################################" + rpm-build: - mkdir -p rpm-build + mkdir -p $@ rpm-build/$(SDIST_TAR_FILE): rpm-build dist/$(SDIST_TAR_FILE) cp packaging/rpm/$(NAME).spec rpm-build/ @@ -419,22 +467,22 @@ rpm-build/$(RPM_NVR).src.rpm: /etc/mock/$(MOCK_CFG).cfg mock-srpm: rpmtar rpm-build/$(RPM_NVR).src.rpm -rpm-build/$(RPM_NVR).noarch.rpm: rpm-build/$(RPM_NVR).src.rpm +rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm: rpm-build/$(RPM_NVR).src.rpm $(MOCK_BIN) -r $(MOCK_CFG) --resultdir rpm-build --rebuild rpm-build/$(RPM_NVR).src.rpm \ --define "tower_version $(VERSION)" --define "tower_release $(RELEASE)" @echo "#############################################" @echo "RPM artifacts:" - @echo rpm-build/$(RPM_NVR).noarch.rpm + @echo rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm @echo "#############################################" -mock-rpm: rpmtar rpm-build/$(RPM_NVR).noarch.rpm +mock-rpm: rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm ifeq ($(OFFICIAL),yes) rpm-build/$(GPG_FILE): rpm-build gpg --export -a "${GPG_KEY}" > "$@" -rpm-sign: rpm-build/$(GPG_FILE) rpmtar rpm-build/$(RPM_NVR).noarch.rpm - rpm --define "_signature gpg" --define "_gpg_name $(GPG_KEY)" --addsign rpm-build/$(RPM_NVR).noarch.rpm +rpm-sign: rpm-build/$(GPG_FILE) rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm + rpm --define "_signature gpg" --define "_gpg_name $(GPG_KEY)" --addsign rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm endif deb-build/$(SDIST_TAR_NAME): @@ -446,14 +494,14 @@ deb-build/$(SDIST_TAR_NAME): debian: sdist deb-build/$(SDIST_TAR_NAME) -deb-build/$(NAME)_$(VERSION)-$(RELEASE)_all.deb: +deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb: cd deb-build/$(SDIST_TAR_NAME) && $(DEBUILD) -b @echo "#############################################" @echo "DEB artifacts:" - @echo deb-build/$(NAME)_$(VERSION)-$(RELEASE)_all.deb + @echo deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb @echo "#############################################" -deb: debian deb-build/$(NAME)_$(VERSION)-$(RELEASE)_all.deb +deb: debian deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb deb-build/$(NAME)_$(VERSION)-$(RELEASE)_source.changes: cd deb-build/$(SDIST_TAR_NAME) && $(DEBUILD) -S @@ -465,7 +513,7 @@ deb-build/$(NAME)_$(VERSION)-$(RELEASE)_source.changes: deb-src: debian deb-build/$(NAME)_$(VERSION)-$(RELEASE)_source.changes deb-upload: deb - $(DPUT_BIN) $(DPUT_OPTS) $(DEB_PPA) deb-build/$(NAME)_$(VERSION)-$(RELEASE)_amd64.changes ; \ + $(DPUT_BIN) $(DPUT_OPTS) $(DEB_PPA) deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).changes ; \ deb-src-upload: deb-src $(DPUT_BIN) $(DPUT_OPTS) $(DEB_PPA) deb-build/$(NAME)_$(VERSION)-$(RELEASE)_source.changes ; \ @@ -473,7 +521,7 @@ deb-src-upload: deb-src reprepro: deb mkdir -p reprepro/conf cp -a packaging/reprepro/* reprepro/conf/ - @DEB=deb-build/$(NAME)_$(VERSION)-$(RELEASE)_all.deb ; \ + @DEB=deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb ; \ for DIST in trusty precise ; do \ echo "Removing '$(NAME)' from the $${DIST} apt repo" ; \ echo reprepro --export=force -b reprepro remove $${DIST} $(NAME) ; \ @@ -505,5 +553,11 @@ virtualbox-centos-7: packaging/packer/output-virtualbox-iso/centos-7.ovf docker-dev: docker build --no-cache=true --rm=true -t ansible/tower_devel:latest tools/docker +# TODO - figure out how to build the front-end and python requirements with +# 'build' +build: + $(PYTHON) setup.py build + +# TODO - only use --install-layout=deb on Debian install: - $(PYTHON) setup.py install egg_info -b "" + $(PYTHON) setup.py install $(SETUP_INSTALL_ARGS) diff --git a/awx/__init__.py b/awx/__init__.py index b0e5709abe..424337a49c 100644 --- a/awx/__init__.py +++ b/awx/__init__.py @@ -4,6 +4,7 @@ import os import sys import warnings +import site __version__ = '2.4.0' @@ -39,7 +40,10 @@ def prepare_env(): # Add local site-packages directory to path. local_site_packages = os.path.join(os.path.dirname(__file__), 'lib', 'site-packages') - sys.path.insert(0, local_site_packages) + site.addsitedir(local_site_packages) + # Work around https://bugs.python.org/issue7744 + # by moving local_site_packages to the front of sys.path + sys.path.insert(0, sys.path.pop()) # Hide DeprecationWarnings when running in production. Need to first load # settings to apply our filter after Django's own warnings filter. from django.conf import settings diff --git a/awx/api/views.py b/awx/api/views.py index 9598d21b42..95a58cca24 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -2839,6 +2839,8 @@ class UnifiedJobStdout(RetrieveAPIView): return response except Exception, e: return Response({"error": "Error generating stdout download file: %s" % str(e)}, status=status.HTTP_400_BAD_REQUEST) + elif request.accepted_renderer.format == 'txt': + return Response(unified_job.result_stdout) else: return super(UnifiedJobStdout, self).retrieve(request, *args, **kwargs) diff --git a/awx/main/management/commands/run_fact_cache_receiver.py b/awx/main/management/commands/run_fact_cache_receiver.py index ebabaee86b..aa3abe1bfd 100644 --- a/awx/main/management/commands/run_fact_cache_receiver.py +++ b/awx/main/management/commands/run_fact_cache_receiver.py @@ -3,6 +3,7 @@ # Python import logging +from threading import Thread from datetime import datetime # Django @@ -76,14 +77,18 @@ class FactCacheReceiver(object): (fact_obj, version_obj) = Fact.add_fact(self.timestamp, facts, host, module) logger.info('Created new fact <%s, %s>' % (fact_obj.id, version_obj.id)) - def run_receiver(self): + def run_receiver(self, use_processing_threads=True): with Socket('fact_cache', 'r') as facts: for message in facts.listen(): if 'host' not in message or 'facts' not in message or 'date_key' not in message: logger.warn('Received invalid message %s' % message) continue logger.info('Received message %s' % message) - self.process_fact_message(message) + if use_processing_threads: + wt = Thread(target=self.process_fact_message, args=(message,)) + wt.start() + else: + self.process_fact_message(message) class Command(NoArgsCommand): ''' diff --git a/awx/main/migrations/0070_v221_changes.py b/awx/main/migrations/0070_v221_changes.py index 698028648e..0bc36f27ac 100644 --- a/awx/main/migrations/0070_v221_changes.py +++ b/awx/main/migrations/0070_v221_changes.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import uuid import os +import subprocess from south.utils import datetime_utils as datetime from south.db import db @@ -20,6 +21,8 @@ class Migration(DataMigration): j.result_stdout_file = stdout_filename j.result_stdout_text = "" j.save() + sed_command = subprocess.Popen(["sed", "-i", "-e", "s/\\\\r\\\\n/\\n/g", stdout_filename]) + sed_command.wait() def backwards(self, orm): pass diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index d86a500528..4fd165339f 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1259,7 +1259,7 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions): if not super(InventoryUpdate, self).can_start: return False - if (self.source != 'custom' and + if (self.source not in ('custom', 'ec2') and not (self.credential and self.credential.active)): return False return True diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index 3bbbe55fa9..b90e29cd85 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -357,6 +357,16 @@ class ProjectUpdate(UnifiedJob, ProjectOptions): def result_stdout(self): return self._result_stdout_raw(redact_sensitive=True, escape_ascii=True) + @property + def result_stdout_raw(self): + return self._result_stdout_raw(redact_sensitive=True) + + def result_stdout_raw_limited(self, start_line=0, end_line=None, redact_sensitive=True): + return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive=redact_sensitive) + + def result_stdout_limited(self, start_line=0, end_line=None, redact_sensitive=True): + return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive=redact_sensitive, escape_ascii=True) + def get_absolute_url(self): return reverse('api:project_update_detail', args=(self.pk,)) diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index b2afb9df54..2a7875d321 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -671,11 +671,11 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique return return_buffer, start_actual, end_actual, absolute_end - def result_stdout_raw_limited(self, start_line=0, end_line=None): - return self._result_stdout_raw_limited(start_line, end_line) + def result_stdout_raw_limited(self, start_line=0, end_line=None, redact_sensitive=False): + return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive) - def result_stdout_limited(self, start_line=0, end_line=None): - return self._result_stdout_raw_limited(start_line, end_line, escape_ascii=True) + def result_stdout_limited(self, start_line=0, end_line=None, redact_sensitive=False): + return self._result_stdout_raw_limited(start_line, end_line, redact_sensitive, escape_ascii=True) @property def celery_task(self): diff --git a/awx/main/tests/base.py b/awx/main/tests/base.py index 5e3669df9e..771eac2fbf 100644 --- a/awx/main/tests/base.py +++ b/awx/main/tests/base.py @@ -627,7 +627,7 @@ class BaseTestMixin(QueueTestMixin, MockCommonlySlowTestMixin): msg += '"%s" found in: "%s"' % (substr, string) self.assertEqual(count, 0, msg) - def check_found(self, string, substr, count, description=None, word_boundary=False): + def check_found(self, string, substr, count=-1, description=None, word_boundary=False): if word_boundary: count_actual = len(re.findall(r'\b%s\b' % re.escape(substr), string)) else: @@ -636,8 +636,11 @@ class BaseTestMixin(QueueTestMixin, MockCommonlySlowTestMixin): msg = '' if description: msg = 'Test "%s".\n' % description - msg += 'Found %d occurances of "%s" instead of %d in: "%s"' % (count_actual, substr, count, string) - self.assertEqual(count_actual, count, msg) + if count == -1: + self.assertTrue(count_actual > 0) + else: + msg += 'Found %d occurances of "%s" instead of %d in: "%s"' % (count_actual, substr, count, string) + self.assertEqual(count_actual, count, msg) def check_job_result(self, job, expected='successful', expect_stdout=True, expect_traceback=False): diff --git a/awx/main/tests/commands/run_fact_cache_receiver.py b/awx/main/tests/commands/run_fact_cache_receiver.py index b0fe0aa2bb..0dac618758 100644 --- a/awx/main/tests/commands/run_fact_cache_receiver.py +++ b/awx/main/tests/commands/run_fact_cache_receiver.py @@ -142,7 +142,7 @@ class RunFactCacheReceiverUnitTest(BaseTest, MongoDBRequired): receiver = FactCacheReceiver() receiver.process_fact_message = MagicMock(name='process_fact_message') - receiver.run_receiver() + receiver.run_receiver(use_processing_threads=False) receiver.process_fact_message.assert_called_once_with(TEST_MSG) diff --git a/awx/main/tests/inventory.py b/awx/main/tests/inventory.py index 27590ec988..b38ca13d82 100644 --- a/awx/main/tests/inventory.py +++ b/awx/main/tests/inventory.py @@ -1665,6 +1665,17 @@ class InventoryUpdatesTest(BaseTransactionTest): inventory_source.save() self.check_inventory_source(inventory_source, initial=False) + def test_update_from_ec2_without_credential(self): + self.create_test_license_file() + group = self.group + group.name = 'ec2' + group.save() + self.group = group + cache_path = tempfile.mkdtemp(prefix='awx_ec2_') + self._temp_paths.append(cache_path) + inventory_source = self.update_inventory_source(self.group, source='ec2') + self.check_inventory_update(inventory_source, should_fail=True) + def test_update_from_ec2_with_nested_groups(self): source_username = getattr(settings, 'TEST_AWS_ACCESS_KEY_ID', '') source_password = getattr(settings, 'TEST_AWS_SECRET_ACCESS_KEY', '') diff --git a/awx/main/tests/views.py b/awx/main/tests/views.py index 629b1e1fbc..6903b1ebf8 100644 --- a/awx/main/tests/views.py +++ b/awx/main/tests/views.py @@ -4,6 +4,7 @@ from django.core.urlresolvers import reverse # Reuse Test code from awx.main.tests.base import BaseLiveServerTest, QueueStartStopTestMixin from awx.main.tests.base import URI +from awx.main.models.projects import * # noqa __all__ = ['UnifiedJobStdoutRedactedTests'] @@ -32,12 +33,20 @@ class UnifiedJobStdoutRedactedTests(BaseLiveServerTest, QueueStartStopTestMixin) self.setup_instances() self.setup_users() self.test_cases = [] + self.negative_test_cases = [] + + proj = self.make_project() for e in TEST_STDOUTS: - e['job'] = self.make_job() - e['job'].result_stdout_text = e['text'] - e['job'].save() + e['project'] = ProjectUpdate(project=proj) + e['project'].result_stdout_text = e['text'] + e['project'].save() self.test_cases.append(e) + for d in TEST_STDOUTS: + d['job'] = self.make_job() + d['job'].result_stdout_text = d['text'] + d['job'].save() + self.negative_test_cases.append(d) # This is more of a functional test than a unit test. # should filter out username and password @@ -49,7 +58,13 @@ class UnifiedJobStdoutRedactedTests(BaseLiveServerTest, QueueStartStopTestMixin) # Ensure the host didn't get redacted self.check_found(response['content'], uri.host, test_data['occurrences'], test_data['description']) - def _get_url_job_stdout(self, job, format='json'): + def check_sensitive_not_redacted(self, test_data, response): + uri = test_data['uri'] + self.assertIsNotNone(response['content']) + self.check_found(response['content'], uri.username, description=test_data['description']) + self.check_found(response['content'], uri.password, description=test_data['description']) + + def _get_url_job_stdout(self, job, url_base, format='json'): formats = { 'json': 'application/json', 'ansi': 'text/plain', @@ -57,22 +72,39 @@ class UnifiedJobStdoutRedactedTests(BaseLiveServerTest, QueueStartStopTestMixin) 'html': 'text/html', } content_type = formats[format] - job_stdout_url = reverse('api:job_stdout', args=(job.pk,)) + "?format=" + format - return self.get(job_stdout_url, expect=200, auth=self.get_super_credentials(), accept=content_type) + project_update_stdout_url = reverse(url_base, args=(job.pk,)) + "?format=" + format + return self.get(project_update_stdout_url, expect=200, auth=self.get_super_credentials(), accept=content_type) def _test_redaction_enabled(self, format): for test_data in self.test_cases: - response = self._get_url_job_stdout(test_data['job'], format=format) + response = self._get_url_job_stdout(test_data['project'], "api:project_update_stdout", format=format) self.check_sensitive_redacted(test_data, response) - def test_redaction_enabled_json(self): + def _test_redaction_disabled(self, format): + for test_data in self.negative_test_cases: + response = self._get_url_job_stdout(test_data['job'], "api:job_stdout", format=format) + self.check_sensitive_not_redacted(test_data, response) + + def test_project_update_redaction_enabled_json(self): self._test_redaction_enabled('json') - def test_redaction_enabled_ansi(self): + def test_project_update_redaction_enabled_ansi(self): self._test_redaction_enabled('ansi') - def test_redaction_enabled_html(self): + def test_project_update_redaction_enabled_html(self): self._test_redaction_enabled('html') - def test_redaction_enabled_txt(self): + def test_project_update_redaction_enabled_txt(self): self._test_redaction_enabled('txt') + + def test_job_redaction_disabled_json(self): + self._test_redaction_disabled('json') + + def test_job_redaction_disabled_ansi(self): + self._test_redaction_disabled('ansi') + + def test_job_redaction_disabled_html(self): + self._test_redaction_disabled('html') + + def test_job_redaction_disabled_txt(self): + self._test_redaction_disabled('txt') diff --git a/awx/main/utils.py b/awx/main/utils.py index 1f75d23aa8..202e9989fe 100644 --- a/awx/main/utils.py +++ b/awx/main/utils.py @@ -446,7 +446,8 @@ def build_proot_temp_dir(): ''' Create a temporary directory for proot to use. ''' - path = tempfile.mkdtemp(prefix='ansible_tower_proot_') + from django.conf import settings + path = tempfile.mkdtemp(prefix='ansible_tower_proot_', dir=settings.AWX_PROOT_BASE_PATH) os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) return path diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 7b0464bec0..ad2412f78c 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -352,6 +352,9 @@ AWX_PROOT_SHOW_PATHS = [] # Number of jobs to show as part of the job template history AWX_JOB_TEMPLATE_HISTORY = 10 +# The directory in which proot will create new temporary directories for its root +AWX_PROOT_BASE_PATH = "/tmp" + # Default list of modules allowed for ad hoc commands. AD_HOC_COMMANDS = [ 'command', diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 3764e1fd6d..f51512da0d 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -35,6 +35,7 @@ git+https://github.com/chrismeyersfsu/gevent-socketio.git@tower_0.3.6#egg=socket git+https://github.com/chrismeyersfsu/python-ipy.git@master#egg=ipy git+https://github.com/chrismeyersfsu/python-keystoneclient.git@1.3.0#egg=keystoneclient-1.3.0 git+https://github.com/chrismeyersfsu/shade.git@tower_0.5.0#egg=shade-tower_0.5.0 +git+https://github.com/chrismeyersfsu/sitecustomize.git#egg=sitecustomize greenlet==0.4.7 httplib2==0.9 idna==2.0 diff --git a/requirements/requirements_dev.txt b/requirements/requirements_dev.txt index 956ee56403..9c0d7f83ae 100644 --- a/requirements/requirements_dev.txt +++ b/requirements/requirements_dev.txt @@ -2,3 +2,5 @@ django-devserver django-debug-toolbar unittest2 +pep8 +flake8 diff --git a/requirements/requirements_jenkins.txt b/requirements/requirements_jenkins.txt index b26f2fb92f..8ded2f06cd 100644 --- a/requirements/requirements_jenkins.txt +++ b/requirements/requirements_jenkins.txt @@ -1,3 +1,4 @@ +-r requirements.txt ansible django-jenkins coverage diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 2cbf2bc050..f90618f97d 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -6,7 +6,7 @@ ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 RUN apt-get update RUN apt-get install -y software-properties-common python-software-properties curl -RUN add-apt-repository -y ppa:chris-lea/redis-server; add-apt-repository -y ppa:chris-lea/zeromq; add-apt-repository ppa:ansible/ansible +RUN add-apt-repository -y ppa:chris-lea/redis-server; add-apt-repository -y ppa:chris-lea/zeromq; add-apt-repository -y ppa:chris-lea/node.js; add-apt-repository ppa:ansible/ansible RUN curl -sL https://deb.nodesource.com/setup_0.12 | bash - RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10 && apt-key adv --fetch-keys http://www.postgresql.org/media/keys/ACCC4CF8.asc RUN echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.0.list && echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" | tee /etc/apt/sources.list.d/postgres-9.4.list diff --git a/tools/license-audit/TowerLicenses.csv b/tools/license-audit/TowerLicenses.csv index dbec5d62b3..5a06eb30ce 100644 --- a/tools/license-audit/TowerLicenses.csv +++ b/tools/license-audit/TowerLicenses.csv @@ -1,7 +1,8 @@ Babel,BSD,http://babel.pocoo.org/,pip -IPy,BSD,https://github.com/autocracy/python-ipy,pip +Django,BSD,http://www.djangoproject.com/,pip Markdown,BSD,http://packages.python.org/Markdown/,pip PrettyTable,BSD,http://code.google.com/p/prettytable/,pip +PyYAML,MIT,http://pyyaml.org/wiki/PyYAML,pip South,Apache 2.0,http://south.aeracode.org/,pip amqp,LGPL 2.1,http://github.com/celery/py-amqp,pip angular,MIT,https://github.com/angular/angular.js.git,js @@ -27,8 +28,12 @@ bootstrap,MIT,https://github.com/twbs/bootstrap.git,js bootstrap-datepicker,Apache 2.0,https://github.com/eternicode/bootstrap-datepicker,js boto,MIT,https://github.com/boto/boto/,pip celery,BSD,http://celeryproject.org,pip +cffi,MIT,http://cffi.readthedocs.org,pip +cliff,Apache 2.0,https://launchpad.net/python-cliff,pip +cmd2,MIT,http://packages.python.org/cmd2/,pip codemirror,MIT,https://github.com/codemirror/CodeMirror.git,js components-font-awesome,SIL Open Font License and MIT,http://fortawesome.github.io/Font-Awesome/,js +cryptography,BSD or Apache 2.0,https://github.com/pyca/cryptography,pip d2to1,BSD,http://pypi.python.org/pypi/d2to1,pip d3,BSD,https://github.com/mbostock/d3.git,js distribute,PSF or ZPL,http://packages.python.org/distribute,pip @@ -37,6 +42,7 @@ django-celery,BSD,http://celeryproject.org,pip django-crum,BSD,https://projects.ninemoreminutes.com/projects/django-crum/,pip django-extensions,MIT,http://github.com/django-extensions/django-extensions,pip django-jsonfield,BSD,http://bitbucket.org/schinckel/django-jsonfield/,pip +django-polymorphic,BSD,https://github.com/chrisglass/django_polymorphic,pip django-qsstats-magic,MIT,http://bitbucket.org/kmike/django-qsstats-magic/,pip django-rest-framework-mongoengine,MIT,https://github.com/umutbozkurt/django-rest-framework-mongoengine,pip django-split-settings,BSD,http://github.com/2general/django-split-settings,pip @@ -46,10 +52,16 @@ djangorestframework,BSD,http://www.django-rest-framework.org,pip dogpile.cache,BSD,http://bitbucket.org/zzzeek/dogpile.cache,pip dogpile.core,BSD,http://bitbucket.org/zzzeek/dogpile.core,pip ember-cli-test-loader,MIT,https://github.com/rjackson/ember-cli-test-loader,js +enum34,BSD,https://pypi.python.org/pypi/enum34,pip +gevent,MIT,http://www.gevent.org/,pip gevent-socketio,BSD,https://github.com/abourget/gevent-socketio,pip gevent-websocket,Apache 2.0,https://bitbucket.org/Jeffrey/gevent-websocket,pip +greenlet,MIT,https://github.com/python-greenlet/greenlet,pip httplib2,MIT,https://github.com/jcgregorio/httplib2,pip +idna,BSD,https://github.com/kjd/idna,pip importlib,PSF,https://pypi.python.org/pypi/importlib,pip +ipaddress,PSF,https://github.com/phihag/ipaddress,pip +ipy,BSD,https://github.com/autocracy/python-ipy,pip iso8601,MIT,https://bitbucket.org/micktwomey/pyiso8601,pip isodate,BSD,http://cheeseshop.python.org/pypi/isodate,pip jQuery.dotdotdot,MIT and GPL (*Ansible licenses via MIT),https://github.com/BeSite/jQuery.dotdotdot,js @@ -59,19 +71,28 @@ jquery-ui,MIT,http://jqueryui.com/,js jqueryui,MIT,http://jqueryui.com/,js js-yaml,MIT,https://github.com/nodeca/js-yaml,js jsonlint,MIT,https://github.com/zaach/jsonlint.git,js +jsonpatch,BSD,https://github.com/stefankoegl/python-json-patch,pip +jsonpointer,BSD,https://github.com/stefankoegl/python-json-pointer,pip +jsonschema,MIT,http://github.com/Julian/jsonschema,pip kapusta-jquery.sparkline,BSD,http://omnipotent.net/jquery.sparkline/,js keyring,PSF and MIT,http://bitbucket.org/kang/python-keyring-lib,pip kombu,BSD,http://kombu.readthedocs.org,pip loader.js,MIT,https://github.com/stefanpenner/loader.js,js lodash,MIT,https://github.com/lodash/lodash,js lrInfiniteScroll,MIT,https://github.com/lorenzofox3/lrInfiniteScroll,js +lxml,BSD,http://lxml.de/,pip mock,BSD,http://www.voidspace.org.uk/python/mock/,pip moment,MIT,http://momentjs.com/,js mongoengine,MIT,http://mongoengine.org/,pip +msgpack-python,Apache 2.0,http://msgpack.org/,pip netaddr,BSD,https://github.com/drkjam/netaddr/,pip +netifaces,MIT,https://bitbucket.org/al45tair/netifaces,pip nvd3,Apache 2.0,http://www.nvd3.org,js ordereddict,MIT,https://pypi.python.org/pypi/ordereddict,pip os-client-config,Apache 2.0,http://www.openstack.org/,pip +os-diskconfig-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_diskconfig_python_novaclient_ext,pip +os-networksv2-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_networksv2_python_novaclient_ext,pip +os-virtual-interfacesv2-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_virtual_interfacesv2_ext,pip os_diskconfig_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_diskconfig_python_novaclient_ext,pip os_networksv2_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_networksv2_python_novaclient_ext,pip os_virtual_interfacesv2_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_virtual_interfacesv2_ext,pip @@ -82,19 +103,29 @@ oslo.utils,Apache 2.0,http://launchpad.net/oslo,pip pbr,Apache 2.0,http://pypi.python.org/pypi/pbr,pip pexpect,ISC,http://pexpect.readthedocs.org/,pip pip,MIT,http://www.pip-installer.org,pip +prettytable,BSD,http://code.google.com/p/prettytable/,pip psphere,Apache 2.0,https://github.com/jkinred/psphere,pip +psycopg2,LGPL with exceptions or ZPL,http://initd.org/psycopg/,pip +pyOpenSSL,Apache 2.0,https://github.com/pyca/pyopenssl,pip +pyasn1,BSD,http://sourceforge.net/projects/pyasn1/,pip +pycparser,BSD,https://github.com/eliben/pycparser,pip +pycrypto,Public domain,http://www.pycrypto.org/,pip +pymongo,Apache 2.0,http://github.com/mongodb/mongo-python-driver,pip +pyparsing,MIT,http://pyparsing.wikispaces.com/,pip pyrax,Apache 2.0,https://github.com/rackspace/pyrax,pip python-cinderclient,Apache 2.0,http://www.openstack.org/,pip python-dateutil,BSD,https://dateutil.readthedocs.org,pip python-glanceclient,Apache 2.0,http://www.openstack.org/,pip python-ironicclient,Apache 2.0,http://www.openstack.org/,pip python-keystoneclient,Apache 2.0,http://www.openstack.org/,pip +python-ldap,Python style,http://www.python-ldap.org/,pip python-neutronclient,Apache 2.0,http://www.openstack.org/,pip python-novaclient,Apache 2.0,https://git.openstack.org/cgit/openstack/python-novaclient,pip python-swiftclient,Apache 2.0,http://www.openstack.org/,pip python-troveclient,Apache 2.0,http://www.openstack.org/,pip pytz,MIT,http://pythonhosted.org/pytz,pip pywinrm,MIT,http://github.com/diyan/pywinrm/,pip +pyzmq,LGPL+BSD,http://github.com/zeromq/pyzmq,pip rackspace-auth-openstack,Apache 2.0,https://github.com/rackerlabs/rackspace-auth-openstack,pip rackspace-novaclient,Apache 2.0,https://github.com/rackerlabs/rackspace-novaclient,pip rax_default_network_flags_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/rax_default_network_flags_python_novaclient_ext,pip @@ -112,7 +143,11 @@ sizzle,MIT,https://github.com/jquery/sizzle.git,js socket.io-client,MIT,https://github.com/Automattic/socket.io-client.git,js stevedore,Apache 2.0,https://github.com/dreamhost/stevedore,pip suds,LGPL 3,https://fedorahosted.org/suds,pip +superlance,BSD,http://supervisord.org,pip timezone-js,Apache 2.0,https://github.com/mde/timezone-js.git,js twitter,MIT,http://getbootstrap.com,js underscore,MIT,https://github.com/jashkenas/underscore,js +warlock,Apache 2.0,http://github.com/bcwaldon/warlock,pip +wheel,MIT,http://bitbucket.org/pypa/wheel/,pip +wsgiref,PSF or ZPL,http://cheeseshop.python.org/pypi/wsgiref,pip xmltodict,MIT,https://github.com/martinblech/xmltodict,pip diff --git a/tools/license-audit/license-audit.py b/tools/license-audit/license-audit.py index c33092a6d7..e5a71ec8e2 100755 --- a/tools/license-audit/license-audit.py +++ b/tools/license-audit/license-audit.py @@ -17,52 +17,55 @@ def usage(): sys.exit(1) def read_requirements(towerpath): - filename = '%s/awx/lib/site-packages/README' % (towerpath,) + filename = '%s/requirements/requirements.txt' % (towerpath,) ret = {} - f = open(filename) - if not f: + try: + f = open(filename) + except: print "failed to open %s" %(filename,) return None data = f.readlines() f.close() for line in data: + if line[0] == '#': + continue if '==' in line: - m = re.match(r"(\S+)==(\S+) \((\S+)",line) + m = re.match(r"(\S+)==(\S+)",line) if m: name = m.group(1) version = m.group(2) - pathname = m.group(3) - if pathname.endswith(',') or pathname.endswith(')'): - pathname = pathname[:-1] - if pathname.endswith('/*'): - pathname = pathname[:-2] item = {} item['name'] = name item['version'] = version - item['path'] = pathname ret[name] = item - return ret - -def get_python(towerpath): - excludes = [ - 'README*', - '*.dist-info', - 'funtests', - 'easy_install.py', - 'oslo', - 'pkg_resources', - '_markerlib' - ] - directory = '%s/awx/lib/site-packages' % (towerpath,) - dirlist = os.listdir(directory) - ret = [] - for item in dirlist: - use = True - for exclude in excludes: - if fnmatch.fnmatch(item, exclude): - use = False - if use: - ret.append(item) + continue + elif line.startswith("git+https"): + l = line.rsplit('/',1) + m = re.match(r"(\S+).git@(\S+)#",l[1]) + if m: + name = m.group(1) + version = m.group(2) + if version.startswith('tower_'): + version = version[6:] + item = {} + if name == 'python-ipy': + name='ipy' + item['name'] = name + item['version'] = version + if len(version) > 20: + # it's a sha1sum, read it off the egg spec + lver = l[1].rsplit('-',1) + if lver[1] == l[1]: + lver = l[1].rsplit('_',1) + item['version'] = lver[1][:-1] + ret[name] = item + continue + else: + item = {} + item['name'] = line[:-1] + item['version'] = '' + ret[name] = item + continue return ret def get_js(towerpath): @@ -116,13 +119,13 @@ def normalize_license(license): license = license.replace('"','') if license == 'None': return 'UNKNOWN' - if license in ['Apache License, Version 2.0', 'Apache License (2.0)', 'Apache License 2.0', 'Apache-2.0', 'Apache License, v2.0']: + if license in ['Apache License, Version 2.0', 'Apache License (2.0)', 'Apache License 2.0', 'Apache-2.0', 'Apache License, v2.0', 'APL2']: return 'Apache 2.0' if license == 'ISC license': return 'ISC' if license == 'MIT License' or license == 'MIT license': return 'MIT' - if license == 'BSD License' or license == 'Simplified BSD': + if license in ['BSD License', 'Simplified BSD', 'BSD-derived (http://www.repoze.org/LICENSE.txt)', 'BSD-like', 'Modified BSD License']: return 'BSD' if license == 'LGPL': return 'LGPL 2.1' @@ -131,6 +134,10 @@ def normalize_license(license): return 'Apache 2.0' if license.find('https://github.com/umutbozkurt/django-rest-framework-mongoengine/blob/master/LICENSE') != -1: return 'MIT' + if license == '"BSD or Apache License, Version 2.0"': + return 'BSD or Apache 2.0' + if license == 'Modified BSD License': + return 'BSD' if license == 'Python Software Foundation License': return 'PSF' return license @@ -183,24 +190,11 @@ if not olddata or not requirements: print "No starting data" sys.exit(1) -# Get directory of vendored things from site-packages... -python_packages = get_python(tower_path) - -# ... and ensure they're noted in the requirements file -ok = True -for package in python_packages: - if not search_requirements(requirements, package): - print "%s not in requirements!" % (package,) - ok = False -if not ok: - sys.exit(1) - - # See if there's pip things in our current license list that we don't have now reqs = requirements.keys() for item in olddata.values(): if item['source'] == 'pip' and item['name'] not in reqs: - print "No longer vendoring %s" %(item['name'],) + print "Potentially no longer vendoring %s" %(item['name'],) # Get directory of vendored JS things from the js dir js_packages = get_js(tower_path) @@ -214,10 +208,12 @@ for item in olddata.values(): # Take the requirements file, and get license information where necessary cs = yolk.pypi.CheeseShop() for req in requirements.values(): - cs_info = cs.release_data(req['name'],req['version']) + # name sanitization + (pname, pvers) = cs.query_versions_pypi(req['name']) + cs_info = cs.release_data(pname,req['version']) if not cs_info: - print "Couldn't find '%s-%s'" %(req['name'],req['version']) - if 'name' not in olddata: + print "Couldn't find '%s==%s'" %(req['name'],req['version']) + if req['name'] not in olddata: print "... and it's not in the current data. This needs fixed!" sys.exit(1) continue @@ -241,7 +237,7 @@ for req in requirements.values(): # Update JS package info for pkg in js: - if 'pkg' in olddata: + if pkg in olddata: data = olddata[pkg] new = js_packages[pkg] if new['license'] != 'UNKNOWN' and new['license'] != data['license']: