From 5e4626254628f9148122ec40141646599163976a Mon Sep 17 00:00:00 2001 From: James Laska Date: Thu, 3 Sep 2015 20:42:43 -0400 Subject: [PATCH 01/32] Remove DIST_FULL from offline-tar-build path Also provide suitable defaults for RPM_DIST and RPM_ARCH when being run on a platform that doesn't have the `rpm` command. --- Makefile | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index c14c44afbf..c4f69bbe41 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,13 @@ DEB_ARCH ?= amd64 RPM_SPECDIR= packaging/rpm RPM_SPEC = $(RPM_SPECDIR)/$(NAME).spec RPM_DIST ?= $(shell rpm --eval '%{?dist}' 2>/dev/null) +ifeq ($(RPM_DIST),) +RPM_DIST = .el6 +endif RPM_ARCH ?= $(shell rpm --eval '%{_arch}' 2>/dev/null) +ifeq ($(RPM_ARCH),) +RPM_ARCH = $(shell uname -m) +endif RPM_NVR = $(NAME)-$(VERSION)-$(RELEASE)$(RPM_DIST) MOCK_BIN ?= mock MOCK_CFG ?= @@ -86,9 +92,9 @@ MOCK_CFG ?= DIST = $(shell echo $(RPM_DIST) | sed -e 's|^\.\(el\)\([0-9]\).*|\1|') DIST_MAJOR = $(shell echo $(RPM_DIST) | sed -e 's|^\.\(el\)\([0-9]\).*|\2|') DIST_FULL = $(DIST)$(DIST_MAJOR) -OFFLINE_TAR_NAME = $(NAME)-offline-$(DIST_FULL)-$(VERSION)-$(RELEASE) +OFFLINE_TAR_NAME = $(NAME)-offline-$(VERSION)-$(RELEASE).$(DIST_FULL) OFFLINE_TAR_FILE = $(OFFLINE_TAR_NAME).tar.gz -OFFLINE_TAR_LINK = $(NAME)-offline-$(DIST_FULL)-latest.tar.gz +OFFLINE_TAR_LINK = $(NAME)-offline-latest.$(DIST_FULL).tar.gz DISTRO := $(shell . /etc/os-release 2>/dev/null && echo $${ID} || echo redhat) ifeq ($(DISTRO),ubuntu) @@ -385,22 +391,19 @@ sdist: minjs dist/$(SDIST_TAR_FILE) offline-tar-build: mkdir -p $@ -offline-tar-build/$(DIST_FULL): - mkdir -p $@ - # TODO - Somehow share implementation with setup_tarball -offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_FILE): - cp -a setup offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_NAME) - cd offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_NAME) && sed -e 's#%NAME%#$(NAME)#;s#%VERSION%#$(VERSION)#;s#%RELEASE%#$(RELEASE)#;' group_vars/all.in > group_vars/all - $(PYTHON) $(DEPS_SCRIPT) -d $(DIST) -r $(DIST_MAJOR) -u $(AW_REPO_URL) -s offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_NAME) -v -v -v - cd offline-tar-build/$(DIST_FULL) && tar -czf $(OFFLINE_TAR_FILE) --exclude "*/all.in" $(OFFLINE_TAR_NAME)/ - ln -sf $(OFFLINE_TAR_FILE) offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_LINK) +offline-tar-build/$(OFFLINE_TAR_FILE): + cp -a setup offline-tar-build/$(OFFLINE_TAR_NAME) + cd offline-tar-build/$(OFFLINE_TAR_NAME) && sed -e 's#%NAME%#$(NAME)#;s#%VERSION%#$(VERSION)#;s#%RELEASE%#$(RELEASE)#;' group_vars/all.in > group_vars/all + $(PYTHON) $(DEPS_SCRIPT) -d $(DIST) -r $(DIST_MAJOR) -u $(AW_REPO_URL) -s offline-tar-build/$(OFFLINE_TAR_NAME) -v -v -v + cd offline-tar-build && tar -czf $(OFFLINE_TAR_FILE) --exclude "*/all.in" $(OFFLINE_TAR_NAME)/ + ln -sf $(OFFLINE_TAR_FILE) offline-tar-build/$(OFFLINE_TAR_LINK) -setup_offline_tarball: offline-tar-build offline-tar-build/$(DIST_FULL) offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_FILE) +setup_offline_tarball: offline-tar-build offline-tar-build/$(OFFLINE_TAR_FILE) @echo "#############################################" @echo "Offline artifacts:" - @echo offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_FILE) - @echo offline-tar-build/$(DIST_FULL)/$(OFFLINE_TAR_LINK) + @echo offline-tar-build/$(OFFLINE_TAR_FILE) + @echo offline-tar-build/$(OFFLINE_TAR_LINK) @echo "#############################################" rpm-build: From b3e20b843dc92e83c335488792fd7e3dc76c1fc4 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Fri, 4 Sep 2015 13:43:24 -0400 Subject: [PATCH 02/32] more carefully add to python path * ensure our path is the first in the sys.path list instead of blindly poping and pushing onto the path --- awx/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/awx/__init__.py b/awx/__init__.py index 2c4a64f58e..08837cdfd5 100644 --- a/awx/__init__.py +++ b/awx/__init__.py @@ -41,9 +41,14 @@ def prepare_env(): local_site_packages = os.path.join(os.path.dirname(__file__), 'lib', 'site-packages') site.addsitedir(local_site_packages) - # Work around https://bugs.python.org/issue7744 - # by moving local_site_packages to the front of sys.path - sys.path.insert(0, sys.path.pop()) + try: + index = sys.path.index(local_site_packages) + sys.path.pop(index) + # Work around https://bugs.python.org/issue7744 + # by moving local_site_packages to the front of sys.path + sys.path.insert(0, local_site_packages) + except ValueError: + pass # Hide DeprecationWarnings when running in production. Need to first load # settings to apply our filter after Django's own warnings filter. from django.conf import settings From b456f2955e6a0118399c2487cd00f76b06daa702 Mon Sep 17 00:00:00 2001 From: James Laska Date: Thu, 3 Sep 2015 21:33:55 -0400 Subject: [PATCH 03/32] Create CHECKSUM files for tarball targets * Renames the setup_offline_tarball -> setup_bundle_tarball * Uses sha256sum and gpg to create a --clearsign CHECKSUM file --- Makefile | 73 +++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index c4f69bbe41..b3ba13ef34 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,15 @@ NODE ?= node DEPS_SCRIPT ?= packaging/offline/deps.py AW_REPO_URL ?= "http://releases.ansible.com/ansible-tower" +# Determine appropriate shasum command +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) + SHASUM_BIN ?= sha256sum +endif +ifeq ($(UNAME_S),Darwin) + SHASUM_BIN ?= shasum -a 256 +endif + # Get the branch information from git GIT_DATE := $(shell git log -n 1 --format="%ai") DATE := $(shell date -u +%Y%m%d%H%M) @@ -26,6 +35,7 @@ endif AWS_INSTANCE_COUNT ?= 0 # GPG signature parameters (BETA key not yet used) +GPG_BIN ?= gpg GPG_RELEASE = 442667A9 GPG_BETA = D7B00447 GPG_RELEASE_FILE = RPM-GPG-KEY-ansible-release @@ -43,13 +53,14 @@ ifeq ($(OFFICIAL),yes) SDIST_TAR_NAME=$(NAME)-$(VERSION) PACKER_BUILD_OPTS=-var-file=vars-release.json else - SETUP_TAR_NAME=$(NAME)-setup-$(VERSION)-$(BUILD) - SDIST_TAR_NAME=$(NAME)-$(VERSION)-$(BUILD) + SETUP_TAR_NAME=$(NAME)-setup-$(VERSION)-$(RELEASE) + SDIST_TAR_NAME=$(NAME)-$(VERSION)-$(RELEASE) PACKER_BUILD_OPTS=-var-file=vars-nightly.json endif SDIST_TAR_FILE=$(SDIST_TAR_NAME).tar.gz SETUP_TAR_FILE=$(SETUP_TAR_NAME).tar.gz SETUP_TAR_LINK=$(NAME)-setup-latest.tar.gz +SETUP_TAR_CHECKSUM=$(NAME)-setup-CHECKSUM # DEB build parameters DEBUILD_BIN ?= debuild @@ -74,8 +85,11 @@ DEB_PPA ?= reprepro DEB_ARCH ?= amd64 # RPM build parameters +MOCK_BIN ?= mock +MOCK_CFG ?= RPM_SPECDIR= packaging/rpm RPM_SPEC = $(RPM_SPECDIR)/$(NAME).spec +# Provide a fallback value for RPM_DIST RPM_DIST ?= $(shell rpm --eval '%{?dist}' 2>/dev/null) ifeq ($(RPM_DIST),) RPM_DIST = .el6 @@ -85,16 +99,15 @@ ifeq ($(RPM_ARCH),) RPM_ARCH = $(shell uname -m) endif RPM_NVR = $(NAME)-$(VERSION)-$(RELEASE)$(RPM_DIST) -MOCK_BIN ?= mock -MOCK_CFG ?= -# Offline TAR build parameters +# TAR Bundle build parameters DIST = $(shell echo $(RPM_DIST) | sed -e 's|^\.\(el\)\([0-9]\).*|\1|') DIST_MAJOR = $(shell echo $(RPM_DIST) | sed -e 's|^\.\(el\)\([0-9]\).*|\2|') DIST_FULL = $(DIST)$(DIST_MAJOR) -OFFLINE_TAR_NAME = $(NAME)-offline-$(VERSION)-$(RELEASE).$(DIST_FULL) +OFFLINE_TAR_NAME = $(NAME)-bundle-$(VERSION)-$(RELEASE).$(DIST_FULL) OFFLINE_TAR_FILE = $(OFFLINE_TAR_NAME).tar.gz -OFFLINE_TAR_LINK = $(NAME)-offline-latest.$(DIST_FULL).tar.gz +OFFLINE_TAR_LINK = $(NAME)-bundle-latest.$(DIST_FULL).tar.gz +OFFLINE_TAR_CHECKSUM=$(NAME)-bundle-CHECKSUM DISTRO := $(shell . /etc/os-release 2>/dev/null && echo $${ID} || echo redhat) ifeq ($(DISTRO),ubuntu) @@ -113,7 +126,7 @@ endif devjs minjs testjs testjs_ci node-tests browser-tests jshint ngdocs sync_ui \ deb deb-src debian reprepro setup_tarball \ virtualbox-ovf virtualbox-centos-7 virtualbox-centos-6 \ - clean-offline setup_offline_tarball + clean-bundle setup_bundle_tarball # Remove setup build files clean-tar: @@ -147,11 +160,11 @@ clean-packer: rm -rf packaging/packer/ansible-tower*-ova rm -f Vagrantfile -clean-offline: - rm -rf offline-tar-build +clean-bundle: + rm -rf setup-bundle-build # Remove temporary build files, compiled Python files. -clean: clean-rpm clean-deb clean-grunt clean-ui clean-tar clean-packer clean-offline +clean: clean-rpm clean-deb clean-grunt clean-ui clean-tar clean-packer clean-bundle rm -rf awx/lib/site-packages rm -rf dist/* rm -rf build $(NAME)-$(VERSION) *.egg-info @@ -371,11 +384,19 @@ tar-build/$(SETUP_TAR_FILE): @cd tar-build && tar -czf $(SETUP_TAR_FILE) --exclude "*/all.in" $(SETUP_TAR_NAME)/ @ln -sf $(SETUP_TAR_FILE) tar-build/$(SETUP_TAR_LINK) -setup_tarball: tar-build/$(SETUP_TAR_FILE) +tar-build/$(SETUP_TAR_CHECKSUM): + @if [ "$(OFFICIAL)" != "yes" ] ; then \ + $(SHASUM_BIN) tar-build/$(NAME)*.tar.gz > $@ ; \ + else \ + $(SHASUM_BIN) tar-build/$(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $@ - ; \ + fi + +setup_tarball: tar-build/$(SETUP_TAR_FILE) tar-build/$(SETUP_TAR_CHECKSUM) @echo "#############################################" @echo "Setup artifacts:" @echo tar-build/$(SETUP_TAR_FILE) @echo tar-build/$(SETUP_TAR_LINK) + @echo tar-build/$(SETUP_TAR_CHECKSUM) @echo "#############################################" release_clean: @@ -387,23 +408,27 @@ dist/$(SDIST_TAR_FILE): sdist: minjs dist/$(SDIST_TAR_FILE) -# Build setup offline tarball -offline-tar-build: +# Build setup bundle tarball +setup-bundle-build: mkdir -p $@ # TODO - Somehow share implementation with setup_tarball -offline-tar-build/$(OFFLINE_TAR_FILE): - cp -a setup offline-tar-build/$(OFFLINE_TAR_NAME) - cd offline-tar-build/$(OFFLINE_TAR_NAME) && sed -e 's#%NAME%#$(NAME)#;s#%VERSION%#$(VERSION)#;s#%RELEASE%#$(RELEASE)#;' group_vars/all.in > group_vars/all - $(PYTHON) $(DEPS_SCRIPT) -d $(DIST) -r $(DIST_MAJOR) -u $(AW_REPO_URL) -s offline-tar-build/$(OFFLINE_TAR_NAME) -v -v -v - cd offline-tar-build && tar -czf $(OFFLINE_TAR_FILE) --exclude "*/all.in" $(OFFLINE_TAR_NAME)/ - ln -sf $(OFFLINE_TAR_FILE) offline-tar-build/$(OFFLINE_TAR_LINK) +setup-bundle-build/$(OFFLINE_TAR_FILE): + cp -a setup setup-bundle-build/$(OFFLINE_TAR_NAME) + cd setup-bundle-build/$(OFFLINE_TAR_NAME) && sed -e 's#%NAME%#$(NAME)#;s#%VERSION%#$(VERSION)#;s#%RELEASE%#$(RELEASE)#;' group_vars/all.in > group_vars/all + $(PYTHON) $(DEPS_SCRIPT) -d $(DIST) -r $(DIST_MAJOR) -u $(AW_REPO_URL) -s setup-bundle-build/$(OFFLINE_TAR_NAME) -v -v -v + cd setup-bundle-build && tar -czf $(OFFLINE_TAR_FILE) --exclude "*/all.in" $(OFFLINE_TAR_NAME)/ + ln -sf $(OFFLINE_TAR_FILE) setup-bundle-build/$(OFFLINE_TAR_LINK) -setup_offline_tarball: offline-tar-build offline-tar-build/$(OFFLINE_TAR_FILE) +setup-bundle-build/$(OFFLINE_TAR_CHECKSUM): + @$(SHASUM_BIN) tar-build/$(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $@ - + +setup_bundle_tarball: setup-bundle-build setup-bundle-build/$(OFFLINE_TAR_FILE) setup-bundle-build/$(OFFLINE_TAR_CHECKSUM) @echo "#############################################" @echo "Offline artifacts:" - @echo offline-tar-build/$(OFFLINE_TAR_FILE) - @echo offline-tar-build/$(OFFLINE_TAR_LINK) + @echo setup-bundle-build/$(OFFLINE_TAR_FILE) + @echo setup-bundle-build/$(OFFLINE_TAR_LINK) + @echo setup-bundle-build/$(OFFLINE_TAR_CHECKSUM) @echo "#############################################" rpm-build: @@ -447,7 +472,7 @@ mock-rpm: rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm ifeq ($(OFFICIAL),yes) rpm-build/$(GPG_FILE): rpm-build - gpg --export -a "${GPG_KEY}" > "$@" + $(GPG_BIN) --export -a "${GPG_KEY}" > "$@" rpm-sign: rpm-build/$(GPG_FILE) rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm rpm --define "_signature gpg" --define "_gpg_name $(GPG_KEY)" --addsign rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm From adb912a8d6f766cf2ae14a705988525b6fa3e217 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Fri, 4 Sep 2015 13:57:21 -0400 Subject: [PATCH 04/32] Merge pull request #404 from chrismeyersfsu/fix-setup_privilage pair --ask-xxx-pass with respective escalation From cb1d60939bc27e07a697761b71b484a8002acd68 Mon Sep 17 00:00:00 2001 From: James Laska Date: Fri, 4 Sep 2015 14:08:35 -0400 Subject: [PATCH 05/32] Renamed packaging/offline -> packaging/bundle Also replaced all instances off `offline` with `bundle`. Also includes minor trailing whitespace cleanup. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b3ba13ef34..bc47171524 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ PACKER ?= packer GRUNT ?= $(shell [ -t 0 ] && echo "grunt" || echo "grunt --no-color") BROCCOLI ?= ./node_modules/.bin/broccoli NODE ?= node -DEPS_SCRIPT ?= packaging/offline/deps.py +DEPS_SCRIPT ?= packaging/bundle/deps.py AW_REPO_URL ?= "http://releases.ansible.com/ansible-tower" # Determine appropriate shasum command From 838383829695e03c12773b8fbb9a05b1f8c76980 Mon Sep 17 00:00:00 2001 From: James Laska Date: Fri, 4 Sep 2015 21:05:59 -0400 Subject: [PATCH 06/32] Fix path in build target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bc47171524..d4c7f5dd9f 100644 --- a/Makefile +++ b/Makefile @@ -421,7 +421,7 @@ setup-bundle-build/$(OFFLINE_TAR_FILE): ln -sf $(OFFLINE_TAR_FILE) setup-bundle-build/$(OFFLINE_TAR_LINK) setup-bundle-build/$(OFFLINE_TAR_CHECKSUM): - @$(SHASUM_BIN) tar-build/$(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $@ - + @$(SHASUM_BIN) setup-bundle-build/$(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $@ - setup_bundle_tarball: setup-bundle-build setup-bundle-build/$(OFFLINE_TAR_FILE) setup-bundle-build/$(OFFLINE_TAR_CHECKSUM) @echo "#############################################" From 4aa6f74437a822284a116e6dea37bf273ebaf0db Mon Sep 17 00:00:00 2001 From: James Laska Date: Fri, 4 Sep 2015 21:08:49 -0400 Subject: [PATCH 07/32] Only sign the CHECKSUM for OFFICIAL builds --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d4c7f5dd9f..edbff632d1 100644 --- a/Makefile +++ b/Makefile @@ -421,7 +421,11 @@ setup-bundle-build/$(OFFLINE_TAR_FILE): ln -sf $(OFFLINE_TAR_FILE) setup-bundle-build/$(OFFLINE_TAR_LINK) setup-bundle-build/$(OFFLINE_TAR_CHECKSUM): - @$(SHASUM_BIN) setup-bundle-build/$(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $@ - + @if [ "$(OFFICIAL)" != "yes" ] ; then \ + $(SHASUM_BIN) setup-bundle-build/$(NAME)*.tar.gz > $@ ; \ + else \ + @$(SHASUM_BIN) setup-bundle-build/$(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $@ - ; \ + fi setup_bundle_tarball: setup-bundle-build setup-bundle-build/$(OFFLINE_TAR_FILE) setup-bundle-build/$(OFFLINE_TAR_CHECKSUM) @echo "#############################################" From b7a67ca17640056cc6d9faa6685f6a55835282bf Mon Sep 17 00:00:00 2001 From: James Laska Date: Fri, 4 Sep 2015 21:14:01 -0400 Subject: [PATCH 08/32] Don't include the build path in CHECKSUM --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index edbff632d1..b0a97ce910 100644 --- a/Makefile +++ b/Makefile @@ -386,9 +386,9 @@ tar-build/$(SETUP_TAR_FILE): tar-build/$(SETUP_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ - $(SHASUM_BIN) tar-build/$(NAME)*.tar.gz > $@ ; \ + cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $@ ; \ else \ - $(SHASUM_BIN) tar-build/$(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $@ - ; \ + cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $@ - ; \ fi setup_tarball: tar-build/$(SETUP_TAR_FILE) tar-build/$(SETUP_TAR_CHECKSUM) @@ -422,9 +422,9 @@ setup-bundle-build/$(OFFLINE_TAR_FILE): setup-bundle-build/$(OFFLINE_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ - $(SHASUM_BIN) setup-bundle-build/$(NAME)*.tar.gz > $@ ; \ + cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $@ ; \ else \ - @$(SHASUM_BIN) setup-bundle-build/$(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $@ - ; \ + cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $@ - ; \ fi setup_bundle_tarball: setup-bundle-build setup-bundle-build/$(OFFLINE_TAR_FILE) setup-bundle-build/$(OFFLINE_TAR_CHECKSUM) From 92d737e721ec0de62388f6e540643637900b16c4 Mon Sep 17 00:00:00 2001 From: James Laska Date: Sat, 5 Sep 2015 08:39:32 -0400 Subject: [PATCH 09/32] Fix path bug when creating CHECKSUM file --- Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index b0a97ce910..88ba7706b7 100644 --- a/Makefile +++ b/Makefile @@ -104,10 +104,10 @@ RPM_NVR = $(NAME)-$(VERSION)-$(RELEASE)$(RPM_DIST) DIST = $(shell echo $(RPM_DIST) | sed -e 's|^\.\(el\)\([0-9]\).*|\1|') DIST_MAJOR = $(shell echo $(RPM_DIST) | sed -e 's|^\.\(el\)\([0-9]\).*|\2|') DIST_FULL = $(DIST)$(DIST_MAJOR) -OFFLINE_TAR_NAME = $(NAME)-bundle-$(VERSION)-$(RELEASE).$(DIST_FULL) +OFFLINE_TAR_NAME = $(NAME)-setup-bundle-$(VERSION)-$(RELEASE).$(DIST_FULL) OFFLINE_TAR_FILE = $(OFFLINE_TAR_NAME).tar.gz -OFFLINE_TAR_LINK = $(NAME)-bundle-latest.$(DIST_FULL).tar.gz -OFFLINE_TAR_CHECKSUM=$(NAME)-bundle-CHECKSUM +OFFLINE_TAR_LINK = $(NAME)-setup-bundle-latest.$(DIST_FULL).tar.gz +OFFLINE_TAR_CHECKSUM=$(NAME)-setup-bundle-CHECKSUM DISTRO := $(shell . /etc/os-release 2>/dev/null && echo $${ID} || echo redhat) ifeq ($(DISTRO),ubuntu) @@ -386,9 +386,9 @@ tar-build/$(SETUP_TAR_FILE): tar-build/$(SETUP_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ - cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $@ ; \ + cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $(notdir $@) ; \ else \ - cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $@ - ; \ + cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $(notdir $@) - ; \ fi setup_tarball: tar-build/$(SETUP_TAR_FILE) tar-build/$(SETUP_TAR_CHECKSUM) @@ -422,9 +422,9 @@ setup-bundle-build/$(OFFLINE_TAR_FILE): setup-bundle-build/$(OFFLINE_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ - cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $@ ; \ + cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $(notdir $@) ; \ else \ - cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $@ - ; \ + cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $(notdir $@) - ; \ fi setup_bundle_tarball: setup-bundle-build setup-bundle-build/$(OFFLINE_TAR_FILE) setup-bundle-build/$(OFFLINE_TAR_CHECKSUM) From 8e603dc18d3262370bf5e7ede711719d7498d734 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 7 Sep 2015 08:09:38 -0400 Subject: [PATCH 10/32] Merge pull request #407 from chrismeyersfsu/feature-yaml_fallback fallback to simple yaml parser if yaml import fail From 3210b8c0dce126f75631786aa73c998b5e72ea82 Mon Sep 17 00:00:00 2001 From: James Laska Date: Mon, 7 Sep 2015 20:33:08 -0400 Subject: [PATCH 11/32] Add helper script to sign using expect --- tools/scripts/sign.exp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 tools/scripts/sign.exp diff --git a/tools/scripts/sign.exp b/tools/scripts/sign.exp new file mode 100755 index 0000000000..d85d77d77e --- /dev/null +++ b/tools/scripts/sign.exp @@ -0,0 +1,17 @@ +#!/usr/bin/expect -f +# +# Helper script to respond to passphrase prompts from the gpg command. +# + +set timeout 600 +set command [join $argv] +set passphrase $env(PASSPHRASE) +puts "# $command" +spawn -noecho {*}$command +expect { + -exact "Enter passphrase: " { + send -- "$passphrase\r" + exp_continue + } + eof { } +} From 7de8f534a17574bb6d62cfbde93936db20b004fd Mon Sep 17 00:00:00 2001 From: James Laska Date: Tue, 8 Sep 2015 13:01:50 -0400 Subject: [PATCH 12/32] Ignore setup-bundle-build directory --- .gitignore | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index b01683f972..f86a281884 100644 --- a/.gitignore +++ b/.gitignore @@ -24,10 +24,10 @@ awx/ui/dist # Python & setuptools __pycache__ build -deb-build -rpm-build -tar-build -/offline_tar-build +/deb-build +/rpm-build +/tar-build +/setup-bundle-build /dist *.egg-info *.py[c,o] From aa6470b73026aa0ef4c7031c3e6abbeb3f70eef5 Mon Sep 17 00:00:00 2001 From: James Laska Date: Tue, 8 Sep 2015 13:02:15 -0400 Subject: [PATCH 13/32] Re-enable package signing --- Makefile | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 88ba7706b7..d6180dc7d7 100644 --- a/Makefile +++ b/Makefile @@ -36,15 +36,19 @@ AWS_INSTANCE_COUNT ?= 0 # GPG signature parameters (BETA key not yet used) GPG_BIN ?= gpg -GPG_RELEASE = 442667A9 -GPG_BETA = D7B00447 -GPG_RELEASE_FILE = RPM-GPG-KEY-ansible-release -GPG_BETA_FILE = RPM-GPG-KEY-ansible-beta +RPM_GPG_RELEASE = 442667A9 +RPM_GPG_RELEASE_FILE = RPM-GPG-KEY-ansible-release +RPM_GPG_BETA = D7B00447 +RPM_GPG_BETA_FILE = RPM-GPG-KEY-ansible-beta +DEB_GPG_RELEASE = 3DD29021 +DEB_GPG_RELEASE_FILE = DEB-GPG-KEY-ansible-release -# Determine GPG key for RPM signing +# Determine GPG key for package signing ifeq ($(OFFICIAL),yes) - GPG_KEY = $(GPG_RELEASE) - GPG_FILE = $(GPG_RELEASE_FILE) + RPM_GPG_KEY = $(RPM_GPG_RELEASE) + RPM_GPG_FILE = $(RPM_GPG_RELEASE_FILE) + DEB_GPG_KEY = $(DEB_GPG_RELEASE) + DEB_GPG_FILE = $(DEB_GPG_RELEASE_FILE) endif # TAR build parameters @@ -69,11 +73,8 @@ DPUT_BIN ?= dput DPUT_OPTS ?= ifeq ($(OFFICIAL),yes) DEB_DIST ?= stable - # Sign OFFICIAL builds using 'DEBSIGN_KEYID' - # DEBSIGN_KEYID is required when signing - ifneq ($(DEBSIGN_KEYID),) - DEBUILD_OPTS += -k$(DEBSIGN_KEYID) - endif + # Sign official builds + DEBUILD_OPTS += -k$(DEB_GPG_KEY) else DEB_DIST ?= unstable # Do not sign development builds @@ -475,11 +476,11 @@ rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm: rpm-build/$(RPM_NVR).src.rpm mock-rpm: rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm ifeq ($(OFFICIAL),yes) -rpm-build/$(GPG_FILE): rpm-build - $(GPG_BIN) --export -a "${GPG_KEY}" > "$@" +rpm-build/$(RPM_GPG_FILE): rpm-build + $(GPG_BIN) --export -a "${RPM_GPG_KEY}" > "$@" -rpm-sign: rpm-build/$(GPG_FILE) rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm - rpm --define "_signature gpg" --define "_gpg_name $(GPG_KEY)" --addsign rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm +rpm-sign: rpm-build/$(RPM_GPG_FILE) rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm + rpm --define "_signature gpg" --define "_gpg_name $(RPM_GPG_KEY)" --addsign rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm endif deb-build/$(SDIST_TAR_NAME): From c02d993e736de8dc2e55bb96eaf9fde538f150e0 Mon Sep 17 00:00:00 2001 From: James Laska Date: Tue, 8 Sep 2015 15:43:04 -0400 Subject: [PATCH 14/32] Return exit code from spawned process --- tools/scripts/sign.exp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/scripts/sign.exp b/tools/scripts/sign.exp index d85d77d77e..30cbe1d975 100755 --- a/tools/scripts/sign.exp +++ b/tools/scripts/sign.exp @@ -15,3 +15,12 @@ expect { } eof { } } + +lassign [wait] pid spawnid os_error_flag retval + +if {$os_error_flag == 0} { + puts "exit status: $retval" +} else { + puts "errno: $retval" +} +exit $retval From a38e8ca88986a867fa4dc5020e88751368e161b3 Mon Sep 17 00:00:00 2001 From: James Laska Date: Tue, 8 Sep 2015 19:51:11 -0400 Subject: [PATCH 15/32] Specify a gpg key for TAR artifacts --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d6180dc7d7..7dbf75d49e 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,7 @@ DEB_GPG_RELEASE_FILE = DEB-GPG-KEY-ansible-release # Determine GPG key for package signing ifeq ($(OFFICIAL),yes) + TAR_GPG_KEY = $(RPM_GPG_RELEASE) RPM_GPG_KEY = $(RPM_GPG_RELEASE) RPM_GPG_FILE = $(RPM_GPG_RELEASE_FILE) DEB_GPG_KEY = $(DEB_GPG_RELEASE) @@ -389,7 +390,7 @@ tar-build/$(SETUP_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $(notdir $@) ; \ else \ - cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $(notdir $@) - ; \ + cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ fi setup_tarball: tar-build/$(SETUP_TAR_FILE) tar-build/$(SETUP_TAR_CHECKSUM) @@ -425,7 +426,7 @@ setup-bundle-build/$(OFFLINE_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $(notdir $@) ; \ else \ - cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(GPG_RELEASE)" -o $(notdir $@) - ; \ + cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ fi setup_bundle_tarball: setup-bundle-build setup-bundle-build/$(OFFLINE_TAR_FILE) setup-bundle-build/$(OFFLINE_TAR_CHECKSUM) From 8abc7b7ae1af7de0ba0749080642e4f8fc11011e Mon Sep 17 00:00:00 2001 From: James Laska Date: Tue, 8 Sep 2015 19:57:16 -0400 Subject: [PATCH 16/32] Disable gpg tty requirement --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7dbf75d49e..c05582f5bc 100644 --- a/Makefile +++ b/Makefile @@ -390,7 +390,7 @@ tar-build/$(SETUP_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $(notdir $@) ; \ else \ - cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ + cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --no-tty --clearsign -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ fi setup_tarball: tar-build/$(SETUP_TAR_FILE) tar-build/$(SETUP_TAR_CHECKSUM) @@ -426,7 +426,7 @@ setup-bundle-build/$(OFFLINE_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $(notdir $@) ; \ else \ - cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ + cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --no-tty --clearsign -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ fi setup_bundle_tarball: setup-bundle-build setup-bundle-build/$(OFFLINE_TAR_FILE) setup-bundle-build/$(OFFLINE_TAR_CHECKSUM) From d183075519370e79041284621d6dee4a30beed7c Mon Sep 17 00:00:00 2001 From: James Laska Date: Tue, 8 Sep 2015 20:05:24 -0400 Subject: [PATCH 17/32] Revert "Disable gpg tty requirement" This reverts commit 205210045749de215988a08ebdc57503e6698b04. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c05582f5bc..7dbf75d49e 100644 --- a/Makefile +++ b/Makefile @@ -390,7 +390,7 @@ tar-build/$(SETUP_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $(notdir $@) ; \ else \ - cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --no-tty --clearsign -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ + cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ fi setup_tarball: tar-build/$(SETUP_TAR_FILE) tar-build/$(SETUP_TAR_CHECKSUM) @@ -426,7 +426,7 @@ setup-bundle-build/$(OFFLINE_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $(notdir $@) ; \ else \ - cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --no-tty --clearsign -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ + cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ fi setup_bundle_tarball: setup-bundle-build setup-bundle-build/$(OFFLINE_TAR_FILE) setup-bundle-build/$(OFFLINE_TAR_CHECKSUM) From 1b6b622d70bd890c5c07289b24b5f8d38ceafd02 Mon Sep 17 00:00:00 2001 From: James Laska Date: Tue, 8 Sep 2015 22:00:27 -0400 Subject: [PATCH 18/32] Ignore case to handle multiple prompts Handles the multiple prompts from debuild and reprepro, each with different cases. --- tools/scripts/sign.exp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/scripts/sign.exp b/tools/scripts/sign.exp index 30cbe1d975..79f534b285 100755 --- a/tools/scripts/sign.exp +++ b/tools/scripts/sign.exp @@ -9,7 +9,7 @@ set passphrase $env(PASSPHRASE) puts "# $command" spawn -noecho {*}$command expect { - -exact "Enter passphrase: " { + -exact -nocase "enter passphrase: " { send -- "$passphrase\r" exp_continue } From 6fa26bd02626d8221517d1f412b0ffd487d259fc Mon Sep 17 00:00:00 2001 From: James Laska Date: Tue, 8 Sep 2015 22:44:29 -0400 Subject: [PATCH 19/32] Properly sign DEBs using reprepro --- Makefile | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 7dbf75d49e..e64aaf4709 100644 --- a/Makefile +++ b/Makefile @@ -484,6 +484,9 @@ rpm-sign: rpm-build/$(RPM_GPG_FILE) rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm rpm --define "_signature gpg" --define "_gpg_name $(RPM_GPG_KEY)" --addsign rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm endif +deb-build: + mkdir -p $@ + deb-build/$(SDIST_TAR_NAME): mkdir -p deb-build tar -C deb-build/ -xvf dist/$(SDIST_TAR_FILE) @@ -491,7 +494,14 @@ deb-build/$(SDIST_TAR_NAME): cp packaging/remove_tower_source.py deb-build/$(SDIST_TAR_NAME)/debian/ sed -ie "s#^$(NAME) (\([^)]*\)) \([^;]*\);#$(NAME) ($(VERSION)-$(RELEASE)) $(DEB_DIST);#" deb-build/$(SDIST_TAR_NAME)/debian/changelog +ifeq ($(OFFICIAL),yes) +debian: sdist deb-build/$(SDIST_TAR_NAME) deb-build/$(DEB_GPG_FILE) + +deb-build/$(DEB_GPG_FILE): deb-build + $(GPG_BIN) --export -a "${DEB_GPG_KEY}" > "$@" +else debian: sdist deb-build/$(SDIST_TAR_NAME) +endif deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb: cd deb-build/$(SDIST_TAR_NAME) && $(DEBUILD) -b @@ -518,19 +528,27 @@ deb-src-upload: deb-src $(DPUT_BIN) $(DPUT_OPTS) $(DEB_PPA) deb-build/$(NAME)_$(VERSION)-$(RELEASE)_source.changes ; \ reprepro: deb - mkdir -p reprepro/conf - cp -a packaging/reprepro/* reprepro/conf/ + mkdir -p $@/conf + cp -a packaging/reprepro/* $@/conf/ + if [ "$(OFFICIAL)" == "yes" ] ; then \ + echo "ask-passphrase" >> $@/conf; \ + sed -i -e 's|^\(Codename:\)|SignWith: $(DEB_GPG_KEY)\n\1|' $@/distributions ; \ + fi @DEB=deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb ; \ for DIST in trusty precise ; do \ echo "Removing '$(NAME)' from the $${DIST} apt repo" ; \ - echo reprepro --export=force -b reprepro remove $${DIST} $(NAME) ; \ + echo reprepro --export=force -b $@ remove $${DIST} $(NAME) ; \ done; \ - reprepro --export=force -b reprepro clearvanished; \ + reprepro --export=force -b $@ clearvanished; \ for DIST in trusty precise ; do \ echo "Adding $${DEB} to the $${DIST} apt repo"; \ - reprepro --keepunreferencedfiles --export=force -b reprepro --ignore=brokenold includedeb $${DIST} $${DEB} ; \ + reprepro --keepunreferencedfiles --export=force -b $@ --ignore=brokenold includedeb $${DIST} $${DEB} ; \ done; \ +# +# Packer build targets +# + amazon-ebs: cd packaging/packer && $(PACKER) build -only $@ $(PACKER_BUILD_OPTS) -var "aws_instance_count=$(AWS_INSTANCE_COUNT)" -var "product_version=$(VERSION)" packer-$(NAME).json @@ -557,6 +575,5 @@ docker-dev: build: $(PYTHON) setup.py build -# TODO - only use --install-layout=deb on Debian install: $(PYTHON) setup.py install $(SETUP_INSTALL_ARGS) From 77f4a280e367597494a01935d76e48c433b40aac Mon Sep 17 00:00:00 2001 From: James Laska Date: Tue, 8 Sep 2015 23:14:31 -0400 Subject: [PATCH 20/32] Correct bash conditional --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index e64aaf4709..e585f28f50 100644 --- a/Makefile +++ b/Makefile @@ -529,10 +529,10 @@ deb-src-upload: deb-src reprepro: deb mkdir -p $@/conf - cp -a packaging/reprepro/* $@/conf/ - if [ "$(OFFICIAL)" == "yes" ] ; then \ - echo "ask-passphrase" >> $@/conf; \ - sed -i -e 's|^\(Codename:\)|SignWith: $(DEB_GPG_KEY)\n\1|' $@/distributions ; \ + cp -a packaging/reprepro $@/conf + if [ "$(OFFICIAL)" = "yes" ] ; then \ + echo "ask-passphrase" >> $@/conf/options; \ + sed -i -e 's|^\(Codename:\)|SignWith: $(DEB_GPG_KEY)\n\1|' $@/conf/distributions ; \ fi @DEB=deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb ; \ for DIST in trusty precise ; do \ From 99444f2737f779696b6d5be9164bdf4f1aa97e00 Mon Sep 17 00:00:00 2001 From: James Laska Date: Wed, 9 Sep 2015 10:57:55 -0400 Subject: [PATCH 21/32] Fix Makefile tabs/spaces --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index e585f28f50..f0dc8786c8 100644 --- a/Makefile +++ b/Makefile @@ -529,11 +529,11 @@ deb-src-upload: deb-src reprepro: deb mkdir -p $@/conf - cp -a packaging/reprepro $@/conf + cp -a packaging/reprepro/* $@/conf/ if [ "$(OFFICIAL)" = "yes" ] ; then \ - echo "ask-passphrase" >> $@/conf/options; \ - sed -i -e 's|^\(Codename:\)|SignWith: $(DEB_GPG_KEY)\n\1|' $@/conf/distributions ; \ - fi + echo "ask-passphrase" >> $@/conf/options; \ + sed -i -e 's|^\(Codename:\)|SignWith: $(DEB_GPG_KEY)\n\1|' $@/conf/distributions ; \ + fi @DEB=deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb ; \ for DIST in trusty precise ; do \ echo "Removing '$(NAME)' from the $${DIST} apt repo" ; \ From 3a914619ce4d7d95465e46607c1f4d713a089139 Mon Sep 17 00:00:00 2001 From: James Laska Date: Wed, 9 Sep 2015 11:01:44 -0400 Subject: [PATCH 22/32] Improvements to the package signing script --- tools/scripts/sign.exp | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tools/scripts/sign.exp b/tools/scripts/sign.exp index 79f534b285..27aefcfcc2 100755 --- a/tools/scripts/sign.exp +++ b/tools/scripts/sign.exp @@ -4,15 +4,33 @@ # set timeout 600 + +# Optionally print usage message +if {[llength $argv] <= 0} { + puts "Usage: sign.exp " + exit 1 +} + +# Process arguments set command [join $argv] -set passphrase $env(PASSPHRASE) -puts "# $command" -spawn -noecho {*}$command + +if { [info exists env(PASSPHRASE) ] } { + set passphrase $env(PASSPHRASE) +} else { + set passphrase "" +} + +# Run the desired command +spawn {*}$command expect { -exact -nocase "enter passphrase: " { send -- "$passphrase\r" exp_continue } + timeout { + puts "[error] expect timeout" + exit 1 + } eof { } } From 06e6980eafdf67a0478eee1aa546d7ee8bbb4fd4 Mon Sep 17 00:00:00 2001 From: James Laska Date: Wed, 9 Sep 2015 11:18:06 -0400 Subject: [PATCH 23/32] Fix expect script hang --- tools/scripts/sign.exp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/scripts/sign.exp b/tools/scripts/sign.exp index 27aefcfcc2..ac279fab42 100755 --- a/tools/scripts/sign.exp +++ b/tools/scripts/sign.exp @@ -23,7 +23,7 @@ if { [info exists env(PASSPHRASE) ] } { # Run the desired command spawn {*}$command expect { - -exact -nocase "enter passphrase: " { + -nocase "enter passphrase: " { send -- "$passphrase\r" exp_continue } From b7ddbc48b1cae2503e679306282e69b5f80a0bdd Mon Sep 17 00:00:00 2001 From: James Laska Date: Wed, 9 Sep 2015 12:13:55 -0400 Subject: [PATCH 24/32] Disable timeout and correct sign.exp error --- tools/scripts/sign.exp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/scripts/sign.exp b/tools/scripts/sign.exp index ac279fab42..c33e70bbde 100755 --- a/tools/scripts/sign.exp +++ b/tools/scripts/sign.exp @@ -3,7 +3,8 @@ # Helper script to respond to passphrase prompts from the gpg command. # -set timeout 600 +# Disable timeout +set timeout -1 # Optionally print usage message if {[llength $argv] <= 0} { @@ -28,7 +29,7 @@ expect { exp_continue } timeout { - puts "[error] expect timeout" + puts "expect timeout" exit 1 } eof { } From 87d8808c2d56cedefb216301e62d2b281128006e Mon Sep 17 00:00:00 2001 From: James Laska Date: Wed, 9 Sep 2015 13:07:09 -0400 Subject: [PATCH 25/32] Fix expect script timeout --- tools/scripts/sign.exp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/scripts/sign.exp b/tools/scripts/sign.exp index c33e70bbde..68167ac1aa 100755 --- a/tools/scripts/sign.exp +++ b/tools/scripts/sign.exp @@ -24,7 +24,7 @@ if { [info exists env(PASSPHRASE) ] } { # Run the desired command spawn {*}$command expect { - -nocase "enter passphrase: " { + -nocase "enter passphrase:" { send -- "$passphrase\r" exp_continue } From 940910b9a3b6876ac23d79e71ab8951248e8fe31 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Wed, 9 Sep 2015 17:16:35 -0400 Subject: [PATCH 26/32] Adding docker-compose development workflow --- .gitignore | 1 + Makefile | 7 +++-- tools/docker-compose.yml | 23 +++++++++++++++ tools/docker-compose/Dockerfile | 24 ++++++++++++++++ tools/docker-compose/README | 2 ++ tools/docker-compose/start_development.sh | 35 +++++++++++++++++++++++ 6 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 tools/docker-compose.yml create mode 100644 tools/docker-compose/Dockerfile create mode 100644 tools/docker-compose/README create mode 100755 tools/docker-compose/start_development.sh diff --git a/.gitignore b/.gitignore index f86a281884..74a3bd3bda 100644 --- a/.gitignore +++ b/.gitignore @@ -83,5 +83,6 @@ nohup.out reports # AWX python libs populated by requirements.txt +awx/lib/.deps_built awx/lib/site-packages diff --git a/Makefile b/Makefile index f0dc8786c8..1b173c1be7 100644 --- a/Makefile +++ b/Makefile @@ -567,9 +567,6 @@ packaging/packer/output-virtualbox-iso/centos-7.ovf: virtualbox-centos-7: packaging/packer/output-virtualbox-iso/centos-7.ovf -docker-dev: - docker build --no-cache=true --rm=true -t ansible/tower_devel:latest tools/docker - # TODO - figure out how to build the front-end and python requirements with # 'build' build: @@ -577,3 +574,7 @@ build: install: $(PYTHON) setup.py install $(SETUP_INSTALL_ARGS) + +# Docker Compose Development environment +docker-compose: + docker-compose -f tools/docker-compose.yml up --no-recreate diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml new file mode 100644 index 0000000000..e882ecd4d2 --- /dev/null +++ b/tools/docker-compose.yml @@ -0,0 +1,23 @@ +tower: + build: ./docker-compose + ports: + - "8080:8080" + - "8013:8013" + links: + - postgres + - redis + - mongo + volumes: + - ../:/tower_devel +postgres: + image: postgres:9.4.1 + # ports: + # - 5432:5432 +redis: + image: redis:3.0.1 + # ports: + # - 6379:6379 +mongo: + image: mongo:3.0 + # ports: + # - 27017:27017 \ No newline at end of file diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile new file mode 100644 index 0000000000..d794df1b30 --- /dev/null +++ b/tools/docker-compose/Dockerfile @@ -0,0 +1,24 @@ +FROM ubuntu:14.04 + +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +RUN apt-get update +RUN apt-get install -y software-properties-common python-software-properties curl +RUN add-apt-repository -y ppa:chris-lea/zeromq; add-apt-repository -y ppa:chris-lea/node.js; add-apt-repository ppa:ansible/ansible +RUN curl -sL https://deb.nodesource.com/setup_0.12 | bash - +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10 && apt-key adv --fetch-keys http://www.postgresql.org/media/keys/ACCC4CF8.asc +RUN echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.0.list && echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" | tee /etc/apt/sources.list.d/postgres-9.4.list +RUN apt-get update +RUN apt-get install -y openssh-server ansible mg vim tmux git mercurial subversion python-dev python-psycopg2 make postgresql-client libpq-dev nodejs python-psutil libxml2-dev libxslt-dev lib32z1-dev libsasl2-dev libldap2-dev libffi-dev libzmq-dev proot python-pip && rm -rf /var/lib/apt/lists/* +RUN /usr/bin/ssh-keygen -q -t rsa -N "" -f /root/.ssh/id_rsa +RUN mkdir -p /etc/tower +RUN mkdir -p /data/db +ADD license /etc/awx/license +ADD license /etc/tower/license +RUN pip2 install honcho +ADD start_development.sh /start_development.sh + +EXPOSE 8013 8080 22 +CMD /start_development.sh diff --git a/tools/docker-compose/README b/tools/docker-compose/README new file mode 100644 index 0000000000..1d1829ec11 --- /dev/null +++ b/tools/docker-compose/README @@ -0,0 +1,2 @@ +docker build --no-cache=true --rm=true -t ansible/tower_devel:latest . +docker run --name tower_test -it --memory="4g" --cpuset="0,1" -v /Users/meyers/ansible/:/tower_devel -p 8013:8013 -p 8080:8080 -p 27017:27017 -p 2222:22 ansible/tower_devel diff --git a/tools/docker-compose/start_development.sh b/tools/docker-compose/start_development.sh new file mode 100755 index 0000000000..f2267a85ce --- /dev/null +++ b/tools/docker-compose/start_development.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# We need to give the databases enough time to start up (switch to using ansible's wait_for here) +sleep 20 + +# In case Tower in the container wants to connect to itself, use "docker exec" to attach to the container otherwise +/etc/init.d/ssh start +ansible -i "127.0.0.1," -c local -v -m postgresql_user -U postgres -a "name=awx-dev password=AWXsome1 login_user=postgres login_host=postgres" all +ansible -i "127.0.0.1," -c local -v -m postgresql_db -U postgres -a "name=awx-dev owner=awx-dev login_user=postgres login_host=postgres" all + +# Move to the source directory so we can bootstrap +if [ -f "/tower_devel/manage.py" ]; then + cd /tower_devel +elif [ -f "/tower_devel/ansible-tower/manage.py" ]; then + cd /tower_devel/ansible-tower +else + echo "Failed to find tower source tree, map your development tree volume" +fi +make develop + +# Check if we need to build dependencies +if [ -f "awx/lib/.deps_built" ]; then + echo "Skipping dependency build - remove awx/lib/.deps_built to force a rebuild" +else + make requirements_dev + touch awx/lib/.deps_built +fi + +# Tower bootstrapping +make version_file +make migrate +make init + +# Start the service +make honcho From 9d5d17fa70b642013c10df123d21752c3ce98fe3 Mon Sep 17 00:00:00 2001 From: James Laska Date: Wed, 9 Sep 2015 18:51:23 -0400 Subject: [PATCH 27/32] Handle rpm --addsign pasphrase prompts --- tools/scripts/sign.exp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/scripts/sign.exp b/tools/scripts/sign.exp index 68167ac1aa..56c4da4b52 100755 --- a/tools/scripts/sign.exp +++ b/tools/scripts/sign.exp @@ -28,6 +28,10 @@ expect { send -- "$passphrase\r" exp_continue } + -nocase "enter pass phrase:" { + send -- "$passphrase\r" + exp_continue + } timeout { puts "expect timeout" exit 1 From 19c847ae10ed993563e6463d892de00b872d3307 Mon Sep 17 00:00:00 2001 From: James Laska Date: Thu, 10 Sep 2015 13:40:24 -0400 Subject: [PATCH 28/32] Use gpg --passphrase for signing CHECKSUM --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1b173c1be7..a1f4d1180c 100644 --- a/Makefile +++ b/Makefile @@ -390,7 +390,7 @@ tar-build/$(SETUP_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $(notdir $@) ; \ else \ - cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ + cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign --batch --passphrase "$(GPG_PASSPHRASE)" -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ fi setup_tarball: tar-build/$(SETUP_TAR_FILE) tar-build/$(SETUP_TAR_CHECKSUM) @@ -424,9 +424,9 @@ setup-bundle-build/$(OFFLINE_TAR_FILE): setup-bundle-build/$(OFFLINE_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ - cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $(notdir $@) ; \ + cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $(notdir $@) ; \ else \ - cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ + cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign --batch --passphrase "$(GPG_PASSPHRASE)" -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ fi setup_bundle_tarball: setup-bundle-build setup-bundle-build/$(OFFLINE_TAR_FILE) setup-bundle-build/$(OFFLINE_TAR_CHECKSUM) From 3f9dd3ab2245e96fedf8d81cf80b749f30185639 Mon Sep 17 00:00:00 2001 From: James Laska Date: Thu, 10 Sep 2015 16:26:59 -0400 Subject: [PATCH 29/32] Simplify GPG keys --- Makefile | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index a1f4d1180c..38f105a31f 100644 --- a/Makefile +++ b/Makefile @@ -36,20 +36,15 @@ AWS_INSTANCE_COUNT ?= 0 # GPG signature parameters (BETA key not yet used) GPG_BIN ?= gpg -RPM_GPG_RELEASE = 442667A9 -RPM_GPG_RELEASE_FILE = RPM-GPG-KEY-ansible-release -RPM_GPG_BETA = D7B00447 -RPM_GPG_BETA_FILE = RPM-GPG-KEY-ansible-beta -DEB_GPG_RELEASE = 3DD29021 -DEB_GPG_RELEASE_FILE = DEB-GPG-KEY-ansible-release +GPG_RELEASE = 442667A9 +GPG_RELEASE_FILE = GPG-KEY-ansible-release +GPG_BETA = D7B00447 +GPG_BETA_FILE = GPG-KEY-ansible-beta # Determine GPG key for package signing ifeq ($(OFFICIAL),yes) - TAR_GPG_KEY = $(RPM_GPG_RELEASE) - RPM_GPG_KEY = $(RPM_GPG_RELEASE) - RPM_GPG_FILE = $(RPM_GPG_RELEASE_FILE) - DEB_GPG_KEY = $(DEB_GPG_RELEASE) - DEB_GPG_FILE = $(DEB_GPG_RELEASE_FILE) + GPG_KEY = $(GPG_RELEASE) + GPG_FILE = $(GPG_RELEASE_FILE) endif # TAR build parameters @@ -75,7 +70,7 @@ DPUT_OPTS ?= ifeq ($(OFFICIAL),yes) DEB_DIST ?= stable # Sign official builds - DEBUILD_OPTS += -k$(DEB_GPG_KEY) + DEBUILD_OPTS += -k$(GPG_KEY) else DEB_DIST ?= unstable # Do not sign development builds @@ -390,7 +385,7 @@ tar-build/$(SETUP_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $(notdir $@) ; \ else \ - cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign --batch --passphrase "$(GPG_PASSPHRASE)" -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ + cd tar-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign --batch --passphrase "$(GPG_PASSPHRASE)" -u "$(GPG_KEY)" -o $(notdir $@) - ; \ fi setup_tarball: tar-build/$(SETUP_TAR_FILE) tar-build/$(SETUP_TAR_CHECKSUM) @@ -426,7 +421,7 @@ setup-bundle-build/$(OFFLINE_TAR_CHECKSUM): @if [ "$(OFFICIAL)" != "yes" ] ; then \ cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz > $(notdir $@) ; \ else \ - cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign --batch --passphrase "$(GPG_PASSPHRASE)" -u "$(TAR_GPG_KEY)" -o $(notdir $@) - ; \ + cd setup-bundle-build && $(SHASUM_BIN) $(NAME)*.tar.gz | $(GPG_BIN) --clearsign --batch --passphrase "$(GPG_PASSPHRASE)" -u "$(GPG_KEY)" -o $(notdir $@) - ; \ fi setup_bundle_tarball: setup-bundle-build setup-bundle-build/$(OFFLINE_TAR_FILE) setup-bundle-build/$(OFFLINE_TAR_CHECKSUM) @@ -477,11 +472,11 @@ rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm: rpm-build/$(RPM_NVR).src.rpm mock-rpm: rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm ifeq ($(OFFICIAL),yes) -rpm-build/$(RPM_GPG_FILE): rpm-build - $(GPG_BIN) --export -a "${RPM_GPG_KEY}" > "$@" +rpm-build/$(GPG_FILE): rpm-build + $(GPG_BIN) --export -a "${GPG_KEY}" > "$@" -rpm-sign: rpm-build/$(RPM_GPG_FILE) rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm - rpm --define "_signature gpg" --define "_gpg_name $(RPM_GPG_KEY)" --addsign rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm +rpm-sign: rpm-build/$(GPG_FILE) rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm + rpm --define "_signature gpg" --define "_gpg_name $(GPG_KEY)" --addsign rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm endif deb-build: @@ -495,10 +490,10 @@ deb-build/$(SDIST_TAR_NAME): sed -ie "s#^$(NAME) (\([^)]*\)) \([^;]*\);#$(NAME) ($(VERSION)-$(RELEASE)) $(DEB_DIST);#" deb-build/$(SDIST_TAR_NAME)/debian/changelog ifeq ($(OFFICIAL),yes) -debian: sdist deb-build/$(SDIST_TAR_NAME) deb-build/$(DEB_GPG_FILE) +debian: sdist deb-build/$(SDIST_TAR_NAME) deb-build/$(GPG_FILE) -deb-build/$(DEB_GPG_FILE): deb-build - $(GPG_BIN) --export -a "${DEB_GPG_KEY}" > "$@" +deb-build/$(GPG_FILE): deb-build + $(GPG_BIN) --export -a "${GPG_KEY}" > "$@" else debian: sdist deb-build/$(SDIST_TAR_NAME) endif @@ -532,7 +527,7 @@ reprepro: deb cp -a packaging/reprepro/* $@/conf/ if [ "$(OFFICIAL)" = "yes" ] ; then \ echo "ask-passphrase" >> $@/conf/options; \ - sed -i -e 's|^\(Codename:\)|SignWith: $(DEB_GPG_KEY)\n\1|' $@/conf/distributions ; \ + sed -i -e 's|^\(Codename:\)|SignWith: $(GPG_KEY)\n\1|' $@/conf/distributions ; \ fi @DEB=deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb ; \ for DIST in trusty precise ; do \ From 32cf468059aadd976dc6a23114b695ad3da18a32 Mon Sep 17 00:00:00 2001 From: James Laska Date: Thu, 10 Sep 2015 19:03:41 -0400 Subject: [PATCH 30/32] Allow customizing npm path So jenkins can use `npm --no-color` --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 38f105a31f..9310c4db34 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ PACKER ?= packer GRUNT ?= $(shell [ -t 0 ] && echo "grunt" || echo "grunt --no-color") BROCCOLI ?= ./node_modules/.bin/broccoli NODE ?= node +NPM_BIN ?= npm DEPS_SCRIPT ?= packaging/bundle/deps.py AW_REPO_URL ?= "http://releases.ansible.com/ansible-tower" @@ -193,7 +194,7 @@ real-requirements_dev: # Install third-party requirements needed for running unittests in jenkins real-requirements_jenkins: pip install -r requirements/requirements_jenkins.txt - npm install csslint jshint + $(NPM_BIN) install csslint jshint # "Install" ansible-tower package in development mode. develop: @@ -345,7 +346,7 @@ sync_ui: node_modules Brocfile.js # Update local npm install node_modules: package.json - npm install + $(NPM_BIN) install touch $@ devjs: node_modules clean-ui Brocfile.js bower.json Gruntfile.js From 8785af996db4fe85f4f22ee975c9e289d0576947 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Fri, 11 Sep 2015 14:39:01 -0400 Subject: [PATCH 31/32] Purge an old supervisor config Apparently awx_supervisor.conf was still hanging around in the debian install. We'll make sure we won't lay that file down and bump the supervisor migration to make sure the file is removed permanently --- config/awx_supervisor.conf | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 config/awx_supervisor.conf diff --git a/config/awx_supervisor.conf b/config/awx_supervisor.conf deleted file mode 100644 index f81156f1d0..0000000000 --- a/config/awx_supervisor.conf +++ /dev/null @@ -1,12 +0,0 @@ -[program:awx-celeryd] -autorestart = true -logfile = /var/log/supervisor/awx-celeryd.log -stopwaitsecs = 600 -log_stdout = true -command = /usr/bin/awx-manage celeryd -B -l info --autoscale=20,2 -user = awx -autostart = true -directory = /var/lib/awx -log_stderr = true -logfile_maxbytes = 50MB -logfile_backups = 999 From 0ac4359becd8dc417f3418ae5fd35b55e4a48f23 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Fri, 11 Sep 2015 14:42:01 -0400 Subject: [PATCH 32/32] Purge superlance We no longer get memmon with our package vendoring changes. Don't think we need it anyway as we've got the socket service fixed. --- requirements/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 9b1d9ee860..a757909a9f 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -102,7 +102,6 @@ six==1.9.0 South==0.8.4 stevedore==1.3.0 suds==0.4 -superlance==0.11 warlock==1.1.0 wheel==0.24.0 wsgiref==0.1.2