From 32a4a941e0b95debc3fcca1f21389902043cb147 Mon Sep 17 00:00:00 2001 From: James Laska Date: Wed, 26 Aug 2015 11:49:09 -0400 Subject: [PATCH 1/4] Gather pip requirements at package build time Previously, requirements were gathered prior to the `sdist` target, and included in the sdist tarball. As some requirements are compiled, this caused problems where the compiled dependencies were linked against the wrong libraries. This pull-request addresses that by moving the requirements gathering into the `dh_prep` and `%build` steps of the packaging process. --- MANIFEST.in | 4 ++-- Makefile | 32 +++++++++++++++++++------------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 5af81ddecc..1825574341 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,9 +6,8 @@ recursive-include awx/ui/templates *.html recursive-include awx/ui/dist * recursive-include awx/playbooks *.yml recursive-include awx/lib/site-packages * +recursive-include requirements *.txt recursive-include config * -recursive-include config/deb * -recursive-include config/rpm * recursive-exclude awx devonly.py* recursive-exclude awx/api/tests * recursive-exclude awx/main/tests * @@ -18,5 +17,6 @@ include tools/scripts/ansible-tower-service include tools/munin_monitors/* include tools/sosreport/* include COPYING +include Makefile prune awx/public prune awx/projects diff --git a/Makefile b/Makefile index c17e51468f..6d8811e3d2 100644 --- a/Makefile +++ b/Makefile @@ -71,11 +71,13 @@ else endif DEBUILD = $(DEBUILD_BIN) $(DEBUILD_OPTS) DEB_PPA ?= reprepro +DEB_ARCH ?= amd64 # RPM build parameters RPM_SPECDIR= packaging/rpm RPM_SPEC = $(RPM_SPECDIR)/$(NAME).spec RPM_DIST ?= $(shell rpm --eval '%{?dist}' 2>/dev/null) +RPM_ARCH ?= $(shell rpm --eval '%{_arch}' 2>/dev/null) RPM_NVR = $(NAME)-$(VERSION)-$(RELEASE)$(RPM_DIST) MOCK_BIN ?= mock MOCK_CFG ?= @@ -149,15 +151,19 @@ push: git push origin master # Install runtime, development and jenkins requirements -requirements requirements_dev requirements_jenkins: %: real-% awx/lib/site-packages/oslo/__init__.py awx/lib/site-packages/dogpile/__init__.py +requirements requirements_dev requirements_jenkins: %: real-% # Create missing __init__.py files awx/lib/site-packages/%/__init__.py: touch $@ # Install third-party requirements needed for development environment. +# NOTE: +# * --target is only supported on newer versions of pip +# * https://github.com/pypa/pip/issues/3056 - the workaround is to override the `install-platlib` +# * --user (in conjunction with PYTHONUSERBASE="awx" may be a better option real-requirements: - pip install -r requirements/requirements.txt --target awx/lib/site-packages/ --ignore-installed + pip install -r requirements/requirements.txt --target awx/lib/site-packages/ --ignore-installed --install-option="--install-platlib=\$$base/lib/python" real-requirements_dev: real-requirements # (cat requirements/requirements.txt requirements/requirements_dev.txt > /tmp/req_dev.txt); @@ -368,7 +374,7 @@ release_clean: dist/$(SDIST_TAR_FILE): BUILD="$(BUILD)" $(PYTHON) setup.py sdist -sdist: minjs requirements dist/$(SDIST_TAR_FILE) +sdist: minjs dist/$(SDIST_TAR_FILE) # Build setup offline tarball offline_tar-build/$(DIST_FULL)/$(OFFLINE_TAR_FILE): @@ -410,22 +416,22 @@ rpm-build/$(RPM_NVR).src.rpm: /etc/mock/$(MOCK_CFG).cfg mock-srpm: rpmtar rpm-build/$(RPM_NVR).src.rpm -rpm-build/$(RPM_NVR).noarch.rpm: rpm-build/$(RPM_NVR).src.rpm +rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm: rpm-build/$(RPM_NVR).src.rpm $(MOCK_BIN) -r $(MOCK_CFG) --resultdir rpm-build --rebuild rpm-build/$(RPM_NVR).src.rpm \ --define "tower_version $(VERSION)" --define "tower_release $(RELEASE)" @echo "#############################################" @echo "RPM artifacts:" - @echo rpm-build/$(RPM_NVR).noarch.rpm + @echo rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm @echo "#############################################" -mock-rpm: rpmtar rpm-build/$(RPM_NVR).noarch.rpm +mock-rpm: rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm ifeq ($(OFFICIAL),yes) rpm-build/$(GPG_FILE): rpm-build gpg --export -a "${GPG_KEY}" > "$@" -rpm-sign: rpm-build/$(GPG_FILE) rpmtar rpm-build/$(RPM_NVR).noarch.rpm - rpm --define "_signature gpg" --define "_gpg_name $(GPG_KEY)" --addsign rpm-build/$(RPM_NVR).noarch.rpm +rpm-sign: rpm-build/$(GPG_FILE) rpmtar rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm + rpm --define "_signature gpg" --define "_gpg_name $(GPG_KEY)" --addsign rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm endif deb-build/$(SDIST_TAR_NAME): @@ -437,14 +443,14 @@ deb-build/$(SDIST_TAR_NAME): debian: sdist deb-build/$(SDIST_TAR_NAME) -deb-build/$(NAME)_$(VERSION)-$(RELEASE)_all.deb: +deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb: cd deb-build/$(SDIST_TAR_NAME) && $(DEBUILD) -b @echo "#############################################" @echo "DEB artifacts:" - @echo deb-build/$(NAME)_$(VERSION)-$(RELEASE)_all.deb + @echo deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb @echo "#############################################" -deb: debian deb-build/$(NAME)_$(VERSION)-$(RELEASE)_all.deb +deb: debian deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb deb-build/$(NAME)_$(VERSION)-$(RELEASE)_source.changes: cd deb-build/$(SDIST_TAR_NAME) && $(DEBUILD) -S @@ -456,7 +462,7 @@ deb-build/$(NAME)_$(VERSION)-$(RELEASE)_source.changes: deb-src: debian deb-build/$(NAME)_$(VERSION)-$(RELEASE)_source.changes deb-upload: deb - $(DPUT_BIN) $(DPUT_OPTS) $(DEB_PPA) deb-build/$(NAME)_$(VERSION)-$(RELEASE)_amd64.changes ; \ + $(DPUT_BIN) $(DPUT_OPTS) $(DEB_PPA) deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).changes ; \ deb-src-upload: deb-src $(DPUT_BIN) $(DPUT_OPTS) $(DEB_PPA) deb-build/$(NAME)_$(VERSION)-$(RELEASE)_source.changes ; \ @@ -464,7 +470,7 @@ deb-src-upload: deb-src reprepro: deb mkdir -p reprepro/conf cp -a packaging/reprepro/* reprepro/conf/ - @DEB=deb-build/$(NAME)_$(VERSION)-$(RELEASE)_all.deb ; \ + @DEB=deb-build/$(NAME)_$(VERSION)-$(RELEASE)_$(DEB_ARCH).deb ; \ for DIST in trusty precise ; do \ echo "Removing '$(NAME)' from the $${DIST} apt repo" ; \ echo reprepro --export=force -b reprepro remove $${DIST} $(NAME) ; \ From 51e2120886a9a652bc4e47becad232d9e7f85c00 Mon Sep 17 00:00:00 2001 From: James Laska Date: Wed, 26 Aug 2015 11:31:37 -0400 Subject: [PATCH 2/4] Properly handle namespace packages (oslo, dogpile) Using `site.addsitedir()` handles namespace packages, where the traditional `sys.path.insert` does not. Kudos to Chris Meyers for this discovery. --- awx/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/awx/__init__.py b/awx/__init__.py index 4b3a8072fa..2c4a64f58e 100644 --- a/awx/__init__.py +++ b/awx/__init__.py @@ -4,6 +4,7 @@ import os import sys import warnings +import site __version__ = '2.3.0' @@ -39,7 +40,10 @@ def prepare_env(): # Add local site-packages directory to path. local_site_packages = os.path.join(os.path.dirname(__file__), 'lib', 'site-packages') - sys.path.insert(0, local_site_packages) + site.addsitedir(local_site_packages) + # Work around https://bugs.python.org/issue7744 + # by moving local_site_packages to the front of sys.path + sys.path.insert(0, sys.path.pop()) # Hide DeprecationWarnings when running in production. Need to first load # settings to apply our filter after Django's own warnings filter. from django.conf import settings From 6512c765641a931c9242f5e120f5c3bfa2a51b1b Mon Sep 17 00:00:00 2001 From: James Laska Date: Wed, 26 Aug 2015 12:15:58 -0400 Subject: [PATCH 3/4] Ignore offline TAR build directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a0026aaaa2..b01683f972 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ build deb-build rpm-build tar-build +/offline_tar-build /dist *.egg-info *.py[c,o] From 265802322d74b95905e708f9364b89d9e5176990 Mon Sep 17 00:00:00 2001 From: James Laska Date: Wed, 26 Aug 2015 16:15:07 -0400 Subject: [PATCH 4/4] Add a new default build target Also override dh_auto_clean to avoid removing the awx/ui/dist during DEB package builds. Update the `.spec` file to call the same Makefile targets used by in `debian/rules`. --- Makefile | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 6d8811e3d2..fde0d07f06 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,15 @@ OFFLINE_TAR_NAME=$(NAME)-offline-$(DIST_FULL)-$(VERSION)-$(RELEASE) OFFLINE_TAR_FILE=$(OFFLINE_TAR_NAME).tar.gz OFFLINE_TAR_LINK=$(NAME)-setup-latest.tar.gz +DISTRO := $(shell source /etc/os-release 2>/dev/null && echo $${ID} || echo redhat) +ifeq ($(DISTRO),ubuntu) + SETUP_INSTALL_ARGS = --skip-build --no-compile --root=$(DESTDIR) -v --install-layout=deb +else + SETUP_INSTALL_ARGS = --skip-build --no-compile --root=$(DESTDIR) -v +endif + +.DEFAULT_GOAL := build + .PHONY: clean rebase push requirements requirements_dev requirements_jenkins \ real-requirements real-requirements_dev real-requirements_jenkins \ develop refresh adduser syncdb migrate dbchange dbshell runserver celeryd \ @@ -153,10 +162,6 @@ push: # Install runtime, development and jenkins requirements requirements requirements_dev requirements_jenkins: %: real-% -# Create missing __init__.py files -awx/lib/site-packages/%/__init__.py: - touch $@ - # Install third-party requirements needed for development environment. # NOTE: # * --target is only supported on newer versions of pip @@ -502,5 +507,11 @@ virtualbox-centos-7: packaging/packer/output-virtualbox-iso/centos-7.ovf docker-dev: docker build --no-cache=true --rm=true -t ansible/tower_devel:latest tools/docker +# TODO - figure out how to build the front-end and python requirements with +# 'build' +build: + $(PYTHON) setup.py build + +# TODO - only use --install-layout=deb on Debian install: - $(PYTHON) setup.py install egg_info -b "" + $(PYTHON) setup.py install $(SETUP_INSTALL_ARGS)