From 4742cede0ddd240b9577a2e82d69f5387ceca2f4 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 16:45:20 -0400 Subject: [PATCH 1/7] Add pip and setuptools to requirements files These packages were getting silently updated, making vendoring these dependencies difficult. This ensures that the expected versions of these packages are installed in the virtualenvs. --- requirements/requirements.in | 2 ++ requirements/requirements.txt | 3 ++- requirements/requirements_ansible.in | 2 ++ requirements/requirements_ansible.txt | 3 ++- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/requirements/requirements.in b/requirements/requirements.in index 5d3b4c0e45..19d222e315 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -49,3 +49,5 @@ slackclient==1.0.2 twilio==5.6.0 uWSGI==2.0.14 xmltodict==0.10.2 +pip==8.1.2 +setuptools==23.0.0 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 0012ad8330..1ff0a862cb 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -197,4 +197,5 @@ xmltodict==0.10.2 zope.interface==4.3.3 # via twisted # The following packages are considered to be unsafe in a requirements file: -# setuptools # via cryptography, django-polymorphic, python-ldap, zope.interface +pip==8.1.2 +setuptools==23.0.0 diff --git a/requirements/requirements_ansible.in b/requirements/requirements_ansible.in index 671eaf1496..c5479b85d9 100644 --- a/requirements/requirements_ansible.in +++ b/requirements/requirements_ansible.in @@ -11,3 +11,5 @@ pyvmomi==6.5 pywinrm[kerberos]==0.2.2 secretstorage==2.3.1 shade==1.13.1 +setuptools==23.0.0 +pip==8.1.2 diff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt index 687378c9aa..763dc872cb 100644 --- a/requirements/requirements_ansible.txt +++ b/requirements/requirements_ansible.txt @@ -128,4 +128,5 @@ wrapt==1.10.8 # via debtcollector, positional xmltodict==0.10.2 # via pywinrm # The following packages are considered to be unsafe in a requirements file: -# setuptools # via cryptography +pip==8.1.2 +setuptools==23.0.0 From 092c0b739e1cc51f3cb58289327855ca77c9cde4 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 17:40:32 -0400 Subject: [PATCH 2/7] Simplify requirements targets AFAIK, there is never a situation where VENV is going to be unset. --- Makefile | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 541a04a571..f56e2eade6 100644 --- a/Makefile +++ b/Makefile @@ -282,14 +282,8 @@ virtualenv_tower: fi requirements_ansible: virtualenv_ansible - if [ "$(VENV_BASE)" ]; then \ - . $(VENV_BASE)/ansible/bin/activate; \ - $(VENV_BASE)/ansible/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt ;\ - $(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt; \ - else \ - pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt ; \ - pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt; \ - fi + $(VENV_BASE)/ansible/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt + $(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt requirements_ansible_dev: if [ "$(VENV_BASE)" ]; then \ @@ -298,21 +292,12 @@ requirements_ansible_dev: # Install third-party requirements needed for Tower's environment. requirements_tower: virtualenv_tower - if [ "$(VENV_BASE)" ]; then \ - . $(VENV_BASE)/tower/bin/activate; \ - $(VENV_BASE)/tower/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt ;\ - $(VENV_BASE)/tower/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt; \ - else \ - pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt ; \ - pip uninstall --yes -r requirements/requirements_tower_uninstall.txt; \ - fi + $(VENV_BASE)/tower/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt + $(VENV_BASE)/tower/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt requirements_tower_dev: - if [ "$(VENV_BASE)" ]; then \ - . $(VENV_BASE)/tower/bin/activate; \ - $(VENV_BASE)/tower/bin/pip install -r requirements/requirements_dev.txt; \ - $(VENV_BASE)/tower/bin/pip uninstall --yes -r requirements/requirements_dev_uninstall.txt; \ - fi + $(VENV_BASE)/tower/bin/pip install -r requirements/requirements_dev.txt + $(VENV_BASE)/tower/bin/pip uninstall --yes -r requirements/requirements_dev_uninstall.txt requirements: requirements_ansible requirements_tower From fbb6650ad0ce18eaabc8e823750416f172e91645 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 17:56:46 -0400 Subject: [PATCH 3/7] Add PIP_OPTIONS This will allow us to conditionally pass in things like --no-index and --find-links, which will be necessary for building in Brew. --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index f56e2eade6..5cfbcde4b5 100644 --- a/Makefile +++ b/Makefile @@ -264,8 +264,8 @@ virtualenv_ansible: fi; \ if [ ! -d "$(VENV_BASE)/ansible" ]; then \ virtualenv --system-site-packages --setuptools $(VENV_BASE)/ansible && \ - $(VENV_BASE)/ansible/bin/pip install --force-reinstall setuptools==23.0.0 && \ - $(VENV_BASE)/ansible/bin/pip install --force-reinstall pip==8.1.2; \ + $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --force-reinstall setuptools==23.0.0 && \ + $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --force-reinstall pip==8.1.2; \ fi; \ fi @@ -276,13 +276,13 @@ virtualenv_tower: fi; \ if [ ! -d "$(VENV_BASE)/tower" ]; then \ virtualenv --system-site-packages --setuptools $(VENV_BASE)/tower && \ - $(VENV_BASE)/tower/bin/pip install --force-reinstall setuptools==23.0.0 && \ - $(VENV_BASE)/tower/bin/pip install --force-reinstall pip==8.1.2; \ + $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --force-reinstall setuptools==23.0.0 && \ + $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --force-reinstall pip==8.1.2; \ fi; \ fi requirements_ansible: virtualenv_ansible - $(VENV_BASE)/ansible/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt + $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt $(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt requirements_ansible_dev: @@ -292,7 +292,7 @@ requirements_ansible_dev: # Install third-party requirements needed for Tower's environment. requirements_tower: virtualenv_tower - $(VENV_BASE)/tower/bin/pip install --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt + $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt $(VENV_BASE)/tower/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt requirements_tower_dev: From c94f294d068ca37384d168d0b088e2d24c165549 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 17:58:41 -0400 Subject: [PATCH 4/7] Use --ignore-installed when creating virtualenvs --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 5cfbcde4b5..8801f80bc9 100644 --- a/Makefile +++ b/Makefile @@ -264,8 +264,8 @@ virtualenv_ansible: fi; \ if [ ! -d "$(VENV_BASE)/ansible" ]; then \ virtualenv --system-site-packages --setuptools $(VENV_BASE)/ansible && \ - $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --force-reinstall setuptools==23.0.0 && \ - $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --force-reinstall pip==8.1.2; \ + $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed setuptools==23.0.0 && \ + $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed pip==8.1.2; \ fi; \ fi @@ -276,8 +276,8 @@ virtualenv_tower: fi; \ if [ ! -d "$(VENV_BASE)/tower" ]; then \ virtualenv --system-site-packages --setuptools $(VENV_BASE)/tower && \ - $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --force-reinstall setuptools==23.0.0 && \ - $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --force-reinstall pip==8.1.2; \ + $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --ignore-installed setuptools==23.0.0 && \ + $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --ignore-installed pip==8.1.2; \ fi; \ fi From f79cd028f06d831877a573ff172c828ef8641dc5 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 19:40:46 -0400 Subject: [PATCH 5/7] Add separate requirements file for setup_requires stuff pip doesnt know how to resolve setup_requires dependencies. we need this for offline pip installs when building in brew --- requirements/requirements_setup_requires.txt | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 requirements/requirements_setup_requires.txt diff --git a/requirements/requirements_setup_requires.txt b/requirements/requirements_setup_requires.txt new file mode 100644 index 0000000000..f9857bb2a6 --- /dev/null +++ b/requirements/requirements_setup_requires.txt @@ -0,0 +1,5 @@ +pbr>=1.8 +setuptools_scm>=1.15.0 +vcversioner>=2.16.0.0 +pytest-runner +isort From 7e1eeb994b84679be23579e93700c23f84b4bda8 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 18:07:36 -0400 Subject: [PATCH 6/7] Add make targets for vendoring python requirements --- Makefile | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Makefile b/Makefile index 8801f80bc9..579e451b93 100644 --- a/Makefile +++ b/Makefile @@ -708,6 +708,30 @@ rpm-build/$(SDIST_TAR_FILE): rpm-build dist/$(SDIST_TAR_FILE) rpmtar: sdist rpm-build/$(SDIST_TAR_FILE) +brewrpmtar: rpm-build/python-deps.tar.gz rpmtar + +rpm-build/python-deps.tar.gz: requirements/vendor rpm-build + tar czf rpm-build/python-deps.tar.gz requirements/vendor + +requirements/vendor: + pip download \ + --no-binary=:all: \ + --requirement=requirements/requirements_ansible.txt \ + --dest=$@ \ + --exists-action=i + + pip download \ + --no-binary=:all: \ + --requirement=requirements/requirements.txt \ + --dest=$@ \ + --exists-action=i + + pip download \ + --no-binary=:all: \ + --requirement=requirements/requirements_setup_requires.txt \ + --dest=$@ \ + --exists-action=i + rpm-build/$(RPM_NVR).src.rpm: /etc/mock/$(MOCK_CFG).cfg $(MOCK_BIN) -r $(MOCK_CFG) --resultdir rpm-build --buildsrpm --spec rpm-build/$(NAME).spec --sources rpm-build \ --define "tower_version $(VERSION)" --define "tower_release $(RELEASE)" $(SCL_DEFINES) From d76d55adafe6313fc264167065ce9c5e3f4072c0 Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Mon, 13 Mar 2017 18:15:32 -0400 Subject: [PATCH 7/7] Add brew-srpm target --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 579e451b93..0d933cff61 100644 --- a/Makefile +++ b/Makefile @@ -742,6 +742,8 @@ mock-srpm: rpmtar rpm-build/$(RPM_NVR).src.rpm @echo rpm-build/$(RPM_NVR).src.rpm @echo "#############################################" +brew-srpm: brewrpmtar mock-srpm + rpm-build/$(RPM_NVR).$(RPM_ARCH).rpm: rpm-build/$(RPM_NVR).src.rpm $(MOCK_BIN) -r $(MOCK_CFG) --resultdir rpm-build --rebuild rpm-build/$(RPM_NVR).src.rpm \ --define "tower_version $(VERSION)" --define "tower_release $(RELEASE)" $(SCL_DEFINES)