External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-05-09 10:27:37 -02:30
Code Issues Packages Projects Releases Wiki Activity

Attempts to simplify build environment

Browse Source 
Remove build isolation flag

Do not use --ignore-installed for any pip install commands

Add Makefile comments

do not use system site packages for awx venv

Consolidate bootstrap pins

Do another upgrade, properly document wheel
This commit is contained in:
AlanCoding
2019-12-16 08:24:33 -05:00
parent 477f566da0
commit 271b3f00b7
6 changed files with 40 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
Makefile
View File

@@ -26,6 +26,9 @@ DEV_DOCKER_TAG_BASE ?= gcr.io/ansible-tower-engineering
# Python packages to install only from source (not from binary wheels) # Python packages to install only from source (not from binary wheels)
# Comma separated list # Comma separated list
SRC_ONLY_PKGS ?= cffi,pycparser,psycopg2,twilio SRC_ONLY_PKGS ?= cffi,pycparser,psycopg2,twilio
# These should be upgraded in the AWX and Ansible venv before attempting
# to install the actual requirements
VENV_BOOTSTRAP ?= pip==19.3.1 setuptools==41.6.0 wheel==0.33.6
# Determine appropriate shasum command # Determine appropriate shasum command
UNAME_S := $(shell uname -s) UNAME_S := $(shell uname -s)
@@ -130,6 +133,8 @@ guard-%:
virtualenv: virtualenv_ansible virtualenv_awx virtualenv: virtualenv_ansible virtualenv_awx
# virtualenv_* targets do not use --system-site-packages to prevent bugs installing packages
# but Ansible venvs are expected to have this, so that must be done after venv creation
virtualenv_ansible: virtualenv_ansible:
if [ "$(VENV_BASE)" ]; then \ if [ "$(VENV_BASE)" ]; then \
if [ ! -d "$(VENV_BASE)" ]; then \ if [ ! -d "$(VENV_BASE)" ]; then \
@@ -137,7 +142,7 @@ virtualenv_ansible:
fi; \ fi; \
if [ ! -d "$(VENV_BASE)/ansible" ]; then \ if [ ! -d "$(VENV_BASE)/ansible" ]; then \
virtualenv -p python $(VENV_BASE)/ansible && \ virtualenv -p python $(VENV_BASE)/ansible && \
$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) pip==19.3.1 setuptools==41.6.0 wheel==0.33.6; \ $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) $(VENV_BOOTSTRAP); \
fi; \ fi; \
fi fi
@@ -148,10 +153,13 @@ virtualenv_ansible_py3:
fi; \ fi; \
if [ ! -d "$(VENV_BASE)/ansible" ]; then \ if [ ! -d "$(VENV_BASE)/ansible" ]; then \
virtualenv -p $(PYTHON) $(VENV_BASE)/ansible; \ virtualenv -p $(PYTHON) $(VENV_BASE)/ansible; \
$(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) pip==19.3.1 setuptools==41.6.0 wheel==0.33.6; \ $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) $(VENV_BOOTSTRAP); \
fi; \ fi; \
fi fi
# poetry and flit are needed for offline install of certain packages
# but are not declared in setup_requires due to complex reasons
# similar to pip, setuptools, and wheel, these are all needed here as a bootstrapping issues
virtualenv_awx: virtualenv_awx:
if [ "$(VENV_BASE)" ]; then \ if [ "$(VENV_BASE)" ]; then \
if [ ! -d "$(VENV_BASE)" ]; then \ if [ ! -d "$(VENV_BASE)" ]; then \
@@ -159,27 +167,30 @@ virtualenv_awx:
fi; \ fi; \
if [ ! -d "$(VENV_BASE)/awx" ]; then \ if [ ! -d "$(VENV_BASE)/awx" ]; then \
virtualenv -p $(PYTHON) $(VENV_BASE)/awx; \ virtualenv -p $(PYTHON) $(VENV_BASE)/awx; \
$(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) pip==19.3.1 setuptools==41.6.0 wheel==0.33.6 && \ $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) $(VENV_BOOTSTRAP) && \
$(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) poetry==0.12.17 flit; \ $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) poetry==0.12.17 flit; \
fi; \ fi; \
fi fi
# --ignore-install flag is not used because *.txt files should specify exact versions
requirements_ansible: virtualenv_ansible requirements_ansible: virtualenv_ansible
if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \ if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \
cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed --no-build-isolation -r /dev/stdin ; \ cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) -r /dev/stdin ; \
else \ else \
cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \ cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \
fi fi
$(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt $(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt
# Same effect as using --system-site-packages flag on venv creation
rm $(shell ls -d $(VENV_BASE)/ansible/lib/python* | head -n 1)/no-global-site-packages.txt rm $(shell ls -d $(VENV_BASE)/ansible/lib/python* | head -n 1)/no-global-site-packages.txt
requirements_ansible_py3: virtualenv_ansible_py3 requirements_ansible_py3: virtualenv_ansible_py3
if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \ if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \
cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) --ignore-installed --no-build-isolation -r /dev/stdin ; \ cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) -r /dev/stdin ; \
else \ else \
cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \ cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \
fi fi
$(VENV_BASE)/ansible/bin/pip3 uninstall --yes -r requirements/requirements_ansible_uninstall.txt $(VENV_BASE)/ansible/bin/pip3 uninstall --yes -r requirements/requirements_ansible_uninstall.txt
# Same effect as using --system-site-packages flag on venv creation
rm $(shell ls -d $(VENV_BASE)/ansible/lib/python* | head -n 1)/no-global-site-packages.txt rm $(shell ls -d $(VENV_BASE)/ansible/lib/python* | head -n 1)/no-global-site-packages.txt
requirements_ansible_dev: requirements_ansible_dev:
@@ -188,15 +199,14 @@ requirements_ansible_dev:
fi fi
# Install third-party requirements needed for AWX's environment. # Install third-party requirements needed for AWX's environment.
# this does not use system site packages intentionally
requirements_awx: virtualenv_awx requirements_awx: virtualenv_awx
if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \ if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \
cat requirements/requirements.txt requirements/requirements_local.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --ignore-installed --no-build-isolation -r /dev/stdin ; \ cat requirements/requirements.txt requirements/requirements_local.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) -r /dev/stdin ; \
else \ else \
cat requirements/requirements.txt requirements/requirements_git.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \ cat requirements/requirements.txt requirements/requirements_git.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \
fi fi
echo "include-system-site-packages = true" >> $(VENV_BASE)/awx/lib/python$(PYTHON_VERSION)/pyvenv.cfg
$(VENV_BASE)/awx/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt $(VENV_BASE)/awx/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt
rm $(shell ls -d $(VENV_BASE)/awx/lib/python* | head -n 1)/no-global-site-packages.txt
requirements_awx_dev: requirements_awx_dev:
$(VENV_BASE)/awx/bin/pip install -r requirements/requirements_dev.txt $(VENV_BASE)/awx/bin/pip install -r requirements/requirements_dev.txt

4
requirements/README.md
View File

@@ -129,9 +129,11 @@ This breaks a very large amount of AWX code that assumes these fields
are returned as dicts. Upgrading this library will require a refactor are returned as dicts. Upgrading this library will require a refactor
to accomidate this change. to accomidate this change.
### pip and setuptools ### pip and setuptools and wheel
The offline installer needs to have functionality confirmed before upgrading these. The offline installer needs to have functionality confirmed before upgrading these.
Versions need to match the versions used in the pip bootstrapping step
in the top-level Makefile.
## Library Notes ## Library Notes

1
requirements/requirements.in
View File

@@ -46,3 +46,4 @@ uWSGI
uwsgitop uwsgitop
pip==19.3.1 # see UPGRADE BLOCKERs pip==19.3.1 # see UPGRADE BLOCKERs
setuptools==41.6.0 # see UPGRADE BLOCKERs setuptools==41.6.0 # see UPGRADE BLOCKERs
wheel==0.33.6 # see UPGRADE BLOCKERs

24
requirements/requirements.txt
View File

@@ -22,10 +22,10 @@ constantly==15.1.0 # via twisted
cryptography==2.8 # via adal, autobahn, azure-keyvault cryptography==2.8 # via adal, autobahn, azure-keyvault
daphne==1.4.2 daphne==1.4.2
defusedxml==0.6.0 # via python3-openid, python3-saml, social-auth-core defusedxml==0.6.0 # via python3-openid, python3-saml, social-auth-core
dictdiffer==0.8.0 # via openshift dictdiffer==0.8.1 # via openshift
django-auth-ldap==2.1.0 django-auth-ldap==2.1.0
django-cors-headers==3.2.0 django-cors-headers==3.2.0
django-crum==0.7.4 django-crum==0.7.5
django-extensions==2.2.5 django-extensions==2.2.5
django-jsonfield==1.2.0 django-jsonfield==1.2.0
django-oauth-toolkit==1.1.3 django-oauth-toolkit==1.1.3
@@ -38,15 +38,15 @@ django-split-settings==1.0.0
django-taggit==1.2.0 django-taggit==1.2.0
django==2.2.8 django==2.2.8
djangorestframework-yaml==1.0.3 djangorestframework-yaml==1.0.3
djangorestframework==3.10.3 djangorestframework==3.11.0
future==0.16.0 # via django-radius future==0.16.0 # via django-radius
gitdb2==2.0.6 # via gitpython gitdb2==2.0.6 # via gitpython
gitpython==3.0.5 gitpython==3.0.5
google-auth==1.7.1 # via kubernetes google-auth==1.9.0 # via kubernetes
hyperlink==19.0.0 # via twisted hyperlink==19.0.0 # via twisted
idna==2.8 # via hyperlink, requests idna==2.8 # via hyperlink, requests
importlib-metadata==1.1.0 # via inflect, irc, jsonschema, kombu importlib-metadata==1.3.0 # via inflect, irc, jsonschema, kombu
importlib-resources==1.0.2 # via jaraco.text importlib-resources==1.0.2 # via jaraco.text
incremental==17.5.0 # via twisted incremental==17.5.0 # via twisted
inflect==3.0.2 # via jaraco.itertools inflect==3.0.2 # via jaraco.itertools
@@ -57,24 +57,24 @@ jaraco.collections==2.1 # via irc
jaraco.functools==2.0 # via irc, jaraco.text, tempora jaraco.functools==2.0 # via irc, jaraco.text, tempora
jaraco.itertools==4.4.2 # via irc jaraco.itertools==4.4.2 # via irc
jaraco.logging==2.0 # via irc jaraco.logging==2.0 # via irc
jaraco.stream==2.0 # via irc jaraco.stream==3.0.0 # via irc
jaraco.text==3.2.0 # via irc, jaraco.collections jaraco.text==3.2.0 # via irc, jaraco.collections
jinja2==2.10.3 jinja2==2.10.3
jsonpickle==1.2 # via asgi-amqp jsonpickle==1.2 # via asgi-amqp
jsonschema==3.2.0 jsonschema==3.2.0
kombu==4.6.6 # via asgi-amqp, celery kombu==4.6.7 # via asgi-amqp, celery
kubernetes==10.0.1 # via openshift kubernetes==10.0.1 # via openshift
lockfile==0.12.2 # via python-daemon lockfile==0.12.2 # via python-daemon
lxml==4.4.2 # via xmlsec lxml==4.4.2 # via xmlsec
markdown==3.1.1 markdown==3.1.1
markupsafe==1.1.1 # via jinja2 markupsafe==1.1.1 # via jinja2
more-itertools==8.0.0 # via irc, jaraco.functools, jaraco.itertools, zipp more-itertools==8.0.2 # via irc, jaraco.functools, jaraco.itertools, zipp
msgpack-python==0.5.6 # via asgi-amqp msgpack-python==0.5.6 # via asgi-amqp
msrest==0.6.10 # via azure-keyvault, msrestazure msrest==0.6.10 # via azure-keyvault, msrestazure
msrestazure==0.6.2 # via azure-keyvault msrestazure==0.6.2 # via azure-keyvault
netaddr==0.7.19 # via pyrad netaddr==0.7.19 # via pyrad
oauthlib==3.1.0 # via django-oauth-toolkit, requests-oauthlib, social-auth-core oauthlib==3.1.0 # via django-oauth-toolkit, requests-oauthlib, social-auth-core
openshift==0.10.0 openshift==0.10.1
pexpect==4.7.0 # via ansible-runner pexpect==4.7.0 # via ansible-runner
pkgconfig==1.5.1 # via xmlsec pkgconfig==1.5.1 # via xmlsec
prometheus-client==0.7.1 prometheus-client==0.7.1
@@ -106,7 +106,7 @@ requests==2.22.0
rsa==4.0 # via google-auth rsa==4.0 # via google-auth
ruamel.yaml.clib==0.2.0 # via ruamel.yaml ruamel.yaml.clib==0.2.0 # via ruamel.yaml
ruamel.yaml==0.16.5 # via openshift ruamel.yaml==0.16.5 # via openshift
six==1.13.0 # via ansible-runner, asgi-amqp, asgiref, autobahn, automat, cryptography, django-extensions, google-auth, isodate, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.stream, jaraco.text, jsonschema, kubernetes, openshift, pygerduty, pyhamcrest, pyrad, pyrsistent, python-dateutil, python-memcached, slackclient, social-auth-app-django, social-auth-core, tacacs-plus, tempora, twilio, txaio, websocket-client six==1.13.0 # via ansible-runner, asgi-amqp, asgiref, autobahn, automat, cryptography, django-extensions, google-auth, isodate, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.text, jsonschema, kubernetes, openshift, pygerduty, pyhamcrest, pyrad, pyrsistent, python-dateutil, python-memcached, slackclient, social-auth-app-django, social-auth-core, tacacs-plus, tempora, twilio, txaio, websocket-client
slackclient==1.1.2 slackclient==1.1.2
smmap2==2.0.5 # via gitdb2 smmap2==2.0.5 # via gitdb2
social-auth-app-django==3.1.0 social-auth-app-django==3.1.0
@@ -114,7 +114,7 @@ social-auth-core==3.2.0
sqlparse==0.3.0 # via django sqlparse==0.3.0 # via django
tacacs_plus==1.0 tacacs_plus==1.0
tempora==1.14.1 # via irc, jaraco.logging tempora==1.14.1 # via irc, jaraco.logging
twilio==6.33.1 twilio==6.35.0
twisted==19.10.0 # via daphne twisted==19.10.0 # via daphne
txaio==18.8.1 # via autobahn txaio==18.8.1 # via autobahn
urllib3==1.25.7 # via kubernetes, requests urllib3==1.25.7 # via kubernetes, requests
@@ -122,6 +122,7 @@ uwsgi==2.0.18
uwsgitop==0.11 uwsgitop==0.11
vine==1.3.0 # via amqp, celery vine==1.3.0 # via amqp, celery
websocket-client==0.56.0 # via kubernetes, slackclient websocket-client==0.56.0 # via kubernetes, slackclient
wheel==0.33.6
xmlsec==1.3.3 # via python3-saml xmlsec==1.3.3 # via python3-saml
zipp==0.6.0 # via importlib-metadata zipp==0.6.0 # via importlib-metadata
zope.interface==4.7.1 # via twisted zope.interface==4.7.1 # via twisted
@@ -129,4 +130,3 @@ zope.interface==4.7.1 # via twisted
# The following packages are considered to be unsafe in a requirements file: # The following packages are considered to be unsafe in a requirements file:
pip==19.3.1 pip==19.3.1
setuptools==41.6.0 setuptools==41.6.0
wheel==0.33.6

3
requirements/requirements_ansible.in
View File

@@ -63,4 +63,5 @@ requests-credssp==1.0.2 # For windows authentication awx/issues/1144
# OpenStack # OpenStack
openstacksdk==0.31.1 openstacksdk==0.31.1
pip==19.3.1 # see upgrade blockers pip==19.3.1 # see upgrade blockers
setuptools==41.6.0 # see upgrade blockers setuptools==41.6.0 # see upgrade blockers
wheel==0.33.6 # see UPGRADE BLOCKERs

9
requirements/requirements_ansible.txt
View File

@@ -42,11 +42,11 @@ bcrypt==3.1.7 # via paramiko
boto3==1.9.223 boto3==1.9.223
boto==2.47.0 boto==2.47.0
botocore==1.12.253 # via boto3, s3transfer botocore==1.12.253 # via boto3, s3transfer
cachetools==3.1.1 # via google-auth cachetools==3.1.1; python_version < "3" # via google-auth
certifi==2019.11.28 # via msrest, requests certifi==2019.11.28 # via msrest, requests
cffi==1.13.2 # via bcrypt, cryptography, pynacl cffi==1.13.2 # via bcrypt, cryptography, pynacl
chardet==3.0.4 # via requests chardet==3.0.4 # via requests
colorama==0.4.1 # via azure-cli-core, knack colorama==0.4.3 # via azure-cli-core, knack
cryptography==2.8 # via adal, azure-keyvault, azure-storage, openstacksdk, paramiko, pyopenssl, requests-credssp, requests-kerberos, requests-ntlm cryptography==2.8 # via adal, azure-keyvault, azure-storage, openstacksdk, paramiko, pyopenssl, requests-credssp, requests-kerberos, requests-ntlm
decorator==4.4.1 # via dogpile.cache, openstacksdk decorator==4.4.1 # via dogpile.cache, openstacksdk
docutils==0.15.2 # via botocore docutils==0.15.2 # via botocore
@@ -80,7 +80,7 @@ openstacksdk==0.31.1
os-service-types==1.7.0 # via keystoneauth1, openstacksdk os-service-types==1.7.0 # via keystoneauth1, openstacksdk
ovirt-engine-sdk-python==4.3.0 ovirt-engine-sdk-python==4.3.0
packaging==19.2 packaging==19.2
paramiko==2.7.0 # via azure-cli-core, ncclient paramiko==2.7.1 # via azure-cli-core, ncclient
pbr==5.4.4 # via keystoneauth1, openstacksdk, os-service-types, stevedore pbr==5.4.4 # via keystoneauth1, openstacksdk, os-service-types, stevedore
psutil==5.4.3 psutil==5.4.3
pyasn1-modules==0.2.7 # via google-auth pyasn1-modules==0.2.7 # via google-auth
@@ -111,10 +111,9 @@ stevedore==1.31.0 # via keystoneauth1
tabulate==0.8.2 # via azure-cli-core, knack tabulate==0.8.2 # via azure-cli-core, knack
typing==3.7.4.1; python_version < "3" # via msrest typing==3.7.4.1; python_version < "3" # via msrest
urllib3==1.25.7 # via botocore, requests urllib3==1.25.7 # via botocore, requests
# wheel==0.30.0 # via azure-cli-core wheel==0.33.6
xmltodict==0.12.0 # via pywinrm xmltodict==0.12.0 # via pywinrm
# The following packages are considered to be unsafe in a requirements file: # The following packages are considered to be unsafe in a requirements file:
pip==19.3.1 pip==19.3.1
setuptools==41.6.0 setuptools==41.6.0
wheel==0.33.6