diff --git a/Makefile b/Makefile index 88774dc2a1..9b7378fc12 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,9 @@ DEV_DOCKER_TAG_BASE ?= gcr.io/ansible-tower-engineering # Python packages to install only from source (not from binary wheels) # Comma separated list SRC_ONLY_PKGS ?= cffi,pycparser,psycopg2,twilio +# These should be upgraded in the AWX and Ansible venv before attempting +# to install the actual requirements +VENV_BOOTSTRAP ?= pip==19.3.1 setuptools==41.6.0 wheel==0.33.6 # Determine appropriate shasum command UNAME_S := $(shell uname -s) @@ -130,6 +133,8 @@ guard-%: virtualenv: virtualenv_ansible virtualenv_awx +# virtualenv_* targets do not use --system-site-packages to prevent bugs installing packages +# but Ansible venvs are expected to have this, so that must be done after venv creation virtualenv_ansible: if [ "$(VENV_BASE)" ]; then \ if [ ! -d "$(VENV_BASE)" ]; then \ @@ -137,7 +142,7 @@ virtualenv_ansible: fi; \ if [ ! -d "$(VENV_BASE)/ansible" ]; then \ virtualenv -p python $(VENV_BASE)/ansible && \ - $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) pip==19.3.1 setuptools==41.6.0 wheel==0.33.6; \ + $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) $(VENV_BOOTSTRAP); \ fi; \ fi @@ -148,10 +153,13 @@ virtualenv_ansible_py3: fi; \ if [ ! -d "$(VENV_BASE)/ansible" ]; then \ virtualenv -p $(PYTHON) $(VENV_BASE)/ansible; \ - $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) pip==19.3.1 setuptools==41.6.0 wheel==0.33.6; \ + $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) $(VENV_BOOTSTRAP); \ fi; \ fi +# poetry and flit are needed for offline install of certain packages +# but are not declared in setup_requires due to complex reasons +# similar to pip, setuptools, and wheel, these are all needed here as a bootstrapping issues virtualenv_awx: if [ "$(VENV_BASE)" ]; then \ if [ ! -d "$(VENV_BASE)" ]; then \ @@ -159,27 +167,30 @@ virtualenv_awx: fi; \ if [ ! -d "$(VENV_BASE)/awx" ]; then \ virtualenv -p $(PYTHON) $(VENV_BASE)/awx; \ - $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) pip==19.3.1 setuptools==41.6.0 wheel==0.33.6 && \ + $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) $(VENV_BOOTSTRAP) && \ $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) poetry==0.12.17 flit; \ fi; \ fi +# --ignore-install flag is not used because *.txt files should specify exact versions requirements_ansible: virtualenv_ansible if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \ - cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed --no-build-isolation -r /dev/stdin ; \ + cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) -r /dev/stdin ; \ else \ cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \ fi $(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt + # Same effect as using --system-site-packages flag on venv creation rm $(shell ls -d $(VENV_BASE)/ansible/lib/python* | head -n 1)/no-global-site-packages.txt requirements_ansible_py3: virtualenv_ansible_py3 if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \ - cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) --ignore-installed --no-build-isolation -r /dev/stdin ; \ + cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) -r /dev/stdin ; \ else \ cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible/bin/pip3 install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \ fi $(VENV_BASE)/ansible/bin/pip3 uninstall --yes -r requirements/requirements_ansible_uninstall.txt + # Same effect as using --system-site-packages flag on venv creation rm $(shell ls -d $(VENV_BASE)/ansible/lib/python* | head -n 1)/no-global-site-packages.txt requirements_ansible_dev: @@ -188,15 +199,14 @@ requirements_ansible_dev: fi # Install third-party requirements needed for AWX's environment. +# this does not use system site packages intentionally requirements_awx: virtualenv_awx if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \ - cat requirements/requirements.txt requirements/requirements_local.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --ignore-installed --no-build-isolation -r /dev/stdin ; \ + cat requirements/requirements.txt requirements/requirements_local.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) -r /dev/stdin ; \ else \ cat requirements/requirements.txt requirements/requirements_git.txt | $(VENV_BASE)/awx/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) -r /dev/stdin ; \ fi - echo "include-system-site-packages = true" >> $(VENV_BASE)/awx/lib/python$(PYTHON_VERSION)/pyvenv.cfg $(VENV_BASE)/awx/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt - rm $(shell ls -d $(VENV_BASE)/awx/lib/python* | head -n 1)/no-global-site-packages.txt requirements_awx_dev: $(VENV_BASE)/awx/bin/pip install -r requirements/requirements_dev.txt diff --git a/requirements/README.md b/requirements/README.md index 1a58c30833..83b15d2774 100644 --- a/requirements/README.md +++ b/requirements/README.md @@ -129,9 +129,11 @@ This breaks a very large amount of AWX code that assumes these fields are returned as dicts. Upgrading this library will require a refactor to accomidate this change. -### pip and setuptools +### pip and setuptools and wheel The offline installer needs to have functionality confirmed before upgrading these. +Versions need to match the versions used in the pip bootstrapping step +in the top-level Makefile. ## Library Notes diff --git a/requirements/requirements.in b/requirements/requirements.in index ca521ad069..1fd9d01bc6 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -46,3 +46,4 @@ uWSGI uwsgitop pip==19.3.1 # see UPGRADE BLOCKERs setuptools==41.6.0 # see UPGRADE BLOCKERs +wheel==0.33.6 # see UPGRADE BLOCKERs \ No newline at end of file diff --git a/requirements/requirements.txt b/requirements/requirements.txt index f3eb577a55..41c532f232 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -22,10 +22,10 @@ constantly==15.1.0 # via twisted cryptography==2.8 # via adal, autobahn, azure-keyvault daphne==1.4.2 defusedxml==0.6.0 # via python3-openid, python3-saml, social-auth-core -dictdiffer==0.8.0 # via openshift +dictdiffer==0.8.1 # via openshift django-auth-ldap==2.1.0 django-cors-headers==3.2.0 -django-crum==0.7.4 +django-crum==0.7.5 django-extensions==2.2.5 django-jsonfield==1.2.0 django-oauth-toolkit==1.1.3 @@ -38,15 +38,15 @@ django-split-settings==1.0.0 django-taggit==1.2.0 django==2.2.8 djangorestframework-yaml==1.0.3 -djangorestframework==3.10.3 +djangorestframework==3.11.0 future==0.16.0 # via django-radius gitdb2==2.0.6 # via gitpython gitpython==3.0.5 -google-auth==1.7.1 # via kubernetes +google-auth==1.9.0 # via kubernetes hyperlink==19.0.0 # via twisted idna==2.8 # via hyperlink, requests -importlib-metadata==1.1.0 # via inflect, irc, jsonschema, kombu +importlib-metadata==1.3.0 # via inflect, irc, jsonschema, kombu importlib-resources==1.0.2 # via jaraco.text incremental==17.5.0 # via twisted inflect==3.0.2 # via jaraco.itertools @@ -57,24 +57,24 @@ jaraco.collections==2.1 # via irc jaraco.functools==2.0 # via irc, jaraco.text, tempora jaraco.itertools==4.4.2 # via irc jaraco.logging==2.0 # via irc -jaraco.stream==2.0 # via irc +jaraco.stream==3.0.0 # via irc jaraco.text==3.2.0 # via irc, jaraco.collections jinja2==2.10.3 jsonpickle==1.2 # via asgi-amqp jsonschema==3.2.0 -kombu==4.6.6 # via asgi-amqp, celery +kombu==4.6.7 # via asgi-amqp, celery kubernetes==10.0.1 # via openshift lockfile==0.12.2 # via python-daemon lxml==4.4.2 # via xmlsec markdown==3.1.1 markupsafe==1.1.1 # via jinja2 -more-itertools==8.0.0 # via irc, jaraco.functools, jaraco.itertools, zipp +more-itertools==8.0.2 # via irc, jaraco.functools, jaraco.itertools, zipp msgpack-python==0.5.6 # via asgi-amqp msrest==0.6.10 # via azure-keyvault, msrestazure msrestazure==0.6.2 # via azure-keyvault netaddr==0.7.19 # via pyrad oauthlib==3.1.0 # via django-oauth-toolkit, requests-oauthlib, social-auth-core -openshift==0.10.0 +openshift==0.10.1 pexpect==4.7.0 # via ansible-runner pkgconfig==1.5.1 # via xmlsec prometheus-client==0.7.1 @@ -106,7 +106,7 @@ requests==2.22.0 rsa==4.0 # via google-auth ruamel.yaml.clib==0.2.0 # via ruamel.yaml ruamel.yaml==0.16.5 # via openshift -six==1.13.0 # via ansible-runner, asgi-amqp, asgiref, autobahn, automat, cryptography, django-extensions, google-auth, isodate, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.stream, jaraco.text, jsonschema, kubernetes, openshift, pygerduty, pyhamcrest, pyrad, pyrsistent, python-dateutil, python-memcached, slackclient, social-auth-app-django, social-auth-core, tacacs-plus, tempora, twilio, txaio, websocket-client +six==1.13.0 # via ansible-runner, asgi-amqp, asgiref, autobahn, automat, cryptography, django-extensions, google-auth, isodate, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.text, jsonschema, kubernetes, openshift, pygerduty, pyhamcrest, pyrad, pyrsistent, python-dateutil, python-memcached, slackclient, social-auth-app-django, social-auth-core, tacacs-plus, tempora, twilio, txaio, websocket-client slackclient==1.1.2 smmap2==2.0.5 # via gitdb2 social-auth-app-django==3.1.0 @@ -114,7 +114,7 @@ social-auth-core==3.2.0 sqlparse==0.3.0 # via django tacacs_plus==1.0 tempora==1.14.1 # via irc, jaraco.logging -twilio==6.33.1 +twilio==6.35.0 twisted==19.10.0 # via daphne txaio==18.8.1 # via autobahn urllib3==1.25.7 # via kubernetes, requests @@ -122,6 +122,7 @@ uwsgi==2.0.18 uwsgitop==0.11 vine==1.3.0 # via amqp, celery websocket-client==0.56.0 # via kubernetes, slackclient +wheel==0.33.6 xmlsec==1.3.3 # via python3-saml zipp==0.6.0 # via importlib-metadata zope.interface==4.7.1 # via twisted @@ -129,4 +130,3 @@ zope.interface==4.7.1 # via twisted # The following packages are considered to be unsafe in a requirements file: pip==19.3.1 setuptools==41.6.0 -wheel==0.33.6 diff --git a/requirements/requirements_ansible.in b/requirements/requirements_ansible.in index 7c9d53abd7..f85ac9c75b 100644 --- a/requirements/requirements_ansible.in +++ b/requirements/requirements_ansible.in @@ -63,4 +63,5 @@ requests-credssp==1.0.2 # For windows authentication awx/issues/1144 # OpenStack openstacksdk==0.31.1 pip==19.3.1 # see upgrade blockers -setuptools==41.6.0 # see upgrade blockers \ No newline at end of file +setuptools==41.6.0 # see upgrade blockers +wheel==0.33.6 # see UPGRADE BLOCKERs \ No newline at end of file diff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt index f06fa16ade..1ea66c0d3d 100644 --- a/requirements/requirements_ansible.txt +++ b/requirements/requirements_ansible.txt @@ -42,11 +42,11 @@ bcrypt==3.1.7 # via paramiko boto3==1.9.223 boto==2.47.0 botocore==1.12.253 # via boto3, s3transfer -cachetools==3.1.1 # via google-auth +cachetools==3.1.1; python_version < "3" # via google-auth certifi==2019.11.28 # via msrest, requests cffi==1.13.2 # via bcrypt, cryptography, pynacl chardet==3.0.4 # via requests -colorama==0.4.1 # via azure-cli-core, knack +colorama==0.4.3 # via azure-cli-core, knack cryptography==2.8 # via adal, azure-keyvault, azure-storage, openstacksdk, paramiko, pyopenssl, requests-credssp, requests-kerberos, requests-ntlm decorator==4.4.1 # via dogpile.cache, openstacksdk docutils==0.15.2 # via botocore @@ -80,7 +80,7 @@ openstacksdk==0.31.1 os-service-types==1.7.0 # via keystoneauth1, openstacksdk ovirt-engine-sdk-python==4.3.0 packaging==19.2 -paramiko==2.7.0 # via azure-cli-core, ncclient +paramiko==2.7.1 # via azure-cli-core, ncclient pbr==5.4.4 # via keystoneauth1, openstacksdk, os-service-types, stevedore psutil==5.4.3 pyasn1-modules==0.2.7 # via google-auth @@ -111,10 +111,9 @@ stevedore==1.31.0 # via keystoneauth1 tabulate==0.8.2 # via azure-cli-core, knack typing==3.7.4.1; python_version < "3" # via msrest urllib3==1.25.7 # via botocore, requests -# wheel==0.30.0 # via azure-cli-core +wheel==0.33.6 xmltodict==0.12.0 # via pywinrm # The following packages are considered to be unsafe in a requirements file: pip==19.3.1 setuptools==41.6.0 -wheel==0.33.6