Merge pull request #4123 from clushie/update_openstacksdk_dependencies

Fix outdated openstacksdk version and add pip-compile helper script

Reviewed-by: https://github.com/softwarefactory-project-zuul[bot]

requirements/README.md

@@ -1,36 +1,17 @@
-The requirements.txt and requirements_ansible.txt files are generated from requirements.in and requirements_ansible.in, respectively, using `pip-tools` `pip-compile`. The following commands should do this if ran inside the tools_awx container.
+The `requirements.txt` and `requirements_ansible.txt` files are generated from `requirements.in` and `requirements_ansible.in`, respectively, using `pip-tools` `pip-compile`.
 
-Run these commands from the root of the awx repo. This will produce python 3 requirements files.
+Run `./updater.sh` command from inside `./requirements` directory of the awx repository.
 
-```
-python3 -m venv /buildit
-source /buildit/bin/activate
-pip install pip-tools
-pip install pip --upgrade
+Make sure you have `patch, awk, python3, python2, python3-venv, python2-virtualenv, pip2, pip3` installed.
 
-pip-compile -U -r --allow-unsafe --output-file requirements/requirements.txt requirements/requirements.in
-pip-compile -U -r --allow-unsafe --output-file requirements/requirements_ansible_py3.txt requirements/requirements_ansible.in
-```
+This script will:
 
-Remove the `docutils` line from `requirements/requirements.txt`.
+  - Update `requirements.txt` based on `requirements.in`
+  - Update/generate `requirements_ansible.txt` based on `requirements_ansible.in`
+    - including an automated patch that adds `python_version < "3"` for Python 2 backward compatibility
+  - Removes the `docutils` dependency line from `requirements.txt` and `requirements_ansible.txt`
 
-The Ansible venv requirements file needs to start with the python 2 version
-as a base. Then we can run the tool again to get the python 3 version.
-Consult the output of the `diff` command and add a conditional switch in those cases.
-
-```
-virtualenv -p python2 /buildit_py2
-source /buildit_py2/bin/activate
-pip install pip-tools
-pip install pip --upgrade
-
-pip-compile -U -r --allow-unsafe --output-file requirements/requirements_ansible.txt requirements/requirements_ansible.in
-diff requirements/requirements_ansible_py3.txt requirements/requirements_ansible.txt
-rm requirements/requirements_ansible_py3.txt
-```
-
-Python 3 exceptions should be added to relevant `requirements_ansible.txt` lines
-after version numbers with the syntax of `; python_version < '3'`.
+You can also upgrade (`pip-compile --upgrade`) the dependencies by running `./updater.sh upgrade`.
 
 ## Licenses and Source Files
 

requirements/requirements.txt

@@ -1,9 +1,3 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile --allow-unsafe --output-file=requirements/requirements.txt requirements/requirements.in
-#
 adal==1.2.1               # via msrestazure
 amqp==2.4.2               # via kombu
 ansible-runner==1.3.4
@@ -44,6 +38,7 @@ django-taggit==0.22.2
 django==1.11.20
 djangorestframework-yaml==1.0.3
 djangorestframework==3.9.4
+
 future==0.16.0            # via django-radius
 hyperlink==19.0.0         # via twisted
 idna==2.8                 # via hyperlink, requests, twisted

requirements/requirements_ansible.in

@@ -63,4 +63,4 @@ pywinrm[kerberos]==0.3.0
 requests
 requests-credssp==1.0.2   # For windows authentication awx/issues/1144
 # OpenStack
-openstacksdk==0.23.0
+openstacksdk==0.31.1

requirements/requirements_ansible.txt

@@ -1,9 +1,3 @@
-#
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    pip-compile -U -r --allow-unsafe --output-file requirements/requirements_ansible.txt requirements/requirements_ansible.in
-#
 adal==1.2.1               # via msrestazure
 apache-libcloud==2.5.0
 appdirs==1.4.3            # via openstacksdk
@@ -52,16 +46,16 @@ certifi==2019.3.9         # via msrest, requests
 cffi==1.12.3              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
 colorama==0.4.1           # via azure-cli-core, knack
-cryptography==2.6.1       # via adal, azure-keyvault, azure-storage, openstacksdk, paramiko, pyopenssl, requests-kerberos, requests-ntlm
+cryptography==2.6.1       # via adal, azure-keyvault, azure-storage, openstacksdk, paramiko, pyopenssl, requests-credssp, requests-kerberos, requests-ntlm
 decorator==4.4.0          # via dogpile.cache, openstacksdk
 docutils==0.14            # via botocore
 dogpile.cache==0.7.1      # via openstacksdk
-enum34==1.1.6; python_version < '3' # via cryptography, knack, msrest, ovirt-engine-sdk-python
-futures==3.2.0; python_version < '3' # via openstacksdk, s3transfer
+enum34==1.1.6; python_version < "3" # via cryptography, knack, msrest, ovirt-engine-sdk-python
+futures==3.2.0; python_version < "3" # via openstacksdk, s3transfer
 google-auth==1.6.2
 humanfriendly==4.18       # via azure-cli-core
 idna==2.8                 # via requests
-ipaddress==1.0.22; python_version < '3' # via cryptography, openstacksdk
+ipaddress==1.0.22; python_version < "3" # via cryptography, openstacksdk
 iso8601==0.1.12           # via keystoneauth1, openstacksdk
 isodate==0.6.0            # via msrest
 jinja2==2.10.1
@@ -72,7 +66,7 @@ keystoneauth1==3.14.0     # via openstacksdk
 knack==0.3.3              # via azure-cli-core
 lxml==4.3.3               # via ncclient, pyvmomi
 markupsafe==1.1.1         # via jinja2
-monotonic==1.5; python_version < '3' # via humanfriendly
+monotonic==1.5; python_version < "3" # via humanfriendly
 msrest==0.6.1
 msrestazure==0.5.0
 munch==2.3.2              # via openstacksdk
@@ -81,17 +75,17 @@ netaddr==0.7.19
 netifaces==0.10.9         # via openstacksdk
 ntlm-auth==1.3.0          # via requests-credssp, requests-ntlm
 oauthlib==3.0.1           # via requests-oauthlib
-openstacksdk==0.23.0
-os-service-types==1.6.0   # via keystoneauth1, openstacksdk
+openstacksdk==0.31.1
+os-service-types==1.7.0   # via keystoneauth1, openstacksdk
 ovirt-engine-sdk-python==4.3.0
 packaging==19.0
 paramiko==2.4.2           # via azure-cli-core, ncclient
 pbr==5.2.0                # via keystoneauth1, openstacksdk, os-service-types, stevedore
 pexpect==4.6.0
 psutil==5.4.3
-ptyprocess==0.5.2         # via pexpect
+ptyprocess==0.5.2
 pyasn1-modules==0.2.5     # via google-auth
-pyasn1==0.4.5             # via paramiko, pyasn1-modules, rsa
+pyasn1==0.4.5             # via paramiko, pyasn1-modules, requests-credssp, rsa
 pycparser==2.19           # via cffi
 pycurl==7.43.0.1
 pygments==2.3.1           # via azure-cli-core, knack
@@ -113,10 +107,10 @@ requestsexceptions==1.4.0  # via openstacksdk
 rsa==4.0                  # via google-auth
 s3transfer==0.1.13        # via boto3
 selectors2==2.0.1         # via ncclient
-six==1.12.0               # via azure-cli-core, bcrypt, cryptography, google-auth, isodate, keystoneauth1, knack, munch, ncclient, openstacksdk, ovirt-engine-sdk-python, packaging, pynacl, pyopenssl, python-dateutil, pyvmomi, pywinrm, stevedore
+six==1.12.0               # via azure-cli-core, bcrypt, cryptography, google-auth, isodate, keystoneauth1, knack, munch, ncclient, openstacksdk, ovirt-engine-sdk-python, packaging, pynacl, pyopenssl, python-dateutil, pyvmomi, pywinrm, requests-credssp, stevedore
 stevedore==1.30.1         # via keystoneauth1
 tabulate==0.8.2           # via azure-cli-core, knack
-typing==3.6.6; python_version < '3' # via msrest
+typing==3.6.6; python_version < "3" # via msrest
 urllib3==1.24.3           # via requests
 wheel==0.30.0             # via azure-cli-core
 xmltodict==0.12.0         # via pywinrm

requirements/updater.sh

@@ -0,0 +1,82 @@
+#!/bin/sh
+set -ue
+
+requirements_in="$(readlink -f ./requirements.in)"
+requirements_ansible_in="$(readlink -f ./requirements_ansible.in)"
+requirements="$(readlink -f ./requirements.txt)"
+requirements_ansible="$(readlink -f ./requirements_ansible.txt)"
+pip_compile="pip-compile --no-header --quiet -r --allow-unsafe"
+
+_cleanup() {
+  cd /
+  test "${KEEP_TMP:-0}" = 1 || rm -rf "${_tmp}"
+}
+
+install_deps() {
+  pip install pip --upgrade
+  pip install pip-tools
+}
+
+generate_requirements_v3() {
+  venv="./venv3"
+  python3 -m venv "${venv}"
+  # shellcheck disable=SC1090
+  . "${venv}/bin/activate"
+
+  install_deps
+
+  ${pip_compile} --output-file requirements.txt "${requirements_in}"
+  ${pip_compile} --output-file requirements_ansible_py3.txt "${requirements_ansible_in}"
+}
+
+generate_requirements_v2() {
+  venv="./venv2"
+  virtualenv -p python2 "${venv}"
+  # shellcheck disable=SC1090
+  PS1="" . "${venv}/bin/activate"
+
+  install_deps
+
+  ${pip_compile} --output-file requirements_ansible.txt "${requirements_ansible_in}"
+}
+
+generate_patch() {
+  a="requirements_ansible_py3.txt"
+  b="requirements_ansible.txt"
+  replace='; python_version < "3" #'
+
+  # most elegant/quick solution I could come up for now
+  out="$(diff --ignore-matching-lines='^#' --unified "${a}" "${b}" | \
+    awk -v replace="${replace}" '{ if (/^+\w/){ $2=replace; print;} else print; }' | \
+    sed 's/ ;/;/g')"
+  test -n "${out}"
+  echo "${out}"
+}
+
+main() {
+  _tmp="$(mktemp -d --suffix .awx-requirements XXXX -p /tmp)"
+  trap _cleanup INT TERM EXIT
+
+  if [ "$1" = "upgrade" ]; then
+      pip_compile="${pip_compile} --upgrade"
+  fi
+
+  cp -vf requirements.txt requirements_ansible.txt "${_tmp}"
+  cp -vf requirements_ansible.txt "${_tmp}/requirements_ansible_py3.txt"
+
+  cd "${_tmp}"
+
+  generate_requirements_v3
+  generate_requirements_v2
+
+  sed -i 's/^docutils.*//g' requirements.txt
+  generate_patch | patch -p4 requirements_ansible_py3.txt
+
+  cp -vf requirements_ansible_py3.txt "${requirements_ansible}"
+  cp -vf requirements.txt "${requirements}"
+
+  _cleanup
+}
+
+# set EVAL=1 in case you want to source this script
+test "${EVAL:-0}" = "1" || main "${1:-}"