Add pip-compile updater.sh script

Signed-off-by: github.com/clushie <47661139+clushie@users.noreply.github.com>

requirements/README.md

@@ -1,36 +1,17 @@
-The requirements.txt and requirements_ansible.txt files are generated from requirements.in and requirements_ansible.in, respectively, using `pip-tools` `pip-compile`. The following commands should do this if ran inside the tools_awx container.
+The `requirements.txt` and `requirements_ansible.txt` files are generated from `requirements.in` and `requirements_ansible.in`, respectively, using `pip-tools` `pip-compile`.
 
-Run these commands from the root of the awx repo. This will produce python 3 requirements files.
+Run `./updater.sh` command from inside `./requirements` directory of the awx repository.
 
-```
+Make sure you have `patch, awk, python3, python2, python3-venv, python2-virtualenv, pip2, pip3` installed.
-python3 -m venv /buildit
-source /buildit/bin/activate
-pip install pip-tools
-pip install pip --upgrade
 
-pip-compile -U -r --allow-unsafe --output-file requirements/requirements.txt requirements/requirements.in
+This script will:
-pip-compile -U -r --allow-unsafe --output-file requirements/requirements_ansible_py3.txt requirements/requirements_ansible.in
-```
 
-Remove the `docutils` line from `requirements/requirements.txt`.
+  - Update `requirements.txt` based on `requirements.in`
+  - Update/generate `requirements_ansible.txt` based on `requirements_ansible.in`
+    - including an automated patch that adds `python_version < "3"` for Python 2 backward compatibility
+  - Removes the `docutils` dependency line from `requirements.txt` and `requirements_ansible.txt`
 
-The Ansible venv requirements file needs to start with the python 2 version
+You can also upgrade (`pip-compile --upgrade`) the dependencies by running `./updater.sh upgrade`.
-as a base. Then we can run the tool again to get the python 3 version.
-Consult the output of the `diff` command and add a conditional switch in those cases.
-
-```
-virtualenv -p python2 /buildit_py2
-source /buildit_py2/bin/activate
-pip install pip-tools
-pip install pip --upgrade
-
-pip-compile -U -r --allow-unsafe --output-file requirements/requirements_ansible.txt requirements/requirements_ansible.in
-diff requirements/requirements_ansible_py3.txt requirements/requirements_ansible.txt
-rm requirements/requirements_ansible_py3.txt
-```
-
-Python 3 exceptions should be added to relevant `requirements_ansible.txt` lines
-after version numbers with the syntax of `; python_version < '3'`.
 
 ## Licenses and Source Files
 

requirements/updater.sh

@@ -0,0 +1,82 @@
+#!/bin/sh
+set -ue
+
+requirements_in="$(readlink -f ./requirements.in)"
+requirements_ansible_in="$(readlink -f ./requirements_ansible.in)"
+requirements="$(readlink -f ./requirements.txt)"
+requirements_ansible="$(readlink -f ./requirements_ansible.txt)"
+pip_compile="pip-compile --no-header --quiet -r --allow-unsafe"
+
+_cleanup() {
+  cd /
+  test "${KEEP_TMP:-0}" = 1 || rm -rf "${_tmp}"
+}
+
+install_deps() {
+  pip install pip --upgrade
+  pip install pip-tools
+}
+
+generate_requirements_v3() {
+  venv="./venv3"
+  python3 -m venv "${venv}"
+  # shellcheck disable=SC1090
+  . "${venv}/bin/activate"
+
+  install_deps
+
+  ${pip_compile} --output-file requirements.txt "${requirements_in}"
+  ${pip_compile} --output-file requirements_ansible_py3.txt "${requirements_ansible_in}"
+}
+
+generate_requirements_v2() {
+  venv="./venv2"
+  virtualenv -p python2 "${venv}"
+  # shellcheck disable=SC1090
+  PS1="" . "${venv}/bin/activate"
+
+  install_deps
+
+  ${pip_compile} --output-file requirements_ansible.txt "${requirements_ansible_in}"
+}
+
+generate_patch() {
+  a="requirements_ansible_py3.txt"
+  b="requirements_ansible.txt"
+  replace='; python_version < "3" #'
+
+  # most elegant/quick solution I could come up for now
+  out="$(diff --ignore-matching-lines='^#' --unified "${a}" "${b}" | \
+    awk -v replace="${replace}" '{ if (/^+\w/){ $2=replace; print;} else print; }' | \
+    sed 's/ ;/;/g')"
+  test -n "${out}"
+  echo "${out}"
+}
+
+main() {
+  _tmp="$(mktemp -d --suffix .awx-requirements XXXX -p /tmp)"
+  trap _cleanup INT TERM EXIT
+
+  if [ "$1" = "upgrade" ]; then
+      pip_compile="${pip_compile} --upgrade"
+  fi
+
+  cp -vf requirements.txt requirements_ansible.txt "${_tmp}"
+  cp -vf requirements_ansible.txt "${_tmp}/requirements_ansible_py3.txt"
+
+  cd "${_tmp}"
+
+  generate_requirements_v3
+  generate_requirements_v2
+
+  sed -i 's/^docutils.*//g' requirements.txt
+  generate_patch | patch -p4 requirements_ansible_py3.txt
+
+  cp -vf requirements_ansible_py3.txt "${requirements_ansible}"
+  cp -vf requirements.txt "${requirements}"
+
+  _cleanup
+}
+
+# set EVAL=1 in case you want to source this script
+test "${EVAL:-0}" = "1" || main "${1:-}"