Merge pull request #3320 from vismay-golwala/custom_venvs

Feature: custom virtual environment directories

Reviewed-by: https://github.com/softwarefactory-project-zuul[bot]

awx/conf/fields.py

@@ -1,4 +1,5 @@
 # Python
+import os
 import logging
 import urllib.parse as urlparse
 from collections import OrderedDict
@@ -96,6 +97,26 @@ class StringListBooleanField(ListField):
         self.fail('type_error', input_type=type(data))
 
 
+class StringListPathField(StringListField):
+
+    default_error_messages = {
+        'type_error': _('Expected list of strings but got {input_type} instead.'),
+        'path_error': _('{path} is not a valid path choice.'),
+    }
+
+    def to_internal_value(self, paths):
+        if isinstance(paths, (list, tuple)):
+            for p in paths:
+                if not isinstance(p, str):
+                    self.fail('type_error', input_type=type(p))
+                if not os.path.exists(p):
+                    self.fail('path_error', path=p)
+
+            return super(StringListPathField, self).to_internal_value(sorted({os.path.normpath(path) for path in paths}))
+        else:
+            self.fail('type_error', input_type=type(paths))
+
+
 class URLField(CharField):
 
     def __init__(self, **kwargs):

awx/conf/tests/unit/test_fields.py

@@ -1,7 +1,7 @@
 import pytest
 
 from rest_framework.fields import ValidationError
-from awx.conf.fields import StringListBooleanField, ListTuplesField
+from awx.conf.fields import StringListBooleanField, StringListPathField, ListTuplesField
 
 
 class TestStringListBooleanField():
@@ -84,3 +84,49 @@ class TestListTuplesField():
         assert e.value.detail[0] == "Expected a list of tuples of max length 2 " \
             "but got {} instead.".format(t)
 
+
+class TestStringListPathField():
+
+    FIELD_VALUES = [
+        ((".", "..", "/"), [".", "..", "/"]),
+        (("/home",), ["/home"]),
+        (("///home///",), ["/home"]),
+        (("/home/././././",), ["/home"]),
+        (("/home", "/home", "/home/"), ["/home"]),
+        (["/home/", "/home/", "/opt/", "/opt/", "/var/"], ["/home", "/opt", "/var"])
+    ]
+
+    FIELD_VALUES_INVALID_TYPE = [
+        1.245,
+        {"a": "b"},
+        ("/home"),
+    ]
+
+    FIELD_VALUES_INVALID_PATH = [
+        "",
+        "~/",
+        "home",
+        "/invalid_path",
+        "/home/invalid_path",
+    ]
+
+    @pytest.mark.parametrize("value_in, value_known", FIELD_VALUES)
+    def test_to_internal_value_valid(self, value_in, value_known):
+        field = StringListPathField()
+        v = field.to_internal_value(value_in)
+        assert v == value_known
+
+    @pytest.mark.parametrize("value", FIELD_VALUES_INVALID_TYPE)
+    def test_to_internal_value_invalid_type(self, value):
+        field = StringListPathField()
+        with pytest.raises(ValidationError) as e:
+            field.to_internal_value(value)
+        assert e.value.detail[0] == "Expected list of strings but got {} instead.".format(type(value))
+
+    @pytest.mark.parametrize("value", FIELD_VALUES_INVALID_PATH)
+    def test_to_internal_value_invalid_path(self, value):
+        field = StringListPathField()
+        with pytest.raises(ValidationError) as e:
+            field.to_internal_value([value])
+        assert e.value.detail[0] == "{} is not a valid path choice.".format(value)
+

awx/main/conf.py

@@ -126,6 +126,17 @@ register(
     category_slug='system',
 )
 
+register(
+    'CUSTOM_VENV_PATHS',
+    field_class=fields.StringListPathField,
+    label=_('Custom virtual environment paths'),
+    help_text=_('Paths where Tower will look for custom virtual environments '
+                '(in addition to /var/lib/awx/venv/). Enter one path per line.'),
+    category=_('System'),
+    category_slug='system',
+    default=[],
+)
+
 register(
     'AD_HOC_COMMANDS',
     field_class=fields.StringListField,

awx/main/tests/unit/utils/test_common.py

@@ -177,9 +177,18 @@ def test_get_custom_venv_choices():
 
     with TemporaryDirectory(dir=settings.BASE_VENV_PATH, prefix='tmp') as temp_dir:
         os.makedirs(os.path.join(temp_dir, 'bin', 'activate'))
-        assert sorted(common.get_custom_venv_choices()) == [
+
+        custom_venv_dir = os.path.join(temp_dir, 'custom')
+        custom_venv_1 = os.path.join(custom_venv_dir, 'venv-1')
+        custom_venv_awx = os.path.join(custom_venv_dir, 'custom', 'awx')
+
+        os.makedirs(os.path.join(custom_venv_1, 'bin', 'activate'))
+        os.makedirs(os.path.join(custom_venv_awx, 'bin', 'activate'))
+
+        assert sorted(common.get_custom_venv_choices([custom_venv_dir])) == [
             bundled_venv,
-            os.path.join(temp_dir, '')
+            os.path.join(temp_dir, ''),
+            os.path.join(custom_venv_1, '')
         ]
 
 

awx/main/utils/common.py

@@ -943,19 +943,22 @@ def get_current_apps():
     return current_apps
 
 
-def get_custom_venv_choices():
+def get_custom_venv_choices(custom_paths=None):
     from django.conf import settings
-    custom_venv_path = settings.BASE_VENV_PATH
-    if os.path.exists(custom_venv_path):
-        return [
-            os.path.join(custom_venv_path, x, '')
-            for x in os.listdir(custom_venv_path)
-            if x != 'awx' and
-            os.path.isdir(os.path.join(custom_venv_path, x)) and
-            os.path.exists(os.path.join(custom_venv_path, x, 'bin', 'activate'))
-        ]
-    else:
-        return []
+    custom_paths = custom_paths or settings.CUSTOM_VENV_PATHS
+    all_venv_paths = [settings.BASE_VENV_PATH] + custom_paths
+    custom_venv_choices = []
+
+    for custom_venv_path in all_venv_paths:
+        if os.path.exists(custom_venv_path):
+            custom_venv_choices.extend([
+                os.path.join(custom_venv_path, x, '')
+                for x in os.listdir(custom_venv_path)
+                if x != 'awx' and
+                os.path.isdir(os.path.join(custom_venv_path, x)) and
+                os.path.exists(os.path.join(custom_venv_path, x, 'bin', 'activate'))
+            ])
+    return custom_venv_choices
 
 
 class OutputEventFilter(object):

awx/settings/defaults.py

@@ -166,6 +166,8 @@ REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST']
 # REMOTE_HOST_HEADERS will be trusted unconditionally')
 PROXY_IP_WHITELIST = []
 
+CUSTOM_VENV_PATHS = []
+
 # Note: This setting may be overridden by database settings.
 STDOUT_MAX_BYTES_DISPLAY = 1048576
 

awx/ui/client/src/configuration/forms/system-form/sub-forms/system-misc.form.js

@@ -54,6 +54,10 @@ export default ['i18n', function(i18n) {
             REMOTE_HOST_HEADERS: {
                 type: 'textarea',
                 reset: 'REMOTE_HOST_HEADERS'
+            },
+            CUSTOM_VENV_PATHS: {
+                type: 'textarea',
+                reset: 'CUSTOM_VENV_PATHS'
             }
         },
 

docs/custom_virtualenvs.md

@@ -16,29 +16,48 @@ virtualenv; this documentation describes the supported way to do so.
 Preparing a New Custom Virtualenv
 =================================
 awx allows a _different_ virtualenv to be specified and used on Job Template
-runs.  To choose a custom virtualenv, first create one in `/var/lib/awx/venv`:
+runs.  To choose a custom virtualenv, first we need to create one. Here, we are
+using `/opt/my-envs/` as the directory to hold custom venvs. But you can use any
+other directory and replace `/opt/my-envs/` with that. Let's create the directory
+first if absent:
 
-    $ sudo virtualenv /var/lib/awx/venv/my-custom-venv
+    $ sudo mkdir /opt/my-envs
+
+Now, we need to tell Tower to look into this directory for custom venvs. For that,
+we can add this directory to the `CUSTOM_VENV_PATHS` setting as:
+
+    $ HTTP PATCH /api/v2/settings/system {'CUSTOM_VENV_PATHS': ["/opt/my-envs/"]}
+
+If we have venvs spanned over multiple directories, we can add all the paths and
+Tower will aggregate venvs from them:
+
+    $ HTTP PATCH /api/v2/settings/system {'CUSTOM_VENV_PATHS': ["/path/1/to/venv/",
+                                                                "/path/2/to/venv/",
+                                                                "/path/3/to/venv/"]}
+
+Now that we have the directory setup, we can create a virtual environment in that using:
+
+    $ sudo virtualenv /opt/my-envs/custom-venv
 
 Multiple versions of Python are supported, though it's important to note that
 the semantics for creating virtualenvs in Python 3 has changed slightly:
 
-    $ sudo python3 -m venv /var/lib/awx/venv/my-custom-venv
+    $ sudo python3 -m venv /opt/my-envs/custom-venv
 
 Your newly created virtualenv needs a few base dependencies to properly run
 playbooks:
 fact gathering):
 
-    $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install psutil
+    $ sudo /opt/my-envs/custom-venv/bin/pip install psutil
 
 From here, you can install _additional_ Python dependencies that you care
 about, such as a per-virtualenv version of ansible itself:
 
-    $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install -U "ansible == X.Y.Z"
+    $ sudo /opt/my-envs/custom-venv/bin/pip install -U "ansible == X.Y.Z"
 
 ...or an additional third-party SDK that's not included with the base awx installation:
 
-    $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install -U python-digitalocean
+    $ sudo /opt/my-envs/custom-venv/bin/pip install -U python-digitalocean
 
 If you want to copy them, the libraries included in awx's default virtualenv
 can be found using `pip freeze`:
@@ -47,7 +66,7 @@ can be found using `pip freeze`:
 
 One important item to keep in mind is that in a clustered awx installation,
 you need to ensure that the same custom virtualenv exists on _every_ local file
-system at `/var/lib/awx/venv/`.  For container-based deployments, this likely
+system at `/opt/my-envs/`.  For container-based deployments, this likely
 means building these steps into your own custom image building workflow, e.g.,
 
 ```diff
@@ -60,8 +79,9 @@ index aa8b304..eb05f91 100644
     fi
  
 +requirements_custom:
-+	virtualenv $(VENV_BASE)/my-custom-env
-+	$(VENV_BASE)/my-custom-env/bin/pip install psutil
++ mkdir -p /opt/my-envs
++	virtualenv /opt/my-envs/my-custom-env
++	/opt/my-envs/my-custom-env/bin/pip install psutil
 +
 diff --git a/installer/image_build/templates/Dockerfile.j2 b/installer/image_build/templates/Dockerfile.j2
 index d69e2c9..a08bae5 100644
@@ -74,6 +94,8 @@ index d69e2c9..a08bae5 100644
 +    VENV_BASE=/var/lib/awx/venv make requirements_custom && \
 ```
 
+Once the AWX API is available, update the `CUSTOM_VENV_PATHS` setting as described in `Preparing a New Custom Virtualenv`.
+
 Kubernetes Custom Virtualenvs
 =============================
 
@@ -87,7 +109,7 @@ Now create an initContainer stanza.  You can subsititute your own custom images
 
     initContainers:
         - image: 'centos:7'
-          name: init-my-custom-venv
+          name: init-custom-venv
           command:
             - sh
             - '-c'
@@ -95,19 +117,20 @@ Now create an initContainer stanza.  You can subsititute your own custom images
               yum install -y ansible python-pip curl python-setuptools epel-release openssl openssl-devel gcc python-devel &&
               curl 'https://bootstrap.pypa.io/get-pip.py' | python &&
               pip install virtualenv &&
-              virtualenv /var/lib/awx/venv/my-custom-venv &&
-              source /var/lib/awx/venv/my-custom-venv/bin/activate &&
-              /var/lib/awx/venv/my-custom-venv/bin/pip install psutil &&
-              /var/lib/awx/venv/my-custom-venv/bin/pip install -U "ansible == X.Y.Z" &&
-              /var/lib/awx/venv/my-custom-venv/bin/pip install -U custom-python-module
+              mkdir -p /opt/my-envs &&
+              virtualenv /opt/my-envs/custom-venv &&
+              source /opt/my-envs/custom-venv/bin/activate &&
+              /opt/my-envs/custom-venv/bin/pip install psutil &&
+              /opt/my-envs/custom-venv/bin/pip install -U "ansible == X.Y.Z" &&
+              /opt/my-envs/custom-venv/bin/pip install -U custom-python-module
           volumeMounts:
-            - mountPath: /var/lib/awx/venv/my-custom-venv
+            - mountPath: /opt/my-envs/custom-venv
               name: custom-venv
 
 Finally in the awx-celery and awx-web containers stanza add the shared volume as a mount.
 
     volumeMounts:
-        - mountPath: /var/lib/awx/venv/my-custom-venv
+        - mountPath: /opt/my-envs/custom-venv
           name: custom-venv
         - mountPath: /etc/tower
           name: awx-application-config
@@ -116,6 +139,8 @@ Finally in the awx-celery and awx-web containers stanza add the shared volume as
           name: awx-confd
           readOnly: true
 
+Once the AWX API is available, update the `CUSTOM_VENV_PATHS` setting as described in `Preparing a New Custom Virtualenv`.
+
 Assigning Custom Virtualenvs
 ============================
 Once you've created a custom virtualenv, you can assign it at the Organization,
@@ -127,7 +152,7 @@ Project, or Job Template level:
 
     Content-Type: application/json
     {
-        'custom_virtualenv': '/var/lib/awx/venv/my-custom-venv'
+        'custom_virtualenv': '/opt/my-envs/custom-venv'
     }
 
 An HTTP `GET` request to `/api/v2/config/` will provide a list of
@@ -135,8 +160,8 @@ detected installed virtualenvs:
 
     {
         "custom_virtualenvs": [
-            "/var/lib/awx/venv/my-custom-venv",
-            "/var/lib/awx/venv/my-other-custom-venv",
+            "/opt/my-envs/custom-venv",
+            "/opt/my-envs/my-other-custom-venv",
         ],
         ...
     }