implement support for per-playbook/project/org virtualenvs

see: https://github.com/ansible/awx/issues/34

awx/api/serializers.py

@@ -896,7 +896,7 @@ class OrganizationSerializer(BaseSerializer):
 
     class Meta:
         model = Organization
-        fields = ('*',)
+        fields = ('*', 'custom_virtualenv',)
 
     def get_related(self, obj):
         res = super(OrganizationSerializer, self).get_related(obj)
@@ -984,7 +984,7 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
     class Meta:
         model = Project
         fields = ('*', 'organization', 'scm_delete_on_next_update', 'scm_update_on_launch',
-                  'scm_update_cache_timeout', 'scm_revision',) + \
+                  'scm_update_cache_timeout', 'scm_revision', 'custom_virtualenv',) + \
                  ('last_update_failed', 'last_updated')  # Backwards compatibility
         read_only_fields = ('scm_delete_on_next_update',)
 
@@ -2530,7 +2530,7 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO
         fields = ('*', 'host_config_key', 'ask_diff_mode_on_launch', 'ask_variables_on_launch', 'ask_limit_on_launch', 'ask_tags_on_launch',
                   'ask_skip_tags_on_launch', 'ask_job_type_on_launch', 'ask_verbosity_on_launch', 'ask_inventory_on_launch',
                   'ask_credential_on_launch', 'survey_enabled', 'become_enabled', 'diff_mode',
-                  'allow_simultaneous')
+                  'allow_simultaneous', 'custom_virtualenv')
 
     def get_related(self, obj):
         res = super(JobTemplateSerializer, self).get_related(obj)

awx/api/views.py

@@ -315,6 +315,7 @@ class ApiV1ConfigView(APIView):
             data.update(dict(
                 project_base_dir = settings.PROJECTS_ROOT,
                 project_local_paths = Project.get_local_path_choices(),
+                custom_virtualenvs = get_custom_venv_choices(),
             ))
 
         return Response(data)

awx/main/migrations/0019_v330_custom_virtualenv.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.7 on 2018-01-09 21:30
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0018_v330_add_additional_stdout_events'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='custom_virtualenv',
+            field=models.CharField(blank=True, default=None, max_length=100, null=True),
+        ),
+        migrations.AddField(
+            model_name='organization',
+            name='custom_virtualenv',
+            field=models.CharField(blank=True, default=None, max_length=100, null=True),
+        ),
+        migrations.AddField(
+            model_name='project',
+            name='custom_virtualenv',
+            field=models.CharField(blank=True, default=None, max_length=100, null=True),
+        ),
+    ]

awx/main/models/jobs.py

@@ -34,7 +34,7 @@ from awx.main.models.notifications import (
 )
 from awx.main.utils import parse_yaml_or_json
 from awx.main.fields import ImplicitRoleField
-from awx.main.models.mixins import ResourceMixin, SurveyJobTemplateMixin, SurveyJobMixin, TaskManagerJobMixin
+from awx.main.models.mixins import ResourceMixin, SurveyJobTemplateMixin, SurveyJobMixin, TaskManagerJobMixin, CustomVirtualEnvMixin
 from awx.main.fields import JSONField, AskForField
 
 
@@ -215,7 +215,7 @@ class JobOptions(BaseModel):
         return needed
 
 
-class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, ResourceMixin):
+class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, ResourceMixin, CustomVirtualEnvMixin):
     '''
     A job template is a reusable job definition for applying a project (with
     playbook) to an inventory source with a given credential.
@@ -509,6 +509,18 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
     def get_ui_url(self):
         return urljoin(settings.TOWER_URL_BASE, "/#/jobs/{}".format(self.pk))
 
+    @property
+    def ansible_virtualenv_path(self):
+        # the order here enforces precedence (it matters)
+        for virtualenv in (
+            self.job_template.custom_virtualenv,
+            self.project.custom_virtualenv,
+            self.project.organization.custom_virtualenv
+        ):
+            if virtualenv:
+                return virtualenv
+        return settings.ANSIBLE_VENV_PATH
+
     @property
     def event_class(self):
         return JobEvent

awx/main/models/mixins.py

@@ -1,26 +1,29 @@
 # Python
+import os
 import json
 from copy import copy, deepcopy
 
 # Django
+from django.conf import settings
 from django.db import models
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.auth.models import User # noqa
 from django.utils.translation import ugettext_lazy as _
+from django.core.exceptions import ValidationError
 
 # AWX
 from awx.main.models.base import prevent_search
 from awx.main.models.rbac import (
     Role, RoleAncestorEntry, get_roles_on_resource
 )
-from awx.main.utils import parse_yaml_or_json
+from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices
 from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted
 from awx.main.fields import JSONField, AskForField
 
 
 __all__ = ['ResourceMixin', 'SurveyJobTemplateMixin', 'SurveyJobMixin',
            'TaskManagerUnifiedJobMixin', 'TaskManagerJobMixin', 'TaskManagerProjectUpdateMixin',
-           'TaskManagerInventoryUpdateMixin',]
+           'TaskManagerInventoryUpdateMixin', 'CustomVirtualEnvMixin']
 
 
 class ResourceMixin(models.Model):
@@ -416,3 +419,23 @@ class TaskManagerProjectUpdateMixin(TaskManagerUpdateOnLaunchMixin):
 class TaskManagerInventoryUpdateMixin(TaskManagerUpdateOnLaunchMixin):
     class Meta:
         abstract = True
+
+
+class CustomVirtualEnvMixin(models.Model):
+    class Meta:
+        abstract = True
+
+    custom_virtualenv = models.CharField(
+        blank=True,
+        null=True,
+        default=None,
+        max_length=100
+    )
+
+    def clean_custom_virtualenv(self):
+        value = self.custom_virtualenv
+        if value and os.path.join(value, '') not in get_custom_venv_choices():
+            raise ValidationError(
+                _('{} is not a valid virtualenv in {}').format(value, settings.BASE_VENV_PATH)
+            )
+        return os.path.join(value or '', '')

awx/main/models/organization.py

@@ -22,12 +22,12 @@ from awx.main.models.rbac import (
     ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
     ROLE_SINGLETON_SYSTEM_AUDITOR,
 )
-from awx.main.models.mixins import ResourceMixin
+from awx.main.models.mixins import ResourceMixin, CustomVirtualEnvMixin
 
 __all__ = ['Organization', 'Team', 'Profile', 'AuthToken']
 
 
-class Organization(CommonModel, NotificationFieldsModel, ResourceMixin):
+class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVirtualEnvMixin):
     '''
     An organization is the basic unit of multi-tenancy divisions
     '''

awx/main/models/projects.py

@@ -24,7 +24,7 @@ from awx.main.models.notifications import (
     JobNotificationMixin,
 )
 from awx.main.models.unified_jobs import * # noqa
-from awx.main.models.mixins import ResourceMixin, TaskManagerProjectUpdateMixin
+from awx.main.models.mixins import ResourceMixin, TaskManagerProjectUpdateMixin, CustomVirtualEnvMixin
 from awx.main.utils import update_scm_url
 from awx.main.utils.ansible import skip_directory, could_be_inventory, could_be_playbook
 from awx.main.fields import ImplicitRoleField
@@ -223,7 +223,7 @@ class ProjectOptions(models.Model):
         return proj_path + '.lock'
 
 
-class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin):
+class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEnvMixin):
     '''
     A project represents a playbook git repo that can access a set of inventories
     '''

awx/main/tasks.py

@@ -626,10 +626,16 @@ class BaseTask(LogErrorsTask):
             '': '',
         }
 
-    def add_ansible_venv(self, env, add_awx_lib=True):
-        env['VIRTUAL_ENV'] = settings.ANSIBLE_VENV_PATH
-        env['PATH'] = os.path.join(settings.ANSIBLE_VENV_PATH, "bin") + ":" + env['PATH']
-        venv_libdir = os.path.join(settings.ANSIBLE_VENV_PATH, "lib")
+    def add_ansible_venv(self, venv_path, env, add_awx_lib=True):
+        env['VIRTUAL_ENV'] = venv_path
+        env['PATH'] = os.path.join(venv_path, "bin") + ":" + env['PATH']
+        venv_libdir = os.path.join(venv_path, "lib")
+
+        if not os.path.exists(venv_libdir):
+            raise RuntimeError(
+                'a valid Python virtualenv does not exist at {}'.format(venv_path)
+            )
+
         env.pop('PYTHONPATH', None)  # default to none if no python_ver matches
         if os.path.isdir(os.path.join(venv_libdir, "python2.7")):
             env['PYTHONPATH'] = os.path.join(venv_libdir, "python2.7", "site-packages") + ":"
@@ -802,6 +808,8 @@ class BaseTask(LogErrorsTask):
             kwargs['private_data_files'] = self.build_private_data_files(instance, **kwargs)
             kwargs['passwords'] = self.build_passwords(instance, **kwargs)
             kwargs['proot_show_paths'] = self.proot_show_paths
+            if getattr(instance, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH) != settings.ANSIBLE_VENV_PATH:
+                kwargs['proot_custom_virtualenv'] = instance.ansible_virtualenv_path
             args = self.build_args(instance, **kwargs)
             safe_args = self.build_safe_args(instance, **kwargs)
             output_replacements = self.build_output_replacements(instance, **kwargs)
@@ -1021,7 +1029,7 @@ class RunJob(BaseTask):
             plugin_dirs.extend(settings.AWX_ANSIBLE_CALLBACK_PLUGINS)
         plugin_path = ':'.join(plugin_dirs)
         env = super(RunJob, self).build_env(job, **kwargs)
-        env = self.add_ansible_venv(env, add_awx_lib=kwargs.get('isolated', False))
+        env = self.add_ansible_venv(job.ansible_virtualenv_path, env, add_awx_lib=kwargs.get('isolated', False))
         # Set environment variables needed for inventory and job event
         # callbacks to work.
         env['JOB_ID'] = str(job.pk)
@@ -1314,7 +1322,7 @@ class RunProjectUpdate(BaseTask):
         Build environment dictionary for ansible-playbook.
         '''
         env = super(RunProjectUpdate, self).build_env(project_update, **kwargs)
-        env = self.add_ansible_venv(env)
+        env = self.add_ansible_venv(settings.ANSIBLE_VENV_PATH, env)
         env['ANSIBLE_RETRY_FILES_ENABLED'] = str(False)
         env['ANSIBLE_ASK_PASS'] = str(False)
         env['ANSIBLE_BECOME_ASK_PASS'] = str(False)
@@ -2055,7 +2063,7 @@ class RunAdHocCommand(BaseTask):
         '''
         plugin_dir = self.get_path_to('..', 'plugins', 'callback')
         env = super(RunAdHocCommand, self).build_env(ad_hoc_command, **kwargs)
-        env = self.add_ansible_venv(env)
+        env = self.add_ansible_venv(settings.ANSIBLE_VENV_PATH, env)
         # Set environment variables needed for inventory and ad hoc event
         # callbacks to work.
         env['AD_HOC_COMMAND_ID'] = str(ad_hoc_command.pk)

awx/main/tests/functional/api/test_job_template.py

@@ -1,3 +1,6 @@
+import os
+
+from backports.tempfile import TemporaryDirectory
 import pytest
 
 # AWX
@@ -7,6 +10,7 @@ from awx.main.models.jobs import Job, JobTemplate
 from awx.main.migrations import _save_password_keys as save_password_keys
 
 # Django
+from django.conf import settings
 from django.apps import apps
 
 
@@ -570,3 +574,31 @@ def test_save_survey_passwords_on_migration(job_template_with_survey_passwords):
     save_password_keys.migrate_survey_passwords(apps, None)
     job = job_template_with_survey_passwords.jobs.all()[0]
     assert job.survey_passwords == {'SSN': '$encrypted$', 'secret_key': '$encrypted$'}
+
+
+@pytest.mark.django_db
+def test_job_template_custom_virtualenv(get, patch, organization_factory, job_template_factory):
+    objs = organization_factory("org", superusers=['admin'])
+    jt = job_template_factory("jt", organization=objs.organization,
+                              inventory='test_inv', project='test_proj').job_template
+
+    with TemporaryDirectory(dir=settings.BASE_VENV_PATH) as temp_dir:
+        admin = objs.superusers.admin
+        os.makedirs(os.path.join(temp_dir, 'bin', 'activate'))
+        url = reverse('api:job_template_detail', kwargs={'pk': jt.id})
+        patch(url, {'custom_virtualenv': temp_dir}, user=admin, expect=200)
+        assert get(url, user=admin).data['custom_virtualenv'] == os.path.join(temp_dir, '')
+
+
+@pytest.mark.django_db
+def test_job_template_invalid_custom_virtualenv(get, patch, organization_factory,
+                                                job_template_factory):
+    objs = organization_factory("org", superusers=['admin'])
+    jt = job_template_factory("jt", organization=objs.organization,
+                              inventory='test_inv', project='test_proj').job_template
+
+    url = reverse('api:job_template_detail', kwargs={'pk': jt.id})
+    resp = patch(url, {'custom_virtualenv': '/foo/bar'}, user=objs.superusers.admin, expect=400)
+    assert resp.data['custom_virtualenv'] == [
+        '/foo/bar is not a valid virtualenv in {}'.format(settings.BASE_VENV_PATH)
+    ]

awx/main/tests/functional/api/test_organizations.py

@@ -2,15 +2,16 @@
 # All Rights Reserved.
 
 # Python
+import os
+
+from backports.tempfile import TemporaryDirectory
+from django.conf import settings
 import pytest
 import mock
 
-
-# Django
-from awx.api.versioning import reverse
-
 # AWX
 from awx.main.models import * # noqa
+from awx.api.versioning import reverse
 
 
 @pytest.mark.django_db
@@ -188,3 +189,21 @@ def test_delete_organization_xfail1(delete, organization, alice):
 @mock.patch('awx.main.access.BaseAccess.check_license', lambda *a, **kw: True)
 def test_delete_organization_xfail2(delete, organization):
     delete(reverse('api:organization_detail', kwargs={'pk': organization.id}), user=None, expect=401)
+
+
+@pytest.mark.django_db
+def test_organization_custom_virtualenv(get, patch, organization, admin):
+    with TemporaryDirectory(dir=settings.BASE_VENV_PATH) as temp_dir:
+        os.makedirs(os.path.join(temp_dir, 'bin', 'activate'))
+        url = reverse('api:organization_detail', kwargs={'pk': organization.id})
+        patch(url, {'custom_virtualenv': temp_dir}, user=admin, expect=200)
+        assert get(url, user=admin).data['custom_virtualenv'] == os.path.join(temp_dir, '')
+
+
+@pytest.mark.django_db
+def test_organization_invalid_custom_virtualenv(get, patch, organization, admin):
+    url = reverse('api:organization_detail', kwargs={'pk': organization.id})
+    resp = patch(url, {'custom_virtualenv': '/foo/bar'}, user=admin, expect=400)
+    assert resp.data['custom_virtualenv'] == [
+        '/foo/bar is not a valid virtualenv in {}'.format(settings.BASE_VENV_PATH)
+    ]

awx/main/tests/functional/api/test_project.py

@@ -1,5 +1,11 @@
+import os
+
+from backports.tempfile import TemporaryDirectory
+from django.conf import settings
 import pytest
 
+from awx.api.versioning import reverse
+
 
 @pytest.mark.django_db
 class TestInsightsCredential:
@@ -13,3 +19,20 @@ class TestInsightsCredential:
               {'credential': scm_credential.id}, admin_user,
               expect=400)
 
+
+@pytest.mark.django_db
+def test_project_custom_virtualenv(get, patch, project, admin):
+    with TemporaryDirectory(dir=settings.BASE_VENV_PATH) as temp_dir:
+        os.makedirs(os.path.join(temp_dir, 'bin', 'activate'))
+        url = reverse('api:project_detail', kwargs={'pk': project.id})
+        patch(url, {'custom_virtualenv': temp_dir}, user=admin, expect=200)
+        assert get(url, user=admin).data['custom_virtualenv'] == os.path.join(temp_dir, '')
+
+
+@pytest.mark.django_db
+def test_project_invalid_custom_virtualenv(get, patch, project, admin):
+    url = reverse('api:project_detail', kwargs={'pk': project.id})
+    resp = patch(url, {'custom_virtualenv': '/foo/bar'}, user=admin, expect=400)
+    assert resp.data['custom_virtualenv'] == [
+        '/foo/bar is not a valid virtualenv in {}'.format(settings.BASE_VENV_PATH)
+    ]

awx/main/tests/functional/models/test_job.py

@@ -0,0 +1,40 @@
+import pytest
+
+from awx.main.models import JobTemplate
+
+
+@pytest.mark.django_db
+def test_awx_virtualenv_from_settings(inventory, project, machine_credential):
+    jt = JobTemplate.objects.create(
+        name='my-jt',
+        inventory=inventory,
+        project=project,
+        playbook='helloworld.yml'
+    )
+    jt.credentials.add(machine_credential)
+    job = jt.create_unified_job()
+    assert job.ansible_virtualenv_path == '/venv/ansible'
+
+
+@pytest.mark.django_db
+def test_awx_custom_virtualenv(inventory, project, machine_credential):
+    jt = JobTemplate.objects.create(
+        name='my-jt',
+        inventory=inventory,
+        project=project,
+        playbook='helloworld.yml'
+    )
+    jt.credentials.add(machine_credential)
+    job = jt.create_unified_job()
+
+    job.project.organization.custom_virtualenv = '/venv/fancy-org'
+    job.project.organization.save()
+    assert job.ansible_virtualenv_path == '/venv/fancy-org'
+
+    job.project.custom_virtualenv = '/venv/fancy-proj'
+    job.project.save()
+    assert job.ansible_virtualenv_path == '/venv/fancy-proj'
+
+    job.job_template.custom_virtualenv = '/venv/fancy-jt'
+    job.job_template.save()
+    assert job.ansible_virtualenv_path == '/venv/fancy-jt'

awx/main/tests/unit/test_tasks.py

@@ -8,6 +8,7 @@ import re
 import shutil
 import tempfile
 
+from backports.tempfile import TemporaryDirectory
 import fcntl
 import mock
 import pytest
@@ -204,6 +205,7 @@ class TestJobExecution:
             mock.patch.object(Project, 'get_project_path', lambda *a, **kw: self.project_path),
             # don't emit websocket statuses; they use the DB and complicate testing
             mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()),
+            mock.patch.object(Job, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH),
             mock.patch('awx.main.expect.run.run_pexpect', self.run_pexpect),
         ]
         for cls in (Job, AdHocCommand):
@@ -348,6 +350,35 @@ class TestGenericRun(TestJobExecution):
         args, cwd, env, stdout = call_args
         assert env['FOO'] == 'BAR'
 
+    def test_valid_custom_virtualenv(self):
+        with TemporaryDirectory(dir=settings.BASE_VENV_PATH) as tempdir:
+            os.makedirs(os.path.join(tempdir, 'lib'))
+            os.makedirs(os.path.join(tempdir, 'bin', 'activate'))
+            venv_patch = mock.patch.object(Job, 'ansible_virtualenv_path', tempdir)
+            self.patches.append(venv_patch)
+            venv_patch.start()
+
+            self.task.run(self.pk)
+
+            assert self.run_pexpect.call_count == 1
+            call_args, _ = self.run_pexpect.call_args_list[0]
+            args, cwd, env, stdout = call_args
+
+            assert env['PATH'].startswith(os.path.join(tempdir, 'bin'))
+            assert env['VIRTUAL_ENV'] == tempdir
+            for path in (settings.ANSIBLE_VENV_PATH, tempdir):
+                assert '--ro-bind {} {}'.format(path, path) in ' '.join(args)
+
+    def test_invalid_custom_virtualenv(self):
+        venv_patch = mock.patch.object(Job, 'ansible_virtualenv_path', '/venv/missing')
+        self.patches.append(venv_patch)
+        venv_patch.start()
+
+        with pytest.raises(Exception):
+            self.task.run(self.pk)
+            tb = self.task.update_model.call_args[-1]['result_traceback']
+            assert 'a valid Python virtualenv does not exist at /venv/missing' in tb
+
 
 class TestAdhocRun(TestJobExecution):
 

awx/main/tests/unit/utils/test_common.py

@@ -8,6 +8,8 @@ from uuid import uuid4
 import json
 import yaml
 
+from backports.tempfile import TemporaryDirectory
+from django.conf import settings
 from django.core.cache import cache
 
 from rest_framework.exceptions import ParseError
@@ -136,7 +138,7 @@ def test_memoize_delete(memoized_function):
     assert memoized_function('john', 'smith') == 'smith'
     assert memoized_function('john', 'smith') == 'smith'
     assert memoized_function.calls['john'] == 1
-    
+
     assert cache.get('myfunction') == {u'john-smith': 'smith'}
 
     common.memoize_delete('myfunction')
@@ -164,3 +166,11 @@ def test_extract_ansible_vars():
     redacted, var_list = common.extract_ansible_vars(json.dumps(my_dict))
     assert var_list == set(['ansible_connetion_setting'])
     assert redacted == {"foobar": "baz"}
+
+
+def test_get_custom_venv_choices():
+    assert common.get_custom_venv_choices() == []
+
+    with TemporaryDirectory(dir=settings.BASE_VENV_PATH) as temp_dir:
+        os.makedirs(os.path.join(temp_dir, 'bin', 'activate'))
+        assert common.get_custom_venv_choices() == [os.path.join(temp_dir, '')]

awx/main/utils/common.py

@@ -47,7 +47,7 @@ __all__ = ['get_object_or_400', 'get_object_or_403', 'camelcase_to_underscore',
            'extract_ansible_vars', 'get_search_fields', 'get_system_task_capacity',
            'wrap_args_with_proot', 'build_proot_temp_dir', 'check_proot_installed', 'model_to_dict',
            'model_instance_diff', 'timestamp_apiformat', 'parse_yaml_or_json', 'RequireDebugTrueOrTest',
-           'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError',]
+           'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError', 'get_custom_venv_choices']
 
 
 def get_object_or_400(klass, *args, **kwargs):
@@ -756,9 +756,11 @@ def wrap_args_with_proot(args, cwd, **kwargs):
         show_paths = [cwd]
     for venv in (
         settings.ANSIBLE_VENV_PATH,
-        settings.AWX_VENV_PATH
+        settings.AWX_VENV_PATH,
+        kwargs.get('proot_custom_virtualenv')
     ):
-        new_args.extend(['--ro-bind', venv, venv])
+        if venv:
+            new_args.extend(['--ro-bind', venv, venv])
     show_paths.extend(getattr(settings, 'AWX_PROOT_SHOW_PATHS', None) or [])
     show_paths.extend(kwargs.get('proot_show_paths', []))
     for path in sorted(set(show_paths)):
@@ -838,6 +840,21 @@ def get_current_apps():
     return current_apps
 
 
+def get_custom_venv_choices():
+    from django.conf import settings
+    custom_venv_path = settings.BASE_VENV_PATH
+    if os.path.exists(custom_venv_path):
+        return [
+            os.path.join(custom_venv_path, x.decode('utf-8'), '')
+            for x in os.listdir(custom_venv_path)
+            if x not in ('awx', 'ansible') and
+            os.path.isdir(os.path.join(custom_venv_path, x)) and
+            os.path.exists(os.path.join(custom_venv_path, x, 'bin', 'activate'))
+        ]
+    else:
+        return []
+
+
 class OutputEventFilter(object):
     '''
     File-like object that looks for encoded job events in stdout data.

awx/settings/development.py

@@ -4,6 +4,7 @@
 # Development settings for AWX project.
 
 # Python
+import os
 import socket
 import copy
 import sys
@@ -123,8 +124,9 @@ for setting in dir(this_module):
 include(optional('/etc/tower/settings.py'), scope=locals())
 include(optional('/etc/tower/conf.d/*.py'), scope=locals())
 
-ANSIBLE_VENV_PATH = "/venv/ansible"
-AWX_VENV_PATH = "/venv/awx"
+BASE_VENV_PATH = "/venv/"
+ANSIBLE_VENV_PATH = os.path.join(BASE_VENV_PATH, "ansible")
+AWX_VENV_PATH = os.path.join(BASE_VENV_PATH, "awx")
 
 # If any local_*.py files are present in awx/settings/, use them to override
 # default settings for development.  If not present, we can still run using

awx/settings/production.py

@@ -4,6 +4,7 @@
 # Production settings for AWX project.
 
 # Python
+import os
 import copy
 import errno
 import sys
@@ -43,10 +44,11 @@ JOBOUTPUT_ROOT = '/var/lib/awx/job_status/'
 SCHEDULE_METADATA_LOCATION = '/var/lib/awx/.tower_cycle'
 
 # Ansible base virtualenv paths and enablement
-ANSIBLE_VENV_PATH = "/var/lib/awx/venv/ansible"
+BASE_VENV_PATH = "/var/lib/awx/venv"
+ANSIBLE_VENV_PATH = os.path.join(BASE_VENV_PATH, "ansible")
 
 # Tower base virtualenv paths and enablement
-AWX_VENV_PATH = "/var/lib/awx/venv/awx"
+AWX_VENV_PATH = os.path.join(BASE_VENV_PATH, "awx")
 
 AWX_ISOLATED_USERNAME = 'awx'
 

docs/custom_virtualenvs.md

@@ -0,0 +1,71 @@
+Managing Custom Python Dependencies
+===================================
+awx installations pre-build a special [Python
+virtualenv](https://pypi.python.org/pypi/virtualenv) which is automatically
+activated for all `ansible-playbook` runs invoked by awx (for example, any time
+a Job Template is launched).  By default, this virtualenv is located at
+`/var/lib/awx/venv/ansible` on the file system.
+
+awx pre-installs a variety of third-party library/SDK support into this
+virtualenv for its integration points with a variety of cloud providers (such
+as EC2, OpenStack, Azure, etc...)
+
+Periodically, awx users want to add additional SDK support into this
+virtualenv; this documentation describes the supported way to do so.
+
+Preparing a New Custom Virtualenv
+=================================
+awx allows a _different_ virtualenv to be specified and used on Job Template
+runs.  To choose a custom virtualenv, first create one in `/var/lib/awx/venv`:
+
+    $ sudo virtualenv /var/lib/awx/venv/my-custom-venv
+
+Your newly created virtualenv needs a few base dependencies to properly run
+playbooks (awx uses memcached as a holding space for playbook artifacts and
+fact gathering):
+
+    $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install python-memcached psutil
+
+From here, you can install _additional_ Python dependencies that you care
+about, such as a per-virtualenv version of ansible itself:
+
+    $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install -U "ansible == X.Y.Z"
+
+...or an additional third-party SDK that's not included with the base awx installation:
+
+    $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install -U python-digitalocean
+
+If you want to copy them, the libraries included in awx's default virtualenv
+can be found using `pip freeze`:
+
+    $ sudo /var/lib/awx/venv/ansible/bin/pip freeze
+
+One important item to keep in mind is that in a clustered awx installation,
+you need to ensure that the same custom virtualenv exists on _every_ local file
+system at `/var/lib/awx/venv/`.  For container-based deployments, this likely
+means building these steps into your own custom image building workflow.
+
+Assigning Custom Virtualenvs
+============================
+Once you've created a custom virtualenv, you can assign it at the Organization,
+Project, or Job Template level:
+
+    PATCH https://awx-host.example.org/api/v2/organizations/N/
+    PATCH https://awx-host.example.org/api/v2/projects/N/
+    PATCH https://awx-host.example.org/api/v2/job_templates/N/
+
+    Content-Type: application/json
+    {
+        'custom_virtualenv': '/var/lib/awx/venv/my-custom-venv'
+    }
+
+An HTTP `GET` request to `/api/v2/config/` will provide a list of
+detected installed virtualenvs:
+
+    {
+        "custom_virtualenvs": [
+            "/var/lib/awx/venv/my-custom-venv",
+            "/var/lib/awx/venv/my-other-custom-venv",
+        ],
+        ...
+    }

requirements/requirements_dev.txt

@@ -15,3 +15,4 @@ flower
 uwsgitop
 jupyter
 matplotlib
+backports.tempfile  # support in unit tests for py32+ tempfile.TemporaryDirectory