add support for custom py3 ansible virtualenvs

Makefile

@@ -127,6 +127,16 @@ virtualenv_ansible:
 		fi; \
 	fi
 
+virtualenv_ansible_py3:
+	if [ "$(VENV_BASE)" ]; then \
+		if [ ! -d "$(VENV_BASE)" ]; then \
+			mkdir $(VENV_BASE); \
+		fi; \
+		if [ ! -d "$(VENV_BASE)/ansible3" ]; then \
+			python3 -m venv --system-site-packages $(VENV_BASE)/ansible3; \
+		fi; \
+	fi
+
 virtualenv_awx:
 	if [ "$(VENV_BASE)" ]; then \
 		if [ ! -d "$(VENV_BASE)" ]; then \
@@ -145,6 +155,11 @@ requirements_ansible: virtualenv_ansible
 	fi
 	$(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt
 
+requirements_ansible_py3: virtualenv_ansible_py3
+	cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible3/bin/pip3 install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) --ignore-installed -r /dev/stdin
+	$(VENV_BASE)/ansible3/bin/pip3 install ansible  # can't inherit from system ansible, it's py2
+	$(VENV_BASE)/ansible3/bin/pip3 uninstall --yes -r requirements/requirements_ansible_uninstall.txt
+
 requirements_ansible_dev:
 	if [ "$(VENV_BASE)" ]; then \
 		$(VENV_BASE)/ansible/bin/pip install pytest mock; \
@@ -172,7 +187,7 @@ requirements_awx_dev:
 
 requirements: requirements_ansible requirements_awx
 
-requirements_dev: requirements requirements_awx_dev requirements_ansible_dev
+requirements_dev: requirements requirements_ansible_py3 requirements_awx_dev requirements_ansible_dev
 
 requirements_test: requirements
 

awx/main/tasks.py

@@ -5,6 +5,7 @@
 from collections import OrderedDict, namedtuple
 import configparser
 import errno
+import fnmatch
 import functools
 import importlib
 import json
@@ -688,6 +689,13 @@ class BaseTask(object):
         '''
         return os.path.abspath(os.path.join(os.path.dirname(__file__), *args))
 
+    def get_path_to_ansible(self, instance, executable='ansible-playbook', **kwargs):
+        venv_path = getattr(instance, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH)
+        venv_exe = os.path.join(venv_path, 'bin', executable)
+        if os.path.exists(venv_exe):
+            return venv_exe
+        return shutil.which(executable)
+
     def build_private_data(self, job, **kwargs):
         '''
         Return SSH private key data (only if stored in DB as ssh_key_data).
@@ -782,8 +790,11 @@ class BaseTask(object):
                 'a valid Python virtualenv does not exist at {}'.format(venv_path)
             )
         env.pop('PYTHONPATH', None)  # default to none if no python_ver matches
-        if os.path.isdir(os.path.join(venv_libdir, "python2.7")):
+        for version in os.listdir(venv_libdir):
-            env['PYTHONPATH'] = os.path.join(venv_libdir, "python2.7", "site-packages") + ":"
+            if fnmatch.fnmatch(version, 'python[23].*'):
+                if os.path.isdir(os.path.join(venv_libdir, version)):
+                    env['PYTHONPATH'] = os.path.join(venv_libdir, version, "site-packages") + ":"
+                    break
         # Add awx/lib to PYTHONPATH.
         if add_awx_lib:
             env['PYTHONPATH'] = env.get('PYTHONPATH', '') + self.get_path_to('..', 'lib') + ':'
@@ -1263,7 +1274,11 @@ class RunJob(BaseTask):
         # it doesn't make sense to rely on ansible-playbook's default of using
         # the current user.
         ssh_username = ssh_username or 'root'
-        args = ['ansible-playbook', '-i', self.build_inventory(job, **kwargs)]
+        args = [
+            self.get_path_to_ansible(job, 'ansible-playbook', **kwargs),
+            '-i',
+            self.build_inventory(job, **kwargs)
+        ]
         if job.job_type == 'check':
             args.append('--check')
         args.extend(['-u', sanitize_jinja(ssh_username)])
@@ -1557,7 +1572,11 @@ class RunProjectUpdate(BaseTask):
         Build command line argument list for running ansible-playbook,
         optionally using ssh-agent for public/private key authentication.
         '''
-        args = ['ansible-playbook', '-i', self.build_inventory(project_update, **kwargs)]
+        args = [
+            self.get_path_to_ansible(project_update, 'ansible-playbook', **kwargs),
+            '-i',
+            self.build_inventory(project_update, **kwargs)
+        ]
         if getattr(settings, 'PROJECT_UPDATE_VVV', False):
             args.append('-vvv')
         else:
@@ -2274,7 +2293,11 @@ class RunAdHocCommand(BaseTask):
         # it doesn't make sense to rely on ansible's default of using the
         # current user.
         ssh_username = ssh_username or 'root'
-        args = ['ansible', '-i', self.build_inventory(ad_hoc_command, **kwargs)]
+        args = [
+            self.get_path_to_ansible(ad_hoc_command, 'ansible', **kwargs),
+            '-i',
+            self.build_inventory(ad_hoc_command, **kwargs)
+        ]
         if ad_hoc_command.job_type == 'check':
             args.append('--check')
         args.extend(['-u', sanitize_jinja(ssh_username)])

awx/main/tests/unit/models/test_survey_models.py

@@ -87,6 +87,7 @@ def job(mocker):
         'launch_type': 'manual',
         'verbosity': 1,
         'awx_meta_vars.return_value': {},
+        'ansible_virtualenv_path': '',
         'inventory.get_script_data.return_value': {}})
     ret.project = mocker.MagicMock(scm_revision='asdf1234')
     return ret

awx/main/tests/unit/utils/test_common.py

@@ -173,12 +173,14 @@ def test_extract_ansible_vars():
 
 def test_get_custom_venv_choices():
     bundled_venv = os.path.join(settings.BASE_VENV_PATH, 'ansible', '')
-    assert common.get_custom_venv_choices() == [bundled_venv]
+    bundled_venv_py3 = os.path.join(settings.BASE_VENV_PATH, 'ansible3', '')
+    assert sorted(common.get_custom_venv_choices()) == [bundled_venv, bundled_venv_py3]
 
     with TemporaryDirectory(dir=settings.BASE_VENV_PATH, prefix='tmp') as temp_dir:
         os.makedirs(os.path.join(temp_dir, 'bin', 'activate'))
         assert sorted(common.get_custom_venv_choices()) == [
             bundled_venv,
+            bundled_venv_py3,
             os.path.join(temp_dir, '')
         ]
 

docs/custom_virtualenvs.md

@@ -20,6 +20,11 @@ runs.  To choose a custom virtualenv, first create one in `/var/lib/awx/venv`:
 
     $ sudo virtualenv /var/lib/awx/venv/my-custom-venv
 
+Multiple versions of Python are supported, though it's important to note that
+the semantics for creating virtualenvs in Python 3 has changed slightly:
+
+    $ sudo python3 -m venv /var/lib/awx/venv/my-custom-venv
+
 Your newly created virtualenv needs a few base dependencies to properly run
 playbooks (awx uses memcached as a holding space for playbook artifacts and
 fact gathering):

requirements/requirements_ansible.txt

@@ -50,7 +50,7 @@ deprecation==2.0          # via openstacksdk
 docutils==0.14            # via botocore
 dogpile.cache==0.6.5      # via openstacksdk
 entrypoints==0.2.3        # via keyring
-enum34==1.1.6             # via cryptography, knack, msrest, ovirt-engine-sdk-python
+enum34==1.1.6; python_version < '3' # via cryptography, knack, msrest, ovirt-engine-sdk-python
 humanfriendly==4.8        # via azure-cli-core
 idna==2.6                 # via cryptography, requests
 ipaddress==1.0.19         # via cryptography, openstacksdk