diff --git a/Makefile b/Makefile index 47c00a2841..f565390067 100644 --- a/Makefile +++ b/Makefile @@ -129,6 +129,16 @@ virtualenv_ansible: fi; \ fi +virtualenv_ansible_py3: + if [ "$(VENV_BASE)" ]; then \ + if [ ! -d "$(VENV_BASE)" ]; then \ + mkdir $(VENV_BASE); \ + fi; \ + if [ ! -d "$(VENV_BASE)/ansible3" ]; then \ + python3 -m venv --system-site-packages $(VENV_BASE)/ansible3; \ + fi; \ + fi + virtualenv_awx: if [ "$(VENV_BASE)" ]; then \ if [ ! -d "$(VENV_BASE)" ]; then \ @@ -147,6 +157,11 @@ requirements_ansible: virtualenv_ansible fi $(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt +requirements_ansible_py3: virtualenv_ansible_py3 + cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible3/bin/pip3 install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) --ignore-installed -r /dev/stdin + $(VENV_BASE)/ansible3/bin/pip3 install ansible # can't inherit from system ansible, it's py2 + $(VENV_BASE)/ansible3/bin/pip3 uninstall --yes -r requirements/requirements_ansible_uninstall.txt + requirements_ansible_dev: if [ "$(VENV_BASE)" ]; then \ $(VENV_BASE)/ansible/bin/pip install pytest mock; \ @@ -174,7 +189,7 @@ requirements_awx_dev: requirements: requirements_ansible requirements_awx -requirements_dev: requirements requirements_awx_dev requirements_ansible_dev +requirements_dev: requirements requirements_ansible_py3 requirements_awx_dev requirements_ansible_dev requirements_test: requirements diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 14790f6ad0..61955b96ec 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -5,6 +5,7 @@ from collections import OrderedDict, namedtuple import configparser import errno +import fnmatch import functools import importlib import json @@ -688,6 +689,13 @@ class BaseTask(object): ''' return os.path.abspath(os.path.join(os.path.dirname(__file__), *args)) + def get_path_to_ansible(self, instance, executable='ansible-playbook', **kwargs): + venv_path = getattr(instance, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH) + venv_exe = os.path.join(venv_path, 'bin', executable) + if os.path.exists(venv_exe): + return venv_exe + return shutil.which(executable) + def build_private_data(self, job, **kwargs): ''' Return SSH private key data (only if stored in DB as ssh_key_data). @@ -782,8 +790,11 @@ class BaseTask(object): 'a valid Python virtualenv does not exist at {}'.format(venv_path) ) env.pop('PYTHONPATH', None) # default to none if no python_ver matches - if os.path.isdir(os.path.join(venv_libdir, "python2.7")): - env['PYTHONPATH'] = os.path.join(venv_libdir, "python2.7", "site-packages") + ":" + for version in os.listdir(venv_libdir): + if fnmatch.fnmatch(version, 'python[23].*'): + if os.path.isdir(os.path.join(venv_libdir, version)): + env['PYTHONPATH'] = os.path.join(venv_libdir, version, "site-packages") + ":" + break # Add awx/lib to PYTHONPATH. if add_awx_lib: env['PYTHONPATH'] = env.get('PYTHONPATH', '') + self.get_path_to('..', 'lib') + ':' @@ -1263,7 +1274,11 @@ class RunJob(BaseTask): # it doesn't make sense to rely on ansible-playbook's default of using # the current user. ssh_username = ssh_username or 'root' - args = ['ansible-playbook', '-i', self.build_inventory(job, **kwargs)] + args = [ + self.get_path_to_ansible(job, 'ansible-playbook', **kwargs), + '-i', + self.build_inventory(job, **kwargs) + ] if job.job_type == 'check': args.append('--check') args.extend(['-u', sanitize_jinja(ssh_username)]) @@ -1557,7 +1572,11 @@ class RunProjectUpdate(BaseTask): Build command line argument list for running ansible-playbook, optionally using ssh-agent for public/private key authentication. ''' - args = ['ansible-playbook', '-i', self.build_inventory(project_update, **kwargs)] + args = [ + self.get_path_to_ansible(project_update, 'ansible-playbook', **kwargs), + '-i', + self.build_inventory(project_update, **kwargs) + ] if getattr(settings, 'PROJECT_UPDATE_VVV', False): args.append('-vvv') else: @@ -2274,7 +2293,11 @@ class RunAdHocCommand(BaseTask): # it doesn't make sense to rely on ansible's default of using the # current user. ssh_username = ssh_username or 'root' - args = ['ansible', '-i', self.build_inventory(ad_hoc_command, **kwargs)] + args = [ + self.get_path_to_ansible(ad_hoc_command, 'ansible', **kwargs), + '-i', + self.build_inventory(ad_hoc_command, **kwargs) + ] if ad_hoc_command.job_type == 'check': args.append('--check') args.extend(['-u', sanitize_jinja(ssh_username)]) diff --git a/awx/main/tests/unit/models/test_survey_models.py b/awx/main/tests/unit/models/test_survey_models.py index 30eecff523..c6751e9b27 100644 --- a/awx/main/tests/unit/models/test_survey_models.py +++ b/awx/main/tests/unit/models/test_survey_models.py @@ -87,6 +87,7 @@ def job(mocker): 'launch_type': 'manual', 'verbosity': 1, 'awx_meta_vars.return_value': {}, + 'ansible_virtualenv_path': '', 'inventory.get_script_data.return_value': {}}) ret.project = mocker.MagicMock(scm_revision='asdf1234') return ret diff --git a/awx/main/tests/unit/utils/test_common.py b/awx/main/tests/unit/utils/test_common.py index d2a1fd8b0e..1f0e464f15 100644 --- a/awx/main/tests/unit/utils/test_common.py +++ b/awx/main/tests/unit/utils/test_common.py @@ -173,12 +173,14 @@ def test_extract_ansible_vars(): def test_get_custom_venv_choices(): bundled_venv = os.path.join(settings.BASE_VENV_PATH, 'ansible', '') - assert common.get_custom_venv_choices() == [bundled_venv] + bundled_venv_py3 = os.path.join(settings.BASE_VENV_PATH, 'ansible3', '') + assert sorted(common.get_custom_venv_choices()) == [bundled_venv, bundled_venv_py3] with TemporaryDirectory(dir=settings.BASE_VENV_PATH, prefix='tmp') as temp_dir: os.makedirs(os.path.join(temp_dir, 'bin', 'activate')) assert sorted(common.get_custom_venv_choices()) == [ bundled_venv, + bundled_venv_py3, os.path.join(temp_dir, '') ] diff --git a/docs/custom_virtualenvs.md b/docs/custom_virtualenvs.md index 37f0173357..b89ac670c1 100644 --- a/docs/custom_virtualenvs.md +++ b/docs/custom_virtualenvs.md @@ -20,6 +20,11 @@ runs. To choose a custom virtualenv, first create one in `/var/lib/awx/venv`: $ sudo virtualenv /var/lib/awx/venv/my-custom-venv +Multiple versions of Python are supported, though it's important to note that +the semantics for creating virtualenvs in Python 3 has changed slightly: + + $ sudo python3 -m venv /var/lib/awx/venv/my-custom-venv + Your newly created virtualenv needs a few base dependencies to properly run playbooks (awx uses memcached as a holding space for playbook artifacts and fact gathering): diff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt index 99fb6c32a3..2cbb52a84a 100644 --- a/requirements/requirements_ansible.txt +++ b/requirements/requirements_ansible.txt @@ -50,7 +50,7 @@ deprecation==2.0 # via openstacksdk docutils==0.14 # via botocore dogpile.cache==0.6.5 # via openstacksdk entrypoints==0.2.3 # via keyring -enum34==1.1.6 # via cryptography, knack, msrest, ovirt-engine-sdk-python +enum34==1.1.6; python_version < '3' # via cryptography, knack, msrest, ovirt-engine-sdk-python humanfriendly==4.8 # via azure-cli-core idna==2.6 # via cryptography, requests ipaddress==1.0.19 # via cryptography, openstacksdk