From 1e8c89f5367c5c4060170a07164600b498fa4159 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 8 Jan 2018 17:51:59 -0500 Subject: [PATCH] implement support for per-playbook/project/org virtualenvs see: https://github.com/ansible/awx/issues/34 --- awx/api/serializers.py | 6 +- awx/api/views.py | 1 + ...0018_v330_add_additional_stdout_events.py} | 0 .../migrations/0019_v330_custom_virtualenv.py | 30 ++++++++ awx/main/models/jobs.py | 16 ++++- awx/main/models/mixins.py | 27 ++++++- awx/main/models/organization.py | 4 +- awx/main/models/projects.py | 4 +- awx/main/tasks.py | 22 ++++-- .../tests/functional/api/test_job_template.py | 32 +++++++++ .../functional/api/test_organizations.py | 27 +++++-- awx/main/tests/functional/api/test_project.py | 23 ++++++ awx/main/tests/functional/models/test_job.py | 40 +++++++++++ awx/main/tests/unit/test_tasks.py | 31 ++++++++ awx/main/tests/unit/utils/test_common.py | 12 +++- awx/main/utils/common.py | 23 +++++- awx/settings/development.py | 6 +- awx/settings/production.py | 6 +- docs/custom_virtualenvs.md | 71 +++++++++++++++++++ requirements/requirements_dev.txt | 1 + 20 files changed, 352 insertions(+), 30 deletions(-) rename awx/main/migrations/{0018_add_additional_stdout_events.py => 0018_v330_add_additional_stdout_events.py} (100%) create mode 100644 awx/main/migrations/0019_v330_custom_virtualenv.py create mode 100644 awx/main/tests/functional/models/test_job.py create mode 100644 docs/custom_virtualenvs.md diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 1ca84dce6b..7f8c34372d 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -896,7 +896,7 @@ class OrganizationSerializer(BaseSerializer): class Meta: model = Organization - fields = ('*',) + fields = ('*', 'custom_virtualenv',) def get_related(self, obj): res = super(OrganizationSerializer, self).get_related(obj) @@ -984,7 +984,7 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer): class Meta: model = Project fields = ('*', 'organization', 'scm_delete_on_next_update', 'scm_update_on_launch', - 'scm_update_cache_timeout', 'scm_revision',) + \ + 'scm_update_cache_timeout', 'scm_revision', 'custom_virtualenv',) + \ ('last_update_failed', 'last_updated') # Backwards compatibility read_only_fields = ('scm_delete_on_next_update',) @@ -2530,7 +2530,7 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO fields = ('*', 'host_config_key', 'ask_diff_mode_on_launch', 'ask_variables_on_launch', 'ask_limit_on_launch', 'ask_tags_on_launch', 'ask_skip_tags_on_launch', 'ask_job_type_on_launch', 'ask_verbosity_on_launch', 'ask_inventory_on_launch', 'ask_credential_on_launch', 'survey_enabled', 'become_enabled', 'diff_mode', - 'allow_simultaneous') + 'allow_simultaneous', 'custom_virtualenv') def get_related(self, obj): res = super(JobTemplateSerializer, self).get_related(obj) diff --git a/awx/api/views.py b/awx/api/views.py index 93b1098f61..aa45f2616d 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -315,6 +315,7 @@ class ApiV1ConfigView(APIView): data.update(dict( project_base_dir = settings.PROJECTS_ROOT, project_local_paths = Project.get_local_path_choices(), + custom_virtualenvs = get_custom_venv_choices(), )) return Response(data) diff --git a/awx/main/migrations/0018_add_additional_stdout_events.py b/awx/main/migrations/0018_v330_add_additional_stdout_events.py similarity index 100% rename from awx/main/migrations/0018_add_additional_stdout_events.py rename to awx/main/migrations/0018_v330_add_additional_stdout_events.py diff --git a/awx/main/migrations/0019_v330_custom_virtualenv.py b/awx/main/migrations/0019_v330_custom_virtualenv.py new file mode 100644 index 0000000000..ba6aeb0167 --- /dev/null +++ b/awx/main/migrations/0019_v330_custom_virtualenv.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2018-01-09 21:30 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0018_v330_add_additional_stdout_events'), + ] + + operations = [ + migrations.AddField( + model_name='jobtemplate', + name='custom_virtualenv', + field=models.CharField(blank=True, default=None, max_length=100, null=True), + ), + migrations.AddField( + model_name='organization', + name='custom_virtualenv', + field=models.CharField(blank=True, default=None, max_length=100, null=True), + ), + migrations.AddField( + model_name='project', + name='custom_virtualenv', + field=models.CharField(blank=True, default=None, max_length=100, null=True), + ), + ] diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index f3c237b2e5..d2ee6fef85 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -34,7 +34,7 @@ from awx.main.models.notifications import ( ) from awx.main.utils import parse_yaml_or_json from awx.main.fields import ImplicitRoleField -from awx.main.models.mixins import ResourceMixin, SurveyJobTemplateMixin, SurveyJobMixin, TaskManagerJobMixin +from awx.main.models.mixins import ResourceMixin, SurveyJobTemplateMixin, SurveyJobMixin, TaskManagerJobMixin, CustomVirtualEnvMixin from awx.main.fields import JSONField, AskForField @@ -215,7 +215,7 @@ class JobOptions(BaseModel): return needed -class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, ResourceMixin): +class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, ResourceMixin, CustomVirtualEnvMixin): ''' A job template is a reusable job definition for applying a project (with playbook) to an inventory source with a given credential. @@ -509,6 +509,18 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana def get_ui_url(self): return urljoin(settings.TOWER_URL_BASE, "/#/jobs/{}".format(self.pk)) + @property + def ansible_virtualenv_path(self): + # the order here enforces precedence (it matters) + for virtualenv in ( + self.job_template.custom_virtualenv, + self.project.custom_virtualenv, + self.project.organization.custom_virtualenv + ): + if virtualenv: + return virtualenv + return settings.ANSIBLE_VENV_PATH + @property def event_class(self): return JobEvent diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index 2f3b20f8f6..f0463ba867 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -1,26 +1,29 @@ # Python +import os import json from copy import copy, deepcopy # Django +from django.conf import settings from django.db import models from django.contrib.contenttypes.models import ContentType from django.contrib.auth.models import User # noqa from django.utils.translation import ugettext_lazy as _ +from django.core.exceptions import ValidationError # AWX from awx.main.models.base import prevent_search from awx.main.models.rbac import ( Role, RoleAncestorEntry, get_roles_on_resource ) -from awx.main.utils import parse_yaml_or_json +from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted from awx.main.fields import JSONField, AskForField __all__ = ['ResourceMixin', 'SurveyJobTemplateMixin', 'SurveyJobMixin', 'TaskManagerUnifiedJobMixin', 'TaskManagerJobMixin', 'TaskManagerProjectUpdateMixin', - 'TaskManagerInventoryUpdateMixin',] + 'TaskManagerInventoryUpdateMixin', 'CustomVirtualEnvMixin'] class ResourceMixin(models.Model): @@ -416,3 +419,23 @@ class TaskManagerProjectUpdateMixin(TaskManagerUpdateOnLaunchMixin): class TaskManagerInventoryUpdateMixin(TaskManagerUpdateOnLaunchMixin): class Meta: abstract = True + + +class CustomVirtualEnvMixin(models.Model): + class Meta: + abstract = True + + custom_virtualenv = models.CharField( + blank=True, + null=True, + default=None, + max_length=100 + ) + + def clean_custom_virtualenv(self): + value = self.custom_virtualenv + if value and os.path.join(value, '') not in get_custom_venv_choices(): + raise ValidationError( + _('{} is not a valid virtualenv in {}').format(value, settings.BASE_VENV_PATH) + ) + return os.path.join(value or '', '') diff --git a/awx/main/models/organization.py b/awx/main/models/organization.py index fbffc73315..cd0ccfc785 100644 --- a/awx/main/models/organization.py +++ b/awx/main/models/organization.py @@ -22,12 +22,12 @@ from awx.main.models.rbac import ( ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR, ) -from awx.main.models.mixins import ResourceMixin +from awx.main.models.mixins import ResourceMixin, CustomVirtualEnvMixin __all__ = ['Organization', 'Team', 'Profile', 'AuthToken'] -class Organization(CommonModel, NotificationFieldsModel, ResourceMixin): +class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVirtualEnvMixin): ''' An organization is the basic unit of multi-tenancy divisions ''' diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index cff74bf2ad..5794d170d7 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -24,7 +24,7 @@ from awx.main.models.notifications import ( JobNotificationMixin, ) from awx.main.models.unified_jobs import * # noqa -from awx.main.models.mixins import ResourceMixin, TaskManagerProjectUpdateMixin +from awx.main.models.mixins import ResourceMixin, TaskManagerProjectUpdateMixin, CustomVirtualEnvMixin from awx.main.utils import update_scm_url from awx.main.utils.ansible import skip_directory, could_be_inventory, could_be_playbook from awx.main.fields import ImplicitRoleField @@ -223,7 +223,7 @@ class ProjectOptions(models.Model): return proj_path + '.lock' -class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin): +class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEnvMixin): ''' A project represents a playbook git repo that can access a set of inventories ''' diff --git a/awx/main/tasks.py b/awx/main/tasks.py index b5e5a2c712..f9183d4a70 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -626,10 +626,16 @@ class BaseTask(LogErrorsTask): '': '', } - def add_ansible_venv(self, env, add_awx_lib=True): - env['VIRTUAL_ENV'] = settings.ANSIBLE_VENV_PATH - env['PATH'] = os.path.join(settings.ANSIBLE_VENV_PATH, "bin") + ":" + env['PATH'] - venv_libdir = os.path.join(settings.ANSIBLE_VENV_PATH, "lib") + def add_ansible_venv(self, venv_path, env, add_awx_lib=True): + env['VIRTUAL_ENV'] = venv_path + env['PATH'] = os.path.join(venv_path, "bin") + ":" + env['PATH'] + venv_libdir = os.path.join(venv_path, "lib") + + if not os.path.exists(venv_libdir): + raise RuntimeError( + 'a valid Python virtualenv does not exist at {}'.format(venv_path) + ) + env.pop('PYTHONPATH', None) # default to none if no python_ver matches if os.path.isdir(os.path.join(venv_libdir, "python2.7")): env['PYTHONPATH'] = os.path.join(venv_libdir, "python2.7", "site-packages") + ":" @@ -802,6 +808,8 @@ class BaseTask(LogErrorsTask): kwargs['private_data_files'] = self.build_private_data_files(instance, **kwargs) kwargs['passwords'] = self.build_passwords(instance, **kwargs) kwargs['proot_show_paths'] = self.proot_show_paths + if getattr(instance, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH) != settings.ANSIBLE_VENV_PATH: + kwargs['proot_custom_virtualenv'] = instance.ansible_virtualenv_path args = self.build_args(instance, **kwargs) safe_args = self.build_safe_args(instance, **kwargs) output_replacements = self.build_output_replacements(instance, **kwargs) @@ -1021,7 +1029,7 @@ class RunJob(BaseTask): plugin_dirs.extend(settings.AWX_ANSIBLE_CALLBACK_PLUGINS) plugin_path = ':'.join(plugin_dirs) env = super(RunJob, self).build_env(job, **kwargs) - env = self.add_ansible_venv(env, add_awx_lib=kwargs.get('isolated', False)) + env = self.add_ansible_venv(job.ansible_virtualenv_path, env, add_awx_lib=kwargs.get('isolated', False)) # Set environment variables needed for inventory and job event # callbacks to work. env['JOB_ID'] = str(job.pk) @@ -1314,7 +1322,7 @@ class RunProjectUpdate(BaseTask): Build environment dictionary for ansible-playbook. ''' env = super(RunProjectUpdate, self).build_env(project_update, **kwargs) - env = self.add_ansible_venv(env) + env = self.add_ansible_venv(settings.ANSIBLE_VENV_PATH, env) env['ANSIBLE_RETRY_FILES_ENABLED'] = str(False) env['ANSIBLE_ASK_PASS'] = str(False) env['ANSIBLE_BECOME_ASK_PASS'] = str(False) @@ -2055,7 +2063,7 @@ class RunAdHocCommand(BaseTask): ''' plugin_dir = self.get_path_to('..', 'plugins', 'callback') env = super(RunAdHocCommand, self).build_env(ad_hoc_command, **kwargs) - env = self.add_ansible_venv(env) + env = self.add_ansible_venv(settings.ANSIBLE_VENV_PATH, env) # Set environment variables needed for inventory and ad hoc event # callbacks to work. env['AD_HOC_COMMAND_ID'] = str(ad_hoc_command.pk) diff --git a/awx/main/tests/functional/api/test_job_template.py b/awx/main/tests/functional/api/test_job_template.py index 83875ae6cc..29f67d8ae8 100644 --- a/awx/main/tests/functional/api/test_job_template.py +++ b/awx/main/tests/functional/api/test_job_template.py @@ -1,3 +1,6 @@ +import os + +from backports.tempfile import TemporaryDirectory import pytest # AWX @@ -7,6 +10,7 @@ from awx.main.models.jobs import Job, JobTemplate from awx.main.migrations import _save_password_keys as save_password_keys # Django +from django.conf import settings from django.apps import apps @@ -570,3 +574,31 @@ def test_save_survey_passwords_on_migration(job_template_with_survey_passwords): save_password_keys.migrate_survey_passwords(apps, None) job = job_template_with_survey_passwords.jobs.all()[0] assert job.survey_passwords == {'SSN': '$encrypted$', 'secret_key': '$encrypted$'} + + +@pytest.mark.django_db +def test_job_template_custom_virtualenv(get, patch, organization_factory, job_template_factory): + objs = organization_factory("org", superusers=['admin']) + jt = job_template_factory("jt", organization=objs.organization, + inventory='test_inv', project='test_proj').job_template + + with TemporaryDirectory(dir=settings.BASE_VENV_PATH) as temp_dir: + admin = objs.superusers.admin + os.makedirs(os.path.join(temp_dir, 'bin', 'activate')) + url = reverse('api:job_template_detail', kwargs={'pk': jt.id}) + patch(url, {'custom_virtualenv': temp_dir}, user=admin, expect=200) + assert get(url, user=admin).data['custom_virtualenv'] == os.path.join(temp_dir, '') + + +@pytest.mark.django_db +def test_job_template_invalid_custom_virtualenv(get, patch, organization_factory, + job_template_factory): + objs = organization_factory("org", superusers=['admin']) + jt = job_template_factory("jt", organization=objs.organization, + inventory='test_inv', project='test_proj').job_template + + url = reverse('api:job_template_detail', kwargs={'pk': jt.id}) + resp = patch(url, {'custom_virtualenv': '/foo/bar'}, user=objs.superusers.admin, expect=400) + assert resp.data['custom_virtualenv'] == [ + '/foo/bar is not a valid virtualenv in {}'.format(settings.BASE_VENV_PATH) + ] diff --git a/awx/main/tests/functional/api/test_organizations.py b/awx/main/tests/functional/api/test_organizations.py index 9fb0e085d2..c3905bd8e8 100644 --- a/awx/main/tests/functional/api/test_organizations.py +++ b/awx/main/tests/functional/api/test_organizations.py @@ -2,15 +2,16 @@ # All Rights Reserved. # Python +import os + +from backports.tempfile import TemporaryDirectory +from django.conf import settings import pytest import mock - -# Django -from awx.api.versioning import reverse - # AWX from awx.main.models import * # noqa +from awx.api.versioning import reverse @pytest.mark.django_db @@ -188,3 +189,21 @@ def test_delete_organization_xfail1(delete, organization, alice): @mock.patch('awx.main.access.BaseAccess.check_license', lambda *a, **kw: True) def test_delete_organization_xfail2(delete, organization): delete(reverse('api:organization_detail', kwargs={'pk': organization.id}), user=None, expect=401) + + +@pytest.mark.django_db +def test_organization_custom_virtualenv(get, patch, organization, admin): + with TemporaryDirectory(dir=settings.BASE_VENV_PATH) as temp_dir: + os.makedirs(os.path.join(temp_dir, 'bin', 'activate')) + url = reverse('api:organization_detail', kwargs={'pk': organization.id}) + patch(url, {'custom_virtualenv': temp_dir}, user=admin, expect=200) + assert get(url, user=admin).data['custom_virtualenv'] == os.path.join(temp_dir, '') + + +@pytest.mark.django_db +def test_organization_invalid_custom_virtualenv(get, patch, organization, admin): + url = reverse('api:organization_detail', kwargs={'pk': organization.id}) + resp = patch(url, {'custom_virtualenv': '/foo/bar'}, user=admin, expect=400) + assert resp.data['custom_virtualenv'] == [ + '/foo/bar is not a valid virtualenv in {}'.format(settings.BASE_VENV_PATH) + ] diff --git a/awx/main/tests/functional/api/test_project.py b/awx/main/tests/functional/api/test_project.py index ddf28f9b52..e554c33280 100644 --- a/awx/main/tests/functional/api/test_project.py +++ b/awx/main/tests/functional/api/test_project.py @@ -1,5 +1,11 @@ +import os + +from backports.tempfile import TemporaryDirectory +from django.conf import settings import pytest +from awx.api.versioning import reverse + @pytest.mark.django_db class TestInsightsCredential: @@ -13,3 +19,20 @@ class TestInsightsCredential: {'credential': scm_credential.id}, admin_user, expect=400) + +@pytest.mark.django_db +def test_project_custom_virtualenv(get, patch, project, admin): + with TemporaryDirectory(dir=settings.BASE_VENV_PATH) as temp_dir: + os.makedirs(os.path.join(temp_dir, 'bin', 'activate')) + url = reverse('api:project_detail', kwargs={'pk': project.id}) + patch(url, {'custom_virtualenv': temp_dir}, user=admin, expect=200) + assert get(url, user=admin).data['custom_virtualenv'] == os.path.join(temp_dir, '') + + +@pytest.mark.django_db +def test_project_invalid_custom_virtualenv(get, patch, project, admin): + url = reverse('api:project_detail', kwargs={'pk': project.id}) + resp = patch(url, {'custom_virtualenv': '/foo/bar'}, user=admin, expect=400) + assert resp.data['custom_virtualenv'] == [ + '/foo/bar is not a valid virtualenv in {}'.format(settings.BASE_VENV_PATH) + ] diff --git a/awx/main/tests/functional/models/test_job.py b/awx/main/tests/functional/models/test_job.py new file mode 100644 index 0000000000..aa3f6f4a03 --- /dev/null +++ b/awx/main/tests/functional/models/test_job.py @@ -0,0 +1,40 @@ +import pytest + +from awx.main.models import JobTemplate + + +@pytest.mark.django_db +def test_awx_virtualenv_from_settings(inventory, project, machine_credential): + jt = JobTemplate.objects.create( + name='my-jt', + inventory=inventory, + project=project, + playbook='helloworld.yml' + ) + jt.credentials.add(machine_credential) + job = jt.create_unified_job() + assert job.ansible_virtualenv_path == '/venv/ansible' + + +@pytest.mark.django_db +def test_awx_custom_virtualenv(inventory, project, machine_credential): + jt = JobTemplate.objects.create( + name='my-jt', + inventory=inventory, + project=project, + playbook='helloworld.yml' + ) + jt.credentials.add(machine_credential) + job = jt.create_unified_job() + + job.project.organization.custom_virtualenv = '/venv/fancy-org' + job.project.organization.save() + assert job.ansible_virtualenv_path == '/venv/fancy-org' + + job.project.custom_virtualenv = '/venv/fancy-proj' + job.project.save() + assert job.ansible_virtualenv_path == '/venv/fancy-proj' + + job.job_template.custom_virtualenv = '/venv/fancy-jt' + job.job_template.save() + assert job.ansible_virtualenv_path == '/venv/fancy-jt' diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 2bd36349d7..f4b5d8bb4e 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -8,6 +8,7 @@ import re import shutil import tempfile +from backports.tempfile import TemporaryDirectory import fcntl import mock import pytest @@ -204,6 +205,7 @@ class TestJobExecution: mock.patch.object(Project, 'get_project_path', lambda *a, **kw: self.project_path), # don't emit websocket statuses; they use the DB and complicate testing mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()), + mock.patch.object(Job, 'ansible_virtualenv_path', settings.ANSIBLE_VENV_PATH), mock.patch('awx.main.expect.run.run_pexpect', self.run_pexpect), ] for cls in (Job, AdHocCommand): @@ -348,6 +350,35 @@ class TestGenericRun(TestJobExecution): args, cwd, env, stdout = call_args assert env['FOO'] == 'BAR' + def test_valid_custom_virtualenv(self): + with TemporaryDirectory(dir=settings.BASE_VENV_PATH) as tempdir: + os.makedirs(os.path.join(tempdir, 'lib')) + os.makedirs(os.path.join(tempdir, 'bin', 'activate')) + venv_patch = mock.patch.object(Job, 'ansible_virtualenv_path', tempdir) + self.patches.append(venv_patch) + venv_patch.start() + + self.task.run(self.pk) + + assert self.run_pexpect.call_count == 1 + call_args, _ = self.run_pexpect.call_args_list[0] + args, cwd, env, stdout = call_args + + assert env['PATH'].startswith(os.path.join(tempdir, 'bin')) + assert env['VIRTUAL_ENV'] == tempdir + for path in (settings.ANSIBLE_VENV_PATH, tempdir): + assert '--ro-bind {} {}'.format(path, path) in ' '.join(args) + + def test_invalid_custom_virtualenv(self): + venv_patch = mock.patch.object(Job, 'ansible_virtualenv_path', '/venv/missing') + self.patches.append(venv_patch) + venv_patch.start() + + with pytest.raises(Exception): + self.task.run(self.pk) + tb = self.task.update_model.call_args[-1]['result_traceback'] + assert 'a valid Python virtualenv does not exist at /venv/missing' in tb + class TestAdhocRun(TestJobExecution): diff --git a/awx/main/tests/unit/utils/test_common.py b/awx/main/tests/unit/utils/test_common.py index 45ed62068b..2d0d139dc1 100644 --- a/awx/main/tests/unit/utils/test_common.py +++ b/awx/main/tests/unit/utils/test_common.py @@ -8,6 +8,8 @@ from uuid import uuid4 import json import yaml +from backports.tempfile import TemporaryDirectory +from django.conf import settings from django.core.cache import cache from rest_framework.exceptions import ParseError @@ -136,7 +138,7 @@ def test_memoize_delete(memoized_function): assert memoized_function('john', 'smith') == 'smith' assert memoized_function('john', 'smith') == 'smith' assert memoized_function.calls['john'] == 1 - + assert cache.get('myfunction') == {u'john-smith': 'smith'} common.memoize_delete('myfunction') @@ -164,3 +166,11 @@ def test_extract_ansible_vars(): redacted, var_list = common.extract_ansible_vars(json.dumps(my_dict)) assert var_list == set(['ansible_connetion_setting']) assert redacted == {"foobar": "baz"} + + +def test_get_custom_venv_choices(): + assert common.get_custom_venv_choices() == [] + + with TemporaryDirectory(dir=settings.BASE_VENV_PATH) as temp_dir: + os.makedirs(os.path.join(temp_dir, 'bin', 'activate')) + assert common.get_custom_venv_choices() == [os.path.join(temp_dir, '')] diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 43e88875b7..97b5ae1253 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -47,7 +47,7 @@ __all__ = ['get_object_or_400', 'get_object_or_403', 'camelcase_to_underscore', 'extract_ansible_vars', 'get_search_fields', 'get_system_task_capacity', 'wrap_args_with_proot', 'build_proot_temp_dir', 'check_proot_installed', 'model_to_dict', 'model_instance_diff', 'timestamp_apiformat', 'parse_yaml_or_json', 'RequireDebugTrueOrTest', - 'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError',] + 'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError', 'get_custom_venv_choices'] def get_object_or_400(klass, *args, **kwargs): @@ -756,9 +756,11 @@ def wrap_args_with_proot(args, cwd, **kwargs): show_paths = [cwd] for venv in ( settings.ANSIBLE_VENV_PATH, - settings.AWX_VENV_PATH + settings.AWX_VENV_PATH, + kwargs.get('proot_custom_virtualenv') ): - new_args.extend(['--ro-bind', venv, venv]) + if venv: + new_args.extend(['--ro-bind', venv, venv]) show_paths.extend(getattr(settings, 'AWX_PROOT_SHOW_PATHS', None) or []) show_paths.extend(kwargs.get('proot_show_paths', [])) for path in sorted(set(show_paths)): @@ -838,6 +840,21 @@ def get_current_apps(): return current_apps +def get_custom_venv_choices(): + from django.conf import settings + custom_venv_path = settings.BASE_VENV_PATH + if os.path.exists(custom_venv_path): + return [ + os.path.join(custom_venv_path, x.decode('utf-8'), '') + for x in os.listdir(custom_venv_path) + if x not in ('awx', 'ansible') and + os.path.isdir(os.path.join(custom_venv_path, x)) and + os.path.exists(os.path.join(custom_venv_path, x, 'bin', 'activate')) + ] + else: + return [] + + class OutputEventFilter(object): ''' File-like object that looks for encoded job events in stdout data. diff --git a/awx/settings/development.py b/awx/settings/development.py index bf36a09cac..682cf21dd8 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -4,6 +4,7 @@ # Development settings for AWX project. # Python +import os import socket import copy import sys @@ -123,8 +124,9 @@ for setting in dir(this_module): include(optional('/etc/tower/settings.py'), scope=locals()) include(optional('/etc/tower/conf.d/*.py'), scope=locals()) -ANSIBLE_VENV_PATH = "/venv/ansible" -AWX_VENV_PATH = "/venv/awx" +BASE_VENV_PATH = "/venv/" +ANSIBLE_VENV_PATH = os.path.join(BASE_VENV_PATH, "ansible") +AWX_VENV_PATH = os.path.join(BASE_VENV_PATH, "awx") # If any local_*.py files are present in awx/settings/, use them to override # default settings for development. If not present, we can still run using diff --git a/awx/settings/production.py b/awx/settings/production.py index eaec67f247..9df01dd720 100644 --- a/awx/settings/production.py +++ b/awx/settings/production.py @@ -4,6 +4,7 @@ # Production settings for AWX project. # Python +import os import copy import errno import sys @@ -43,10 +44,11 @@ JOBOUTPUT_ROOT = '/var/lib/awx/job_status/' SCHEDULE_METADATA_LOCATION = '/var/lib/awx/.tower_cycle' # Ansible base virtualenv paths and enablement -ANSIBLE_VENV_PATH = "/var/lib/awx/venv/ansible" +BASE_VENV_PATH = "/var/lib/awx/venv" +ANSIBLE_VENV_PATH = os.path.join(BASE_VENV_PATH, "ansible") # Tower base virtualenv paths and enablement -AWX_VENV_PATH = "/var/lib/awx/venv/awx" +AWX_VENV_PATH = os.path.join(BASE_VENV_PATH, "awx") AWX_ISOLATED_USERNAME = 'awx' diff --git a/docs/custom_virtualenvs.md b/docs/custom_virtualenvs.md new file mode 100644 index 0000000000..7ecb70316d --- /dev/null +++ b/docs/custom_virtualenvs.md @@ -0,0 +1,71 @@ +Managing Custom Python Dependencies +=================================== +awx installations pre-build a special [Python +virtualenv](https://pypi.python.org/pypi/virtualenv) which is automatically +activated for all `ansible-playbook` runs invoked by awx (for example, any time +a Job Template is launched). By default, this virtualenv is located at +`/var/lib/awx/venv/ansible` on the file system. + +awx pre-installs a variety of third-party library/SDK support into this +virtualenv for its integration points with a variety of cloud providers (such +as EC2, OpenStack, Azure, etc...) + +Periodically, awx users want to add additional SDK support into this +virtualenv; this documentation describes the supported way to do so. + +Preparing a New Custom Virtualenv +================================= +awx allows a _different_ virtualenv to be specified and used on Job Template +runs. To choose a custom virtualenv, first create one in `/var/lib/awx/venv`: + + $ sudo virtualenv /var/lib/awx/venv/my-custom-venv + +Your newly created virtualenv needs a few base dependencies to properly run +playbooks (awx uses memcached as a holding space for playbook artifacts and +fact gathering): + + $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install python-memcached psutil + +From here, you can install _additional_ Python dependencies that you care +about, such as a per-virtualenv version of ansible itself: + + $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install -U "ansible == X.Y.Z" + +...or an additional third-party SDK that's not included with the base awx installation: + + $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install -U python-digitalocean + +If you want to copy them, the libraries included in awx's default virtualenv +can be found using `pip freeze`: + + $ sudo /var/lib/awx/venv/ansible/bin/pip freeze + +One important item to keep in mind is that in a clustered awx installation, +you need to ensure that the same custom virtualenv exists on _every_ local file +system at `/var/lib/awx/venv/`. For container-based deployments, this likely +means building these steps into your own custom image building workflow. + +Assigning Custom Virtualenvs +============================ +Once you've created a custom virtualenv, you can assign it at the Organization, +Project, or Job Template level: + + PATCH https://awx-host.example.org/api/v2/organizations/N/ + PATCH https://awx-host.example.org/api/v2/projects/N/ + PATCH https://awx-host.example.org/api/v2/job_templates/N/ + + Content-Type: application/json + { + 'custom_virtualenv': '/var/lib/awx/venv/my-custom-venv' + } + +An HTTP `GET` request to `/api/v2/config/` will provide a list of +detected installed virtualenvs: + + { + "custom_virtualenvs": [ + "/var/lib/awx/venv/my-custom-venv", + "/var/lib/awx/venv/my-other-custom-venv", + ], + ... + } diff --git a/requirements/requirements_dev.txt b/requirements/requirements_dev.txt index 328a4eecd9..eb0b6a8bd5 100644 --- a/requirements/requirements_dev.txt +++ b/requirements/requirements_dev.txt @@ -15,3 +15,4 @@ flower uwsgitop jupyter matplotlib +backports.tempfile # support in unit tests for py32+ tempfile.TemporaryDirectory