Merge pull request #10378 from jbradberry/control-plane-ee

Control plane EE

SUMMARY
Per discussion, we are now going to have privileged execution environments for use on the control plane, and these will not be allowed to be edited or deleted.
related ansible/tower#5016
TODO

 reinstate the restrictive RBAC for managed_by_tower=True EEs
 convert any EEs automatically installed prior into managed_by_tower=False EEs
 change the resolver so that ordinary jobs do not get a managed EE, and project updates only get a managed EE
 allow sysadmin users to edit the pull field for managed EEs
 automatically disassociate EEs that get deleted from the DEFAULT_EXECUTION_ENVIRONMENT setting #10363

ISSUE TYPE

Bugfix Pull Request

COMPONENT NAME

API

Reviewed-by: Alan Rominger <arominge@redhat.com>
Reviewed-by: Jeff Bradberry <None>
Reviewed-by: Shane McDonald <me@shanemcd.com>
Reviewed-by: Seth Foster <None>
Reviewed-by: Bianca Henderson <beeankha@gmail.com>

Makefile

@@ -315,7 +315,7 @@ test_collection:
 	if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi && \
-	pip install ansible && \
+	pip install ansible-core && \
 	py.test $(COLLECTION_TEST_DIRS) -v
 	# The python path needs to be modified so that the tests can find Ansible within the container
 	# First we will use anything expility set as PYTHONPATH

awx/api/metadata.py

@@ -24,7 +24,7 @@ from rest_framework.request import clone_request
 from awx.api.fields import ChoiceNullField
 from awx.main.fields import JSONField, ImplicitRoleField
 from awx.main.models import NotificationTemplate
-from awx.main.tasks import AWXReceptorJob
+from awx.main.utils.execution_environments import get_default_pod_spec
 
 # Polymorphic
 from polymorphic.models import PolymorphicModel
@@ -211,7 +211,7 @@ class Metadata(metadata.SimpleMetadata):
                     continue
 
                 if field == "pod_spec_override":
-                    meta['default'] = AWXReceptorJob().pod_definition
+                    meta['default'] = get_default_pod_spec()
 
                 # Add type choices if available from the serializer.
                 if field == 'type' and hasattr(serializer, 'get_type_choices'):

awx/api/views/__init__.py

@@ -695,6 +695,7 @@ class TeamAccessList(ResourceAccessList):
 
 class ExecutionEnvironmentList(ListCreateAPIView):
 
+    always_allow_superuser = False
     model = models.ExecutionEnvironment
     serializer_class = serializers.ExecutionEnvironmentSerializer
     swagger_topic = "Execution Environments"
@@ -702,10 +703,22 @@ class ExecutionEnvironmentList(ListCreateAPIView):
 
 class ExecutionEnvironmentDetail(RetrieveUpdateDestroyAPIView):
 
+    always_allow_superuser = False
     model = models.ExecutionEnvironment
     serializer_class = serializers.ExecutionEnvironmentSerializer
     swagger_topic = "Execution Environments"
 
+    def update(self, request, *args, **kwargs):
+        instance = self.get_object()
+        fields_to_check = ['name', 'description', 'organization', 'image', 'credential']
+        if instance.managed_by_tower and request.user.can_access(models.ExecutionEnvironment, 'change', instance):
+            for field in fields_to_check:
+                left = getattr(instance, field, None)
+                right = request.data.get(field, None)
+                if left != right:
+                    raise PermissionDenied(_("Only the 'pull' field can be edited for managed execution environments."))
+        return super().update(request, *args, **kwargs)
+
 
 class ExecutionEnvironmentJobTemplateList(SubListAPIView):
 

awx/conf/settings.py

@@ -23,6 +23,7 @@ import cachetools
 # AWX
 from awx.main.utils import encrypt_field, decrypt_field
 from awx.conf import settings_registry
+from awx.conf.fields import PrimaryKeyRelatedField
 from awx.conf.models import Setting
 from awx.conf.migrations._reencrypt import decrypt_field as old_decrypt_field
 
@@ -420,9 +421,9 @@ class SettingsWrapper(UserSettingsHolder):
             raise ImproperlyConfigured('Setting "{}" is read only.'.format(name))
 
         try:
-            data = field.to_representation(value)
+            data = None if value is None and isinstance(field, PrimaryKeyRelatedField) else field.to_representation(value)
             setting_value = field.run_validation(data)
-            db_value = field.to_representation(setting_value)
+            db_value = None if setting_value is None and isinstance(field, PrimaryKeyRelatedField) else field.to_representation(setting_value)
         except Exception as e:
             logger.exception('Unable to assign value "%r" to setting "%s".', value, name, exc_info=True)
             raise e

awx/main/access.py

@@ -465,7 +465,7 @@ class BaseAccess(object):
             if display_method == 'schedule':
                 user_capabilities['schedule'] = user_capabilities['start']
                 continue
-            elif display_method == 'delete' and not isinstance(obj, (User, UnifiedJob, CredentialInputSource)):
+            elif display_method == 'delete' and not isinstance(obj, (User, UnifiedJob, CredentialInputSource, ExecutionEnvironment)):
                 user_capabilities['delete'] = user_capabilities['edit']
                 continue
             elif display_method == 'copy' and isinstance(obj, (Group, Host)):
@@ -1370,6 +1370,8 @@ class ExecutionEnvironmentAccess(BaseAccess):
         return self.check_related('organization', Organization, data, obj=obj, mandatory=True, role_field='execution_environment_admin_role')
 
     def can_delete(self, obj):
+        if obj.managed_by_tower:
+            raise PermissionDenied
         return self.can_change(obj, None)
 
 

awx/main/migrations/0145_deregister_managed_ee_objs.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.2.16 on 2021-06-07 19:36
+
+from django.db import migrations
+
+
+def forwards(apps, schema_editor):
+    ExecutionEnvironment = apps.get_model('main', 'ExecutionEnvironment')
+    for row in ExecutionEnvironment.objects.filter(managed_by_tower=True):
+        row.managed_by_tower = False
+        row.save(update_fields=['managed_by_tower'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0144_event_partitions'),
+    ]
+
+    operations = [
+        migrations.RunPython(forwards),
+    ]

awx/main/models/projects.py

@@ -32,7 +32,7 @@ from awx.main.models.jobs import Job
 from awx.main.models.mixins import ResourceMixin, TaskManagerProjectUpdateMixin, CustomVirtualEnvMixin, RelatedJobsMixin
 from awx.main.utils import update_scm_url, polymorphic
 from awx.main.utils.ansible import skip_directory, could_be_inventory, could_be_playbook
-from awx.main.utils.execution_environments import get_default_execution_environment
+from awx.main.utils.execution_environments import get_control_plane_execution_environment
 from awx.main.fields import ImplicitRoleField
 from awx.main.models.rbac import (
     ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
@@ -185,11 +185,11 @@ class ProjectOptions(models.Model):
 
     def resolve_execution_environment(self):
         """
-        Project updates, themselves, will use the default execution environment.
+        Project updates, themselves, will use the control plane execution environment.
         Jobs using the project can use the default_environment, but the project updates
         are not flexible enough to allow customizing the image they use.
         """
-        return get_default_execution_environment()
+        return get_control_plane_execution_environment()
 
     def get_project_path(self, check_if_exists=True):
         local_path = os.path.basename(self.local_path)

awx/main/signals.py

@@ -31,6 +31,7 @@ from crum.signals import current_user_getter
 # AWX
 from awx.main.models import (
     ActivityStream,
+    ExecutionEnvironment,
     Group,
     Host,
     InstanceGroup,
@@ -623,6 +624,12 @@ def deny_orphaned_approvals(sender, instance, **kwargs):
         approval.deny()
 
 
+@receiver(pre_delete, sender=ExecutionEnvironment)
+def remove_default_ee(sender, instance, **kwargs):
+    if instance.id == getattr(settings.DEFAULT_EXECUTION_ENVIRONMENT, 'id', None):
+        settings.DEFAULT_EXECUTION_ENVIRONMENT = None
+
+
 @receiver(post_save, sender=Session)
 def save_user_session_membership(sender, **kwargs):
     session = kwargs.get('instance', None)

awx/main/tasks.py

@@ -98,7 +98,7 @@ from awx.main.utils import (
     parse_yaml_or_json,
     cleanup_new_process,
 )
-from awx.main.utils.execution_environments import get_default_execution_environment, get_default_pod_spec, CONTAINER_ROOT, to_container_path
+from awx.main.utils.execution_environments import get_default_pod_spec, CONTAINER_ROOT, to_container_path
 from awx.main.utils.ansible import read_ansible_config
 from awx.main.utils.external_logging import reconfigure_rsyslog
 from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja
@@ -2879,7 +2879,7 @@ class TransmitterThread(threading.Thread):
 
 
 class AWXReceptorJob:
-    def __init__(self, task=None, runner_params=None):
+    def __init__(self, task, runner_params=None):
         self.task = task
         self.runner_params = runner_params
         self.unit_id = None
@@ -3034,10 +3034,7 @@ class AWXReceptorJob:
 
     @property
     def pod_definition(self):
-        if self.task and self.task.instance.execution_environment:
+        ee = self.task.instance.execution_environment
-            ee = self.task.instance.execution_environment
-        else:
-            ee = get_default_execution_environment()
 
         default_pod_spec = get_default_pod_spec()
 

awx/main/tests/functional/api/test_instance_group.py

@@ -105,7 +105,9 @@ def test_delete_instance_group_jobs_running(delete, instance_group_jobs_running,
 
 
 @pytest.mark.django_db
-def test_delete_rename_tower_instance_group_prevented(delete, options, tower_instance_group, instance_group, user, patch, execution_environment):
+def test_delete_rename_tower_instance_group_prevented(
+    delete, options, tower_instance_group, instance_group, user, patch, control_plane_execution_environment, default_job_execution_environment
+):
     url = reverse("api:instance_group_detail", kwargs={'pk': tower_instance_group.pk})
     super_user = user('bob', True)
 

awx/main/tests/functional/conftest.py

@@ -824,5 +824,10 @@ def slice_job_factory(slice_jt_factory):
 
 
 @pytest.fixture
-def execution_environment():
+def control_plane_execution_environment():
-    return ExecutionEnvironment.objects.create(name="test-ee", description="test-ee", managed_by_tower=True)
+    return ExecutionEnvironment.objects.create(name="Control Plane EE", managed_by_tower=True)
+
+
+@pytest.fixture
+def default_job_execution_environment():
+    return ExecutionEnvironment.objects.create(name="Default Job EE", managed_by_tower=False)

awx/main/tests/functional/task_management/test_container_groups.py

@@ -35,7 +35,8 @@ def test_containerized_job(containerized_job):
 
 
 @pytest.mark.django_db
-def test_kubectl_ssl_verification(containerized_job, execution_environment):
+def test_kubectl_ssl_verification(containerized_job, default_job_execution_environment):
+    containerized_job.execution_environment = default_job_execution_environment
     cred = containerized_job.instance_group.credential
     cred.inputs['verify_ssl'] = True
     key_material = subprocess.run('openssl genrsa 2> /dev/null', shell=True, check=True, stdout=subprocess.PIPE)

awx/main/tests/functional/test_execution_environments.py

@@ -1,14 +0,0 @@
-import pytest
-
-from awx.main.models import ExecutionEnvironment
-
-
-@pytest.mark.django_db
-def test_execution_environment_creation(execution_environment, organization):
-    execution_env = ExecutionEnvironment.objects.create(
-        name='Hello Environment', image='', organization=organization, managed_by_tower=False, credential=None, pull='missing'
-    )
-    assert type(execution_env) is type(execution_environment)
-    assert execution_env.organization == organization
-    assert execution_env.name == 'Hello Environment'
-    assert execution_env.pull == 'missing'

awx/main/tests/functional/test_inventory_source_injectors.py

@@ -182,7 +182,8 @@ def create_reference_data(source_dir, env, content):
 @pytest.mark.django_db
 @pytest.mark.parametrize('this_kind', CLOUD_PROVIDERS)
 def test_inventory_update_injected_content(this_kind, inventory, fake_credential_factory):
-    ExecutionEnvironment.objects.create(name='test EE', managed_by_tower=True)
+    ExecutionEnvironment.objects.create(name='Control Plane EE', managed_by_tower=True)
+    ExecutionEnvironment.objects.create(name='Default Job EE', managed_by_tower=False)
 
     injector = InventorySource.injectors[this_kind]
     if injector.plugin_name is None:

awx/main/tests/unit/test_tasks.py

@@ -588,7 +588,8 @@ class TestGenericRun:
 @pytest.mark.django_db
 class TestAdhocRun(TestJobExecution):
     def test_options_jinja_usage(self, adhoc_job, adhoc_update_model_wrapper):
-        ExecutionEnvironment.objects.create(name='test EE', managed_by_tower=True)
+        ExecutionEnvironment.objects.create(name='Control Plane EE', managed_by_tower=True)
+        ExecutionEnvironment.objects.create(name='Default Job EE', managed_by_tower=False)
 
         adhoc_job.module_args = '{{ ansible_ssh_pass }}'
         adhoc_job.websocket_emit_status = mock.Mock()

awx/main/utils/execution_environments.py

@@ -6,13 +6,20 @@ from django.conf import settings
 from awx.main.models.execution_environments import ExecutionEnvironment
 
 
-def get_default_execution_environment():
+def get_control_plane_execution_environment():
-    if settings.DEFAULT_EXECUTION_ENVIRONMENT is not None:
-        return settings.DEFAULT_EXECUTION_ENVIRONMENT
     return ExecutionEnvironment.objects.filter(organization=None, managed_by_tower=True).first()
 
 
+def get_default_execution_environment():
+    if settings.DEFAULT_EXECUTION_ENVIRONMENT is not None:
+        return settings.DEFAULT_EXECUTION_ENVIRONMENT
+    return ExecutionEnvironment.objects.filter(organization=None, managed_by_tower=False).first()
+
+
 def get_default_pod_spec():
+    ee = get_default_execution_environment()
+    if ee is None:
+        raise RuntimeError("Unable to find an execution environment.")
 
     return {
         "apiVersion": "v1",
@@ -21,7 +28,7 @@ def get_default_pod_spec():
         "spec": {
             "containers": [
                 {
-                    "image": get_default_execution_environment().image,
+                    "image": ee.image,
                     "name": 'worker',
                     "args": ['ansible-runner', 'worker', '--private-data-dir=/runner'],
                 }

awx_collection/test/awx/conftest.py

@@ -265,7 +265,7 @@ def silence_warning():
 
 @pytest.fixture
 def execution_environment():
-    return ExecutionEnvironment.objects.create(name="test-ee", description="test-ee", managed_by_tower=True)
+    return ExecutionEnvironment.objects.create(name="test-ee", description="test-ee", managed_by_tower=False)
 
 
 @pytest.fixture(scope='session', autouse=True)