Merge pull request #9741 from shanemcd/fix-project-inventories

Fix inventories-from-projects when running in Kubernetes

Related: #9704
Will also require a new release of the operator which will contain ansible/awx-operator#155

Reviewed-by: Elijah DeLee <kdelee@redhat.com>
Reviewed-by: Chris Meyers <None>
Reviewed-by: Bianca Henderson <beeankha@gmail.com>

awx/main/management/commands/create_preload_data.py

@@ -68,12 +68,12 @@ class Command(BaseCommand):
                     print('Demo Credential, Inventory, and Job Template added.')
                     changed = True
 
-                default_ee = settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE
-                ee, created = ExecutionEnvironment.objects.get_or_create(name='Default EE', defaults={'image': default_ee, 'managed_by_tower': True})
+                for ee in reversed(settings.DEFAULT_EXECUTION_ENVIRONMENTS):
+                    _, created = ExecutionEnvironment.objects.get_or_create(name=ee['name'], defaults={'image': ee['image'], 'managed_by_tower': True})
 
                 if created:
                     changed = True
-                    print('Default Execution Environment registered.')
+                    print('Default Execution Environment(s) registered.')
 
         if changed:
             print('(changed: True)')

awx/main/management/commands/inventory_import.py

@@ -29,6 +29,7 @@ from awx.main.utils.safe_yaml import sanitize_jinja
 # other AWX imports
 from awx.main.models.rbac import batch_role_ancestor_rebuilding
 from awx.main.utils import ignore_inventory_computed_fields, get_licenser
+from awx.main.utils.execution_environments import get_execution_environment_default
 from awx.main.signals import disable_activity_stream
 from awx.main.constants import STANDARD_INVENTORY_UPDATE_ENV
 from awx.main.utils.pglock import advisory_lock
@@ -90,7 +91,7 @@ class AnsibleInventoryLoader(object):
         bargs.extend(['-v', '{0}:{0}:Z'.format(self.source)])
         for key, value in STANDARD_INVENTORY_UPDATE_ENV.items():
             bargs.extend(['-e', '{0}={1}'.format(key, value)])
-        bargs.extend([settings.AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE])
+        bargs.extend([get_execution_environment_default().image])
         bargs.extend(['ansible-inventory', '-i', self.source])
         bargs.extend(['--playbook-dir', functioning_dir(self.source)])
         if self.verbosity:

awx/main/models/inventory.py

@@ -1227,6 +1227,10 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin,
         null=True,
     )
 
+    @property
+    def is_container_group_task(self):
+        return bool(self.instance_group and self.instance_group.is_container_group)
+
     def _get_parent_field_name(self):
         return 'inventory_source'
 

awx/main/models/mixins.py

@@ -21,6 +21,7 @@ from django.utils.translation import ugettext_lazy as _
 from awx.main.models.base import prevent_search
 from awx.main.models.rbac import Role, RoleAncestorEntry, get_roles_on_resource
 from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser, polymorphic
+from awx.main.utils.execution_environments import get_execution_environment_default
 from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted
 from awx.main.utils.polymorphic import build_polymorphic_ctypes_map
 from awx.main.fields import JSONField, AskForField
@@ -461,13 +462,6 @@ class ExecutionEnvironmentMixin(models.Model):
         help_text=_('The container image to be used for execution.'),
     )
 
-    def get_execution_environment_default(self):
-        from awx.main.models.execution_environments import ExecutionEnvironment
-
-        if settings.DEFAULT_EXECUTION_ENVIRONMENT is not None:
-            return settings.DEFAULT_EXECUTION_ENVIRONMENT
-        return ExecutionEnvironment.objects.filter(organization=None, managed_by_tower=True).first()
-
     def resolve_execution_environment(self):
         """
         Return the execution environment that should be used when creating a new job.
@@ -482,7 +476,7 @@ class ExecutionEnvironmentMixin(models.Model):
             if self.inventory.organization.default_environment is not None:
                 return self.inventory.organization.default_environment
 
-        return self.get_execution_environment_default()
+        return get_execution_environment_default()
 
 
 class CustomVirtualEnvMixin(models.Model):

awx/main/tasks.py

@@ -97,6 +97,7 @@ from awx.main.utils import (
     deepmerge,
     parse_yaml_or_json,
 )
+from awx.main.utils.execution_environments import get_execution_environment_default
 from awx.main.utils.ansible import read_ansible_config
 from awx.main.utils.external_logging import reconfigure_rsyslog
 from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja
@@ -2505,7 +2506,7 @@ class RunInventoryUpdate(BaseTask):
         args.append(container_location)
 
         args.append('--output')
-        args.append(os.path.join('/runner', 'artifacts', 'output.json'))
+        args.append(os.path.join('/runner', 'artifacts', str(inventory_update.id), 'output.json'))
 
         if os.path.isdir(source_location):
             playbook_dir = container_location
@@ -3010,7 +3011,7 @@ class AWXReceptorJob:
             return self._run_internal(receptor_ctl)
         finally:
             # Make sure to always release the work unit if we established it
-            if self.unit_id is not None:
+            if self.unit_id is not None and not settings.AWX_CONTAINER_GROUP_KEEP_POD:
                 receptor_ctl.simple_command(f"work release {self.unit_id}")
 
     def _run_internal(self, receptor_ctl):
@@ -3126,11 +3127,23 @@ class AWXReceptorJob:
 
     @property
     def pod_definition(self):
+        if self.task:
+            ee = self.task.instance.resolve_execution_environment()
+        else:
+            ee = get_execution_environment_default()
+
         default_pod_spec = {
             "apiVersion": "v1",
             "kind": "Pod",
             "metadata": {"namespace": settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE},
-            "spec": {"containers": [{"image": settings.AWX_CONTAINER_GROUP_DEFAULT_IMAGE, "name": 'worker', "args": ['ansible-runner', 'worker']}]},
+            "spec": {
+                "containers": [
+                    {
+                        "image": ee.image,
+                        "name": 'worker',
+                    }
+                ],
+            },
         }
 
         pod_spec_override = {}

awx/main/tests/functional/api/test_instance_group.py

@@ -140,7 +140,7 @@ def test_delete_instance_group_jobs_running(delete, instance_group_jobs_running,
 
 
 @pytest.mark.django_db
-def test_delete_rename_tower_instance_group_prevented(delete, options, tower_instance_group, instance_group, user, patch):
+def test_delete_rename_tower_instance_group_prevented(delete, options, tower_instance_group, instance_group, user, patch, execution_environment):
     url = reverse("api:instance_group_detail", kwargs={'pk': tower_instance_group.pk})
     super_user = user('bob', True)
 

awx/main/tests/functional/conftest.py

@@ -829,5 +829,5 @@ def slice_job_factory(slice_jt_factory):
 
 
 @pytest.fixture
-def execution_environment(organization):
-    return ExecutionEnvironment.objects.create(name="test-ee", description="test-ee", organization=organization)
+def execution_environment():
+    return ExecutionEnvironment.objects.create(name="test-ee", description="test-ee", managed_by_tower=True)

awx/main/tests/functional/task_management/test_container_groups.py

@@ -1,10 +1,11 @@
 import subprocess
 import base64
+from collections import namedtuple
 
 from unittest import mock  # noqa
 import pytest
 
-from awx.main.scheduler.kubernetes import PodManager
+from awx.main.tasks import AWXReceptorJob
 from awx.main.utils import (
     create_temporary_fifo,
 )
@@ -34,7 +35,7 @@ def test_containerized_job(containerized_job):
 
 
 @pytest.mark.django_db
-def test_kubectl_ssl_verification(containerized_job):
+def test_kubectl_ssl_verification(containerized_job, execution_environment):
     cred = containerized_job.instance_group.credential
     cred.inputs['verify_ssl'] = True
     key_material = subprocess.run('openssl genrsa 2> /dev/null', shell=True, check=True, stdout=subprocess.PIPE)
@@ -46,6 +47,8 @@ def test_kubectl_ssl_verification(containerized_job):
     cert = subprocess.run(cmd.strip(), shell=True, check=True, stdout=subprocess.PIPE)
     cred.inputs['ssl_ca_cert'] = cert.stdout
     cred.save()
-    pm = PodManager(containerized_job)
-    ca_data = pm.kube_config['clusters'][0]['cluster']['certificate-authority-data']
+    RunJob = namedtuple('RunJob', ['instance', 'build_execution_environment_params'])
+    rj = RunJob(instance=containerized_job, build_execution_environment_params=lambda x: {})
+    receptor_job = AWXReceptorJob(rj, runner_params={'settings': {}})
+    ca_data = receptor_job.kube_config['clusters'][0]['cluster']['certificate-authority-data']
     assert cert.stdout == base64.b64decode(ca_data.encode())

awx/main/tests/unit/scheduler/test_kubernetes.py

@@ -1,49 +0,0 @@
-import pytest
-from django.conf import settings
-
-from awx.main.models import (
-    InstanceGroup,
-    Job,
-    JobTemplate,
-    Project,
-    Inventory,
-)
-from awx.main.scheduler.kubernetes import PodManager
-
-
-@pytest.fixture
-def container_group():
-    instance_group = InstanceGroup(name='container-group', id=1)
-
-    return instance_group
-
-
-@pytest.fixture
-def job(container_group):
-    return Job(pk=1, id=1, project=Project(), instance_group=container_group, inventory=Inventory(), job_template=JobTemplate(id=1, name='foo'))
-
-
-def test_default_pod_spec(job):
-    default_image = PodManager(job).pod_definition['spec']['containers'][0]['image']
-    assert default_image == settings.AWX_CONTAINER_GROUP_DEFAULT_IMAGE
-
-
-def test_custom_pod_spec(job):
-    job.instance_group.pod_spec_override = """
-    spec:
-      containers:
-        - image: my-custom-image
-    """
-    custom_image = PodManager(job).pod_definition['spec']['containers'][0]['image']
-    assert custom_image == 'my-custom-image'
-
-
-def test_pod_manager_namespace_property(job):
-    pm = PodManager(job)
-    assert pm.namespace == settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE
-
-    job.instance_group.pod_spec_override = """
-    metadata:
-      namespace: my-namespace
-    """
-    assert PodManager(job).namespace == 'my-namespace'

awx/main/utils/execution_environments.py

@@ -0,0 +1,9 @@
+from django.conf import settings
+
+from awx.main.models.execution_environments import ExecutionEnvironment
+
+
+def get_execution_environment_default():
+    if settings.DEFAULT_EXECUTION_ENVIRONMENT is not None:
+        return settings.DEFAULT_EXECUTION_ENVIRONMENT
+    return ExecutionEnvironment.objects.filter(organization=None, managed_by_tower=True).first()

awx/settings/defaults.py

@@ -68,17 +68,12 @@ DATABASES = {
 # the K8S cluster where awx itself is running)
 IS_K8S = False
 
-# TODO: remove this setting in favor of a default execution environment
-AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE = 'quay.io/ansible/awx-ee'
-
+AWX_CONTAINER_GROUP_KEEP_POD = False
 AWX_CONTAINER_GROUP_K8S_API_TIMEOUT = 10
 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRIES = 100
 AWX_CONTAINER_GROUP_POD_LAUNCH_RETRY_DELAY = 5
 AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE = os.getenv('MY_POD_NAMESPACE', 'default')
 
-# TODO: remove this setting in favor of a default execution environment
-AWX_CONTAINER_GROUP_DEFAULT_IMAGE = AWX_EXECUTION_ENVIRONMENT_DEFAULT_IMAGE
-
 # Internationalization
 # https://docs.djangoproject.com/en/dev/topics/i18n/
 #
@@ -182,8 +177,15 @@ REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST']
 PROXY_IP_ALLOWED_LIST = []
 
 CUSTOM_VENV_PATHS = []
+
+# Warning: this is a placeholder for a configure tower-in-tower setting
+# This should not be set via a file.
 DEFAULT_EXECUTION_ENVIRONMENT = None
 
+# This list is used for creating default EEs when running awx-manage create_preload_data.
+# Should be ordered from highest to lowest precedence.
+DEFAULT_EXECUTION_ENVIRONMENTS = [{'name': 'AWX EE 0.1.1', 'image': 'quay.io/ansible/awx-ee:0.1.1'}]
+
 # Note: This setting may be overridden by database settings.
 STDOUT_MAX_BYTES_DISPLAY = 1048576
 

awx_collection/test/awx/conftest.py

@@ -16,7 +16,7 @@ from requests.models import Response, PreparedRequest
 import pytest
 
 from awx.main.tests.functional.conftest import _request
-from awx.main.models import Organization, Project, Inventory, JobTemplate, Credential, CredentialType
+from awx.main.models import Organization, Project, Inventory, JobTemplate, Credential, CredentialType, ExecutionEnvironment
 
 from django.db import transaction
 
@@ -261,3 +261,8 @@ def silence_warning():
     """Warnings use global variable, same as deprecations."""
     with mock.patch('ansible.module_utils.basic.AnsibleModule.warn') as this_mock:
         yield this_mock
+
+
+@pytest.fixture
+def execution_environment():
+    return ExecutionEnvironment.objects.create(name="test-ee", description="test-ee", managed_by_tower=True)

awx_collection/test/awx/test_completeness.py

@@ -157,7 +157,7 @@ def determine_state(module_id, endpoint, module, parameter, api_option, module_o
     return 'OK'
 
 
-def test_completeness(collection_import, request, admin_user, job_template):
+def test_completeness(collection_import, request, admin_user, job_template, execution_environment):
     option_comparison = {}
     # Load a list of existing module files from disk
     base_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))