Merge pull request #4189 from shanemcd/toc

Container Groups

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

.gitignore

@@ -133,9 +133,12 @@ awx/lib/site-packages
 venv/*
 use_dev_supervisor.txt
 
-.idea/*
 
 # Ansible module tests
 awx_collection_test_venv/
 awx_collection/*.tar.gz
 awx_collection/galaxy.yml
+
+.idea/*
+*.unison.tmp
+*.#

Makefile

@@ -100,20 +100,22 @@ clean-languages:
 	find . -type f -regex ".*\.mo$$" -delete
 
 # Remove temporary build files, compiled Python files.
-clean: clean-ui clean-dist
+clean: clean-ui clean-api clean-dist
 	rm -rf awx/public
 	rm -rf awx/lib/site-packages
 	rm -rf awx/job_status
 	rm -rf awx/job_output
 	rm -rf reports
-	rm -f awx/awx_test.sqlite3*
-	rm -rf requirements/vendor
 	rm -rf tmp
 	rm -rf $(I18N_FLAG_FILE)
 	mkdir tmp
+
+clean-api:
 	rm -rf build $(NAME)-$(VERSION) *.egg-info
 	find . -type f -regex ".*\.py[co]$$" -delete
 	find . -type d -name "__pycache__" -delete
+	rm -f awx/awx_test.sqlite3*
+	rm -rf requirements/vendor
 
 # convenience target to assert environment variables are defined
 guard-%:

awx/api/metadata.py

@@ -22,6 +22,7 @@ from rest_framework.request import clone_request
 # AWX
 from awx.main.fields import JSONField, ImplicitRoleField
 from awx.main.models import InventorySource, NotificationTemplate
+from awx.main.scheduler.kubernetes import PodManager
 
 
 class Metadata(metadata.SimpleMetadata):
@@ -200,6 +201,9 @@ class Metadata(metadata.SimpleMetadata):
                 if not isinstance(meta, dict):
                     continue
 
+                if field == "pod_spec_override":
+                    meta['default'] = PodManager().pod_definition
+
                 # Add type choices if available from the serializer.
                 if field == 'type' and hasattr(serializer, 'get_type_choices'):
                     meta['choices'] = serializer.get_type_choices()

awx/api/serializers.py

@@ -116,7 +116,7 @@ SUMMARIZABLE_FK_FIELDS = {
     'project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'),
     'source_project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'),
     'project_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed',),
-    'credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'),
+    'credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'kubernetes', 'credential_type_id'),
     'job': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'elapsed', 'type'),
     'job_template': DEFAULT_SUMMARY_FIELDS,
     'workflow_job_template': DEFAULT_SUMMARY_FIELDS,
@@ -135,7 +135,7 @@ SUMMARIZABLE_FK_FIELDS = {
     'source_script': ('name', 'description'),
     'role': ('id', 'role_field'),
     'notification_template': DEFAULT_SUMMARY_FIELDS,
-    'instance_group': {'id', 'name', 'controller_id'},
+    'instance_group': ('id', 'name', 'controller_id', 'is_containerized'),
     'insights_credential': DEFAULT_SUMMARY_FIELDS,
     'source_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'),
     'target_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'),
@@ -2515,7 +2515,7 @@ class CredentialSerializer(BaseSerializer):
 
     class Meta:
         model = Credential
-        fields = ('*', 'organization', 'credential_type', 'inputs', 'kind', 'cloud')
+        fields = ('*', 'organization', 'credential_type', 'inputs', 'kind', 'cloud', 'kubernetes')
         extra_kwargs = {
             'credential_type': {
                 'label': _('Credential Type'),
@@ -4730,6 +4730,11 @@ class InstanceGroupSerializer(BaseSerializer):
                     'Isolated groups have a designated controller group.'),
         read_only=True
     )
+    is_containerized = serializers.BooleanField(
+        help_text=_('Indicates whether instances in this group are containerized.'
+                    'Containerized groups have a designated Openshift or Kubernetes cluster.'),
+        read_only=True
+    )
     # NOTE: help_text is duplicated from field definitions, no obvious way of
     # both defining field details here and also getting the field's help_text
     policy_instance_percentage = serializers.IntegerField(
@@ -4755,8 +4760,9 @@ class InstanceGroupSerializer(BaseSerializer):
         fields = ("id", "type", "url", "related", "name", "created", "modified",
                   "capacity", "committed_capacity", "consumed_capacity",
                   "percent_capacity_remaining", "jobs_running", "jobs_total",
-                  "instances", "controller", "is_controller", "is_isolated",
+                  "instances", "controller", "is_controller", "is_isolated", "is_containerized", "credential",
-                  "policy_instance_percentage", "policy_instance_minimum", "policy_instance_list")
+                  "policy_instance_percentage", "policy_instance_minimum", "policy_instance_list",
+                  "pod_spec_override", "summary_fields")
 
     def get_related(self, obj):
         res = super(InstanceGroupSerializer, self).get_related(obj)
@@ -4764,6 +4770,9 @@ class InstanceGroupSerializer(BaseSerializer):
         res['instances'] = self.reverse('api:instance_group_instance_list', kwargs={'pk': obj.pk})
         if obj.controller_id:
             res['controller'] = self.reverse('api:instance_group_detail', kwargs={'pk': obj.controller_id})
+        if obj.credential:
+            res['credential'] = self.reverse('api:credential_detail', kwargs={'pk': obj.credential_id})
+
         return res
 
     def validate_policy_instance_list(self, value):
@@ -4783,6 +4792,11 @@ class InstanceGroupSerializer(BaseSerializer):
             raise serializers.ValidationError(_('tower instance group name may not be changed.'))
         return value
 
+    def validate_credential(self, value):
+        if value and not value.kubernetes:
+            raise serializers.ValidationError(_('Only Kubernetes credentials can be associated with an Instance Group'))
+        return value
+
     def get_capacity_dict(self):
         # Store capacity values (globally computed) in the context
         if 'capacity_map' not in self.context:

awx/main/isolated/manager.py

@@ -11,7 +11,9 @@ from django.conf import settings
 import ansible_runner
 
 import awx
-from awx.main.utils import get_system_task_capacity
+from awx.main.utils import (
+    get_system_task_capacity
+)
 from awx.main.queue import CallbackQueueDispatcher
 
 logger = logging.getLogger('awx.isolated.manager')
@@ -29,7 +31,7 @@ def set_pythonpath(venv_libdir, env):
 
 class IsolatedManager(object):
 
-    def __init__(self, cancelled_callback=None, check_callback=None):
+    def __init__(self, cancelled_callback=None, check_callback=None, pod_manager=None):
         """
         :param cancelled_callback:  a callable - which returns `True` or `False`
                                     - signifying if the job has been prematurely
@@ -40,6 +42,24 @@ class IsolatedManager(object):
         self.idle_timeout = max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT)
         self.started_at = None
         self.captured_command_artifact = False
+        self.instance = None
+        self.pod_manager = pod_manager
+
+    def build_inventory(self, hosts):
+        if self.instance and self.instance.is_containerized:
+            inventory = {'all': {'hosts': {}}}
+            for host in hosts:
+                inventory['all']['hosts'][host] = {
+                    "ansible_connection": "kubectl",
+                    "ansible_kubectl_config": self.pod_manager.kube_config
+                }
+        else:
+            inventory = '\n'.join([
+                '{} ansible_ssh_user={}'.format(host, settings.AWX_ISOLATED_USERNAME)
+                for host in hosts
+            ])
+
+        return inventory
 
     def build_runner_params(self, hosts, verbosity=1):
         env = dict(os.environ.items())
@@ -69,17 +89,12 @@ class IsolatedManager(object):
             else:
                 playbook_logger.info(runner_obj.stdout.read())
 
-        inventory = '\n'.join([
-            '{} ansible_ssh_user={}'.format(host, settings.AWX_ISOLATED_USERNAME)
-            for host in hosts
-        ])
-
         return {
             'project_dir': os.path.abspath(os.path.join(
                 os.path.dirname(awx.__file__),
                 'playbooks'
             )),
-            'inventory': inventory,
+            'inventory': self.build_inventory(hosts),
             'envvars': env,
             'finished_callback': finished_callback,
             'verbosity': verbosity,
@@ -153,6 +168,11 @@ class IsolatedManager(object):
         runner_obj = self.run_management_playbook('run_isolated.yml',
                                                   self.private_data_dir,
                                                   extravars=extravars)
+
+        if runner_obj.status == 'failed':
+            self.instance.result_traceback = runner_obj.stdout.read()
+            self.instance.save(update_fields=['result_traceback'])
+
         return runner_obj.status, runner_obj.rc
 
     def check(self, interval=None):
@@ -175,6 +195,7 @@ class IsolatedManager(object):
         rc = None
         last_check = time.time()
         dispatcher = CallbackQueueDispatcher()
+
         while status == 'failed':
             canceled = self.cancelled_callback() if self.cancelled_callback else False
             if not canceled and time.time() - last_check < interval:
@@ -279,7 +300,6 @@ class IsolatedManager(object):
 
 
     def cleanup(self):
-        # If the job failed for any reason, make a last-ditch effort at cleanup
         extravars = {
             'private_data_dir': self.private_data_dir,
             'cleanup_dirs': [
@@ -393,6 +413,7 @@ class IsolatedManager(object):
             [instance.execution_node],
             verbosity=min(5, self.instance.verbosity)
         )
+
         status, rc = self.dispatch(playbook, module, module_args)
         if status == 'successful':
             status, rc = self.check()

awx/main/managers.py

@@ -221,8 +221,9 @@ class InstanceGroupManager(models.Manager):
             elif t.status == 'running':
                 # Subtract capacity from all groups that contain the instance
                 if t.execution_node not in instance_ig_mapping:
-                    logger.warning('Detected %s running inside lost instance, '
+                    if not t.is_containerized:
-                                   'may still be waiting for reaper.', t.log_format)
+                        logger.warning('Detected %s running inside lost instance, '
+                                       'may still be waiting for reaper.', t.log_format)
                     if t.instance_group:
                         impacted_groups = [t.instance_group.name]
                     else:

awx/main/migrations/0096_v360_container_groups.py

@@ -0,0 +1,38 @@
+# Generated by Django 2.2.4 on 2019-09-16 23:50
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+from awx.main.models import CredentialType
+from awx.main.utils.common import set_current_apps
+
+
+def create_new_credential_types(apps, schema_editor):
+    set_current_apps(apps)
+    CredentialType.setup_tower_managed_defaults()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0095_v360_increase_instance_version_length'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='instancegroup',
+            name='credential',
+            field=models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='instancegroups', to='main.Credential'),
+        ),
+        migrations.AddField(
+            model_name='instancegroup',
+            name='pod_spec_override',
+            field=models.TextField(blank=True, default=''),
+        ),
+        migrations.AlterField(
+            model_name='credentialtype',
+            name='kind',
+            field=models.CharField(choices=[('ssh', 'Machine'), ('vault', 'Vault'), ('net', 'Network'), ('scm', 'Source Control'), ('cloud', 'Cloud'), ('token', 'Personal Access Token'), ('insights', 'Insights'), ('external', 'External'), ('kubernetes', 'Kubernetes')], max_length=32),
+        ),
+        migrations.RunPython(create_new_credential_types)
+    ]

awx/main/models/ad_hoc_commands.py

@@ -150,6 +150,14 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
     def supports_isolation(cls):
         return True
 
+    @property
+    def is_containerized(self):
+        return bool(self.instance_group and self.instance_group.is_containerized)
+
+    @property
+    def can_run_containerized(self):
+        return True
+
     def get_absolute_url(self, request=None):
         return reverse('api:ad_hoc_command_detail', kwargs={'pk': self.pk}, request=request)
 

awx/main/models/credential/__init__.py

@@ -135,6 +135,10 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
     def cloud(self):
         return self.credential_type.kind == 'cloud'
 
+    @property
+    def kubernetes(self):
+        return self.credential_type.kind == 'kubernetes'
+
     def get_absolute_url(self, request=None):
         return reverse('api:credential_detail', kwargs={'pk': self.pk}, request=request)
 
@@ -325,6 +329,7 @@ class CredentialType(CommonModelNameNotUnique):
         ('token', _('Personal Access Token')),
         ('insights', _('Insights')),
         ('external', _('External')),
+        ('kubernetes', _('Kubernetes')),
     )
 
     kind = models.CharField(
@@ -1118,6 +1123,38 @@ ManagedCredentialType(
 )
 
 
+ManagedCredentialType(
+    namespace='kubernetes_bearer_token',
+    kind='kubernetes',
+    name=ugettext_noop('OpenShift or Kubernetes API Bearer Token'),
+    inputs={
+        'fields': [{
+            'id': 'host',
+            'label': ugettext_noop('OpenShift or Kubernetes API Endpoint'),
+            'type': 'string',
+            'help_text': ugettext_noop('The OpenShift or Kubernetes API Endpoint to authenticate with.')
+        },{
+            'id': 'bearer_token',
+            'label': ugettext_noop('API authentication bearer token.'),
+            'type': 'string',
+            'secret': True,
+        },{
+            'id': 'verify_ssl',
+            'label': ugettext_noop('Verify SSL'),
+            'type': 'boolean',
+            'default': True,
+        },{
+            'id': 'ssl_ca_cert',
+            'label': ugettext_noop('Certificate Authority data'),
+            'type': 'string',
+            'secret': True,
+            'multiline': True,
+        }],
+        'required': ['host', 'bearer_token'],
+    }
+)
+
+
 class CredentialInputSource(PrimordialModel):
 
     class Meta:

awx/main/models/ha.py

@@ -18,7 +18,7 @@ from awx import __version__ as awx_application_version
 from awx.api.versioning import reverse
 from awx.main.managers import InstanceManager, InstanceGroupManager
 from awx.main.fields import JSONField
-from awx.main.models.base import BaseModel, HasEditsMixin
+from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search
 from awx.main.models.unified_jobs import UnifiedJob
 from awx.main.utils import get_cpu_capacity, get_mem_capacity, get_system_task_capacity
 from awx.main.models.mixins import RelatedJobsMixin
@@ -176,6 +176,18 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
         null=True,
         on_delete=models.CASCADE
     )
+    credential = models.ForeignKey(
+        'Credential',
+        related_name='%(class)ss',
+        blank=True,
+        null=True,
+        default=None,
+        on_delete=models.SET_NULL,
+    )
+    pod_spec_override = prevent_search(models.TextField(
+        blank=True,
+        default='',
+    ))
     policy_instance_percentage = models.IntegerField(
         default=0,
         help_text=_("Percentage of Instances to automatically assign to this group")
@@ -218,6 +230,10 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
     def is_isolated(self):
         return bool(self.controller)
 
+    @property
+    def is_containerized(self):
+        return bool(self.credential and self.credential.kubernetes)
+
     '''
     RelatedJobsMixin
     '''
@@ -271,7 +287,8 @@ def schedule_policy_task():
 @receiver(post_save, sender=InstanceGroup)
 def on_instance_group_saved(sender, instance, created=False, raw=False, **kwargs):
     if created or instance.has_policy_changes():
-        schedule_policy_task()
+        if not instance.is_containerized:
+            schedule_policy_task()
 
 
 @receiver(post_save, sender=Instance)
@@ -282,7 +299,8 @@ def on_instance_saved(sender, instance, created=False, raw=False, **kwargs):
 
 @receiver(post_delete, sender=InstanceGroup)
 def on_instance_group_deleted(sender, instance, using, **kwargs):
-    schedule_policy_task()
+    if not instance.is_containerized:
+        schedule_policy_task()
 
 
 @receiver(post_delete, sender=Instance)

awx/main/models/jobs.py

@@ -718,6 +718,14 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
             return "$hidden due to Ansible no_log flag$"
         return artifacts
 
+    @property
+    def can_run_containerized(self):
+        return any([ig for ig in self.preferred_instance_groups if ig.is_containerized])
+
+    @property
+    def is_containerized(self):
+        return bool(self.instance_group and self.instance_group.is_containerized)
+
     @property
     def preferred_instance_groups(self):
         if self.project is not None and self.project.organization is not None:

awx/main/models/unified_jobs.py

@@ -714,6 +714,10 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
     def supports_isolation(cls):
         return False
 
+    @property
+    def can_run_containerized(self):
+        return False
+
     def _get_parent_field_name(self):
         return 'unified_job_template' # Override in subclasses.
 
@@ -1425,3 +1429,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
 
     def is_isolated(self):
         return bool(self.controller_node)
+
+    @property
+    def is_containerized(self):
+        return False

awx/main/scheduler/__init__.py

@@ -1,4 +1,6 @@
 # Copyright (c) 2017 Ansible, Inc.
 #
 
-from awx.main.scheduler.task_manager import TaskManager # noqa
+from .task_manager import TaskManager
+
+__all__ = ['TaskManager']

awx/main/scheduler/kubernetes.py

@@ -0,0 +1,153 @@
+import os
+import stat
+import time
+import yaml
+import tempfile
+from base64 import b64encode
+
+from django.conf import settings
+from kubernetes import client, config
+from django.utils.functional import cached_property
+
+from awx.main.utils.common import parse_yaml_or_json
+
+
+class PodManager(object):
+
+    def __init__(self, task=None):
+        self.task = task
+
+    def deploy(self):
+        if not self.credential.kubernetes:
+            raise RuntimeError('Pod deployment cannot occur without a Kubernetes credential')
+
+
+        self.kube_api.create_namespaced_pod(body=self.pod_definition,
+                                            namespace=self.namespace,
+                                            _request_timeout=settings.AWX_CONTAINER_GROUP_DEFAULT_LAUNCH_TIMEOUT)
+
+        # We don't do any fancy timeout logic here because it is handled
+        # at a higher level in the job spawning process. See
+        # settings.AWX_ISOLATED_LAUNCH_TIMEOUT and settings.AWX_ISOLATED_CONNECTION_TIMEOUT
+        while True:
+            pod = self.kube_api.read_namespaced_pod(name=self.pod_name,
+                                                    namespace=self.namespace,
+                                                    _request_timeout=settings.AWX_CONTAINER_GROUP_DEFAULT_LAUNCH_TIMEOUT)
+            if pod.status.phase != 'Pending':
+                break
+            time.sleep(1)
+
+        if pod.status.phase == 'Running':
+            return pod
+        else:
+            raise RuntimeError(f"Unhandled Pod phase: {pod.status.phase}")
+
+
+    def delete(self):
+        return self.kube_api.delete_namespaced_pod(name=self.pod_name,
+                                                   namespace=self.namespace,
+                                                   _request_timeout=settings.AWX_CONTAINER_GROUP_DEFAULT_LAUNCH_TIMEOUT)
+
+    @property
+    def namespace(self):
+        return self.pod_definition['metadata']['namespace']
+
+    @property
+    def credential(self):
+        return self.task.instance_group.credential
+
+    @cached_property
+    def kube_config(self):
+        return generate_tmp_kube_config(self.credential, self.namespace)
+
+    @cached_property
+    def kube_api(self):
+        my_client = config.new_client_from_config(config_file=self.kube_config)
+        return client.CoreV1Api(api_client=my_client)
+
+    @property
+    def pod_name(self):
+        return f"job-{self.task.id}"
+
+    @property
+    def pod_definition(self):
+        default_pod_spec = {
+            "apiVersion": "v1",
+            "kind": "Pod",
+            "metadata": {
+                "namespace": settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE
+            },
+            "spec": {
+                "containers": [{
+                    "image": settings.AWX_CONTAINER_GROUP_DEFAULT_IMAGE,
+                    "tty": True,
+                    "stdin": True,
+                    "imagePullPolicy": "Always",
+                    "args": [
+                        'sleep', 'infinity'
+                    ]
+                }]
+            }
+        }
+
+        pod_spec_override = {}
+        if self.task and self.task.instance_group.pod_spec_override:
+            pod_spec_override = parse_yaml_or_json(
+                self.task.instance_group.pod_spec_override)
+        pod_spec = {**default_pod_spec, **pod_spec_override}
+
+        if self.task:
+            pod_spec['metadata']['name'] = self.pod_name
+            pod_spec['spec']['containers'][0]['name'] = self.pod_name
+
+        return pod_spec
+
+
+def generate_tmp_kube_config(credential, namespace):
+    host_input = credential.get_input('host')
+    config = {
+        "apiVersion": "v1",
+        "kind": "Config",
+        "preferences": {},
+        "clusters": [
+            {
+                "name": host_input,
+                "cluster": {
+                    "server": host_input
+                }
+            }
+        ],
+        "users": [
+            {
+                "name": host_input,
+                "user": {
+                    "token": credential.get_input('bearer_token')
+                }
+            }
+        ],
+        "contexts": [
+            {
+                "name": host_input,
+                "context": {
+                    "cluster": host_input,
+                    "user": host_input,
+                    "namespace": namespace
+                }
+            }
+        ],
+        "current-context": host_input
+    }
+
+    if credential.get_input('verify_ssl'):
+        config["clusters"][0]["cluster"]["certificate-authority-data"] = b64encode(
+            credential.get_input('ssl_ca_cert').encode() # encode to bytes
+        ).decode() # decode the base64 data into a str
+    else:
+        config["clusters"][0]["cluster"]["insecure-skip-tls-verify"] = True
+
+    fd, path = tempfile.mkstemp(prefix='kubeconfig')
+    with open(path, 'wb') as temp:
+        temp.write(yaml.dump(config).encode())
+        temp.flush()
+        os.chmod(temp.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+    return path

awx/main/scheduler/task_manager.py

@@ -251,6 +251,8 @@ class TaskManager():
                 task.controller_node = controller_node
                 logger.debug('Submitting isolated {} to queue {} controlled by {}.'.format(
                              task.log_format, task.execution_node, controller_node))
+            elif rampart_group.is_containerized:
+                task.instance_group = rampart_group
             else:
                 task.instance_group = rampart_group
                 if instance is not None:
@@ -447,7 +449,7 @@ class TaskManager():
             for rampart_group in preferred_instance_groups:
                 if idle_instance_that_fits is None:
                     idle_instance_that_fits = rampart_group.find_largest_idle_instance()
-                if self.get_remaining_capacity(rampart_group.name) <= 0:
+                if not rampart_group.is_containerized and self.get_remaining_capacity(rampart_group.name) <= 0:
                     logger.debug("Skipping group {} capacity <= 0".format(rampart_group.name))
                     continue
 
@@ -456,10 +458,11 @@ class TaskManager():
                     logger.debug("Starting dependent {} in group {} instance {}".format(
                                  task.log_format, rampart_group.name, execution_instance.hostname))
                 elif not execution_instance and idle_instance_that_fits:
-                    execution_instance = idle_instance_that_fits
+                    if not rampart_group.is_containerized:
-                    logger.debug("Starting dependent {} in group {} on idle instance {}".format(
+                        execution_instance = idle_instance_that_fits
-                                 task.log_format, rampart_group.name, execution_instance.hostname))
+                        logger.debug("Starting dependent {} in group {} on idle instance {}".format(
-                if execution_instance:
+                                     task.log_format, rampart_group.name, execution_instance.hostname))
+                if execution_instance or rampart_group.is_containerized:
                     self.graph[rampart_group.name]['graph'].add_job(task)
                     tasks_to_fail = [t for t in dependency_tasks if t != task]
                     tasks_to_fail += [dependent_task]
@@ -492,10 +495,16 @@ class TaskManager():
                 self.start_task(task, None, task.get_jobs_fail_chain(), None)
                 continue
             for rampart_group in preferred_instance_groups:
+                if task.can_run_containerized and rampart_group.is_containerized:
+                    self.graph[rampart_group.name]['graph'].add_job(task)
+                    self.start_task(task, rampart_group, task.get_jobs_fail_chain(), None)
+                    found_acceptable_queue = True
+                    break
+
                 if idle_instance_that_fits is None:
                     idle_instance_that_fits = rampart_group.find_largest_idle_instance()
                 remaining_capacity = self.get_remaining_capacity(rampart_group.name)
-                if remaining_capacity <= 0:
+                if not rampart_group.is_containerized and self.get_remaining_capacity(rampart_group.name) <= 0:
                     logger.debug("Skipping group {}, remaining_capacity {} <= 0".format(
                                  rampart_group.name, remaining_capacity))
                     continue
@@ -505,10 +514,11 @@ class TaskManager():
                     logger.debug("Starting {} in group {} instance {} (remaining_capacity={})".format(
                                  task.log_format, rampart_group.name, execution_instance.hostname, remaining_capacity))
                 elif not execution_instance and idle_instance_that_fits:
-                    execution_instance = idle_instance_that_fits
+                    if not rampart_group.is_containerized:
-                    logger.debug("Starting {} in group {} instance {} (remaining_capacity={})".format(
+                        execution_instance = idle_instance_that_fits
-                                 task.log_format, rampart_group.name, execution_instance.hostname, remaining_capacity))
+                        logger.debug("Starting {} in group {} instance {} (remaining_capacity={})".format(
-                if execution_instance:
+                                     task.log_format, rampart_group.name, execution_instance.hostname, remaining_capacity))
+                if execution_instance or rampart_group.is_containerized:
                     self.graph[rampart_group.name]['graph'].add_job(task)
                     self.start_task(task, rampart_group, task.get_jobs_fail_chain(), execution_instance)
                     found_acceptable_queue = True

awx/main/tasks.py

@@ -251,6 +251,9 @@ def apply_cluster_membership_policies():
         # On a differential basis, apply instances to non-isolated groups
         with transaction.atomic():
             for g in actual_groups:
+                if g.obj.is_containerized:
+                    logger.debug('Skipping containerized group {} for policy calculation'.format(g.obj.name))
+                    continue
                 instances_to_add = set(g.instances) - set(g.prior_instances)
                 instances_to_remove = set(g.prior_instances) - set(g.instances)
                 if instances_to_add:
@@ -878,7 +881,7 @@ class BaseTask(object):
                 settings.AWX_PROOT_SHOW_PATHS
 
             pi_path = settings.AWX_PROOT_BASE_PATH
-            if not self.instance.is_isolated():
+            if not self.instance.is_isolated() and not self.instance.is_containerized:
                 pi_path = tempfile.mkdtemp(
                     prefix='ansible_runner_pi_',
                     dir=settings.AWX_PROOT_BASE_PATH
@@ -1168,6 +1171,7 @@ class BaseTask(object):
 
         try:
             isolated = self.instance.is_isolated()
+            containerized = self.instance.is_containerized
             self.instance.send_notification_templates("running")
             private_data_dir = self.build_private_data_dir(self.instance)
             self.pre_run_hook(self.instance, private_data_dir)
@@ -1261,7 +1265,7 @@ class BaseTask(object):
                 if not params[v]:
                     del params[v]
 
-            if self.instance.is_isolated() is True:
+            if self.instance.is_isolated() or containerized:
                 module_args = None
                 if 'module_args' in params:
                     # if it's adhoc, copy the module args
@@ -1272,10 +1276,22 @@ class BaseTask(object):
                     params.pop('inventory'),
                     os.path.join(private_data_dir, 'inventory')
                 )
+                pod_manager = None
+                if containerized:
+                    from awx.main.scheduler.kubernetes import PodManager # Avoid circular import
+                    params['envvars'].pop('HOME')
+                    pod_manager = PodManager(self.instance)
+                    self.cleanup_paths.append(pod_manager.kube_config)
+                    pod_manager.deploy()
+                    self.instance.execution_node = pod_manager.pod_name
+                    self.instance.save(update_fields=['execution_node'])
+
+
                 ansible_runner.utils.dump_artifacts(params)
                 isolated_manager_instance = isolated_manager.IsolatedManager(
                     cancelled_callback=lambda: self.update_model(self.instance.pk).cancel_flag,
                     check_callback=self.check_handler,
+                    pod_manager=pod_manager
                 )
                 status, rc = isolated_manager_instance.run(self.instance,
                                                            private_data_dir,
@@ -1600,6 +1616,8 @@ class RunJob(BaseTask):
         '''
         Return whether this task should use proot.
         '''
+        if job.is_containerized:
+            return False
         return getattr(settings, 'AWX_PROOT_ENABLED', False)
 
     def pre_run_hook(self, job, private_data_dir):
@@ -1660,6 +1678,7 @@ class RunJob(BaseTask):
             if job.is_isolated() is True:
                 pu_ig = pu_ig.controller
                 pu_en = settings.CLUSTER_HOST_ID
+
             sync_metafields = dict(
                 launch_type="sync",
                 job_type='run',
@@ -1720,8 +1739,13 @@ class RunJob(BaseTask):
                 os.path.join(private_data_dir, 'artifacts', str(job.id), 'fact_cache'),
                 fact_modification_times,
             )
-        if isolated_manager_instance:
+        if isolated_manager_instance and not job.is_containerized:
             isolated_manager_instance.cleanup()
+
+        if job.is_containerized:
+            from awx.main.scheduler.kubernetes import PodManager # prevent circular import
+            PodManager(job).delete()
+
         try:
             inventory = job.inventory
         except Inventory.DoesNotExist:
@@ -2537,6 +2561,8 @@ class RunAdHocCommand(BaseTask):
         '''
         Return whether this task should use proot.
         '''
+        if ad_hoc_command.is_containerized:
+            return False
         return getattr(settings, 'AWX_PROOT_ENABLED', False)
 
     def final_run_hook(self, adhoc_job, status, private_data_dir, fact_modification_times, isolated_manager_instance=None):

awx/main/tests/functional/conftest.py

@@ -203,6 +203,13 @@ def organization(instance):
     return Organization.objects.create(name="test-org", description="test-org-desc")
 
 
+@pytest.fixture
+def credentialtype_kube():
+    kube = CredentialType.defaults['kubernetes_bearer_token']()
+    kube.save()
+    return kube
+
+
 @pytest.fixture
 def credentialtype_ssh():
     ssh = CredentialType.defaults['ssh']()
@@ -336,6 +343,12 @@ def other_external_credential(credentialtype_external):
                                      inputs={'url': 'http://testhost.com', 'token': 'secret2'})
 
 
+@pytest.fixture
+def kube_credential(credentialtype_kube):
+    return Credential.objects.create(credential_type=credentialtype_kube, name='kube-cred',
+                                     inputs={'host': 'my.cluster', 'bearer_token': 'my-token', 'verify_ssl': False})
+
+
 @pytest.fixture
 def inventory(organization):
     return organization.inventories.create(name="test-inv")

awx/main/tests/functional/task_management/test_container_groups.py

@@ -0,0 +1,56 @@
+import subprocess
+import yaml
+import base64
+
+from unittest import mock # noqa
+import pytest
+
+from awx.main.scheduler.kubernetes import PodManager
+from awx.main.utils import (
+    create_temporary_fifo,
+)
+
+
+@pytest.fixture
+def containerized_job(default_instance_group, kube_credential, job_template_factory):
+    default_instance_group.credential = kube_credential
+    default_instance_group.save()
+    objects = job_template_factory('jt', organization='org1', project='proj',
+                                   inventory='inv', credential='cred',
+                                   jobs=['my_job'])
+    jt = objects.job_template
+    jt.instance_groups.add(default_instance_group)
+
+    j1 = objects.jobs['my_job']
+    j1.instance_group = default_instance_group
+    j1.status = 'pending'
+    j1.save()
+    return j1
+
+
+@pytest.mark.django_db
+def test_containerized_job(containerized_job):
+    assert containerized_job.is_containerized
+    assert containerized_job.instance_group.is_containerized
+    assert containerized_job.instance_group.credential.kubernetes
+
+
+@pytest.mark.django_db
+def test_kubectl_ssl_verification(containerized_job):
+    cred = containerized_job.instance_group.credential
+    cred.inputs['verify_ssl'] = True
+    key_material = subprocess.run('openssl genrsa 2> /dev/null',
+                                  shell=True, check=True,
+                                  stdout=subprocess.PIPE)
+    key = create_temporary_fifo(key_material.stdout)
+    cmd = f"""
+    openssl req -x509 -sha256 -new -nodes \
+      -key {key} -subj '/C=US/ST=North Carolina/L=Durham/O=Ansible/OU=AWX Development/CN=awx.localhost'
+    """
+    cert = subprocess.run(cmd.strip(), shell=True, check=True, stdout=subprocess.PIPE)
+    cred.inputs['ssl_ca_cert'] = cert.stdout
+    cred.save()
+    pm = PodManager(containerized_job)
+    config = yaml.load(open(pm.kube_config), Loader=yaml.FullLoader)
+    ca_data = config['clusters'][0]['cluster']['certificate-authority-data']
+    assert cert.stdout == base64.b64decode(ca_data.encode())

awx/main/tests/functional/test_credential.py

@@ -87,6 +87,7 @@ def test_default_cred_types():
         'hashivault_kv',
         'hashivault_ssh',
         'insights',
+        'kubernetes_bearer_token',
         'net',
         'openstack',
         'rhv',

awx/main/tests/unit/scheduler/test_kubernetes.py

@@ -0,0 +1,55 @@
+import pytest
+from unittest import mock
+from django.conf import settings
+
+from awx.main.models import (
+    InstanceGroup,
+    Job,
+    JobTemplate,
+    Project,
+    Inventory,
+)
+from awx.main.scheduler.kubernetes import PodManager
+
+
+@pytest.fixture
+def container_group():
+    instance_group = mock.Mock(InstanceGroup(name='container-group'))
+
+    return instance_group
+
+
+@pytest.fixture
+def job(container_group):
+    return Job(pk=1,
+               id=1,
+               project=Project(),
+               instance_group=container_group,
+               inventory=Inventory(),
+               job_template=JobTemplate(id=1, name='foo'))
+
+
+def test_default_pod_spec(job):
+    default_image = PodManager(job).pod_definition['spec']['containers'][0]['image']
+    assert default_image == settings.AWX_CONTAINER_GROUP_DEFAULT_IMAGE
+
+
+def test_custom_pod_spec(job):
+    job.instance_group.pod_spec_override = """
+    spec:
+      containers:
+        - image: my-custom-image
+    """
+    custom_image = PodManager(job).pod_definition['spec']['containers'][0]['image']
+    assert custom_image == 'my-custom-image'
+
+
+def test_pod_manager_namespace_property(job):
+    pm = PodManager(job)
+    assert pm.namespace == settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE
+
+    job.instance_group.pod_spec_override = """
+    metadata:
+      namespace: my-namespace
+    """
+    assert PodManager(job).namespace == 'my-namespace'

awx/main/tests/unit/test_capacity.py

@@ -11,6 +11,7 @@ class FakeObject(object):
 
 class Job(FakeObject):
     task_impact = 43
+    is_containerized = False
 
     def log_format(self):
         return 'job 382 (fake)'

awx/playbooks/check_isolated.yml

@@ -20,12 +20,42 @@
         mode: pull
         delete: yes
         recursive: yes
+      when: ansible_kubectl_config is not defined
 
     - name: Copy daemon log from the isolated host
       synchronize:
         src: "{{src}}/daemon.log"
         dest: "{{src}}/daemon.log"
         mode: pull
+      when: ansible_kubectl_config is not defined
+
+    - name: Copy artifacts from pod
+      synchronize:
+        src: "{{src}}/artifacts/"
+        dest: "{{src}}/artifacts/"
+        mode: pull
+        delete: yes
+        recursive: yes
+        set_remote_user: no
+        rsync_opts:
+          - "--rsh=$RSH"
+      environment:
+        RSH: "oc rsh --config={{ ansible_kubectl_config }}"
+      delegate_to: localhost
+      when: ansible_kubectl_config is defined
+
+    - name: Copy daemon log from pod
+      synchronize:
+        src: "{{src}}/daemon.log"
+        dest: "{{src}}/daemon.log"
+        mode: pull
+        set_remote_user: no
+        rsync_opts:
+          - "--rsh=$RSH"
+      environment:
+        RSH: "oc rsh --config={{ ansible_kubectl_config }}"
+      delegate_to: localhost
+      when: ansible_kubectl_config is defined
 
     - name: Fail if previous check determined that process is not alive.
       fail:

awx/playbooks/run_isolated.yml

@@ -11,12 +11,25 @@
     secret: "{{ lookup('pipe', 'cat ' + src + '/env/ssh_key') }}"
 
   tasks:
-
     - name: synchronize job environment with isolated host
       synchronize:
-        copy_links: true
+        copy_links: yes
-        src: "{{src}}"
+        src: "{{ src }}"
-        dest: "{{dest}}"
+        dest: "{{ dest }}"
+      when: ansible_kubectl_config is not defined
+
+    - name: synchronize job environment with remote job container
+      synchronize:
+        copy_links: yes
+        src: "{{ src }}"
+        dest: "{{ dest }}"
+        set_remote_user: no
+        rsync_opts:
+          - "--rsh=$RSH"
+      environment:
+        RSH: "oc rsh --config={{ ansible_kubectl_config }}"
+      delegate_to: localhost
+      when: ansible_kubectl_config is defined
 
     - local_action: stat path="{{src}}/env/ssh_key"
       register: key

awx/settings/defaults.py

@@ -67,6 +67,10 @@ DATABASES = {
     }
 }
 
+AWX_CONTAINER_GROUP_DEFAULT_LAUNCH_TIMEOUT = 10
+AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE = 'default'
+AWX_CONTAINER_GROUP_DEFAULT_IMAGE = 'ansible/ansible-runner'
+
 # Internationalization
 # https://docs.djangoproject.com/en/dev/topics/i18n/
 #

awx/settings/development.py

@@ -83,8 +83,7 @@ AWX_PROOT_ENABLED = True
 
 AWX_ISOLATED_USERNAME = 'root'
 AWX_ISOLATED_CHECK_INTERVAL = 1
-AWX_ISOLATED_PERIODIC_CHECK = 5
+AWX_ISOLATED_PERIODIC_CHECK = 30
-AWX_ISOLATED_LAUNCH_TIMEOUT = 30
 
 # Disable Pendo on the UI for development/test.
 # Note: This setting may be overridden by database settings.

awx/ui/client/features/jobs/routes/instanceGroupJobs.route.js

@@ -5,7 +5,7 @@ import jobsListController from '../jobsList.controller';
 const jobsListTemplate = require('~features/jobs/jobsList.view.html');
 const listContainerTemplate = require('~src/instance-groups/jobs/instanceGroupsJobsListContainer.partial.html');
 
-export default {
+const instanceGroupJobsRoute = {
     name: 'instanceGroups.jobs',
     url: '/:instance_group_id/jobs',
     ncyBreadcrumb: {
@@ -63,6 +63,69 @@ export default {
         SearchBasePath: [
             '$stateParams',
             ($stateParams) => `api/v2/instance_groups/${$stateParams.instance_group_id}/jobs`
-        ]
+        ],
     }
 };
+
+const containerGroupJobsRoute = {
+    name: 'instanceGroups.containerGroupJobs',
+    url: '/container_groups/:instance_group_id/jobs',
+    ncyBreadcrumb: {
+        parent: 'instanceGroups.editContainerGroup',
+        label: N_('JOBS')
+    },
+    params: {
+        job_search: {
+            value: {
+                page_size: '10',
+                order_by: '-finished'
+            },
+            dynamic: true
+        }
+    },
+    views: {
+        'containerGroupJobs@instanceGroups': {
+            templateUrl: listContainerTemplate,
+            controller: listContainerController,
+            controllerAs: 'vm'
+        },
+        'jobsList@instanceGroups.containerGroupJobs': {
+            templateUrl: jobsListTemplate,
+            controller: jobsListController,
+            controllerAs: 'vm'
+        },
+    },
+    resolve: {
+        resolvedModels: [
+            'UnifiedJobModel',
+            (UnifiedJob) => {
+                const models = [
+                    new UnifiedJob(['options']),
+                ];
+                return Promise.all(models);
+            },
+        ],
+        Dataset: [
+            '$stateParams',
+            'Wait',
+            'QuerySet',
+            ($stateParams, Wait, qs) => {
+                const groupId = $stateParams.instance_group_id;
+
+                const searchParam = $stateParams.job_search;
+
+                const searchPath = `api/v2/instance_groups/${groupId}/jobs`;
+
+                Wait('start');
+                return qs.search(searchPath, searchParam)
+                    .finally(() => Wait('stop'));
+            }
+        ],
+        SearchBasePath: [
+            '$stateParams',
+            ($stateParams) => `api/v2/instance_groups/${$stateParams.instance_group_id}/jobs`
+        ],
+    }
+};
+
+export { instanceGroupJobsRoute, containerGroupJobsRoute };

awx/ui/client/features/output/details.component.js

@@ -553,12 +553,16 @@ function getInstanceGroupDetails () {
         return null;
     }
 
-    const label = strings.get('labels.INSTANCE_GROUP');
     const value = $filter('sanitize')(instanceGroup.name);
-    const link = `/#/instance_groups/${instanceGroup.id}`;
+
+    let label = strings.get('labels.INSTANCE_GROUP');
+    let link = `/#/instance_groups/${instanceGroup.id}`;
+    if (instanceGroup.is_containerized) {
+        label = strings.get('labels.CONTAINER_GROUP');
+        link = `/#/instance_groups/container_group/${instanceGroup.id}`;
+    }
 
     let isolated = null;
-
     if (instanceGroup.is_isolated) {
         isolated = strings.get('details.ISOLATED');
     }

awx/ui/client/features/output/output.strings.js

@@ -63,6 +63,7 @@ function OutputStrings (BaseString) {
         FORKS: t.s('Forks'),
         HOST_LIMIT_ERROR: t.s('Host Limit Error'),
         INSTANCE_GROUP: t.s('Instance Group'),
+        CONTAINER_GROUP: t.s('Container Group'),
         INVENTORY: t.s('Inventory'),
         INVENTORY_SCM: t.s('Source Project'),
         JOB_EXPLANATION: t.s('Explanation'),

awx/ui/client/lib/components/input/lookup.directive.js

@@ -88,17 +88,14 @@ function AtInputLookupController (baseInputController, $q, $state) {
             scope.state._value = null;
             return vm.check({ isValid: true });
         }
-
         searchParams = searchParams || { [search.key]: scope.state._displayValue };
 
         return model.search(searchParams, search.config)
             .then(found => {
                 if (!found) {
                     vm.reset();
-
                     return;
                 }
-
                 scope[scope.state._resource] = model.get('id');
                 scope.state._value = model.get('id');
                 scope.state._displayValue = model.get('name');

awx/ui/client/lib/components/list/_index.less

@@ -241,6 +241,10 @@
     align-items: center;
 }
 
+.at-Row-rightSide{
+    justify-content: flex-end;
+}
+
 .at-Row-container--wrapped {
     display: flex;
     flex-wrap: wrap;

awx/ui/client/lib/components/switch/switch.partial.html

@@ -3,4 +3,4 @@
         <span class="atSwitch-slider"></span>
         <i class="fa fa-check"></i>
     </span>
-</span>
+</span>

awx/ui/client/src/instance-groups/add-edit/edit-instance-group.controller.js

@@ -2,6 +2,14 @@ function EditController ($rootScope, $state, models, strings) {
     const vm = this || {};
     const { instanceGroup } = models;
 
+    if (instanceGroup.get('is_containerized')) {
+        return $state.go(
+            'instanceGroups.editContainerGroup',
+            { instance_group_id: instanceGroup.get('id') },
+            { reload: true }
+        );
+    }
+
     $rootScope.breadcrumb.instance_group_name = instanceGroup.get('name');
 
     vm.mode = 'edit';
@@ -51,4 +59,4 @@ EditController.$inject = [
     'InstanceGroupsStrings'
 ];
 
-export default EditController;
+export default EditController;

awx/ui/client/src/instance-groups/container-groups/add-container-group.controller.js

@@ -0,0 +1,81 @@
+function AddContainerGroupController(ToJSON, $scope, $state, models, strings, i18n, DataSet) {
+  const vm = this || {};
+  const {
+    instanceGroup,
+    credential
+  } = models;
+
+  vm.mode = 'add';
+  vm.strings = strings;
+  vm.panelTitle = strings.get('state.ADD_CONTAINER_GROUP_BREADCRUMB_LABEL');
+  vm.lookUpTitle = strings.get('container.LOOK_UP_TITLE');
+
+  vm.form = instanceGroup.createFormSchema('post');
+  vm.form.name.required = true;
+
+  vm.form.credential = {
+    type: 'field',
+    label: i18n._('Credential'),
+    id: 'credential'
+  };
+  vm.form.credential._resource = 'credential';
+  vm.form.credential._route = "instanceGroups.addContainerGroup.credentials";
+  vm.form.credential._model = credential;
+  vm.form.credential._placeholder = strings.get('container.CREDENTIAL_PLACEHOLDER');
+  vm.form.credential.required = true;
+
+  vm.form.extraVars = {
+    label: strings.get('container.POD_SPEC_LABEL'),
+    value: DataSet.data.actions.POST.pod_spec_override.default,
+    name: 'extraVars',
+    toggleLabel: strings.get('container.POD_SPEC_TOGGLE'),
+  };
+
+  vm.tab = {
+    details: { _active: true },
+    instances: {_disabled: true },
+    jobs: {_disabled: true }
+};
+
+  $scope.variables = vm.form.extraVars.value;
+  $scope.name = vm.form.extraVars.name;
+  vm.panelTitle = strings.get('container.PANEL_TITLE');
+
+
+  $scope.$watch('credential', () => {
+    if ($scope.credential) {
+        vm.form.credential._idFromModal= $scope.credential;
+      }
+  });
+  vm.form.save = (data) => {
+    data.pod_spec_override = null;
+    if (vm.form.extraVars.isOpen) {
+      data.pod_spec_override = vm.form.extraVars.value;
+    }
+    return instanceGroup.request('post', { data: data }).then((res) => {
+      $state.go('instanceGroups.editContainerGroup', { instance_group_id: res.data.id }, { reload: true });
+    });
+  };
+  vm.form.extraVars.isOpen = false;
+  vm.toggle = () => {
+    if (vm.form.extraVars.isOpen === true) {
+      vm.form.extraVars.isOpen = false;
+    } else {
+      vm.form.extraVars.isOpen = true;
+    }
+    return vm.form.extraVars.isOpen;
+  };
+}
+
+
+AddContainerGroupController.$inject = [
+  'ToJSON',
+  '$scope',
+  '$state',
+  'resolvedModels',
+  'InstanceGroupsStrings',
+  'i18n',
+  'DataSet'
+];
+
+export default AddContainerGroupController;

awx/ui/client/src/instance-groups/container-groups/add-container-group.view.html

@@ -0,0 +1,44 @@
+<div ui-view="credentials"></div>
+<a class="containerGroups-messageBar-link"href="https://docs.ansible.com/ansible-tower/latest/html/userguide/instance_groups.html" target="_blank" style="color: white">
+  <div class="Section-messageBar">
+      <i class="Section-messageBar-warning fa fa-warning"></i>
+      <span class="Section-messageBar-text">This feature is tech preview, and is subject to change in a future release. Click here for documentation.</span>
+  </div>
+</a>
+<at-panel>
+  <at-panel-heading title="{{:: vm.panelTitle }}"></at-panel-heading>
+  <at-tab-group>
+      <at-tab state="vm.tab.details">{{:: vm.strings.get('tab.DETAILS') }}</at-tab>
+      <at-tab state="vm.tab.jobs">{{:: vm.strings.get('tab.JOBS') }}</at-tab>
+  </at-tab-group>
+  <at-panel-body>
+    <at-form state="vm.form" autocomplete="off">
+      <at-input-text col="4" tab="1" state="vm.form.name"></at-input-text>
+      <at-input-lookup col="4" tab="3" state="vm.form.credential" id="containter_groups_credential"></at-input-lookup>
+      <div class="at-Row-toggle Form-formGroup--fullWidth ">
+        <label class="Form-inputLabelContainer" for="vm.form.extraVars.isOpen">
+            <span class="Form-inputLabel" translate>
+                {{ vm.form.extraVars.toggleLabel }}
+            </span>
+        </label>
+        <div ng-class="{'ContainerGroups-codeMirror': vm.form.extraVars.isOpen }">
+          <at-switch on-toggle="vm.toggle(instance)" switch-on="vm.form.extraVars.isOpen"
+          switch-disabled="vm.rowAction.toggle._disabled"></at-switch>
+        </div>
+      <at-code-mirror
+        ng-if="vm.form.extraVars.isOpen"
+        col="4" tab="3"
+        class="Form-formGroup--fullWidth"
+        variables="vm.form.extraVars.value"
+        label="{{ vm.form.extraVars.label }}"
+        name="{{ vm.form.extraVars.name }}"
+      >
+      </at-code-mirror>
+    </div>
+      <at-action-group col="12" pos="right">
+        <at-form-action type="cancel" to="instanceGroups"></at-form-action>
+        <at-form-action type="save"></at-form-action>
+      </at-action-group>
+    </at-form>
+  </at-panel-body>
+</at-panel>

awx/ui/client/src/instance-groups/container-groups/edit-container-group.controller.js

@@ -0,0 +1,137 @@
+function EditContainerGroupController($rootScope, $scope, $state, models, strings, i18n, EditContainerGroupDataset) {
+  const vm = this || {};
+  const {
+    instanceGroup,
+    credential
+  } = models;
+
+  if (!instanceGroup.get('is_containerized')) {
+      return $state.go(
+          'instanceGroups.edit',
+          { instance_group_id: instanceGroup.get('id') },
+          { reload: true }
+      );
+  }
+
+  $rootScope.breadcrumb.instance_group_name = instanceGroup.get('name');
+
+  vm.mode = 'edit';
+  vm.strings = strings;
+  vm.panelTitle = EditContainerGroupDataset.data.name;
+  vm.lookUpTitle = strings.get('container.LOOK_UP_TITLE');
+
+  vm.form = instanceGroup.createFormSchema('post');
+  vm.form.name.required = true;
+  vm.form.credential = {
+    type: 'field',
+    label: i18n._('Credential'),
+    id: 'credential'
+  };
+  vm.form.credential._resource = 'credential';
+  vm.form.credential._route = "instanceGroups.editContainerGroup.credentials";
+  vm.form.credential._model = credential;
+  vm.form.credential._displayValue = EditContainerGroupDataset.data.summary_fields.credential.name;
+  vm.form.credential.required = true;
+  vm.form.credential._value = EditContainerGroupDataset.data.summary_fields.credential.id;
+
+  vm.tab = {
+    details: {
+            _active: true,
+            _go: 'instanceGroups.editContainerGroup',
+            _params: { instance_group_id: instanceGroup.get('id') }
+    },
+    instances: {
+        _go: 'instanceGroups.containerGroupInstances',
+        _params: { instance_group_id: instanceGroup.get('id') }
+    },
+    jobs: {
+        _go: 'instanceGroups.containerGroupJobs',
+        _params: { instance_group_id: instanceGroup.get('id') }
+    }
+};
+
+  vm.form.extraVars = {
+    label: strings.get('container.POD_SPEC_LABEL'),
+    value: EditContainerGroupDataset.data.pod_spec_override || instanceGroup.model.OPTIONS.actions.PUT.pod_spec_override.default,
+    name: 'extraVars',
+    toggleLabel: strings.get('container.POD_SPEC_TOGGLE')
+  };
+
+  function sanitizeVars (str) {
+    // Quick function to test if the host vars are a json-object-string,
+    // by testing if they can be converted to a JSON object w/o error.
+    function IsJsonString (varStr) {
+      try {
+        JSON.parse(varStr);
+      } catch (e) {
+        return false;
+      }
+      return true;
+    }
+
+    if (typeof str === 'undefined') {
+      return '---';
+    }
+    if (typeof str !== 'string') {
+      const yamlStr = jsyaml.safeDump(str);
+      // jsyaml.safeDump doesn't process an empty object correctly
+      if (yamlStr === '{}\n') {
+        return '---';
+      }
+      return yamlStr;
+    }
+    if (str === '' || str === '{}') {
+      return '---';
+    } else if (IsJsonString(str)) {
+      str = JSON.parse(str);
+      return jsyaml.safeDump(str);
+    }
+    return str;
+  }
+
+  const podSpecValue = sanitizeVars(EditContainerGroupDataset.data.pod_spec_override);
+  const defaultPodSpecValue = sanitizeVars(instanceGroup.model.OPTIONS.actions.PUT.pod_spec_override.default);
+
+  if ((podSpecValue !== '---') && podSpecValue && podSpecValue.trim() !== defaultPodSpecValue.trim()) {
+    vm.form.extraVars.isOpen = true;
+  } else {
+    vm.form.extraVars.isOpen = false;
+  }
+
+  $scope.$watch('credential', () => {
+    if ($scope.credential) {
+      vm.form.credential._idFromModal= $scope.credential;
+      }
+  });
+  vm.form.save = (data) => {
+    if (!vm.form.extraVars.isOpen) {
+      data.pod_spec_override = null;
+    } else {
+      data.pod_spec_override = vm.form.extraVars.value;
+    }
+    return instanceGroup.request('put', { data: data }).then((res) => {
+      $state.go('instanceGroups.editContainerGroup', { instance_group_id: res.data.id }, { reload: true });
+    } );
+  };
+
+  vm.toggle = () => {
+    if (vm.form.extraVars.isOpen === true) {
+      vm.form.extraVars.isOpen = false;
+    } else {
+      vm.form.extraVars.isOpen = true;
+    }
+    return vm.form.extraVars.isOpen;
+  };
+}
+
+EditContainerGroupController.$inject = [
+  '$rootScope',
+  '$scope',
+  '$state',
+  'resolvedModels',
+  'InstanceGroupsStrings',
+  'i18n',
+  'EditContainerGroupDataset'
+];
+
+export default EditContainerGroupController;

awx/ui/client/src/instance-groups/instance-group.block.less

@@ -1,4 +1,18 @@
 .InstanceGroups {
+    .at-Row-actions{
+        justify-content: flex-start;
+        width: 300px;
+       & > capacity-bar:only-child{
+            margin-left: 0px;
+            margin-top: 5px
+        }
+    }
+    .at-RowAction{
+        margin: 0;
+    }
+    .at-Row-links{
+        justify-content: flex-start;
+    }
 
     .BreadCrumb-menuLinkImage:hover {
         color: @default-link;
@@ -74,4 +88,47 @@
             color: @at-white;
         }
     }
+    .at-Row-toggle{
+        margin-top: 20px;
+        padding-left: 15px;
+    }
+
+    .ContainerGroups-codeMirror{
+        margin-bottom: 10px;
+    }
+
+    .at-Row-container{
+    flex-wrap: wrap;
+    }
+    .containerGroups-messageBar-link:hover{
+        text-decoration: underline;
+    }
+
+    @media screen and (max-width: 1060px) and (min-width: 769px){
+        .at-Row-links {
+            justify-content: flex-start;
+            flex-wrap: wrap;
+        }
+    }
+
+    @media screen and (min-width: 1061px){
+        .at-Row-actions{
+            justify-content: flex-end;
+            & > capacity-bar:only-child {
+                margin-right: 30px;
+            }
+        }
+        .instanceGroupsList-details{
+            display: flex;
+        }
+        .at-Row-links {
+            justify-content: flex-end;
+            display: flex;
+            width: 445px;
+        }
+    }
 }
+
+
+
+

awx/ui/client/src/instance-groups/instance-groups.partial.html

@@ -2,6 +2,10 @@
     <aw-limit-panels max-panels="2" panel-container="instance-groups-panel"></aw-limit-panels>
     <div ui-view="add"></div>
     <div ui-view="edit"></div>
+    <div ui-view="addContainerGroup"></div>
+    <div ui-view="editContainerGroup"></div>
+    <div ui-view="containerGroupInstances"></div>
+    <div ui-view="containerGroupJobs"></div>
 
     <div ui-view="instanceJobsContainer"></div>
 

awx/ui/client/src/instance-groups/instance-groups.strings.js

@@ -1,13 +1,16 @@
-function InstanceGroupsStrings (BaseString) {
+function InstanceGroupsStrings(BaseString) {
     BaseString.call(this, 'instanceGroups');
 
-    const { t } = this;
+    const {
+        t
+    } = this;
     const ns = this.instanceGroups;
 
     ns.state = {
         INSTANCE_GROUPS_BREADCRUMB_LABEL: t.s('INSTANCE GROUPS'),
         INSTANCES_BREADCRUMB_LABEL: t.s('INSTANCES'),
-        ADD_BREADCRUMB_LABEL: t.s('CREATE INSTANCE GROUP')
+        ADD_BREADCRUMB_LABEL: t.s('CREATE INSTANCE GROUP'),
+        ADD_CONTAINER_GROUP_BREADCRUMB_LABEL: t.s('CREATE CONTAINER GROUP')
     };
 
     ns.list = {
@@ -33,7 +36,8 @@ function InstanceGroupsStrings (BaseString) {
     };
 
     ns.instance = {
-        PANEL_TITLE: t.s('SELECT INSTANCE')
+        PANEL_TITLE: t.s('SELECT INSTANCE'),
+        BADGE_TEXT: t.s('Instance Group')
     };
 
     ns.capacityBar = {
@@ -62,6 +66,15 @@ function InstanceGroupsStrings (BaseString) {
     ns.alert = {
         MISSING_PARAMETER: t.s('Instance Group parameter is missing.'),
     };
+    ns.container = {
+        PANEL_TITLE: t.s('Add Container Group'),
+        LOOK_UP_TITLE: t.s('Add Credential'),
+        CREDENTIAL_PLACEHOLDER: t.s('SELECT A CREDENTIAL'),
+        POD_SPEC_LABEL: t.s('Pod Spec Override'),
+        BADGE_TEXT: t.s('Container Group'),
+        POD_SPEC_TOGGLE: t.s('Customize Pod Spec')
+
+    };
 }
 
 InstanceGroupsStrings.$inject = ['BaseStringService'];

awx/ui/client/src/instance-groups/instances/instance-modal.controller.js

@@ -1,4 +1,4 @@
-function InstanceModalController ($scope, $state, Dataset, models, strings, ProcessErrors, Wait) {
+function InstanceModalController ($scope, $state, Dataset, models, strings, ProcessErrors, Wait, routeData) {
     const { instanceGroup } = models;
     const vm = this || {};
     let relatedInstanceIds = [];
@@ -116,11 +116,11 @@ function InstanceModalController ($scope, $state, Dataset, models, strings, Proc
     };
 
     vm.onSaveSuccess = () => {
-        $state.go('instanceGroups.instances', {}, {reload: 'instanceGroups.instances'});
+        $state.go(`${routeData}`, {}, {reload: `${routeData}`});
     };
 
     vm.dismiss = () => {
-        $state.go('instanceGroups.instances');
+        $state.go(`${routeData}`);
     };
 
     vm.toggleRow = (row) => {
@@ -163,7 +163,8 @@ InstanceModalController.$inject = [
     'resolvedModels',
     'InstanceGroupsStrings',
     'ProcessErrors',
-    'Wait'
+    'Wait',
+    'routeData'
 ];
 
 export default InstanceModalController;

awx/ui/client/src/instance-groups/instances/instance-modal.partial.html

@@ -62,7 +62,7 @@
             <div class="at-ActionGroup">
                 <div class="pull-right">
                     <button class="btn at-ButtonHollow--default"
-                        ng-click="$state.go('instanceGroups.instances')">
+                        ng-click=vm.dismiss()>
                         {{:: vm.strings.get('CANCEL') }}
                     </button>
                     <button class="btn at-Button--success"

awx/ui/client/src/instance-groups/instances/instances-list.partial.html

@@ -23,7 +23,7 @@
             <div class="at-List-toolbarAction">
                 <button
                     type="button"
-                    ng-click="$state.go('instanceGroups.instances.modal.add')"
+                    ng-click= "vm.addInstances()"
                     class="at-Button--add"
                     id="button-add"
                     ng-show="vm.isSuperuser"

awx/ui/client/src/instance-groups/instances/instances.controller.js

@@ -8,6 +8,41 @@ function InstancesController ($scope, $state, $http, $transitions, models, strin
     vm.policy_instance_list = instanceGroup.get('policy_instance_list');
     vm.isSuperuser = $scope.$root.user_is_superuser;
 
+    let tabs = {};
+    let addInstancesRoute ="";
+    if ($state.is("instanceGroups.instances")) {
+        tabs={ state: {
+                    details: {
+                        _go: 'instanceGroups.edit'
+                    },
+                    instances: {
+                        _active: true,
+                        _go: 'instanceGroups.instances'
+                    },
+                    jobs: {
+                        _go: 'instanceGroups.jobs'
+                    }
+                }
+            };
+        addInstancesRoute = 'instanceGroups.instances.modal.add';
+    } else if ($state.is("instanceGroups.containerGroupInstances")) {
+        tabs={
+            state: {
+                details: {
+                    _go: 'instanceGroups.editContainerGroup'
+                },
+                instances: {
+                    _active: true,
+                    _go: 'instanceGroups.containerGroupInstances'
+                },
+                jobs: {
+                    _go: 'instanceGroups.containerGroupJobs'
+                }
+            }
+        };
+        addInstancesRoute = 'instanceGroups.containerGroupInstances.modal.add';
+    }
+
     vm.list = {
         name: 'instances',
         iterator: 'instance',
@@ -21,6 +56,12 @@ function InstancesController ($scope, $state, $http, $transitions, models, strin
         value: 'hostname'
     };
 
+    vm.addInstances = () => {
+
+        return $state.go(`${addInstancesRoute}`);
+    };
+
+
     vm.toolbarSortValue = toolbarSortDefault;
     vm.toolbarSortOptions = [
         toolbarSortDefault,
@@ -64,21 +105,14 @@ function InstancesController ($scope, $state, $http, $transitions, models, strin
         }, { notify: false, location: 'replace' });
     };
 
-    vm.tab = {
+    const tabObj = {};
-        details: {
+    const params = { instance_group_id: instanceGroup.get('id') };
-            _go: 'instanceGroups.edit',
+
-            _params: { instance_group_id: vm.instance_group_id }
+    tabObj.details = { _go: tabs.state.details._go, _params: params };
-        },
+    tabObj.instances = { _go: tabs.state.instances._go, _params: params, _active: true };
-        instances: {
+    tabObj.jobs = { _go: tabs.state.jobs._go, _params: params };
-            _active: true,
+    vm.tab = tabObj;
-            _go: 'instanceGroups.instances',
+
-            _params: { instance_group_id: vm.instance_group_id }
-        },
-        jobs: {
-            _go: 'instanceGroups.jobs',
-            _params: { instance_group_id: vm.instance_group_id }
-        }
-    };
 
     vm.tooltips = {
         add: strings.get('tooltips.ASSOCIATE_INSTANCES')
@@ -107,7 +141,6 @@ function InstancesController ($scope, $state, $http, $transitions, models, strin
             url: instance.url,
             data
         };
-
         $http(req).then(vm.onSaveSuccess)
             .catch(({data, status}) => {
                 ProcessErrors($scope, data, status, null, {
@@ -157,7 +190,7 @@ InstancesController.$inject = [
     'resolvedModels',
     'InstanceGroupsStrings',
     'Dataset',
-    'ProcessErrors'
+    'ProcessErrors',
 ];
 
 export default InstancesController;

awx/ui/client/src/instance-groups/jobs/instanceGroupsJobsListContainer.controller.js

@@ -1,29 +1,49 @@
 
 function InstanceGroupJobsContainerController ($scope, strings, $state) {
     const vm = this || {};
-
     const instanceGroupId = $state.params.instance_group_id;
 
+    let tabs = {};
+    if ($state.is('instanceGroups.jobs')) {
+        tabs =  {
+            state: {
+                details: {
+                    _go: 'instanceGroups.edit'
+                },
+                instances: {
+                    _go: 'instanceGroups.instances'
+                },
+                jobs: {
+                    _active: true,
+                    _go: 'instanceGroups.jobs'
+                }
+            }
+        };
+    } else if ($state.is('instanceGroups.containerGroupJobs')) {
+        tabs =  {
+            state: {
+                details: {
+                    _go: 'instanceGroups.editContainerGroup'
+                },
+                instances: {
+                    _go: 'instanceGroups.containerGroupInstances'
+                },
+                jobs: {
+                    _active: true,
+                    _go: 'instanceGroups.containerGroupJobs'
+                }
+            }
+        };
+    }
+
     vm.panelTitle = strings.get('jobs.PANEL_TITLE');
     vm.strings = strings;
+    const tabObj = {};
 
-    vm.tab = {
+    tabObj.details = { _go: tabs.state.details._go, _params: { instance_group_id: parseInt(instanceGroupId) } };
-        details: {
+    tabObj.instances = { _go: tabs.state.instances._go, _params: { instance_group_id: parseInt(instanceGroupId) } };
-            _go: 'instanceGroups.edit',
+    tabObj.jobs = { _go: tabs.state.jobs._go, _params: { instance_group_id: parseInt(instanceGroupId) }, _active: true };
-            _params: { instance_group_id: instanceGroupId },
+    vm.tab = tabObj;
-            _label: strings.get('tab.DETAILS')
-        },
-        instances: {
-            _go: 'instanceGroups.instances',
-            _params: { instance_group_id: instanceGroupId },
-            _label: strings.get('tab.INSTANCES')
-        },
-        jobs: {
-            _active: true,
-            _params: { instance_group_id: instanceGroupId },
-            _label: strings.get('tab.JOBS')
-        }
-    };
 
     $scope.$on('updateCount', (e, count) => {
         if (typeof count === 'number') {

awx/ui/client/src/instance-groups/jobs/instanceGroupsJobsListContainer.partial.html

@@ -3,7 +3,7 @@
 
     <at-tab-group class="at-TabGroup--padBelow">
         <at-tab state="vm.tab.details">{{:: vm.strings.get('tab.DETAILS') }}</at-tab>
-        <at-tab state="vm.tab.instances">{{:: vm.strings.get('tab.INSTANCES') }}</at-tab>
+        <at-tab ng-if="$state.is('instanceGroups.jobs')" state="vm.tab.instances">{{:: vm.strings.get('tab.INSTANCES') }}</at-tab>
         <at-tab state="vm.tab.jobs">{{:: vm.strings.get('tab.JOBS') }}</at-tab>
     </at-tab-group>
     <div ui-view="jobsList"></div>

awx/ui/client/src/instance-groups/list/instance-groups-list.partial.html

@@ -18,19 +18,20 @@
                 search-tags="searchTags">
             </smart-search>
             <div class="at-List-toolbarAction">
-                <button
+                <button type="button" class="at-Button--add" id="button-add" ng-show="vm.isSuperuser"
-                    type="button"
+                    data-toggle="dropdown" data-placement="top" aria-haspopup="true" aria-expanded="false">
-                    ui-sref="instanceGroups.add"
-                    class="at-Button--add"
-                    id="button-add"
-                    ng-show="vm.isSuperuser"
-                    aw-tool-tip="{{vm.tooltips.add}}"
-                    data-placement="top"
-                    aria-haspopup="true"
-                    aria-expanded="false">
                 </button>
+                <div class="dropdown-menu dropdown-menu-right" aria-labelledby="button-add">
+                    <a class="dropdown-item" ui-sref="instanceGroups.add">
+                        {{:: vm.strings.get('state.ADD_BREADCRUMB_LABEL') }}
+                    </a>
+                    <a class="dropdown-item" ui-sref="instanceGroups.addContainerGroup">
+                        {{:: vm.strings.get('state.ADD_CONTAINER_GROUP_BREADCRUMB_LABEL') }}
+                    </a>
+                </div>
             </div>
         </div>
+        </div>
         <at-list-toolbar
             ng-if="instance_groups.length > 0"
             sort-only="true"
@@ -40,46 +41,72 @@
         </at-list-toolbar>
         <at-list results="instance_groups">
             <at-row ng-repeat="instance_group in instance_groups"
-                ng-class="{'at-Row--active': (instance_group.id === vm.activeId)}">
+                ng-class="{'at-Row--active': (instance_group.id === vm.activeId)}" >
-
                 <div class="at-Row-items">
-                    <at-row-item
+                    <div class="at-Row-container">
-                        header-value="{{ instance_group.name }}"
+                        <div class="at-Row-content">
-                        header-state="instanceGroups.edit({instance_group_id:{{instance_group.id}}})"
+                            <at-row-item
-                        header-tag="{{ instance_group.is_isolated ? vm.strings.get('list.ROW_ITEM_LABEL_ISOLATED') : '' }}"
+                                ng-if="!instance_group.credential"
-                    >
+                                header-value="{{ instance_group.name }}"
-                    </at-row-item>
+                                header-state="instanceGroups.edit({instance_group_id:{{instance_group.id}}})"
-
+                                header-tag="{{ instance_group.is_isolated ? vm.strings.get('list.ROW_ITEM_LABEL_ISOLATED') : '' }}"
-                    <div class="at-Row--inline">
+                            >
-                        <at-row-item
+                            </at-row-item>
-                            label-value="{{:: vm.strings.get('list.ROW_ITEM_LABEL_INSTANCES') }}"
+                            <at-row-item
-                            label-state="instanceGroups.instances({instance_group_id: {{ instance_group.id }}})"
+                                ng-if="instance_group.credential"
-                            value="{{ instance_group.instances }}"
+                                header-value="{{ instance_group.name }}"
-                            inline="true"
+                                header-state="instanceGroups.editContainerGroup({instance_group_id:{{instance_group.id}}})"
-                            badge="true">
+                                header-tag="{{ instance_group.is_isolated ? vm.strings.get('list.ROW_ITEM_LABEL_ISOLATED') : '' }}"
-                        </at-row-item>
+                            >
-                        <at-row-item
+                            </at-row-item>
-                            label-value="{{:: vm.strings.get('list.ROW_ITEM_LABEL_RUNNING_JOBS') }}"
+                            <div class="at-RowItem--labels" ng-if="instance_group.credential">
-                            label-state="instanceGroups.jobs({instance_group_id: {{ instance_group.id }}, job_search: {status__in: ['running,waiting']}})"
+                                <div class="LabelList-tagContainer">
-                            value="{{ instance_group.jobs_running }}"
+                                    <div class="LabelList-tag" ng-class="{'LabelList-tag--deletable' : (showDelete && template.summary_fields.user_capabilities.edit)}">
-                            inline="true"
+                                        <span class="LabelList-name">{{vm.strings.get('container.BADGE_TEXT') }}</span>
-                            badge="true">
+                                    </div>
-                        </at-row-item>
+                                </div>
-                        <at-row-item
+                            </div>
-                            label-value="{{:: vm.strings.get('list.ROW_ITEM_LABEL_TOTAL_JOBS') }}"
+                            <div class="at-RowItem--labels" ng-if="!instance_group.credential">
-                            label-state="instanceGroups.jobs({instance_group_id: {{ instance_group.id }}})"
+                                    <div class="LabelList-tagContainer">
-                            value="{{ instance_group.jobs_total }}"
+                                        <div class="LabelList-tag" ng-class="{'LabelList-tag--deletable' : (showDelete && template.summary_fields.user_capabilities.edit)}">
-                            inline="true"
+                                            <span class="LabelList-name">{{vm.strings.get('instance.BADGE_TEXT') }}</span>
-                            badge="true">
+                                        </div>
-                        </at-row-item>
+                                    </div>
+                                </div>
+                        </div>
+                        <div class="instanceGroupsList-details">
+                        <div class="at-Row-links">
+                            <at-row-item
+                                ng-if="!instance_group.credential"
+                                label-value="{{:: vm.strings.get('list.ROW_ITEM_LABEL_INSTANCES') }}"
+                                label-state="instanceGroups.instances({instance_group_id: {{ instance_group.id }}})"
+                                value="{{ instance_group.instances }}"
+                                inline="true"
+                                badge="true">
+                            </at-row-item>
+                            <at-row-item
+                                label-value="{{:: vm.strings.get('list.ROW_ITEM_LABEL_RUNNING_JOBS') }}"
+                                label-state="instanceGroups.jobs({instance_group_id: {{ instance_group.id }}, job_search: {status__in: ['running,waiting']}})"
+                                value="{{ instance_group.jobs_running }}"
+                                inline="true"
+                                badge="true">
+                            </at-row-item>
+                            <at-row-item
+                                label-value="{{:: vm.strings.get('list.ROW_ITEM_LABEL_TOTAL_JOBS') }}"
+                                label-state="instanceGroups.jobs({instance_group_id: {{ instance_group.id }}})"
+                                value="{{ instance_group.jobs_total }}"
+                                inline="true"
+                                badge="true">
+                            </at-row-item>
+                        </div>
+                        <div class="at-Row-actions" >
+                            <capacity-bar ng-show="!instance_group.credential" label-value="{{:: vm.strings.get('list.ROW_ITEM_LABEL_USED_CAPACITY') }}" capacity="instance_group.consumed_capacity" total-capacity="instance_group.capacity"></capacity-bar>
+                            <at-row-action ng-class="{'at-Row-actions-noCredential': !instance_group.credential}" icon="fa-trash" ng-click="vm.deleteInstanceGroup(instance_group)" ng-if="vm.rowAction.trash(instance_group)">
+                            </at-row-action>
+                        </div>
+                        </div>
                     </div>
                 </div>
-
-                <div class="at-Row-actions">
-                    <capacity-bar label-value="{{:: vm.strings.get('list.ROW_ITEM_LABEL_USED_CAPACITY') }}" capacity="instance_group.consumed_capacity" total-capacity="instance_group.capacity"></capacity-bar>
-                    <at-row-action icon="fa-trash" ng-click="vm.deleteInstanceGroup(instance_group)" ng-if="vm.rowAction.trash(instance_group)">
-                    </at-row-action>
-                </div>
             </at-row>
         </at-list>
     </at-panel-body>

awx/ui/client/src/instance-groups/main.js

@@ -1,5 +1,10 @@
-import { templateUrl } from '../shared/template-url/template-url.factory';
+import {
+    templateUrl
+} from '../shared/template-url/template-url.factory';
 import CapacityAdjuster from './capacity-adjuster/capacity-adjuster.directive';
+import AddContainerGroup from './container-groups/add-container-group.view.html';
+import EditContainerGroupController from './container-groups/edit-container-group.controller';
+import AddContainerGroupController from './container-groups/add-container-group.controller';
 import CapacityBar from './capacity-bar/capacity-bar.directive';
 import instanceGroupsMultiselect from '../shared/instance-groups-multiselect/instance-groups.directive';
 import instanceGroupsModal from '../shared/instance-groups-multiselect/instance-groups-modal/instance-groups-modal.directive';
@@ -22,37 +27,58 @@ import service from './instance-groups.service';
 
 import InstanceGroupsStrings from './instance-groups.strings';
 
-import instanceGroupJobsRoute from '~features/jobs/routes/instanceGroupJobs.route.js';
+import {instanceGroupJobsRoute, containerGroupJobsRoute} from '~features/jobs/routes/instanceGroupJobs.route.js';
 import instanceJobsRoute from '~features/jobs/routes/instanceJobs.route.js';
 
+
 const MODULE_NAME = 'instanceGroups';
 
-function InstanceGroupsResolve ($q, $stateParams, InstanceGroup, Instance, ProcessErrors, strings) {
+function InstanceGroupsResolve($q, $stateParams, InstanceGroup, Credential, Instance, ProcessErrors, strings) {
     const instanceGroupId = $stateParams.instance_group_id;
     const instanceId = $stateParams.instance_id;
     let promises = {};
 
     if (!instanceGroupId && !instanceId) {
         promises.instanceGroup = new InstanceGroup(['get', 'options']);
+        promises.credential = new Credential(['get', 'options']);
         return $q.all(promises);
     }
 
     if (instanceGroupId && instanceId) {
         promises.instance = new Instance(['get', 'options'], [instanceId, instanceId])
-            .then((instance) => instance.extend('get', 'jobs', {params: {page_size: "10", order_by: "-finished"}}));
+        .then((instance) => instance.extend('get', 'jobs', {
+            params: {
+                page_size: "10",
+                order_by: "-finished"
+            }
+        }));
         return $q.all(promises);
     }
 
     promises.instanceGroup = new InstanceGroup(['get', 'options'], [instanceGroupId, instanceGroupId])
-            .then((instanceGroup) =>  instanceGroup.extend('get', 'jobs', {params: {page_size: "10", order_by: "-finished"}}))
+    .then((instanceGroup) => instanceGroup.extend('get', 'jobs', {
-            .then((instanceGroup) =>  instanceGroup.extend('get', 'instances'));
+        params: {
+            page_size: "10",
+            order_by: "-finished"
+        }
+    }))
+    .then((instanceGroup) => instanceGroup.extend('get', 'instances'));
+
+    promises.credential = new Credential();
 
     return $q.all(promises)
         .then(models => models)
-        .catch(({ data, status, config }) => {
+        .catch(({
+            data,
+            status,
+            config
+        }) => {
             ProcessErrors(null, data, status, null, {
                 hdr: strings.get('error.HEADER'),
-                msg: strings.get('error.CALL', { path: `${config.url}`, status })
+                msg: strings.get('error.CALL', {
+                    path: `${config.url}`,
+                    status
+                })
             });
             return $q.reject();
         });
@@ -62,12 +88,13 @@ InstanceGroupsResolve.$inject = [
     '$q',
     '$stateParams',
     'InstanceGroupModel',
+    'CredentialModel',
     'InstanceModel',
     'ProcessErrors',
     'InstanceGroupsStrings'
 ];
 
-function InstanceGroupsRun ($stateExtender, strings) {
+function InstanceGroupsRun($stateExtender, strings) {
     $stateExtender.addState({
         name: 'instanceGroups',
         url: '/instance_groups',
@@ -100,7 +127,7 @@ function InstanceGroupsRun ($stateExtender, strings) {
         resolve: {
             resolvedModels: InstanceGroupsResolve,
             Dataset: ['InstanceGroupList', 'QuerySet', '$stateParams', 'GetBasePath',
-                function(list, qs, $stateParams, GetBasePath) {
+                function (list, qs, $stateParams, GetBasePath) {
                     let path = GetBasePath(list.basePath) || GetBasePath(list.name);
                     return qs.search(path, $stateParams[`${list.iterator}_search`]);
                 }
@@ -143,6 +170,157 @@ function InstanceGroupsRun ($stateExtender, strings) {
             ]
         }
     });
+    $stateExtender.addState({
+        name: 'instanceGroups.addContainerGroup',
+        url: '/container_group',
+        views: {
+            'addContainerGroup@instanceGroups': {
+                templateUrl: AddContainerGroup,
+                controller: AddContainerGroupController,
+                controllerAs: 'vm'
+            }
+        },
+        resolve: {
+            resolvedModels: InstanceGroupsResolve,
+            DataSet: ['Rest', 'GetBasePath', (Rest, GetBasePath) => {
+                Rest.setUrl(`${GetBasePath('instance_groups')}`);
+                return Rest.options();
+            }]
+        },
+        ncyBreadcrumb: {
+            label: strings.get('state.ADD_CONTAINER_GROUP_BREADCRUMB_LABEL')
+        },
+    });
+
+    $stateExtender.addState({
+        name: 'instanceGroups.addContainerGroup.credentials',
+        url: '/credential?selected',
+        searchPrefix: 'credential',
+        params: {
+            credential_search: {
+                value: {
+                    credential_type__kind: 'kubernetes',
+                    order_by: 'name',
+                    page_size: 5,
+                },
+                dynamic: true,
+                squash: ''
+            }
+        },
+        data: {
+            basePath: 'credentials',
+            formChildState: true
+        },
+        ncyBreadcrumb: {
+            skip: true
+        },
+        views: {
+            'credentials@instanceGroups.addContainerGroup': {
+                templateProvider: (ListDefinition, generateList) => {
+                    const html = generateList.build({
+                        mode: 'lookup',
+                        list: ListDefinition,
+                        input_type: 'radio'
+                    });
+                    return `<lookup-modal>${html}</lookup-modal>`;
+                }
+            }
+        },
+        resolve: {
+            ListDefinition: ['CredentialList', list => list],
+            Dataset: ['ListDefinition', 'QuerySet', '$stateParams', 'GetBasePath', (list, qs, $stateParams, GetBasePath) => {
+
+
+                const searchPath = GetBasePath('credentials');
+                return qs.search(
+                    searchPath,
+                    $stateParams[`${list.iterator}_search`]
+                );
+            }]
+        },
+        onExit ($state) {
+            if ($state.transition) {
+                $('#form-modal').modal('hide');
+                $('.modal-backdrop').remove();
+                $('body').removeClass('modal-open');
+            }
+        }
+    });
+
+    $stateExtender.addState({
+        name: 'instanceGroups.editContainerGroup',
+        url: '/container_group/:instance_group_id',
+        views: {
+            'editContainerGroup@instanceGroups': {
+                templateUrl: AddContainerGroup,
+                controller: EditContainerGroupController,
+                controllerAs: 'vm'
+            }
+        },
+
+        resolve: {
+            resolvedModels: InstanceGroupsResolve,
+            EditContainerGroupDataset: ['GetBasePath', 'QuerySet', '$stateParams',
+                function (GetBasePath, qs, $stateParams) {
+                let path = `${GetBasePath('instance_groups')}${$stateParams.instance_group_id}`;
+                    return qs.search(path, $stateParams);
+                }
+            ],
+        },
+        ncyBreadcrumb: {
+            label: '{{breadcrumb.instance_group_name}}'
+        },
+    });
+
+    $stateExtender.addState({
+        name: 'instanceGroups.editContainerGroup.credentials',
+        url: '/credential?selected',
+        searchPrefix: 'credential',
+        params: {
+            credential_search: {
+                value: {
+                    credential_type__kind: 'kubernetes',
+                    order_by: 'name',
+                    page_size: 5,
+                },
+                dynamic: true,
+                squash: ''
+            }
+        },
+        data: {
+            basePath: 'credentials',
+            formChildState: true
+        },
+        views: {
+            'credentials@instanceGroups.editContainerGroup': {
+                templateProvider: (ListDefinition, generateList) => {
+                    const html = generateList.build({
+                        mode: 'lookup',
+                        list: ListDefinition,
+                        input_type: 'radio'
+                    });
+                    return `<lookup-modal>${html}</lookup-modal>`;
+                }
+            }
+        },
+        resolve: {
+            ListDefinition: ['CredentialList', list => list],
+            Dataset: ['ListDefinition', 'QuerySet', '$stateParams', 'GetBasePath', (list, qs, $stateParams, GetBasePath) => {
+                const searchPath = GetBasePath('credentials');
+                return qs.search(
+                    searchPath,
+                    $stateParams[`${list.iterator}_search`]
+                );
+            }]
+        },
+        onExit ($state) {
+            if ($state.transition) {
+                $('#form-modal').modal('hide');
+                $('.modal-backdrop').remove();
+                $('body').removeClass('modal-open');
+            }
+        }
+    });
 
     $stateExtender.addState({
         name: 'instanceGroups.edit',
@@ -207,11 +385,11 @@ function InstanceGroupsRun ($stateExtender, strings) {
         resolve: {
             resolvedModels: InstanceGroupsResolve,
             Dataset: ['GetBasePath', 'QuerySet', '$stateParams',
-                function(GetBasePath, qs, $stateParams) {
+                function (GetBasePath, qs, $stateParams) {
                     let path = `${GetBasePath('instance_groups')}${$stateParams.instance_group_id}/instances`;
                     return qs.search(path, $stateParams[`instance_search`]);
                 }
-            ]
+            ],
         }
     });
 
@@ -255,21 +433,26 @@ function InstanceGroupsRun ($stateExtender, strings) {
         resolve: {
             resolvedModels: InstanceGroupsResolve,
             Dataset: ['GetBasePath', 'QuerySet', '$stateParams',
-                function(GetBasePath, qs, $stateParams) {
+                function (GetBasePath, qs, $stateParams) {
                     let path = `${GetBasePath('instances')}`;
                     return qs.search(path, $stateParams[`add_instance_search`]);
                 }
-            ]
+            ],
+            routeData: [function () {
+                return "instanceGroups.instances";
+            }]
         }
     });
 
     $stateExtender.addState(instanceJobsRoute);
     $stateExtender.addState(instanceGroupJobsRoute);
+    $stateExtender.addState(containerGroupJobsRoute);
 }
 
 InstanceGroupsRun.$inject = [
     '$stateExtender',
-    'InstanceGroupsStrings'
+    'InstanceGroupsStrings',
+    'Rest'
 ];
 
 angular.module(MODULE_NAME, [])

awx/ui/client/src/shared/lookup/lookup-modal.directive.js

@@ -74,7 +74,7 @@ export default ['templateUrl', 'i18n', function(templateUrl, i18n) {
                 }
             }
 
-            $scope.saveForm = function() {
+            $scope.saveForm = function () {
                 eventService.remove(listeners);
                 let list = $scope.list;
                 if($scope.currentSelection.name !== null) {
@@ -89,7 +89,7 @@ export default ['templateUrl', 'i18n', function(templateUrl, i18n) {
                 $state.go('^');
             };
 
-            $scope.toggle_row = function(selectedRow) {
+            $scope.toggle_row = function (selectedRow) {
                 let list = $scope.list;
                 let count = 0;
                 $scope[list.name].forEach(function(row) {

awxkit/awxkit/api/pages/credentials.py

@@ -80,6 +80,7 @@ credential_type_name_to_config_kind_map = {
     'ansible tower': 'tower',
     'google compute engine': 'gce',
     'insights': 'insights',
+    'openshift or kubernetes api bearer token': 'kubernetes',
     'microsoft azure classic (deprecated)': 'azure_classic',
     'microsoft azure resource manager': 'azure_rm',
     'network': 'net',

docs/container_groups.md

@@ -0,0 +1,62 @@
+# Container Groups
+
+In a traditional AWX installation, jobs (ansible-playbook runs) are executed
+either directly on a member of the cluster or on a pre-provisioned "isolated"
+node. 
+
+The concept of a Container Group (working name) allows for job environments to
+be provisioned on-demand as a Pod that exists only for the duration of the
+playbook run. This is known as the ephemeral execution model and ensures a clean
+environment for every job run. 
+
+In some cases it is desireable to have the execution environment be "always-on",
+this is is done by manually creating an instance through the AWX API or UI. 
+
+
+## Configuration
+
+A `ContainerGroup` is simply an `InstanceGroup` that has an associated Credential
+that allows for connecting to an OpenShift or Kubernetes cluster.
+
+To create a new type, add a new `ManagedCredentialType` to
+`awx/main/models/credential/__init__.py` where `kind='kubernetes'`.
+
+### Create Credential
+
+A `Credential` must be created where the associated `CredentialType` is one of:
+
+- `openshift_username_password`
+- `openshift_token`
+- `kubernetes_bearer_token`
+
+### Create a Container Groupp
+
+Once this `Credential` has been associated with an `InstanceGroup`, the
+`InstanceGroup.kubernetes` property will return `True`.
+
+#### Pod Customization
+
+There will be a very simple default pod spec that lives in code.
+
+A custom YAML document may be provided which will be merged on top of the
+default pod spec.
+
+This will allow the UI to implement whatever fields necessary, because
+any custom fields (think 'image' or 'namespace') can be "serialized" as valid
+`Pod` JSON or YAML. A full list of options can be found in the Kubernetes
+documentation
+[here](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#pod-v1-core).
+
+```bash
+cat > api_request.json <<EOF
+{
+  "pod_spec_override": "spec:\n  containers:\n    - image: my-custom-image"
+}
+EOF
+
+curl -Lk --user 'admin:password' \
+     -X PATCH \
+     -d @api_request.json \
+     -H 'Content-Type: application/json' \
+     https://localhost:8043/api/v2/instance_groups/2/
+```

docs/container_groups/README.md

@@ -0,0 +1,72 @@
+# Container Groups
+
+In a traditional AWX installation, jobs (ansible-playbook runs) are executed
+either directly on a member of the cluster or on a pre-provisioned "isolated"
+node. 
+
+The concept of a Container Group (working name) allows for job environments to
+be provisioned on-demand as a Pod that exists only for the duration of the
+playbook run. This is known as the ephemeral execution model and ensures a clean
+environment for every job run.
+
+## Configuration
+
+A `ContainerGroup` is simply an `InstanceGroup` that has an associated Credential
+that allows for connecting to an OpenShift or Kubernetes cluster.
+
+To create a new type, add a new `ManagedCredentialType` to
+`awx/main/models/credential/__init__.py` where `kind='kubernetes'`.
+
+### Create Credential
+
+A `Credential` must be created where the associated `CredentialType` is one of:
+
+- `kubernetes_bearer_token`
+
+Other credential types (such as username/password) may be added in the future.
+
+### Create a Container Groupp
+
+Once this `Credential` has been associated with an `InstanceGroup`, the
+`InstanceGroup.kubernetes` property will return `True`.
+
+#### Pod Customization
+
+There will be a very simple default pod spec that lives in code.
+
+A custom YAML document may be provided. This will allow the UI to implement
+whatever fields necessary, because any custom fields (think 'image' or
+'namespace') can be "serialized" as valid `Pod` JSON or YAML. A full list of
+options can be found in the Kubernetes documentation
+[here](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.15/#pod-v1-core).
+
+```bash
+cat > api_request.json <<EOF
+{
+  "apiVersion": "v1",
+  "kind": "Pod",
+  "metadata": {
+    "namespace": "my-namespace"
+  },
+  "spec": {
+    "containers": [
+      {
+        "args": [
+          "sleep",
+          "infinity"
+        ],
+        "image": "my-custom-image",
+        "stdin": true,
+        "tty": true
+      }
+    ]
+  }
+}
+EOF
+
+curl -Lk --user 'admin:password' \
+     -X PATCH \
+     -d @api_request.json \
+     -H 'Content-Type: application/json' \
+     https://localhost:8043/api/v2/instance_groups/2/
+```

docs/container_groups/service-account.yml

@@ -0,0 +1,42 @@
+#
+# Create service account / role / role-binding:
+# oc -n my-namespace apply -f service-account.yml
+#
+# Obtain API token and CA cert:
+#
+# $ SECRET_NAME=$(oc -n my-namespace get sa awx -o json | jq -r '.secrets[0].name')
+# $ oc -n my-namespace get secret $SECRET_NAME  -o json | jq -r '.data["token"]  | @base64d'
+# $ oc -n my-namespace get secret $SECRET_NAME  -o json | jq -r '.data["ca.crt"] | @base64d'
+#
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: awx
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: pod-manager
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["pods"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+- apiGroups: [""]
+  resources: ["pods/exec"]
+  verbs: ["create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: awx-pod-manager
+subjects:
+- kind: ServiceAccount
+  name: awx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pod-manager

docs/licenses/dictdiffer.txt

@@ -0,0 +1,28 @@
+Dictdiffer is free software; you can redistribute it and/or modify it
+under the terms of the MIT License quoted below.
+
+Copyright (C) 2013 Fatih Erikli.
+Copyright (C) 2013, 2014 CERN.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+In applying this license, CERN does not waive the privileges and
+immunities granted to it by virtue of its status as an
+Intergovernmental Organization or submit itself to any jurisdiction.

docs/licenses/kubernetes.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2014 The Kubernetes Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

docs/licenses/openshift.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

docs/licenses/python-string-utils.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Davide Zanotti
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

docs/licenses/ruamel.yaml.txt

@@ -0,0 +1,21 @@
+ The MIT License (MIT)
+
+ Copyright (c) 2014-2019 Anthon van der Neut, Ruamel bvba
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.

requirements/requirements.in

@@ -27,6 +27,7 @@ irc==16.2
 jinja2==2.10.1
 jsonschema==2.6.0
 Markdown==2.6.11    # used for formatting API help
+openshift==0.9.0
 ordereddict==1.1
 prometheus_client==0.6.0
 psutil==5.4.3

requirements/requirements.txt

@@ -14,8 +14,9 @@ azure-keyvault==1.1.0
 azure-nspkg==3.0.2        # via azure-keyvault
 billiard==3.6.0.0         # via celery
 boto==2.47.0
+cachetools==3.1.1         # via google-auth
 celery==4.3.0
-certifi==2019.3.9         # via msrest, requests
+certifi==2019.3.9         # via kubernetes, msrest, requests
 cffi==1.12.3              # via cryptography
 channels==1.1.8
 chardet==3.0.4            # via requests
@@ -23,6 +24,7 @@ constantly==15.1.0        # via twisted
 cryptography==2.6.1       # via adal, azure-keyvault, pyopenssl, service-identity
 daphne==1.3.0
 defusedxml==0.5.0         # via python3-openid, python3-saml, social-auth-core
+dictdiffer==0.8.0         # via openshift
 django-auth-ldap==1.7.0
 django-cors-headers==2.4.0
 django-crum==0.7.2
@@ -42,6 +44,7 @@ djangorestframework==3.9.4
 future==0.16.0            # via django-radius
 gitdb2==2.0.5             # via gitpython
 gitpython==2.1.11
+google-auth==1.6.3        # via kubernetes
 hyperlink==19.0.0         # via twisted
 idna==2.8                 # via hyperlink, requests, twisted
 incremental==17.5.0       # via twisted
@@ -59,6 +62,7 @@ jinja2==2.10.1
 jsonpickle==1.2           # via asgi-amqp
 jsonschema==2.6.0
 kombu==4.5.0              # via asgi-amqp, celery
+kubernetes==9.0.0         # via openshift
 lockfile==0.12.2          # via python-daemon
 lxml==4.3.3               # via xmlsec
 markdown==2.6.11
@@ -69,6 +73,7 @@ msrest==0.6.6             # via azure-keyvault, msrestazure
 msrestazure==0.6.0        # via azure-keyvault
 netaddr==0.7.19           # via pyrad
 oauthlib==3.0.1           # via django-oauth-toolkit, requests-oauthlib, social-auth-core
+openshift==0.9.0
 ordereddict==1.1
 pexpect==4.6.0            # via ansible-runner
 pkgconfig==1.5.1          # via xmlsec
@@ -76,8 +81,8 @@ prometheus_client==0.6.0
 psutil==5.4.3
 psycopg2==2.8.2
 ptyprocess==0.6.0         # via pexpect
-pyasn1-modules==0.2.5     # via python-ldap, service-identity
+pyasn1-modules==0.2.5     # via google-auth, python-ldap, service-identity
-pyasn1==0.4.5             # via pyasn1-modules, python-ldap, service-identity
+pyasn1==0.4.5             # via pyasn1-modules, python-ldap, rsa, service-identity
 pycparser==2.19           # via cffi
 pygerduty==0.37.0
 pyhamcrest==1.9.0         # via twisted
@@ -91,16 +96,19 @@ python-dateutil==2.7.2
 python-ldap==3.2.0        # via django-auth-ldap
 python-memcached==1.59
 python-radius==1.0
+python-string-utils==0.6.0  # via openshift
 python3-openid==3.1.0     # via social-auth-core
 python3-saml==1.4.0
 pytz==2019.1              # via celery, django, irc, tempora, twilio
-pyyaml==5.1               # via ansible-runner, djangorestframework-yaml
+pyyaml==5.1               # via ansible-runner, djangorestframework-yaml, kubernetes
 requests-futures==0.9.7
-requests-oauthlib==1.2.0  # via msrest, social-auth-core
+requests-oauthlib==1.2.0  # via kubernetes, msrest, social-auth-core
 requests==2.21.0
+rsa==4.0                  # via google-auth
+ruamel.yaml==0.15.99      # via openshift
 service-identity==18.1.0  # via twisted
 simplejson==3.16.0        # via uwsgitop
-six==1.12.0               # via ansible-runner, asgi-amqp, asgiref, autobahn, automat, cryptography, django-extensions, irc, isodate, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.stream, pygerduty, pyhamcrest, pyopenssl, pyrad, python-dateutil, python-memcached, slackclient, social-auth-app-django, social-auth-core, tacacs-plus, tempora, twilio, txaio, websocket-client
+six==1.12.0               # via ansible-runner, asgi-amqp, asgiref, autobahn, automat, cryptography, django-extensions, google-auth, irc, isodate, jaraco.classes, jaraco.collections, jaraco.itertools, jaraco.logging, jaraco.stream, kubernetes, openshift, pygerduty, pyhamcrest, pyopenssl, pyrad, python-dateutil, python-memcached, slackclient, social-auth-app-django, social-auth-core, tacacs-plus, tempora, twilio, txaio, websocket-client
 slackclient==1.1.2
 smmap2==2.0.5             # via gitdb2
 social-auth-app-django==2.1.0
@@ -112,11 +120,11 @@ twilio==6.10.4
 twisted[tls]==19.2.0
 txaio==18.8.1             # via autobahn
 typing==3.6.6             # via django-extensions
-urllib3==1.24.3           # via requests
+urllib3==1.24.3           # via kubernetes, requests
 uwsgi==2.0.17
 uwsgitop==0.10.0
 vine==1.3.0               # via amqp, celery
-websocket-client==0.56.0  # via slackclient
+websocket-client==0.56.0  # via kubernetes, slackclient
 xmlsec==1.3.3             # via python3-saml
 zope.interface==4.6.0     # via twisted
 

requirements/requirements_setup_requires.txt

@@ -1,5 +1,5 @@
 pbr>=1.8
-setuptools_scm>=1.15.0,<3.0
+setuptools_scm>=3.1.0
 vcversioner>=2.16.0.0
 pytest-runner
 isort

tools/docker-compose/Dockerfile

@@ -124,6 +124,14 @@ RUN cd sqlite-autoconf-3290000 && ./configure && make && make install
 RUN mv -f /usr/local/lib/libsqlite3.so.0 /lib64/
 RUN mv -f /usr/local/lib/libsqlite3.so.0.8.6 /lib64/
 
+# Install OpenShift CLI
+RUN cd /usr/local/bin && \
+    curl -L https://github.com/openshift/origin/releases/download/v3.9.0/openshift-origin-client-tools-v3.9.0-191fece-linux-64bit.tar.gz | \
+    tar -xz --strip-components=1 --wildcards --no-anchored 'oc'
+
+ADD tools/docker-compose/google-cloud-sdk.repo /etc/yum.repos.d/
+RUN yum install -y kubectl
+
 RUN yum -y remove cyrus-sasl-devel \
   gcc \
   gcc-c++ \

tools/docker-compose/google-cloud-sdk.repo

@@ -0,0 +1,8 @@
+[google-cloud-sdk]
+name=Google Cloud SDK
+baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64
+enabled=1
+gpgcheck=1
+repo_gpgcheck=1
+gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
+       https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg