Merge pull request #1058 from ansible/scalable_clustering

Implement Container Cluster-based dynamic scaling

Makefile

@@ -23,7 +23,7 @@ COMPOSE_HOST ?= $(shell hostname)
 
 VENV_BASE ?= /venv
 SCL_PREFIX ?=
-CELERY_SCHEDULE_FILE ?= /celerybeat-schedule
+CELERY_SCHEDULE_FILE ?= /var/lib/awx/beat.db
 
 DEV_DOCKER_TAG_BASE ?= gcr.io/ansible-tower-engineering
 # Python packages to install only from source (not from binary wheels)
@@ -216,13 +216,11 @@ init:
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
 	$(MANAGEMENT_COMMAND) provision_instance --hostname=$(COMPOSE_HOST); \
-	$(MANAGEMENT_COMMAND) register_queue --queuename=tower --hostnames=$(COMPOSE_HOST);\
+	$(MANAGEMENT_COMMAND) register_queue --queuename=tower --instance_percent=100;\
 	if [ "$(AWX_GROUP_QUEUES)" == "tower,thepentagon" ]; then \
 		$(MANAGEMENT_COMMAND) provision_instance --hostname=isolated; \
 		$(MANAGEMENT_COMMAND) register_queue --queuename='thepentagon' --hostnames=isolated --controller=tower; \
 		$(MANAGEMENT_COMMAND) generate_isolated_key | ssh -o "StrictHostKeyChecking no" root@isolated 'cat > /root/.ssh/authorized_keys'; \
-	elif [ "$(AWX_GROUP_QUEUES)" != "tower" ]; then \
-		$(MANAGEMENT_COMMAND) register_queue --queuename=$(firstword $(subst $(comma), ,$(AWX_GROUP_QUEUES))) --hostnames=$(COMPOSE_HOST); \
 	fi;
 
 # Refresh development environment after pulling new code.
@@ -326,7 +324,7 @@ celeryd:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	celery worker -A awx -l DEBUG -B -Ofair --autoscale=100,4 --schedule=$(CELERY_SCHEDULE_FILE) -Q tower_scheduler,tower_broadcast_all,$(COMPOSE_HOST),$(AWX_GROUP_QUEUES) -n celery@$(COMPOSE_HOST) --pidfile /tmp/celery_pid
+	celery worker -A awx -l DEBUG -B -Ofair --autoscale=100,4 --schedule=$(CELERY_SCHEDULE_FILE) -Q tower_broadcast_all -n celery@$(COMPOSE_HOST) --pidfile /tmp/celery_pid
 
 # Run to start the zeromq callback receiver
 receiver:

awx/api/serializers.py

@@ -3977,8 +3977,10 @@ class InstanceSerializer(BaseSerializer):
 
     class Meta:
         model = Instance
-        fields = ("id", "type", "url", "related", "uuid", "hostname", "created", "modified",
+        read_only_fields = ('uuid', 'hostname', 'version')
-                  "version", "capacity", "consumed_capacity", "percent_capacity_remaining", "jobs_running")
+        fields = ("id", "type", "url", "related", "uuid", "hostname", "created", "modified", 'capacity_adjustment',
+                  "version", "capacity", "consumed_capacity", "percent_capacity_remaining", "jobs_running",
+                  "cpu", "memory", "cpu_capacity", "mem_capacity", "enabled")
 
     def get_related(self, obj):
         res = super(InstanceSerializer, self).get_related(obj)
@@ -4011,7 +4013,8 @@ class InstanceGroupSerializer(BaseSerializer):
         model = InstanceGroup
         fields = ("id", "type", "url", "related", "name", "created", "modified",
                   "capacity", "committed_capacity", "consumed_capacity",
-                  "percent_capacity_remaining", "jobs_running", "instances", "controller")
+                  "percent_capacity_remaining", "jobs_running", "instances", "controller",
+                  "policy_instance_percentage", "policy_instance_minimum", "policy_instance_list")
 
     def get_related(self, obj):
         res = super(InstanceGroupSerializer, self).get_related(obj)

awx/api/views.py

@@ -57,7 +57,7 @@ import pytz
 from wsgiref.util import FileWrapper
 
 # AWX
-from awx.main.tasks import send_notifications
+from awx.main.tasks import send_notifications, handle_ha_toplogy_changes
 from awx.main.access import get_user_queryset
 from awx.main.ha import is_ha_environment
 from awx.api.authentication import TokenGetAuthentication
@@ -148,6 +148,41 @@ class UnifiedJobDeletionMixin(object):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
+class InstanceGroupMembershipMixin(object):
+    '''
+    Manages signaling celery to reload its queue configuration on Instance Group membership changes
+    '''
+    def attach(self, request, *args, **kwargs):
+        response = super(InstanceGroupMembershipMixin, self).attach(request, *args, **kwargs)
+        sub_id, res = self.attach_validate(request)
+        if status.is_success(response.status_code):
+            if self.parent_model is Instance:
+                ig_obj = get_object_or_400(self.model, pk=sub_id)
+                inst_name = ig_obj.hostname
+            else:
+                ig_obj = self.get_parent_object()
+                inst_name = get_object_or_400(self.model, pk=sub_id).hostname
+            if inst_name not in ig_obj.policy_instance_list:
+                ig_obj.policy_instance_list.append(inst_name)
+                ig_obj.save()
+        return response
+
+    def unattach(self, request, *args, **kwargs):
+        response = super(InstanceGroupMembershipMixin, self).unattach(request, *args, **kwargs)
+        sub_id, res = self.attach_validate(request)
+        if status.is_success(response.status_code):
+            if self.parent_model is Instance:
+                ig_obj = get_object_or_400(self.model, pk=sub_id)
+                inst_name = self.get_parent_object().hostname
+            else:
+                ig_obj = self.get_parent_object()
+                inst_name = get_object_or_400(self.model, pk=sub_id).hostname
+            if inst_name in ig_obj.policy_instance_list:
+                ig_obj.policy_instance_list.pop(ig_obj.policy_instance_list.index(inst_name))
+                ig_obj.save()
+        return response
+
+
 class ApiRootView(APIView):
 
     authentication_classes = []
@@ -525,7 +560,7 @@ class InstanceList(ListAPIView):
     new_in_320 = True
 
 
-class InstanceDetail(RetrieveAPIView):
+class InstanceDetail(RetrieveUpdateAPIView):
 
     view_name = _("Instance Detail")
     model = Instance
@@ -533,6 +568,20 @@ class InstanceDetail(RetrieveAPIView):
     new_in_320 = True
 
 
+    def update(self, request, *args, **kwargs):
+        r = super(InstanceDetail, self).update(request, *args, **kwargs)
+        if status.is_success(r.status_code):
+            obj = self.get_object()
+            if obj.enabled:
+                obj.refresh_capacity()
+            else:
+                obj.capacity = 0
+            obj.save()
+            handle_ha_toplogy_changes.apply_async()
+            r.data = InstanceSerializer(obj, context=self.get_serializer_context()).to_representation(obj)
+        return r
+
+
 class InstanceUnifiedJobsList(SubListAPIView):
 
     view_name = _("Instance Running Jobs")
@@ -548,7 +597,7 @@ class InstanceUnifiedJobsList(SubListAPIView):
         return qs
 
 
-class InstanceInstanceGroupsList(SubListAPIView):
+class InstanceInstanceGroupsList(InstanceGroupMembershipMixin, SubListCreateAttachDetachAPIView):
 
     view_name = _("Instance's Instance Groups")
     model = InstanceGroup
@@ -558,7 +607,7 @@ class InstanceInstanceGroupsList(SubListAPIView):
     relationship = 'rampart_groups'
 
 
-class InstanceGroupList(ListAPIView):
+class InstanceGroupList(ListCreateAPIView):
 
     view_name = _("Instance Groups")
     model = InstanceGroup
@@ -566,7 +615,7 @@ class InstanceGroupList(ListAPIView):
     new_in_320 = True
 
 
-class InstanceGroupDetail(RetrieveAPIView):
+class InstanceGroupDetail(RetrieveUpdateDestroyAPIView):
 
     view_name = _("Instance Group Detail")
     model = InstanceGroup
@@ -584,7 +633,7 @@ class InstanceGroupUnifiedJobsList(SubListAPIView):
     new_in_320 = True
 
 
-class InstanceGroupInstanceList(SubListAPIView):
+class InstanceGroupInstanceList(InstanceGroupMembershipMixin, SubListAttachDetachAPIView):
 
     view_name = _("Instance Group's Instances")
     model = Instance

awx/main/access.py

@@ -424,6 +424,18 @@ class InstanceAccess(BaseAccess):
         return Instance.objects.filter(
             rampart_groups__in=self.user.get_queryset(InstanceGroup)).distinct()
 
+
+    def can_attach(self, obj, sub_obj, relationship, data,
+                   skip_sub_obj_read_check=False):
+        if relationship == 'rampart_groups' and isinstance(sub_obj, InstanceGroup):
+            return self.user.is_superuser
+        return super(InstanceAccess, self).can_attach(obj, sub_obj, relationship, *args, **kwargs)
+
+    def can_unattach(self, obj, sub_obj, relationship, data=None):
+        if relationship == 'rampart_groups' and isinstance(sub_obj, InstanceGroup):
+            return self.user.is_superuser
+        return super(InstanceAccess, self).can_unattach(obj, sub_obj, relationship, *args, **kwargs)
+
     def can_add(self, data):
         return False
 
@@ -444,13 +456,13 @@ class InstanceGroupAccess(BaseAccess):
             organization__in=Organization.accessible_pk_qs(self.user, 'admin_role'))
 
     def can_add(self, data):
-        return False
+        return self.user.is_superuser
 
     def can_change(self, obj, data):
-        return False
+        return self.user.is_superuser
 
     def can_delete(self, obj):
-        return False
+        return self.user.is_superuser
 
 
 class UserAccess(BaseAccess):

awx/main/management/commands/register_queue.py

@@ -17,6 +17,10 @@ class Command(BaseCommand):
                             help='Comma-Delimited Hosts to add to the Queue')
         parser.add_argument('--controller', dest='controller', type=str,
                             default='', help='The controlling group (makes this an isolated group)')
+        parser.add_argument('--instance_percent', dest='instance_percent', type=int, default=0,
+                            help='The percentage of active instances that will be assigned to this group'),
+        parser.add_argument('--instance_minimum', dest='instance_minimum', type=int, default=0,
+                            help='The minimum number of instance that will be retained for this group from available instances')
 
     def handle(self, **options):
         queuename = options.get('queuename')
@@ -38,7 +42,9 @@ class Command(BaseCommand):
                     changed = True
             else:
                 print("Creating instance group {}".format(queuename))
-                ig = InstanceGroup(name=queuename)
+                ig = InstanceGroup(name=queuename,
+                                   policy_instance_percentage=options.get('instance_percent'),
+                                   policy_instance_minimum=options.get('instance_minimum'))
                 if control_ig:
                     ig.controller = control_ig
                 ig.save()
@@ -60,5 +66,7 @@ class Command(BaseCommand):
                     sys.exit(1)
                 else:
                     print("Instance already registered {}".format(instance[0].hostname))
+            ig.policy_instance_list = instance_list
+            ig.save()
             if changed:
                 print('(changed: True)')

awx/main/managers.py

@@ -2,12 +2,9 @@
 # All Rights Reserved.
 
 import sys
-from datetime import timedelta
 import logging
 
 from django.db import models
-from django.utils.timezone import now
-from django.db.models import Sum
 from django.conf import settings
 
 from awx.main.utils.filters import SmartFilter
@@ -93,11 +90,6 @@ class InstanceManager(models.Manager):
         """Return count of active Tower nodes for licensing."""
         return self.all().count()
 
-    def total_capacity(self):
-        sumval = self.filter(modified__gte=now() - timedelta(seconds=settings.AWX_ACTIVE_NODE_TIME)) \
-                     .aggregate(total_capacity=Sum('capacity'))['total_capacity']
-        return max(50, sumval)
-
     def my_role(self):
         # NOTE: TODO: Likely to repurpose this once standalone ramparts are a thing
         return "tower"

awx/main/migrations/0020_v330_instancegroup_policies.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+from decimal import Decimal
+import awx.main.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0019_v330_custom_virtualenv'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='instancegroup',
+            name='policy_instance_list',
+            field=awx.main.fields.JSONField(default=[], help_text='List of exact-match Instances that will always be automatically assigned to this group',
+                                            blank=True),
+        ),
+        migrations.AddField(
+            model_name='instancegroup',
+            name='policy_instance_minimum',
+            field=models.IntegerField(default=0, help_text='Static minimum number of Instances to automatically assign to this group'),
+        ),
+        migrations.AddField(
+            model_name='instancegroup',
+            name='policy_instance_percentage',
+            field=models.IntegerField(default=0, help_text='Percentage of Instances to automatically assign to this group'),
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='capacity_adjustment',
+            field=models.DecimalField(decimal_places=2, default=Decimal('1.0'), max_digits=3),
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='cpu',
+            field=models.IntegerField(default=0, editable=False)
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='memory',
+            field=models.BigIntegerField(default=0, editable=False)
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='cpu_capacity',
+            field=models.IntegerField(default=0, editable=False)
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='mem_capacity',
+            field=models.IntegerField(default=0, editable=False)
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='enabled',
+            field=models.BooleanField(default=True)
+        )
+    ]

awx/main/models/ad_hoc_commands.py

@@ -184,7 +184,7 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
         # NOTE: We sorta have to assume the host count matches and that forks default to 5
         from awx.main.models.inventory import Host
         count_hosts = Host.objects.filter( enabled=True, inventory__ad_hoc_commands__pk=self.pk).count()
-        return min(count_hosts, 5 if self.forks == 0 else self.forks) * 10
+        return min(count_hosts, 5 if self.forks == 0 else self.forks) + 1
 
     def copy(self):
         data = {}

awx/main/models/ha.py

@@ -1,8 +1,10 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved.
 
-from django.db import models
+from decimal import Decimal
-from django.db.models.signals import post_save
+
+from django.db import models, connection
+from django.db.models.signals import post_save, post_delete
 from django.dispatch import receiver
 from django.utils.translation import ugettext_lazy as _
 from django.conf import settings
@@ -10,12 +12,15 @@ from django.utils.timezone import now, timedelta
 
 from solo.models import SingletonModel
 
+from awx import __version__ as awx_application_version
 from awx.api.versioning import reverse
 from awx.main.managers import InstanceManager, InstanceGroupManager
+from awx.main.fields import JSONField
 from awx.main.models.inventory import InventoryUpdate
 from awx.main.models.jobs import Job
 from awx.main.models.projects import ProjectUpdate
 from awx.main.models.unified_jobs import UnifiedJob
+from awx.main.utils import get_cpu_capacity, get_mem_capacity, get_system_task_capacity
 
 __all__ = ('Instance', 'InstanceGroup', 'JobOrigin', 'TowerScheduleState',)
 
@@ -38,6 +43,30 @@ class Instance(models.Model):
         default=100,
         editable=False,
     )
+    capacity_adjustment = models.DecimalField(
+        default=Decimal(1.0),
+        max_digits=3,
+        decimal_places=2,
+    )
+    enabled = models.BooleanField(
+        default=True
+    )
+    cpu = models.IntegerField(
+        default=0,
+        editable=False,
+    )
+    memory = models.BigIntegerField(
+        default=0,
+        editable=False,
+    )
+    cpu_capacity = models.IntegerField(
+        default=0,
+        editable=False,
+    )
+    mem_capacity = models.IntegerField(
+        default=0,
+        editable=False,
+    )
 
     class Meta:
         app_label = 'main'
@@ -63,6 +92,23 @@ class Instance(models.Model):
             grace_period = settings.AWX_ISOLATED_PERIODIC_CHECK * 2
         return self.modified < ref_time - timedelta(seconds=grace_period)
 
+    def is_controller(self):
+        return Instance.objects.filter(rampart_groups__controller__instances=self).exists()
+
+
+    def refresh_capacity(self):
+        cpu = get_cpu_capacity()
+        mem = get_mem_capacity()
+        self.capacity = get_system_task_capacity(self.capacity_adjustment)
+        self.cpu = cpu[0]
+        self.memory = mem[0]
+        self.cpu_capacity = cpu[1]
+        self.mem_capacity = mem[1]
+        self.version = awx_application_version
+        self.save(update_fields=['capacity', 'version', 'modified', 'cpu',
+                                 'memory', 'cpu_capacity', 'mem_capacity'])
+
+    
 
 class InstanceGroup(models.Model):
     """A model representing a Queue/Group of AWX Instances."""
@@ -85,6 +131,19 @@ class InstanceGroup(models.Model):
         default=None,
         null=True
     )
+    policy_instance_percentage = models.IntegerField(
+        default=0,
+        help_text=_("Percentage of Instances to automatically assign to this group")
+    )
+    policy_instance_minimum = models.IntegerField(
+        default=0,
+        help_text=_("Static minimum number of Instances to automatically assign to this group")
+    )
+    policy_instance_list = JSONField(
+        default=[],
+        blank=True,
+        help_text=_("List of exact-match Instances that will always be automatically assigned to this group")
+    )
 
     def get_absolute_url(self, request=None):
         return reverse('api:instance_group_detail', kwargs={'pk': self.pk}, request=request)
@@ -119,6 +178,32 @@ class JobOrigin(models.Model):
         app_label = 'main'
 
 
+@receiver(post_save, sender=InstanceGroup)
+def on_instance_group_saved(sender, instance, created=False, raw=False, **kwargs):
+    if created:
+        from awx.main.tasks import apply_cluster_membership_policies
+        connection.on_commit(lambda: apply_cluster_membership_policies.apply_async())
+
+
+@receiver(post_save, sender=Instance)
+def on_instance_saved(sender, instance, created=False, raw=False, **kwargs):
+    if created:
+        from awx.main.tasks import apply_cluster_membership_policies
+        connection.on_commit(lambda: apply_cluster_membership_policies.apply_async())
+
+
+@receiver(post_delete, sender=InstanceGroup)
+def on_instance_group_deleted(sender, instance, using, **kwargs):
+    from awx.main.tasks import apply_cluster_membership_policies
+    connection.on_commit(lambda: apply_cluster_membership_policies.apply_async())
+
+
+@receiver(post_delete, sender=Instance)
+def on_instance_deleted(sender, instance, using, **kwargs):
+    from awx.main.tasks import apply_cluster_membership_policies
+    connection.on_commit(lambda: apply_cluster_membership_policies.apply_async())
+
+
 # Unfortunately, the signal can't just be connected against UnifiedJob; it
 # turns out that creating a model's subclass doesn't fire the signal for the
 # superclass model.

awx/main/models/inventory.py

@@ -1602,7 +1602,7 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin,
 
     @property
     def task_impact(self):
-        return 50
+        return 1
 
     # InventoryUpdate credential required
     # Custom and SCM InventoryUpdate credential not required

awx/main/models/jobs.py

@@ -623,7 +623,7 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
             count_hosts = 1
         else:
             count_hosts = Host.objects.filter(inventory__jobs__pk=self.pk).count()
-        return min(count_hosts, 5 if self.forks == 0 else self.forks) * 10
+        return min(count_hosts, 5 if self.forks == 0 else self.forks) + 1
 
     @property
     def successful_hosts(self):
@@ -1190,7 +1190,7 @@ class SystemJob(UnifiedJob, SystemJobOptions, JobNotificationMixin):
 
     @property
     def task_impact(self):
-        return 150
+        return 5
 
     @property
     def preferred_instance_groups(self):

awx/main/models/projects.py

@@ -492,7 +492,7 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
 
     @property
     def task_impact(self):
-        return 0 if self.job_type == 'run' else 20
+        return 0 if self.job_type == 'run' else 1
 
     @property
     def result_stdout(self):

awx/main/scheduler/tasks.py

@@ -21,12 +21,12 @@ class LogErrorsTask(Task):
         super(LogErrorsTask, self).on_failure(exc, task_id, args, kwargs, einfo)
 
 
-@shared_task
+@shared_task(base=LogErrorsTask)
 def run_job_launch(job_id):
     TaskManager().schedule()
 
 
-@shared_task
+@shared_task(base=LogErrorsTask)
 def run_job_complete(job_id):
     TaskManager().schedule()
 

awx/main/tasks.py

@@ -2,7 +2,7 @@
 # All Rights Reserved.
 
 # Python
-from collections import OrderedDict
+from collections import OrderedDict, namedtuple
 import ConfigParser
 import cStringIO
 import functools
@@ -25,8 +25,8 @@ except Exception:
     psutil = None
 
 # Celery
-from celery import Task, shared_task
+from celery import Task, shared_task, Celery
-from celery.signals import celeryd_init, worker_process_init, worker_shutdown
+from celery.signals import celeryd_init, worker_process_init, worker_shutdown, worker_ready, celeryd_after_setup
 
 # Django
 from django.conf import settings
@@ -53,16 +53,17 @@ from awx.main.queue import CallbackQueueDispatcher
 from awx.main.expect import run, isolated_manager
 from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, update_scm_url,
                             check_proot_installed, build_proot_temp_dir, get_licenser,
-                            wrap_args_with_proot, get_system_task_capacity, OutputEventFilter,
+                            wrap_args_with_proot, OutputEventFilter, ignore_inventory_computed_fields,
-                            ignore_inventory_computed_fields, ignore_inventory_group_removal,
+                            ignore_inventory_group_removal, get_type_for_model, extract_ansible_vars)
-                            get_type_for_model, extract_ansible_vars)
 from awx.main.utils.reload import restart_local_services, stop_local_services
+from awx.main.utils.pglock import advisory_lock
+from awx.main.utils.ha import update_celery_worker_routes, register_celery_worker_queues
 from awx.main.utils.handlers import configure_external_logger
 from awx.main.consumers import emit_channel_notification
 from awx.conf import settings_registry
 
 __all__ = ['RunJob', 'RunSystemJob', 'RunProjectUpdate', 'RunInventoryUpdate',
-           'RunAdHocCommand', 'handle_work_error', 'handle_work_success',
+           'RunAdHocCommand', 'handle_work_error', 'handle_work_success', 'apply_cluster_membership_policies',
            'update_inventory_computed_fields', 'update_host_smart_inventory_memberships',
            'send_notifications', 'run_administrative_checks', 'purge_old_stdout_files']
 
@@ -130,6 +131,56 @@ def inform_cluster_of_shutdown(*args, **kwargs):
         logger.exception('Encountered problem with normal shutdown signal.')
 
 
+@shared_task(bind=True, queue='tower_instance_router', base=LogErrorsTask)
+def apply_cluster_membership_policies(self):
+    with advisory_lock('cluster_policy_lock', wait=True):
+        considered_instances = Instance.objects.all().order_by('id')
+        total_instances = considered_instances.count()
+        filtered_instances = []
+        actual_groups = []
+        actual_instances = []
+        Group = namedtuple('Group', ['obj', 'instances'])
+        Node = namedtuple('Instance', ['obj', 'groups'])
+        # Process policy instance list first, these will represent manually managed instances
+        # that will not go through automatic policy determination
+        for ig in InstanceGroup.objects.all():
+            logger.info("Considering group {}".format(ig.name))
+            ig.instances.clear()
+            group_actual = Group(obj=ig, instances=[])
+            for i in ig.policy_instance_list:
+                inst = Instance.objects.filter(hostname=i)
+                if not inst.exists():
+                    continue
+                inst = inst[0]
+                logger.info("Policy List, adding {} to {}".format(inst.hostname, ig.name))
+                group_actual.instances.append(inst.id)
+                ig.instances.add(inst)
+                filtered_instances.append(inst)
+            actual_groups.append(group_actual)
+        # Process Instance minimum policies next, since it represents a concrete lower bound to the
+        # number of instances to make available to instance groups
+        actual_instances = [Node(obj=i, groups=[]) for i in filter(lambda x: x not in filtered_instances, considered_instances)]
+        logger.info("Total instances not directly associated: {}".format(total_instances))
+        for g in sorted(actual_groups, cmp=lambda x,y: len(x.instances) - len(y.instances)):
+            for i in sorted(actual_instances, cmp=lambda x,y: len(x.groups) - len(y.groups)):
+                if len(g.instances) >= g.obj.policy_instance_minimum:
+                    break
+                logger.info("Policy minimum, adding {} to {}".format(i.obj.hostname, g.obj.name))
+                g.obj.instances.add(i.obj)
+                g.instances.append(i.obj.id)
+                i.groups.append(g.obj.id)
+        # Finally process instance policy percentages
+        for g in sorted(actual_groups, cmp=lambda x,y: len(x.instances) - len(y.instances)):
+            for i in sorted(actual_instances, cmp=lambda x,y: len(x.groups) - len(y.groups)):
+                if 100 * float(len(g.instances)) / len(actual_instances) >= g.obj.policy_instance_percentage:
+                    break
+                logger.info("Policy percentage, adding {} to {}".format(i.obj.hostname, g.obj.name))
+                g.instances.append(i.obj.id)
+                g.obj.instances.add(i.obj)
+                i.groups.append(g.obj.id)
+        handle_ha_toplogy_changes.apply_async()
+
+
 @shared_task(queue='tower_broadcast_all', bind=True, base=LogErrorsTask)
 def handle_setting_changes(self, setting_keys):
     orig_len = len(setting_keys)
@@ -147,6 +198,45 @@ def handle_setting_changes(self, setting_keys):
             break
 
 
+@shared_task(bind=True, queue='tower_broadcast_all', base=LogErrorsTask)
+def handle_ha_toplogy_changes(self):
+    instance = Instance.objects.me()
+    logger.debug("Reconfigure celeryd queues task on host {}".format(self.request.hostname))
+    awx_app = Celery('awx')
+    awx_app.config_from_object('django.conf:settings', namespace='CELERY')
+    (instance, removed_queues, added_queues) = register_celery_worker_queues(awx_app, self.request.hostname)
+    logger.info("Workers on tower node '{}' removed from queues {} and added to queues {}"
+                .format(instance.hostname, removed_queues, added_queues))
+    updated_routes = update_celery_worker_routes(instance, settings)
+    logger.info("Worker on tower node '{}' updated celery routes {} all routes are now {}"
+                .format(instance.hostname, updated_routes, self.app.conf.CELERY_TASK_ROUTES))
+
+
+@worker_ready.connect
+def handle_ha_toplogy_worker_ready(sender, **kwargs):
+    logger.debug("Configure celeryd queues task on host {}".format(sender.hostname))
+    (instance, removed_queues, added_queues) = register_celery_worker_queues(sender.app, sender.hostname)
+    logger.info("Workers on tower node '{}' unsubscribed from queues {} and subscribed to queues {}"
+                .format(instance.hostname, removed_queues, added_queues))
+
+
+@celeryd_init.connect
+def handle_update_celery_routes(sender=None, conf=None, **kwargs):
+    conf = conf if conf else sender.app.conf
+    logger.debug("Registering celery routes for {}".format(sender))
+    instance = Instance.objects.me()
+    added_routes = update_celery_worker_routes(instance, conf)
+    logger.info("Workers on tower node '{}' added routes {} all routes are now {}"
+                .format(instance.hostname, added_routes, conf.CELERY_TASK_ROUTES))
+
+
+@celeryd_after_setup.connect
+def handle_update_celery_hostname(sender, instance, **kwargs):
+    tower_instance = Instance.objects.me()
+    instance.hostname = 'celery@{}'.format(tower_instance.hostname)
+    logger.warn("Set hostname to {}".format(instance.hostname))
+
+
 @shared_task(queue='tower', base=LogErrorsTask)
 def send_notifications(notification_list, job_id=None):
     if not isinstance(notification_list, list):
@@ -215,6 +305,7 @@ def cluster_node_heartbeat(self):
     instance_list = list(Instance.objects.filter(rampart_groups__controller__isnull=True).distinct())
     this_inst = None
     lost_instances = []
+
     for inst in list(instance_list):
         if inst.hostname == settings.CLUSTER_HOST_ID:
             this_inst = inst
@@ -224,11 +315,15 @@ def cluster_node_heartbeat(self):
             instance_list.remove(inst)
     if this_inst:
         startup_event = this_inst.is_lost(ref_time=nowtime)
-        if this_inst.capacity == 0:
+        if this_inst.capacity == 0 and this_inst.enabled:
             logger.warning('Rejoining the cluster as instance {}.'.format(this_inst.hostname))
-        this_inst.capacity = get_system_task_capacity()
+        if this_inst.enabled:
-        this_inst.version = awx_application_version
+            this_inst.refresh_capacity()
-        this_inst.save(update_fields=['capacity', 'version', 'modified'])
+            handle_ha_toplogy_changes.apply_async()
+        elif this_inst.capacity != 0 and not this_inst.enabled:
+            this_inst.capacity = 0
+            this_inst.save(update_fields=['capacity'])
+            handle_ha_toplogy_changes.apply_async()
         if startup_event:
             return
     else:
@@ -237,7 +332,7 @@ def cluster_node_heartbeat(self):
     for other_inst in instance_list:
         if other_inst.version == "":
             continue
-        if Version(other_inst.version.split('-', 1)[0]) > Version(awx_application_version) and not settings.DEBUG:
+        if Version(other_inst.version.split('-', 1)[0]) > Version(awx_application_version.split('-', 1)[0]) and not settings.DEBUG:
             logger.error("Host {} reports version {}, but this node {} is at {}, shutting down".format(other_inst.hostname,
                                                                                                        other_inst.version,
                                                                                                        this_inst.hostname,
@@ -254,6 +349,10 @@ def cluster_node_heartbeat(self):
             other_inst.save(update_fields=['capacity'])
             logger.error("Host {} last checked in at {}, marked as lost.".format(
                 other_inst.hostname, other_inst.modified))
+            if settings.AWX_AUTO_DEPROVISION_INSTANCES:
+                deprovision_hostname = other_inst.hostname
+                other_inst.delete()
+                logger.info("Host {} Automatically Deprovisioned.".format(deprovision_hostname))
         except DatabaseError as e:
             if 'did not affect any rows' in str(e):
                 logger.debug('Another instance has marked {} as lost'.format(other_inst.hostname))

awx/main/tests/factories/fixtures.py

@@ -35,8 +35,9 @@ def mk_instance(persisted=True, hostname='instance.example.org'):
     return Instance.objects.get_or_create(uuid=settings.SYSTEM_UUID, hostname=hostname)[0]
 
 
-def mk_instance_group(name='tower', instance=None):
+def mk_instance_group(name='tower', instance=None, minimum=0, percentage=0):
-    ig, status = InstanceGroup.objects.get_or_create(name=name)
+    ig, status = InstanceGroup.objects.get_or_create(name=name, policy_instance_minimum=minimum,
+                                                     policy_instance_percentage=percentage)
     if instance is not None:
         if type(instance) == list:
             for i in instance:

awx/main/tests/factories/tower.py

@@ -135,8 +135,8 @@ def create_instance(name, instance_groups=None):
     return mk_instance(hostname=name)
 
 
-def create_instance_group(name, instances=None):
+def create_instance_group(name, instances=None, minimum=0, percentage=0):
-    return mk_instance_group(name=name, instance=instances)
+    return mk_instance_group(name=name, instance=instances, minimum=minimum, percentage=percentage)
 
 
 def create_survey_spec(variables=None, default_type='integer', required=True, min=None, max=None):

awx/main/tests/functional/task_management/test_rampart_groups.py

@@ -2,6 +2,8 @@ import pytest
 import mock
 from datetime import timedelta
 from awx.main.scheduler import TaskManager
+from awx.main.models import InstanceGroup
+from awx.main.tasks import apply_cluster_membership_policies
 
 
 @pytest.mark.django_db
@@ -151,3 +153,34 @@ def test_failover_group_run(instance_factory, default_instance_group, mocker,
             tm.schedule()
             mock_job.assert_has_calls([mock.call(j1, ig1, []), mock.call(j1_1, ig2, [])])
             assert mock_job.call_count == 2
+
+
+@pytest.mark.django_db
+def test_instance_group_basic_policies(instance_factory, instance_group_factory):
+    i0 = instance_factory("i0")
+    i1 = instance_factory("i1")
+    i2 = instance_factory("i2")
+    i3 = instance_factory("i3")
+    i4 = instance_factory("i4")
+    ig0 = instance_group_factory("ig0")
+    ig1 = instance_group_factory("ig1", minimum=2)
+    ig2 = instance_group_factory("ig2", percentage=50)
+    ig3 = instance_group_factory("ig3", percentage=50)
+    ig0.policy_instance_list.append(i0.hostname)
+    ig0.save()
+    apply_cluster_membership_policies()
+    ig0 = InstanceGroup.objects.get(id=ig0.id)
+    ig1 = InstanceGroup.objects.get(id=ig1.id)
+    ig2 = InstanceGroup.objects.get(id=ig2.id)
+    ig3 = InstanceGroup.objects.get(id=ig3.id)
+    assert len(ig0.instances.all()) == 1
+    assert i0 in ig0.instances.all() 
+    assert len(InstanceGroup.objects.get(id=ig1.id).instances.all()) == 2
+    assert i1 in ig1.instances.all()
+    assert i2 in ig1.instances.all()
+    assert len(InstanceGroup.objects.get(id=ig2.id).instances.all()) == 2
+    assert i3 in ig2.instances.all()
+    assert i4 in ig2.instances.all()
+    assert len(InstanceGroup.objects.get(id=ig3.id).instances.all()) == 2
+    assert i1 in ig3.instances.all()
+    assert i2 in ig3.instances.all()

awx/main/tests/functional/test_jobs.py

@@ -1,12 +1,11 @@
-from awx.main.models import (
-    Job,
-    Instance
-)
-from django.test.utils import override_settings
 import pytest
-
+import mock
 import json
 
+from awx.main.models import Job, Instance
+from awx.main.tasks import cluster_node_heartbeat
+from django.test.utils import override_settings
+
 
 @pytest.mark.django_db
 def test_orphan_unified_job_creation(instance, inventory):
@@ -20,13 +19,19 @@ def test_orphan_unified_job_creation(instance, inventory):
 
 
 @pytest.mark.django_db
+@mock.patch('awx.main.utils.common.get_cpu_capacity', lambda: (2,8))
+@mock.patch('awx.main.utils.common.get_mem_capacity', lambda: (8000,62))
+@mock.patch('awx.main.tasks.handle_ha_toplogy_changes.apply_async', lambda: True)
 def test_job_capacity_and_with_inactive_node():
-    Instance.objects.create(hostname='test-1', capacity=50)
+    i = Instance.objects.create(hostname='test-1')
-    assert Instance.objects.total_capacity() == 50
+    i.refresh_capacity()
-    Instance.objects.create(hostname='test-2', capacity=50)
+    assert i.capacity == 62
-    assert Instance.objects.total_capacity() == 100
+    i.enabled = False
-    with override_settings(AWX_ACTIVE_NODE_TIME=0):
+    i.save()
-        assert Instance.objects.total_capacity() < 100
+    with override_settings(CLUSTER_HOST_ID=i.hostname):
+        cluster_node_heartbeat()
+        i = Instance.objects.get(id=i.id)
+        assert i.capacity == 0
 
 
 @pytest.mark.django_db

awx/main/tests/unit/utils/test_ha.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2017 Ansible Tower by Red Hat
+# All Rights Reserved.
+
+# python
+import pytest
+import mock
+
+# AWX
+from awx.main.utils.ha import (
+    _add_remove_celery_worker_queues,
+    update_celery_worker_routes,
+)
+
+
+@pytest.fixture
+def conf():
+    class Conf():
+        CELERY_TASK_ROUTES = dict()
+        CELERYBEAT_SCHEDULE = dict()
+    return Conf()
+
+
+class TestAddRemoveCeleryWorkerQueues():
+    @pytest.fixture
+    def instance_generator(self, mocker):
+        def fn(groups=['east', 'west', 'north', 'south'], hostname='east-1'):
+            instance = mocker.MagicMock()
+            instance.hostname = hostname
+            instance.rampart_groups = mocker.MagicMock()
+            instance.rampart_groups.values_list = mocker.MagicMock(return_value=groups)
+
+            return instance
+        return fn
+
+    @pytest.fixture
+    def worker_queues_generator(self, mocker):
+        def fn(queues=['east', 'west']):
+            return [dict(name=n, alias='') for n in queues]
+        return fn
+
+    @pytest.fixture
+    def mock_app(self, mocker):
+        app = mocker.MagicMock()
+        app.control = mocker.MagicMock()
+        app.control.cancel_consumer = mocker.MagicMock()
+        return app
+
+    @pytest.mark.parametrize("static_queues,_worker_queues,groups,hostname,added_expected,removed_expected", [
+        (['east', 'west'], ['east', 'west', 'east-1'], [], 'east-1', [], []),
+        ([], ['east', 'west', 'east-1'], ['east', 'west'], 'east-1', [], []),
+        ([], ['east', 'west'], ['east', 'west'], 'east-1', ['east-1'], []),
+        ([], [], ['east', 'west'], 'east-1', ['east', 'west', 'east-1'], []),
+        ([], ['china', 'russia'], ['east', 'west'], 'east-1', ['east', 'west', 'east-1'], ['china', 'russia']),
+    ])
+    def test__add_remove_celery_worker_queues_noop(self, mock_app,
+                                                   instance_generator, 
+                                                   worker_queues_generator, 
+                                                   static_queues, _worker_queues, 
+                                                   groups, hostname,
+                                                   added_expected, removed_expected):
+        added_expected.append('tower_instance_router')
+        instance = instance_generator(groups=groups, hostname=hostname)
+        worker_queues = worker_queues_generator(_worker_queues)
+        with mock.patch('awx.main.utils.ha.settings.AWX_CELERY_QUEUES_STATIC', static_queues):
+            (added_queues, removed_queues) = _add_remove_celery_worker_queues(mock_app, instance, worker_queues, hostname)
+            assert set(added_queues) == set(added_expected)
+            assert set(removed_queues) == set(removed_expected)
+
+
+class TestUpdateCeleryWorkerRoutes():
+
+    @pytest.mark.parametrize("is_controller,expected_routes", [
+        (False, { 
+            'awx.main.tasks.cluster_node_heartbeat': {'queue': 'east-1', 'routing_key': 'east-1'},
+            'awx.main.tasks.purge_old_stdout_files': {'queue': 'east-1', 'routing_key': 'east-1'}
+        }),
+        (True, { 
+            'awx.main.tasks.cluster_node_heartbeat': {'queue': 'east-1', 'routing_key': 'east-1'},
+            'awx.main.tasks.purge_old_stdout_files': {'queue': 'east-1', 'routing_key': 'east-1'},
+            'awx.main.tasks.awx_isolated_heartbeat': {'queue': 'east-1', 'routing_key': 'east-1'},
+        }),
+    ])
+    def test_update_celery_worker_routes(self, mocker, conf, is_controller, expected_routes):
+        instance = mocker.MagicMock()
+        instance.hostname = 'east-1'
+        instance.is_controller = mocker.MagicMock(return_value=is_controller)
+
+        assert update_celery_worker_routes(instance, conf) == expected_routes
+        assert conf.CELERY_TASK_ROUTES == expected_routes
+
+    def test_update_celery_worker_routes_deleted(self, mocker, conf):
+        instance = mocker.MagicMock()
+        instance.hostname = 'east-1'
+        instance.is_controller = mocker.MagicMock(return_value=False)
+        conf.CELERY_TASK_ROUTES = {'awx.main.tasks.awx_isolated_heartbeat': 'foobar'}
+
+        update_celery_worker_routes(instance, conf)
+        assert 'awx.main.tasks.awx_isolated_heartbeat' not in conf.CELERY_TASK_ROUTES
+

awx/main/utils/common.py

@@ -20,6 +20,8 @@ import six
 import psutil
 from StringIO import StringIO
 
+from decimal import Decimal
+
 # Decorator
 from decorator import decorator
 
@@ -45,7 +47,7 @@ __all__ = ['get_object_or_400', 'get_object_or_403', 'camelcase_to_underscore',
            'ignore_inventory_computed_fields', 'ignore_inventory_group_removal',
            '_inventory_updates', 'get_pk_from_dict', 'getattrd', 'NoDefaultProvided',
            'get_current_apps', 'set_current_apps', 'OutputEventFilter',
-           'extract_ansible_vars', 'get_search_fields', 'get_system_task_capacity',
+           'extract_ansible_vars', 'get_search_fields', 'get_system_task_capacity', 'get_cpu_capacity', 'get_mem_capacity',
            'wrap_args_with_proot', 'build_proot_temp_dir', 'check_proot_installed', 'model_to_dict',
            'model_instance_diff', 'timestamp_apiformat', 'parse_yaml_or_json', 'RequireDebugTrueOrTest',
            'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError', 'get_custom_venv_choices']
@@ -632,19 +634,52 @@ def parse_yaml_or_json(vars_str, silent_failure=True):
     return vars_dict
 
 
-@memoize()
+def get_cpu_capacity():
-def get_system_task_capacity():
+    from django.conf import settings
+    settings_forkcpu = getattr(settings, 'SYSTEM_TASK_FORKS_CPU', None)
+    env_forkcpu = os.getenv('SYSTEM_TASK_FORKS_CPU', None)
+    cpu = psutil.cpu_count()
+
+    if env_forkcpu:
+        forkcpu = int(env_forkcpu)
+    elif settings_forkcpu:
+        forkcpu = int(settings_forkcpu)
+    else:
+        forkcpu = 4
+    return (cpu, cpu * forkcpu)
+
+
+def get_mem_capacity():
+    from django.conf import settings
+    settings_forkmem = getattr(settings, 'SYSTEM_TASK_FORKS_MEM', None)
+    env_forkmem = os.getenv('SYSTEM_TASK_FORKS_MEM', None)
+    if env_forkmem:
+        forkmem = int(env_forkmem)
+    elif settings_forkmem:
+        forkmem = int(settings_forkmem)
+    else:
+        forkmem = 100
+
+    mem = psutil.virtual_memory().total
+    return (mem, max(1, ((mem / 1024 / 1024) - 2048) / forkmem))
+
+
+def get_system_task_capacity(scale=Decimal(1.0)):
     '''
     Measure system memory and use it as a baseline for determining the system's capacity
     '''
     from django.conf import settings
-    if hasattr(settings, 'SYSTEM_TASK_CAPACITY'):
+    settings_forks = getattr(settings, 'SYSTEM_TASK_FORKS_CAPACITY', None)
-        return settings.SYSTEM_TASK_CAPACITY
+    env_forks = os.getenv('SYSTEM_TASK_FORKS_CAPACITY', None)
-    mem = psutil.virtual_memory()
+
-    total_mem_value = mem.total / 1024 / 1024
+    if env_forks:
-    if total_mem_value <= 2048:
+        return int(env_forks)
-        return 50
+    elif settings_forks:
-    return 50 + ((total_mem_value / 1024) - 2) * 75
+        return int(settings_forks)
+
+    _, cpu_cap = get_cpu_capacity()
+    _, mem_cap = get_mem_capacity()
+    return min(mem_cap, cpu_cap) + ((max(mem_cap, cpu_cap) - min(mem_cap, cpu_cap)) * scale)
 
 
 _inventory_updates = threading.local()

awx/main/utils/ha.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2017 Ansible Tower by Red Hat
+# All Rights Reserved.
+
+# Django
+from django.conf import settings
+
+# AWX
+from awx.main.models import Instance
+
+
+def _add_remove_celery_worker_queues(app, instance, worker_queues, worker_name):
+    removed_queues = []
+    added_queues = []
+    ig_names = set(instance.rampart_groups.values_list('name', flat=True))
+    ig_names.add("tower_instance_router")
+    worker_queue_names = set([q['name'] for q in worker_queues])
+
+
+    # Remove queues that aren't in the instance group
+    for queue in worker_queues:
+        if queue['name'] in settings.AWX_CELERY_QUEUES_STATIC or \
+                queue['alias'] in settings.AWX_CELERY_QUEUES_STATIC:
+            continue
+
+        if queue['name'] not in ig_names | set([instance.hostname]) or not instance.enabled:
+            app.control.cancel_consumer(queue['name'], reply=True, destination=[worker_name])
+            removed_queues.append(queue['name'])
+
+    # Add queues for instance and instance groups
+    for queue_name in ig_names | set([instance.hostname]):
+        if queue_name not in worker_queue_names:
+            app.control.add_consumer(queue_name, reply=True, destination=[worker_name])
+            added_queues.append(queue_name)
+
+    return (added_queues, removed_queues)
+
+
+def update_celery_worker_routes(instance, conf):
+    tasks = [
+        'awx.main.tasks.cluster_node_heartbeat',
+        'awx.main.tasks.purge_old_stdout_files',
+    ]
+    routes_updated = {}
+    # Instance is, effectively, a controller node
+    if instance.is_controller():
+        tasks.append('awx.main.tasks.awx_isolated_heartbeat')
+    else:
+        if 'awx.main.tasks.awx_isolated_heartbeat' in conf.CELERY_TASK_ROUTES:
+            del conf.CELERY_TASK_ROUTES['awx.main.tasks.awx_isolated_heartbeat']
+
+    for t in tasks:
+        conf.CELERY_TASK_ROUTES[t] = {'queue': instance.hostname, 'routing_key': instance.hostname}
+        routes_updated[t] = conf.CELERY_TASK_ROUTES[t]
+
+    return routes_updated
+
+
+def register_celery_worker_queues(app, celery_worker_name):
+    instance = Instance.objects.me()
+    added_queues = []
+    removed_queues = []
+
+    celery_host_queues = app.control.inspect([celery_worker_name]).active_queues()
+
+    celery_worker_queues = celery_host_queues[celery_worker_name] if celery_host_queues else []
+    (added_queues, removed_queues) = _add_remove_celery_worker_queues(app, instance, celery_worker_queues, celery_worker_name)
+
+    return (instance, removed_queues, added_queues)
+

awx/settings/defaults.py

@@ -392,6 +392,18 @@ EMAIL_HOST_USER = ''
 EMAIL_HOST_PASSWORD = ''
 EMAIL_USE_TLS = False
 
+# The number of seconds to sleep between status checks for jobs running on isolated nodes
+AWX_ISOLATED_CHECK_INTERVAL = 30
+
+# The timeout (in seconds) for launching jobs on isolated nodes
+AWX_ISOLATED_LAUNCH_TIMEOUT = 600
+
+# Ansible connection timeout (in seconds) for communicating with isolated instances
+AWX_ISOLATED_CONNECTION_TIMEOUT = 10
+
+# The time (in seconds) between the periodic isolated heartbeat status check
+AWX_ISOLATED_PERIODIC_CHECK = 600
+
 # Memcached django cache configuration
 # CACHES = {
 #     'default': {
@@ -420,6 +432,7 @@ DEVSERVER_DEFAULT_PORT = '8013'
 # Set default ports for live server tests.
 os.environ.setdefault('DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:9013-9199')
 
+BROKER_POOL_LIMIT = None
 CELERY_BROKER_URL = 'amqp://guest:guest@localhost:5672//'
 CELERY_EVENT_QUEUE_TTL = 5
 CELERY_TASK_DEFAULT_QUEUE = 'tower'
@@ -435,18 +448,10 @@ CELERY_BEAT_MAX_LOOP_INTERVAL = 60
 CELERY_RESULT_BACKEND = 'django-db'
 CELERY_IMPORTS = ('awx.main.scheduler.tasks',)
 CELERY_TASK_QUEUES = (
-    Queue('default', Exchange('default'), routing_key='default'),
     Queue('tower', Exchange('tower'), routing_key='tower'),
-    Queue('tower_scheduler', Exchange('scheduler', type='topic'), routing_key='tower_scheduler.job.#', durable=False),
     Broadcast('tower_broadcast_all')
 )
-CELERY_TASK_ROUTES = {
+CELERY_TASK_ROUTES = {}
-    'awx.main.scheduler.tasks.run_task_manager': {'queue': 'tower', 'routing_key': 'tower'},
-    'awx.main.scheduler.tasks.run_job_launch': {'queue': 'tower_scheduler', 'routing_key': 'tower_scheduler.job.launch'},
-    'awx.main.scheduler.tasks.run_job_complete': {'queue': 'tower_scheduler', 'routing_key': 'tower_scheduler.job.complete'},
-    'awx.main.tasks.cluster_node_heartbeat': {'queue': 'default', 'routing_key': 'cluster.heartbeat'},
-    'awx.main.tasks.purge_old_stdout_files': {'queue': 'default', 'routing_key': 'cluster.heartbeat'},
-}
 
 CELERY_BEAT_SCHEDULE = {
     'tower_scheduler': {
@@ -474,11 +479,21 @@ CELERY_BEAT_SCHEDULE = {
     'task_manager': {
         'task': 'awx.main.scheduler.tasks.run_task_manager',
         'schedule': timedelta(seconds=20),
-        'options': {'expires': 20,}
+        'options': {'expires': 20}
     },
+    'isolated_heartbeat': {
+        'task': 'awx.main.tasks.awx_isolated_heartbeat',
+        'schedule': timedelta(seconds=AWX_ISOLATED_PERIODIC_CHECK),
+        'options': {'expires': AWX_ISOLATED_PERIODIC_CHECK * 2},
+    }
 }
 AWX_INCONSISTENT_TASK_INTERVAL = 60 * 3
 
+# Celery queues that will always be listened to by celery workers
+# Note: Broadcast queues have unique, auto-generated names, with the alias
+# property value of the original queue name.
+AWX_CELERY_QUEUES_STATIC = ['tower_broadcast_all',]
+
 # Django Caching Configuration
 if is_testing():
     CACHES = {
@@ -624,20 +639,8 @@ AWX_PROOT_BASE_PATH = "/tmp"
 # Note: This setting may be overridden by database settings.
 AWX_ANSIBLE_CALLBACK_PLUGINS = ""
 
-# Time at which an HA node is considered active
+# Automatically remove nodes that have missed their heartbeats after some time
-AWX_ACTIVE_NODE_TIME = 7200
+AWX_AUTO_DEPROVISION_INSTANCES = False
-
-# The number of seconds to sleep between status checks for jobs running on isolated nodes
-AWX_ISOLATED_CHECK_INTERVAL = 30
-
-# The timeout (in seconds) for launching jobs on isolated nodes
-AWX_ISOLATED_LAUNCH_TIMEOUT = 600
-
-# Ansible connection timeout (in seconds) for communicating with isolated instances
-AWX_ISOLATED_CONNECTION_TIMEOUT = 10
-
-# The time (in seconds) between the periodic isolated heartbeat status check
-AWX_ISOLATED_PERIODIC_CHECK = 600
 
 # Enable Pendo on the UI, possible values are 'off', 'anonymous', and 'detailed'
 # Note: This setting may be overridden by database settings.
@@ -1118,9 +1121,11 @@ LOGGING = {
         },
         'awx.main.tasks': {
             'handlers': ['task_system'],
+            'propagate': False
         },
         'awx.main.scheduler': {
             'handlers': ['task_system'],
+            'propagate': False
         },
         'awx.main.consumers': {
             'handlers': ['null']

awx/settings/development.py

@@ -138,15 +138,6 @@ except ImportError:
     sys.exit(1)
 
 CLUSTER_HOST_ID = socket.gethostname()
-CELERY_TASK_ROUTES['awx.main.tasks.cluster_node_heartbeat'] = {'queue': CLUSTER_HOST_ID, 'routing_key': CLUSTER_HOST_ID}
-# Production only runs this schedule on controlling nodes
-# but development will just run it on all nodes
-CELERY_TASK_ROUTES['awx.main.tasks.awx_isolated_heartbeat'] = {'queue': CLUSTER_HOST_ID, 'routing_key': CLUSTER_HOST_ID}
-CELERY_BEAT_SCHEDULE['isolated_heartbeat'] = {
-    'task': 'awx.main.tasks.awx_isolated_heartbeat',
-    'schedule': timedelta(seconds = AWX_ISOLATED_PERIODIC_CHECK),
-    'options': {'expires': AWX_ISOLATED_PERIODIC_CHECK * 2,}
-}
 
 # Supervisor service name dictionary used for programatic restart
 SERVICE_NAME_DICT = {

awx/settings/local_settings.py.docker_compose

@@ -198,6 +198,27 @@ LOGGING['handlers']['syslog'] = {
     'formatter': 'simple',
 }
 
+LOGGING['loggers']['django.request']['handlers'] = ['console']
+LOGGING['loggers']['rest_framework.request']['handlers'] = ['console']
+LOGGING['loggers']['awx']['handlers'] = ['console']
+LOGGING['loggers']['awx.main.commands.run_callback_receiver']['handlers'] = ['console']
+LOGGING['loggers']['awx.main.commands.inventory_import']['handlers'] = ['console']
+LOGGING['loggers']['awx.main.tasks']['handlers'] = ['console']
+LOGGING['loggers']['awx.main.scheduler']['handlers'] = ['console']
+LOGGING['loggers']['django_auth_ldap']['handlers'] = ['console']
+LOGGING['loggers']['social']['handlers'] = ['console']
+LOGGING['loggers']['system_tracking_migrations']['handlers'] = ['console']
+LOGGING['loggers']['rbac_migrations']['handlers'] = ['console']
+LOGGING['loggers']['awx.isolated.manager.playbooks']['handlers'] = ['console']
+LOGGING['handlers']['callback_receiver'] = {'class': 'logging.NullHandler'}
+LOGGING['handlers']['fact_receiver'] = {'class': 'logging.NullHandler'}
+LOGGING['handlers']['task_system'] = {'class': 'logging.NullHandler'}
+LOGGING['handlers']['tower_warnings'] = {'class': 'logging.NullHandler'}
+LOGGING['handlers']['rbac_migrations'] = {'class': 'logging.NullHandler'}
+LOGGING['handlers']['system_tracking_migrations'] = {'class': 'logging.NullHandler'}
+LOGGING['handlers']['management_playbooks'] = {'class': 'logging.NullHandler'}
+
+
 # Enable the following lines to also log to a file.
 #LOGGING['handlers']['file'] = {
 #    'class': 'logging.FileHandler',

awx/ui/client/lib/components/components.strings.js

@@ -71,16 +71,13 @@ function ComponentsStrings (BaseString) {
         INVENTORY_SCRIPTS: t.s('Inventory Scripts'),
         NOTIFICATIONS: t.s('Notifications'),
         MANAGEMENT_JOBS: t.s('Management Jobs'),
+        INSTANCES: t.s('Instances'),
         INSTANCE_GROUPS: t.s('Instance Groups'),
         SETTINGS: t.s('Settings'),
         FOOTER_ABOUT: t.s('About'),
         FOOTER_COPYRIGHT: t.s('Copyright © 2017 Red Hat, Inc.')
     };
 
-    ns.capacityBar = {
-        IS_OFFLINE: t.s('Unavailable to run jobs.')
-    };
-
     ns.relaunch = {
         DEFAULT: t.s('Relaunch using the same parameters'),
         HOSTS: t.s('Relaunch using host parameters'),

awx/ui/client/lib/components/index.js

@@ -12,6 +12,7 @@ import inputLookup from '~components/input/lookup.directive';
 import inputMessage from '~components/input/message.directive';
 import inputSecret from '~components/input/secret.directive';
 import inputSelect from '~components/input/select.directive';
+import inputSlider from '~components/input/slider.directive';
 import inputText from '~components/input/text.directive';
 import inputTextarea from '~components/input/textarea.directive';
 import inputTextareaSecret from '~components/input/textarea-secret.directive';
@@ -54,6 +55,7 @@ angular
     .directive('atInputMessage', inputMessage)
     .directive('atInputSecret', inputSecret)
     .directive('atInputSelect', inputSelect)
+    .directive('atInputSlider', inputSlider)
     .directive('atInputText', inputText)
     .directive('atInputTextarea', inputTextarea)
     .directive('atInputTextareaSecret', inputTextareaSecret)

awx/ui/client/lib/components/input/_index.less

@@ -218,3 +218,47 @@
     min-height: @at-height-textarea;
     padding: 6px @at-padding-input 0 @at-padding-input;
 }
+
+.at-InputSlider {
+    display: flex;
+    padding: 5px 0;
+
+    p {
+        color: @at-color-form-label;
+        font-size: @at-font-size-help-text;
+        font-weight: @at-font-weight-body;
+        margin: 0 0 0 10px;
+        padding: 0;
+        width: 50px;
+    }
+
+    input[type=range] {
+        -webkit-appearance: none;
+        width: 100%;
+        background: transparent;
+        height: 20px;
+        border-right: 1px solid @at-color-input-slider-track;
+        border-left: 1px solid @at-color-input-slider-track;
+
+        &:focus {
+            outline: none;
+        }
+
+        &::-webkit-slider-runnable-track {
+            background: @at-color-input-slider-track;
+            cursor: pointer;
+            height: 1px;
+            width: 100%;
+        }
+        &::-webkit-slider-thumb {
+            -webkit-appearance: none;
+            background: @at-color-input-slider-thumb;
+            border-radius: 50%;
+            border: none;
+            cursor: pointer;
+            height: 16px;
+            margin-top: -7px;
+            width: 16px;
+        }
+    }
+}

awx/ui/client/lib/components/input/lookup.directive.js

@@ -118,6 +118,10 @@ function AtInputLookupController (baseInputController, $q, $state) {
 
         vm.searchAfterDebounce();
     };
+
+    vm.removeTag = (tagToRemove) => {
+        _.remove(scope.state._value, (tag) => tag === tagToRemove);
+    };
 }
 
 AtInputLookupController.$inject = [

awx/ui/client/lib/components/input/lookup.partial.html

@@ -11,17 +11,30 @@
                 </button>
             </span>
             <input type="text"
-               class="form-control at-Input"
+                class="form-control at-Input"
-               ng-class="{ 'at-Input--rejected': state._rejected }"
+                ng-class="{ 'at-Input--rejected': state._rejected }"
-               ng-model="state._displayValue"
+                ng-model="state._displayValue"
-               ng-attr-tabindex="{{ tab || undefined }}"
+                ng-attr-tabindex="{{ tab || undefined }}"
-               ng-attr-placeholder="{{::state._placeholder || undefined }}"
+                ng-attr-placeholder="{{::state._placeholder || undefined }}"
-               ng-change="vm.searchOnInput()"
+                ng-change="vm.searchOnInput()"
-               ng-disabled="state._disabled || form.disabled" />
+                ng-hide="state._lookupTags"
+                ng-disabled="state._disabled || form.disabled">
+
+            <span class="form-control Form-textInput Form-textInput--variableHeight LabelList-lookupTags"
+                ng-if="state._lookupTags">
+                <div class="LabelList-tagContainer" ng-repeat="tag in state._value track by $index">
+                <div class="LabelList-deleteContainer" ng-click="vm.removeTag(tag)">
+                    <i class="fa fa-times LabelList-tagDelete"></i>
+                </div>
+                <div class="LabelList-tag LabelList-tag--deletable">
+                    <span ng-if="tag.hostname" class="LabelList-name">{{ tag.hostname }}</span>
+                    <span ng-if="!tag.hostname" class="LabelList-name">{{ tag }}</span>
+                </div>
+            </span>
         </div>
 
         <at-input-message></at-input-message>
     </div>
 
     <div ui-view="{{ state._resource }}"></div>
-</div>
+    </div>

awx/ui/client/lib/components/input/slider.directive.js

@@ -0,0 +1,38 @@
+const templateUrl = require('~components/input/slider.partial.html');
+
+function atInputSliderLink (scope, element, attrs, controllers) {
+    const [formController, inputController] = controllers;
+
+    inputController.init(scope, element, formController);
+}
+
+function atInputSliderController (baseInputController) {
+    const vm = this || {};
+
+    vm.init = (_scope_, _element_, form) => {
+        baseInputController.call(vm, 'input', _scope_, _element_, form);
+
+        vm.check();
+    };
+}
+
+atInputSliderController.$inject = ['BaseInputController'];
+
+function atInputSlider () {
+    return {
+        restrict: 'E',
+        require: ['^^atForm', 'atInputSlider'],
+        replace: true,
+        templateUrl,
+        controller: atInputSliderController,
+        controllerAs: 'vm',
+        link: atInputSliderLink,
+        scope: {
+            state: '=?',
+            col: '@',
+            tab: '@'
+        }
+    };
+}
+
+export default atInputSlider;

awx/ui/client/lib/components/input/slider.partial.html

@@ -0,0 +1,13 @@
+<div class="col-sm-{{::col}} at-InputContainer">
+    <div class="form-group at-u-flat">
+        <at-input-label></at-input-label>
+        <div class="at-InputSlider">
+            <input type="range"
+                    ng-model="state._value"
+                    min="0"
+                    max="100"
+                    ng-change="vm.slide(state._value)"/>
+            <p>{{ state._value }}%</p>
+        </div>
+    </div>
+</div>

awx/ui/client/lib/components/list/_index.less

@@ -86,12 +86,35 @@
     border-top: @at-border-default-width solid @at-color-list-border;
 }
 
+.at-Row--rowLayout {
+    display: flex;
+    flex-direction: row;
+
+    .at-RowItem {
+        margin-right: @at-space-4x;
+
+        &-label {
+            width: auto;
+        }
+    }
+}
+
+.at-RowStatus {
+    align-self: flex-start;
+    margin: 0 10px 0 0;
+}
+
+.at-Row-firstColumn {
+    margin-right: @at-space-4x;
+}
+
 .at-Row-actions {
     display: flex;
 }
 
 .at-Row-items {
     align-self: flex-start;
+    flex: 1;
 }
 
 .at-RowItem {
@@ -101,10 +124,19 @@
 }
 
 .at-RowItem--isHeader {
+    color: @at-color-body-text;
     margin-bottom: @at-margin-bottom-list-header;
     line-height: @at-line-height-list-row-item-header;
 }
 
+.at-RowItem--isHeaderLink {
+    color: @at-blue;
+    cursor: pointer;
+}
+.at-RowItem--isHeaderLink:hover {
+    color: @at-blue-hover;
+}
+
 .at-RowItem--labels {
     line-height: @at-line-height-list-row-item-labels;
 }
@@ -146,8 +178,26 @@
 
 .at-RowItem-label {
     text-transform: uppercase;
+    width: auto;
     width: @at-width-list-row-item-label;
     color: @at-color-list-row-item-label;
+    font-size: @at-font-size;
+}
+
+.at-RowItem-value {
+    font-size: @at-font-size-3x;
+}
+
+.at-RowItem-badge {
+    background-color: @at-gray-848992;
+    border-radius: @at-border-radius;
+    color: @at-white;
+    font-size: 11px;
+    font-weight: normal;
+    height: 14px;
+    line-height: 10px;
+    margin: 0 10px;
+    padding: 2px 10px;
 }
 
 .at-RowAction {
@@ -180,6 +230,11 @@
     background-color: @at-color-list-row-action-hover-danger;
 }
 
+.at-Row .at-Row-checkbox {
+    align-self: start;
+    margin: 2px 20px 0 0;
+}
+
 @media screen and (max-width: @at-breakpoint-compact-list) {
     .at-Row-actions {
         flex-direction: column;

awx/ui/client/lib/components/list/row-item.directive.js

@@ -7,10 +7,13 @@ function atRowItem () {
         transclude: true,
         templateUrl,
         scope: {
+            badge: '@',
             headerValue: '@',
             headerLink: '@',
             headerTag: '@',
             labelValue: '@',
+            labelLink: '@',
+            labelState: '@',
             value: '@',
             valueLink: '@',
             smartStatus: '=?',

awx/ui/client/lib/components/list/row-item.partial.html

@@ -9,13 +9,19 @@
         <div class="at-RowItem-tag at-RowItem-tag--header" ng-if="headerTag">
             {{ headerTag }}
         </div>
-        <div class="at-RowItem-label" ng-if="labelValue">
+        <div class="at-RowItem-label" ng-if="labelValue && labelLink">
+            <a ng-href="{{ labelLink }}">{{ labelValue }}</a>
+        </div>
+        <div class="at-RowItem-label" ng-if="labelValue && !labelLink && !labelState">
             {{ labelValue }}
         </div>
+        <div class="at-RowItem-label" ng-if="labelValue && labelState">
+            <a ui-sref="{{ labelState }}"  ui-sref-opts="{reload: true, notify: true}">{{ labelValue }}</a>
+        </div>
         <div class="at-RowItem-value" ng-if="value && valueLink">
             <a ng-href="{{ valueLink }}">{{ value }}</a>
         </div>
-        <div class="at-RowItem-value" ng-if="value && !valueLink"
+        <div class="at-RowItem-value" ng-class="{'at-RowItem-badge': badge}" ng-if="value && !valueLink"
             ng-bind-html="value">
         </div>
         <aw-smart-status jobs="smartStatus.summary_fields.recent_jobs"

awx/ui/client/lib/components/tabs/_index.less

@@ -26,3 +26,7 @@
         cursor: not-allowed;
     }
 }
+
+.at-TabGroup + .at-Panel-body {
+    margin-top: 20px;
+}

awx/ui/client/lib/components/tabs/tab.partial.html

@@ -1,6 +1,7 @@
 <button class="btn at-ButtonHollow--default at-Tab"
         ng-attr-disabled="{{ state._disabled || undefined }}"
         ng-class="{ 'at-Tab--active': state._active, 'at-Tab--disabled': state._disabled }"
+        ng-hide="{{ state._hide }}"
         ng-click="vm.go()">
     <ng-transclude></ng-transclude>
 </button>

awx/ui/client/lib/models/Base.js

@@ -129,6 +129,10 @@ function httpPost (config = {}) {
         data: config.data
     };
 
+    if (config.url) {
+        req.url = `${this.path}${config.url}`;
+    }
+
     return $http(req)
         .then(res => {
             this.model.GET = res.data;
@@ -323,7 +327,7 @@ function has (method, keys) {
     return value !== undefined && value !== null;
 }
 
-function extend (method, related) {
+function extend (method, related, config = {}) {
     if (!related) {
         related = method;
         method = 'GET';
@@ -337,6 +341,8 @@ function extend (method, related) {
             url: this.get(`related.${related}`)
         };
 
+        Object.assign(req, config);
+
         return $http(req)
             .then(({ data }) => {
                 this.set(method, `related.${related}`, data);

awx/ui/client/lib/models/Instance.js

@@ -0,0 +1,47 @@
+let Base;
+
+function createFormSchema (method, config) {
+    if (!config) {
+        config = method;
+        method = 'GET';
+    }
+
+    const schema = Object.assign({}, this.options(`actions.${method.toUpperCase()}`));
+
+    if (config && config.omit) {
+        config.omit.forEach(key => delete schema[key]);
+    }
+
+    Object.keys(schema).forEach(key => {
+        schema[key].id = key;
+
+        if (this.has(key)) {
+            schema[key]._value = this.get(key);
+        }
+    });
+
+    return schema;
+}
+
+function InstanceModel (method, resource, config) {
+    // Base takes two args: resource and settings
+    // resource is the string endpoint
+    Base.call(this, 'instances');
+
+    this.Constructor = InstanceModel;
+    this.createFormSchema = createFormSchema.bind(this);
+
+    return this.create(method, resource, config);
+}
+
+function InstanceModelLoader (BaseModel) {
+    Base = BaseModel;
+
+    return InstanceModel;
+}
+
+InstanceModelLoader.$inject = [
+    'BaseModel'
+];
+
+export default InstanceModelLoader;

awx/ui/client/lib/models/InstanceGroup.js

@@ -0,0 +1,47 @@
+let Base;
+
+function createFormSchema (method, config) {
+    if (!config) {
+        config = method;
+        method = 'GET';
+    }
+
+    const schema = Object.assign({}, this.options(`actions.${method.toUpperCase()}`));
+
+    if (config && config.omit) {
+        config.omit.forEach(key => delete schema[key]);
+    }
+
+    Object.keys(schema).forEach(key => {
+        schema[key].id = key;
+
+        if (this.has(key)) {
+            schema[key]._value = this.get(key);
+        }
+    });
+
+    return schema;
+}
+
+function InstanceGroupModel (method, resource, config) {
+    // Base takes two args: resource and settings
+    // resource is the string endpoint
+    Base.call(this, 'instance_groups');
+
+    this.Constructor = InstanceGroupModel;
+    this.createFormSchema = createFormSchema.bind(this);
+
+    return this.create(method, resource, config);
+}
+
+function InstanceGroupModelLoader (BaseModel) {
+    Base = BaseModel;
+
+    return InstanceGroupModel;
+}
+
+InstanceGroupModelLoader.$inject = [
+    'BaseModel'
+];
+
+export default InstanceGroupModelLoader;

awx/ui/client/lib/models/index.js

@@ -9,6 +9,8 @@ import Organization from '~models/Organization';
 import Project from '~models/Project';
 import JobTemplate from '~models/JobTemplate';
 import WorkflowJobTemplateNode from '~models/WorkflowJobTemplateNode';
+import Instance from '~models/Instance';
+import InstanceGroup from '~models/InstanceGroup';
 import InventorySource from '~models/InventorySource';
 import Inventory from '~models/Inventory';
 import InventoryScript from '~models/InventoryScript';
@@ -32,6 +34,8 @@ angular
     .service('ProjectModel', Project)
     .service('JobTemplateModel', JobTemplate)
     .service('WorkflowJobTemplateNodeModel', WorkflowJobTemplateNode)
+    .service('InstanceModel', Instance)
+    .service('InstanceGroupModel', InstanceGroup)
     .service('InventorySourceModel', InventorySource)
     .service('InventoryModel', Inventory)
     .service('InventoryScriptModel', InventoryScript)

awx/ui/client/lib/services/base-string.service.js

@@ -60,6 +60,8 @@ function BaseStringService (namespace) {
     this.CANCEL = t.s('CANCEL');
     this.SAVE = t.s('SAVE');
     this.OK = t.s('OK');
+    this.ON = t.s('ON');
+    this.OFF = t.s('OFF');
     this.deleteResource = {
         HEADER: t.s('Delete'),
         USED_BY: resourceType => t.s('The {{ resourceType }} is currently being used by other resources.', { resourceType }),

awx/ui/client/lib/theme/_global.less

@@ -16,6 +16,16 @@
     }
 }
 
+.at-Button--add {
+    &:extend(.at-Button--success all);
+    &:before {
+        content: "+";
+        font-size: 20px;
+    }
+    border-color: transparent;
+    margin-left: @at-space-2x;
+}
+
 .at-Button--info {
     .at-mixin-Button();
     .at-mixin-ButtonColor('at-color-info', 'at-color-default');

awx/ui/client/lib/theme/_mixins.less

@@ -21,6 +21,7 @@
 }
 
 .at-mixin-Button () {
+    border-radius: @at-border-radius;
     height: @at-height-input;
     padding: @at-padding-button-vertical @at-padding-button-horizontal;
     font-size: @at-font-size-body;

awx/ui/client/lib/theme/_variables.less

@@ -147,6 +147,8 @@
 @at-color-input-icon: @at-gray-b7;
 @at-color-input-placeholder: @at-gray-848992;
 @at-color-input-text: @at-gray-161b1f;
+@at-color-input-slider-thumb: @at-blue;
+@at-color-input-slider-track: @at-gray-b7;
 
 @at-color-icon-dismiss: @at-gray-d7;
 @at-color-icon-popover: @at-gray-848992;

awx/ui/client/lib/theme/index.less

@@ -72,7 +72,9 @@
 @import '../../src/home/dashboard/lists/dashboard-list.block.less';
 @import '../../src/home/dashboard/dashboard.block.less';
 @import '../../src/instance-groups/capacity-bar/capacity-bar.block.less';
+@import '../../src/instance-groups/capacity-adjuster/capacity-adjuster.block.less';
 @import '../../src/instance-groups/instance-group.block.less';
+@import '../../src/instance-groups/instances/instance-modal.block.less';
 @import '../../src/inventories-hosts/inventories/insights/insights.block.less';
 @import '../../src/inventories-hosts/inventories/list/host-summary-popover/host-summary-popover.block.less';
 @import '../../src/inventories-hosts/inventories/related/hosts/related-groups-labels/relatedGroupsLabelsList.block.less';

awx/ui/client/src/app.js

@@ -97,7 +97,6 @@ angular
         users.name,
         projects.name,
         scheduler.name,
-        instanceGroups.name,
 
         'Utilities',
         'templates',
@@ -105,6 +104,7 @@ angular
         'AWDirectives',
         'features',
 
+        instanceGroups,
         atFeatures,
         atLibComponents,
         atLibModels,

awx/ui/client/src/instance-groups/add-edit/add-edit-instance-groups.view.html

@@ -0,0 +1,30 @@
+<at-panel>
+    <at-panel-heading>
+        {{ vm.panelTitle }}
+    </at-panel-heading>
+
+    <at-tab-group>
+        <at-tab state="vm.tab.details">{{:: vm.strings.get('tab.DETAILS') }}</at-tab>
+        <at-tab state="vm.tab.instances">{{:: vm.strings.get('tab.INSTANCES') }}</at-tab>
+        <at-tab state="vm.tab.jobs">{{:: vm.strings.get('tab.JOBS') }}</at-tab>
+    </at-tab-group>
+
+    <at-panel-body>
+        <at-form state="vm.form" autocomplete="off">
+
+            <at-input-text col="4" tab="1" state="vm.form.name"></at-input-text>
+
+            <at-input-text col="4" tab="3" state="vm.form.policy_instance_minimum"></at-input-text>
+
+            <at-input-slider col="4" tab="3" state="vm.form.policy_instance_percentage"></at-input-slider>
+
+            <at-input-lookup col="4" tab="4" state="vm.form.policy_instance_list"></at-input-lookup>
+            <div ui-view="modal"></div>
+
+            <at-action-group col="12" pos="right">
+                <at-form-action type="cancel" to="instanceGroups"></at-form-action>
+                <at-form-action type="save"></at-form-action>
+            </at-action-group>
+        </at-form>
+    </at-panel-body>
+</at-panel>

awx/ui/client/src/instance-groups/add-edit/add-instance-group.controller.js

@@ -0,0 +1,44 @@
+function AddController ($scope, $state, models, strings) {
+    const vm = this || {};
+    const { instanceGroup, instance } = models;
+
+    vm.mode = 'add';
+    vm.strings = strings;
+    vm.panelTitle = strings.get('state.ADD_BREADCRUMB_LABEL');
+
+    vm.tab = {
+        details: { _active: true },
+        instances: {_disabled: true },
+        jobs: {_disabled: true }
+    };
+
+    vm.form = instanceGroup.createFormSchema('post');
+
+    // Default policy instance percentage value is 0
+    vm.form.policy_instance_percentage._value = 0;
+
+    vm.form.policy_instance_list._lookupTags = true;
+    vm.form.policy_instance_list._model = instance;
+    vm.form.policy_instance_list._placeholder = "Policy Instance List";
+    vm.form.policy_instance_list._resource = 'instances';
+    vm.form.policy_instance_list._route = 'instanceGroups.add.modal.instances';
+    vm.form.policy_instance_list._value = [];
+
+    vm.form.save = data => {
+        data.policy_instance_list = data.policy_instance_list.map(instance => instance.hostname);
+        return instanceGroup.request('post', { data });
+    };
+
+    vm.form.onSaveSuccess = res => {
+        $state.go('instanceGroups.edit', { instance_group_id: res.data.id }, { reload: true });
+    };
+}
+
+AddController.$inject = [
+    '$scope',
+    '$state',
+    'resolvedModels',
+    'InstanceGroupsStrings'
+];
+
+export default AddController;

awx/ui/client/src/instance-groups/add-edit/edit-instance-group.controller.js

@@ -0,0 +1,54 @@
+function EditController ($rootScope, $state, models, strings) {
+    const vm = this || {};
+    const { instanceGroup, instance } = models;
+
+    $rootScope.breadcrumb.instance_group_name = instanceGroup.get('name');
+
+    vm.mode = 'edit';
+    vm.strings = strings;
+    vm.panelTitle = instanceGroup.get('name');
+
+    vm.tab = {
+        details: {
+            _active: true,
+            _go: 'instanceGroups.edit',
+            _params: { instance_group_id: instanceGroup.get('id') }
+        },
+        instances: {
+            _go: 'instanceGroups.instances',
+            _params: { instance_group_id: instanceGroup.get('id') }
+        },
+        jobs: {
+            _go: 'instanceGroups.jobs',
+            _params: { instance_group_id: instanceGroup.get('id') }
+        }
+    };
+
+    vm.form = instanceGroup.createFormSchema('put');
+
+    vm.form.policy_instance_list._lookupTags = true;
+    vm.form.policy_instance_list._model = instance;
+    vm.form.policy_instance_list._placeholder = "Policy Instance List";
+    vm.form.policy_instance_list._resource = 'instances';
+    vm.form.policy_instance_list._route = 'instanceGroups.edit.modal.instances';
+    vm.form.policy_instance_list._value = instanceGroup.get('policy_instance_list');
+
+    vm.form.save = data => {
+        instanceGroup.unset('policy_instance_list');
+        data.policy_instance_list = data.policy_instance_list.map(instance => instance.hostname || instance);
+        return instanceGroup.request('put', { data });
+    };
+
+    vm.form.onSaveSuccess = res => {
+        $state.go('instanceGroups.edit', { instance_group_id: res.data.id }, { reload: true });
+    };
+}
+
+EditController.$inject = [
+    '$rootScope',
+    '$state',
+    'resolvedModels',
+    'InstanceGroupsStrings'
+];
+
+export default EditController;

awx/ui/client/src/instance-groups/add-edit/instance-list-policy.directive.js

@@ -0,0 +1,84 @@
+const templateUrl = require('./instance-list-policy.partial.html');
+
+function InstanceListPolicyLink (scope, el, attrs, controllers) {
+    const instancePolicyController = controllers[0];
+    const formController = controllers[1];
+    const models = scope.$resolve.resolvedModels;
+
+    instancePolicyController.init(formController, models);
+}
+
+
+function InstanceListPolicyController ($scope, $state, strings) {
+    const vm = this || {};
+    let form;
+    let instance;
+    let instanceGroup;
+
+    vm.init = (_form_, _models_) => {
+        form = _form_;
+        ({ instance, instanceGroup} = _models_);
+
+        vm.strings = strings;
+        vm.instanceGroupId = instanceGroup.get('id');
+        vm.defaultParams = { page_size: '10', order_by: 'hostname' };
+
+        if (vm.instanceGroupId === undefined) {
+            vm.setInstances();
+        } else {
+            vm.setRelatedInstances();
+        }
+    };
+
+    vm.setInstances = () => {
+        vm.instances = instance.get('results').map(instance => {
+            instance.isSelected = false;
+            return instance;
+        });
+    };
+
+    vm.setRelatedInstances = () => {
+        vm.instanceGroupName = instanceGroup.get('name');
+        vm.relatedInstances = instanceGroup.get('policy_instance_list');
+
+        vm.instances = instance.get('results').map(instance => {
+            instance.isSelected = vm.relatedInstances.includes(instance.hostname);
+            return instance;
+        });
+    };
+
+    $scope.$watch('vm.instances', function() {
+        vm.selectedRows = _.filter(vm.instances, 'isSelected');
+        vm.deselectedRows = _.filter(vm.instances, 'isSelected', false);
+     }, true);
+
+    vm.submit = () => {
+        form.components
+            .filter(component => component.category === 'input')
+            .filter(component => component.state.id === 'policy_instance_list')
+            .forEach(component => {
+                component.state._value = vm.selectedRows;
+            });
+
+        $state.go("^.^");
+    };
+}
+
+InstanceListPolicyController.$inject = [
+    '$scope',
+    '$state',
+    'InstanceGroupsStrings'
+];
+
+function instanceListPolicy () {
+    return {
+        restrict: 'E',
+        link: InstanceListPolicyLink,
+        controller: InstanceListPolicyController,
+        controllerAs: 'vm',
+        require: ['instanceListPolicy', '^atForm'],
+        templateUrl
+    };
+}
+
+export default instanceListPolicy;

awx/ui/client/src/instance-groups/add-edit/instance-list-policy.partial.html

@@ -0,0 +1,53 @@
+<div id="instance-modal" class="modal-dialog">
+    <at-panel on-dismiss="^.^">
+        <at-panel-heading>
+            {{:: vm.strings.get('instance.PANEL_TITLE') }}
+        </at-panel-heading>
+        <multi-select-preview selected-rows='vm.selectedRows' available-rows='vm.instances'></multi-select-preview>
+        <at-panel-body>
+            <div class="at-List-toolbar">
+                <smart-search
+                    class="at-List-search"
+                    django-model="instances"
+                    base-path="instances"
+                    iterator="instance"
+                    default-params="vm.defaultParams"
+                    list="list"
+                    dataset="vm.instances"
+                    collection="collection"
+                    search-tags="searchTags">
+                </smart-search>
+            </div>
+            <at-list results='vm.instances'>
+                <at-row ng-repeat="instance in vm.instances"
+                    ng-class="{'at-Row--active': (instance.id === vm.activeId)}">
+
+                    <input type="checkbox"
+                        class="at-Row-checkbox"
+                        ng-model="instance.isSelected"
+                        ng-checked="instance.isSelected"
+                        ng-attr-tabindex="{{ tab || undefined }}"
+                        ng-disabled="state._disabled" />
+
+                    <div class="at-Row-items" style="flex: 1">
+                        <at-row-item
+                            header-value="{{ instance.hostname }}">
+                        </at-row-item>
+                    </div>
+                </at-row>
+            </at-list>
+            <div class="at-ActionGroup">
+                <div class="pull-right">
+                    <button class="btn at-ButtonHollow--default"
+                        ng-click="$state.go('^.^')">
+                        {{:: vm.strings.get('CANCEL') }}
+                    </button>
+                    <button class="btn at-Button--success"
+                        ng-click="vm.submit()">
+                        {{:: vm.strings.get('SAVE') }}
+                    </button>
+                </div>
+            </div>
+        </at-panel-body>
+    </at-panel>
+</div>

awx/ui/client/src/instance-groups/capacity-adjuster/capacity-adjuster.block.less

@@ -0,0 +1,11 @@
+.CapacityAdjuster {
+    .at-InputSlider {
+        align-items: center;
+        margin-right: @at-space-4x;
+    }
+
+    .at-InputSlider p {
+        white-space: nowrap;
+        margin: 0 10px;
+    }
+}

awx/ui/client/src/instance-groups/capacity-adjuster/capacity-adjuster.directive.js

@@ -0,0 +1,45 @@
+function CapacityAdjuster (templateUrl) {
+    return {
+        scope: {
+            state: '='
+        },
+        templateUrl: templateUrl('instance-groups/capacity-adjuster/capacity-adjuster'),
+        restrict: 'E',
+        replace: true,
+        link: function(scope) {
+            const adjustment_values = [{
+                label: 'CPU',
+                value: scope.state.cpu_capacity,
+            },{
+                label: 'RAM',
+                value: scope.state.mem_capacity
+            }];
+
+            scope.min_capacity = _.min(adjustment_values, 'value');
+            scope.max_capacity = _.max(adjustment_values, 'value');
+
+        },
+        controller: function($http) {
+            const vm = this || {};
+
+            vm.slide = (state) => {
+                const data = {
+                    "capacity_adjustment": `${state.capacity_adjustment}`
+                };
+                const req = {
+                    method: 'PUT',
+                    url: state.url,
+                    data
+                };
+                $http(req);
+            };
+        },
+        controllerAs: 'vm'
+    };
+}
+
+CapacityAdjuster.$inject = [
+    'templateUrl'
+];
+
+export default CapacityAdjuster;

awx/ui/client/src/instance-groups/capacity-adjuster/capacity-adjuster.partial.html

@@ -0,0 +1,13 @@
+<div class="CapacityAdjuster">
+    <div class="at-InputSlider">
+        <p>{{min_capacity.label}} {{min_capacity.value}}</p>
+            <input string-to-number
+            type="range"
+            ng-model="state.capacity_adjustment"
+            min="0"
+            max="1"
+            step="0.1"
+            ng-change="vm.slide(state)"/>
+        <p>{{max_capacity.label}} {{max_capacity.value}}</p>
+    </div>
+</div>

awx/ui/client/src/instance-groups/capacity-bar/capacity-bar.block.less

@@ -1,21 +1,22 @@
 capacity-bar {
-
-    width: 50%;
-    margin-right: 25px;
-    min-width: 100px;
-    display: flex;
     align-items: center;
+    color: @at-color-body-background-dark;
+    display: flex;
+    font-size: @at-font-size;
+    min-width: 100px;
+    white-space: nowrap;
 
     .CapacityBar {
         background-color: @default-bg;
-        display: flex;
-        flex: 0 0 auto;
-        height: 10px;
-        border: 1px solid @default-link;
-        width: 100%;
         border-radius: 100vw;
+        border: 1px solid @default-link;
+        display: flex;
+        flex: 1;
+        height: 10px;
+        margin-right: @at-space-2x;
+        min-width: 100px;
         overflow: hidden;
-        margin-right: 10px;
+        width: 100%;
     }
 
     .CapacityBar-remaining {
@@ -28,14 +29,21 @@ capacity-bar {
     }
 
     .CapacityBar--offline {
-        border-color: @d7grey;
+        color: @at-red;
+        border-color:  @at-gray-a9;
 
         .CapacityBar-remaining {
-            background-color: @d7grey;
+            background-color:  @at-gray-b7;
         }
     }
 
+    .Capacity-details--label {
+        margin-right: @at-space-2x;
+        text-align: right;
+        text-transform: uppercase;
+    }
+
     .Capacity-details--percentage {
-        color: @default-data-txt;
+        width: 40px;
     }
 }

awx/ui/client/src/instance-groups/capacity-bar/capacity-bar.directive.js

@@ -1,44 +1,52 @@
-export default ['templateUrl', 'ComponentsStrings',
+function CapacityBar (templateUrl, strings) {
-    function (templateUrl, strings) {
+    return {
-        return {
+        scope: {
-            scope: {
+            capacity: '=',
-                capacity: '=',
+            totalCapacity: '=',
-                totalCapacity: '='
+            labelValue: '@',
-            },
+            badge: '='
-            templateUrl: templateUrl('instance-groups/capacity-bar/capacity-bar'),
+        },
-            restrict: 'E',
+        templateUrl: templateUrl('instance-groups/capacity-bar/capacity-bar'),
-            link: function(scope) {
+        restrict: 'E',
-                scope.isOffline = false;
+        link: function(scope) {
+            scope.isOffline = false;
 
-                scope.$watch('totalCapacity', function(val) {
+            scope.$watch('totalCapacity', function(val) {
-                    if (val === 0) {
+                if (val === 0) {
-                        scope.isOffline = true;
+                    scope.isOffline = true;
-                        scope.offlineTip = strings.get(`capacityBar.IS_OFFLINE`);
+                    scope.labelValue = strings.get(`capacityBar.IS_OFFLINE_LABEL`);
-                    } else {
+                    scope.offlineTip = strings.get(`capacityBar.IS_OFFLINE`);
-                        scope.isOffline = false;
+                } else {
-                        scope.offlineTip = null;
+                    scope.isOffline = false;
-                    }
+                    scope.offlineTip = null;
-                }, true);
+                }
+            }, true);
 
-                scope.$watch('capacity', function() {
+            scope.$watch('capacity', function() {
-                    if (scope.totalCapacity !== 0) {
+                if (scope.totalCapacity !== 0) {
-                        var percentageCapacity = Math
+                    var percentageCapacity = Math
-                            .round(scope.capacity / scope.totalCapacity * 1000) / 10;
+                        .round(scope.capacity / scope.totalCapacity * 1000) / 10;
 
-                        scope.CapacityStyle = {
+                    scope.CapacityStyle = {
-                            'flex-grow': percentageCapacity * 0.01
+                        'flex-grow': percentageCapacity * 0.01
-                        };
+                    };
 
-                        scope.consumedCapacity = `${percentageCapacity}%`;
+                    scope.consumedCapacity = `${percentageCapacity}%`;
-                    } else {
+                } else {
-                        scope.CapacityStyle = {
+                    scope.CapacityStyle = {
-                            'flex-grow': 1
+                        'flex-grow': 1
-                        };
+                    };
 
-                        scope.consumedCapacity = null;
+                    scope.consumedCapacity = null;
-                    }
+                }
-                }, true);
+            }, true);
-            }
+        }
-        };
+    };
-    }
+}
+
+CapacityBar.$inject = [
+    'templateUrl',
+    'InstanceGroupsStrings'
 ];
+
+export default CapacityBar;

awx/ui/client/src/instance-groups/capacity-bar/capacity-bar.partial.html

@@ -1,11 +1,20 @@
+<span class="Capacity-details--label" ng-class="{'CapacityBar--offline': isOffline}">
+    {{ labelValue }}
+</span>
+
 <div class="CapacityBar"
     ng-class="{'CapacityBar--offline': isOffline}"
+    ng-if="!badge"
     aw-tool-tip="{{ offlineTip }}"
     data-tip-watch="offlineTip"
     data-placement="top"
     data-trigger="hover"
     data-container="body">
-        <div class="CapacityBar-remaining" ng-style="CapacityStyle"></div>
+    <div class="CapacityBar-remaining" ng-style="CapacityStyle"></div>
-        <div class="CapacityBar-consumed"></div>
+    <div class="CapacityBar-consumed"></div>
 </div>
-<span class="Capacity-details--percentage" ng-show="consumedCapacity">{{ consumedCapacity }}</span>
+
+<span class="Capacity-details--percentage"
+    ng-class="{'badge List-titleBadge': badge}">
+    {{ consumedCapacity }}
+</span>

awx/ui/client/src/instance-groups/capacity-bar/main.js

@@ -1,5 +0,0 @@
-import capacityBar from './capacity-bar.directive';
-
-export default
-    angular.module('capacityBarDirective', [])
-        .directive('capacityBar', capacityBar);

awx/ui/client/src/instance-groups/instance-group.partial.html

@@ -1,33 +0,0 @@
-<div class="Panel">
-    <div class="row Form-tabRow">
-        <div class="col-xs-12">
-            <div class="List-header">
-                <div class="List-title">
-                    <div class="List-titleText">{{ instanceGroupName }}</div>
-                </div>
-                <div class="List-details">
-                    <div class="Capacity-details">
-                        <p class="Capacity-details--label" translate>Used Capacity</p>
-                        <capacity-bar capacity="instanceGroupCapacity" total-capacity="instanceGroupTotalCapacity"></capacity-bar>
-                    </div>
-                    <div class="RunningJobs-details">
-                        <p class="RunningJobs-details--label" translate>Running Jobs</p>
-                        <span class="badge List-titleBadge">
-                            {{ instanceGroupJobsRunning }}
-                        </span>
-                    </div>
-                </div>
-                <div class="List-exitHolder">
-                    <button class="List-exit" ng-click="$state.go('instanceGroups')">
-                        <i class="fa fa-times-circle"></i>
-                    </button>
-                </div>
-            </div>
-            <div class="Form-tabHolder">
-                <div class="Form-tab Form-tab--notitle" ng-click="$state.go('instanceGroups.instances.list', {instance_group_id: $stateParams.instance_group_id})" ng-class="{'is-selected': $state.includes('instanceGroups.instances.list')}" translate>INSTANCES</div>
-                <div class="Form-tab Form-tab--notitle" ng-click="$state.go('instanceGroups.instances.jobs', {instance_group_id: $stateParams.instance_group_id})" ng-class="{'is-selected': $state.includes('instanceGroups.instances.jobs')}" translate>JOBS</div>
-            </div>
-        </div>
-    </div>
-    <div ui-view="list"></div>
-</div>

awx/ui/client/src/instance-groups/instance-groups.partial.html

@@ -1,11 +1,13 @@
 <div class="tab-pane InstanceGroups" id="instance-groups-panel">
     <aw-limit-panels max-panels="2" panel-container="instance-groups-panel"></aw-limit-panels>
+    <div ui-view="add"></div>
+    <div ui-view="edit"></div>
 
     <div ui-view="instanceJobs"></div>
 
     <div ui-view="instances"></div>
 
-    <div ng-cloak id="htmlTemplate" class="Panel">
+    <div ui-view="jobs"></div>
-        <div ui-view="list"></div>
+
-    </div>
+    <div ui-view="list"></div>
 </div>

awx/ui/client/src/instance-groups/instance-groups.route.js

@@ -1,41 +0,0 @@
-import {templateUrl} from '../shared/template-url/template-url.factory';
-import { N_ } from '../i18n';
-
-export default {
-    name: 'instanceGroups',
-    url: '/instance_groups',
-    searchPrefix: 'instance_group',
-    ncyBreadcrumb: {
-        label: N_('INSTANCE GROUPS')
-    },
-    params: {
-        instance_group_search: {
-            value: {
-                page_size: '20',
-                order_by: 'name'
-            },
-            dynamic: true
-        }
-    },
-    data: {
-        alwaysShowRefreshButton: true,
-    },
-    views: {
-        '@': {
-            templateUrl: templateUrl('./instance-groups/instance-groups'),
-        },
-        'list@instanceGroups': {
-            templateUrl: templateUrl('./instance-groups/list/instance-groups-list'),
-            controller: 'InstanceGroupsList'
-
-        }
-    },
-    resolve: {
-        Dataset: ['InstanceGroupList', 'QuerySet', '$stateParams', 'GetBasePath',
-            function(list, qs, $stateParams, GetBasePath) {
-                let path = GetBasePath(list.basePath) || GetBasePath(list.name);
-                return qs.search(path, $stateParams[`${list.iterator}_search`]);
-            }
-        ]
-    }
-};

awx/ui/client/src/instance-groups/instance-groups.strings.js

@@ -0,0 +1,30 @@
+function InstanceGroupsStrings (BaseString) {
+    BaseString.call(this, 'instanceGroups');
+
+    const { t } = this;
+    const ns = this.instanceGroups;
+
+    ns.state = {
+        ADD_BREADCRUMB_LABEL: t.s('CREATE INSTANCE GROUP'),
+        EDIT_BREADCRUMB_LABEL: t.s('EDIT INSTANCE GROUP')
+    };
+
+    ns.tab = {
+        DETAILS: t.s('DETAILS'),
+        INSTANCES: t.s('INSTANCES'),
+        JOBS: t.s('JOBS')
+    };
+
+    ns.instance = {
+        PANEL_TITLE: t.s('SELECT INSTANCE')
+    };
+
+    ns.capacityBar = {
+        IS_OFFLINE: t.s('Unavailable to run jobs.'),
+        IS_OFFLINE_LABEL: t.s('Unavailable')
+    };
+}
+
+InstanceGroupsStrings.$inject = ['BaseStringService'];
+
+export default InstanceGroupsStrings;

awx/ui/client/src/instance-groups/instances/instance-jobs/instance-jobs-list.route.js

@@ -1,41 +0,0 @@
-import { N_ } from '../../../i18n';
-
-export default {
-    name: 'instanceGroups.instances.list.job.list',
-    url: '/jobs',
-    searchPrefix: 'instance_job',
-    ncyBreadcrumb: {
-        parent: 'instanceGroups.instances.list',
-        label: N_('{{ breadcrumb.instance_name }}')
-    },
-    params: {
-        instance_job_search: {
-            value: {
-                page_size: '20',
-                order_by: '-finished',
-                not__launch_type: 'sync'
-            },
-            dynamic: true
-        }
-    },
-    views: {
-        'list@instanceGroups.instances.list.job': {
-            templateProvider: function(InstanceJobsList, generateList) {
-                let html = generateList.build({
-                    list: InstanceJobsList
-                });
-                return html;
-            },
-        controller: 'InstanceJobsController'
-        }
-    },
-
-    resolve: {
-        Dataset: ['InstanceJobsList', 'QuerySet', '$stateParams', 'GetBasePath',
-            function(list, qs, $stateParams, GetBasePath) {
-                let path = `${GetBasePath('instances')}${$stateParams.instance_id}/jobs`;
-                return qs.search(path, $stateParams[`${list.iterator}_search`]);
-            }
-        ],
-    }
-};

awx/ui/client/src/instance-groups/instances/instance-jobs/instance-jobs.controller.js

@@ -1,82 +1,76 @@
-export default ['$scope','InstanceJobsList', 'GetBasePath', 'Rest', 'Dataset','Find', '$state', '$q',
-    function($scope, InstanceJobsList, GetBasePath, Rest, Dataset, Find, $state, $q) {
 
-        let list = InstanceJobsList;
+function InstanceJobsController ($scope, $filter, $state, model, strings, jobStrings) {
+    const vm = this || {};
+    const { instance } = model;
 
-        init();
+    init();
 
-        function init(){
+    function init(){
-            $scope.optionsDefer = $q.defer();
+        vm.strings = strings;
-            $scope.list = list;
+        vm.jobStrings = jobStrings;
-            $scope[`${list.iterator}_dataset`] = Dataset.data;
+        vm.queryset = { page_size: '10', order_by: '-finished'};
-            $scope[list.name] = $scope[`${list.iterator}_dataset`].results;
+        vm.jobs = instance.get('related.jobs.results');
-        }
+        vm.dataset = instance.get('related.jobs');
+        vm.count = instance.get('related.jobs.count');
+        vm.panelTitle = `${jobStrings.get('list.PANEL_TITLE')} | ${instance.get('hostname')}`;
 
-        $scope.$on(`${list.iterator}_options`, function(event, data){
+        vm.tab = {
-            $scope.options = data.data.actions.GET;
+            details: {_hide: true},
-            optionsRequestDataProcessing();
+            instances: {_hide: true},
-        });
+            jobs: {_hide: true}
-
-        // iterate over the list and add fields like type label, after the
-        // OPTIONS request returns, or the list is sorted/paginated/searched
-        function optionsRequestDataProcessing(){
-
-            if($scope[list.name] && $scope[list.name].length > 0) {
-                $scope[list.name].forEach(function(item, item_idx) {
-                    var itm = $scope[list.name][item_idx];
-
-                    if(item.summary_fields && item.summary_fields.source_workflow_job &&
-                        item.summary_fields.source_workflow_job.id){
-                            item.workflow_result_link = `/#/workflows/${item.summary_fields.source_workflow_job.id}`;
-                    }
-
-                    // Set the item type label
-                    if (list.fields.type && $scope.options &&
-                            $scope.options.hasOwnProperty('type')) {
-                                $scope.options.type.choices.forEach(function(choice) {
-                                    if (choice[0] === item.type) {
-                                    itm.type_label = choice[1];
-                                }
-                            });
-                        }
-                        buildTooltips(itm);
-                });
-            }
-        }
-
-        function buildTooltips(job) {
-            job.status_tip = 'Job ' + job.status + ". Click for details.";
-        }
-
-        $scope.viewjobResults = function(job) {
-            var goTojobResults = function(state) {
-                $state.go(state, { id: job.id }, { reload: true });
-            };
-            switch (job.type) {
-                case 'job':
-                    goTojobResults('jobResult');
-                    break;
-                case 'ad_hoc_command':
-                    goTojobResults('adHocJobStdout');
-                    break;
-                case 'system_job':
-                    goTojobResults('managementJobStdout');
-                    break;
-                case 'project_update':
-                    goTojobResults('scmUpdateStdout');
-                    break;
-                case 'inventory_update':
-                    goTojobResults('inventorySyncStdout');
-                    break;
-                case 'workflow_job':
-                    goTojobResults('workflowResults');
-                    break;
-            }
         };
-
-        $scope.$watchCollection(`${$scope.list.name}`, function() {
-                optionsRequestDataProcessing();
-            }
-        );
     }
+
+    vm.getTime = function(time) {
+        let val = "";
+        if (time) {
+            val += $filter('longDate')(time);
+        }
+        if (val === "") {
+            val = undefined;
+        }
+        return val;
+    };
+
+    $scope.isSuccessful = function (status) {
+        return (status === "successful");
+    };
+
+    $scope.viewjobResults = function(job) {
+        var goTojobResults = function(state) {
+            $state.go(state, { id: job.id }, { reload: true });
+        };
+        switch (job.type) {
+            case 'job':
+                goTojobResults('jobResult');
+                break;
+            case 'ad_hoc_command':
+                goTojobResults('adHocJobStdout');
+                break;
+            case 'system_job':
+                goTojobResults('managementJobStdout');
+                break;
+            case 'project_update':
+                goTojobResults('scmUpdateStdout');
+                break;
+            case 'inventory_update':
+                goTojobResults('inventorySyncStdout');
+                break;
+            case 'workflow_job':
+                goTojobResults('workflowResults');
+                break;
+        }
+    };
+
+}
+
+InstanceJobsController.$inject = [
+    '$scope',
+    '$filter',
+    '$state',
+    'resolvedModels',
+    'InstanceGroupsStrings',
+    'JobStrings'
 ];
+
+export default InstanceJobsController;

awx/ui/client/src/instance-groups/instances/instance-jobs/instance-jobs.list.js

@@ -1,78 +0,0 @@
-export default ['i18n', function(i18n) {
-    return {
-
-        name: 'instance_jobs',
-        iterator: 'instance_job',
-        index: false,
-        hover: false,
-        well: false,
-        emptyListText: i18n._('No jobs have yet run.'),
-        title: false,
-        basePath: 'api/v2/instances/{{$stateParams.instance_id}}/jobs',
-
-        fields: {
-            status: {
-                label: '',
-                columnClass: 'col-lg-1 col-md-1 col-sm-2 col-xs-2 List-staticColumn--smallStatus',
-                dataTipWatch: 'instance_job.status_tip',
-                awToolTip: "{{ instance_job.status_tip }}",
-                awTipPlacement: "right",
-                dataTitle: "{{ instance_job.status_popover_title }}",
-                icon: 'icon-job-{{ instance_job.status }}',
-                iconOnly: true,
-                ngClick:"viewjobResults(instance_job)",
-                nosort: true
-            },
-            id: {
-                label: i18n._('ID'),
-                ngClick:"viewjobResults(instance_job)",
-                columnClass: 'col-lg-1 col-md-1 col-sm-2 col-xs-2 List-staticColumnAdjacent',
-                awToolTip: "{{ instance_job.status_tip }}",
-                dataPlacement: 'top',
-                noLink: true
-            },
-            name: {
-                label: i18n._('Name'),
-                columnClass: 'col-lg-2 col-md-3 col-sm-4 col-xs-6',
-                ngClick: "viewjobResults(instance_job)",
-                nosort: true,
-                badgePlacement: 'right',
-                badgeCustom: true,
-                badgeIcon: `<a href="{{ instance_job.workflow_result_link }}"
-                    aw-tool-tip="{{'View workflow results'|translate}}"
-                    data-placement="top"
-                    data-original-title="" title="">
-                    <i class="WorkflowBadge"
-                        ng-show="instance_job.launch_type === 'workflow' ">
-                        W
-                    </i>
-                </a>`
-            },
-            type: {
-                label: i18n._('Type'),
-                ngBind: 'instance_job.type_label',
-                link: false,
-                columnClass: "col-lg-2 hidden-md hidden-sm hidden-xs",
-                nosort: true
-            },
-            finished: {
-                label: i18n._('Finished'),
-                noLink: true,
-                filter: "longDate",
-                columnClass: "col-lg-2 col-md-3 col-sm-3 hidden-xs",
-                key: true,
-                desc: true,
-                nosort: true
-            },
-            labels: {
-                label: i18n._('Labels'),
-                type: 'labels',
-                nosort: true,
-                showDelete: false,
-                columnClass: 'List-tableCell col-lg-4 col-md-4 hidden-sm hidden-xs',
-                sourceModel: 'labels',
-                sourceField: 'name',
-            },
-        }
-    };
-}];

awx/ui/client/src/instance-groups/instances/instance-jobs/instance-jobs.partial.html

@@ -1,32 +0,0 @@
-<div class="Panel">
-    <div class="row Form-tabRow">
-        <div class="col-xs-12">
-            <div class="List-header">
-                <div class="List-title">
-                    <div class="List-titleText">{{ instanceName }}</div>
-                </div>
-                <div class="List-details">
-                    <div class="Capacity-details">
-                        <p class="Capacity-details--label" translate>Used Capacity</p>
-                        <capacity-bar capacity="instanceCapacity" total-capacity="instanceTotalCapacity"></capacity-bar>
-                    </div>
-                    <div class="RunningJobs-details">
-                        <p class="RunningJobs-details--label" translate>Running Jobs</p>
-                        <span class="badge List-titleBadge">
-                            {{ instanceJobsRunning }}
-                        </span>
-                    </div>
-                </div>
-                <div class="List-exitHolder">
-                    <button class="List-exit" ng-click="$state.go('instanceGroups.instances.list')">
-                        <i class="fa fa-times-circle"></i>
-                    </button>
-                </div>
-            </div>
-            <div class="Form-tabHolder">
-                <div class="Form-tab Form-tab--notitle is-selected" translate>JOBS</div>
-            </div>
-        </div>
-    </div>
-    <div class="instance-jobs-list" ui-view="list"></div>
-</div>

awx/ui/client/src/instance-groups/instances/instance-jobs/instance-jobs.route.js

@@ -1,38 +0,0 @@
-import { templateUrl } from '../../../shared/template-url/template-url.factory';
-
-export default {
-    name: 'instanceGroups.instances.list.job',
-    url: '/:instance_id',
-    abstract: true,
-    ncyBreadcrumb: {
-        skip: true
-    },
-    views: {
-        'instanceJobs@instanceGroups': {
-            templateUrl: templateUrl('./instance-groups/instances/instance-jobs/instance-jobs'),
-            controller: function($scope, $rootScope, instance) {
-                $scope.instanceName = instance.hostname;
-                $scope.instanceCapacity = instance.consumed_capacity;
-                $scope.instanceTotalCapacity = instance.capacity;
-                $scope.instanceJobsRunning = instance.jobs_running;
-                $rootScope.breadcrumb.instance_name = instance.hostname;
-            }
-        }
-    },
-    resolve: {
-        instance: ['GetBasePath', 'Rest', 'ProcessErrors', '$stateParams', function(GetBasePath, Rest, ProcessErrors, $stateParams) {
-        let url = GetBasePath('instances') + $stateParams.instance_id;
-        Rest.setUrl(url);
-        return Rest.get()
-            .then(({data}) => {
-                return data;
-            })
-            .catch(({data, status}) => {
-                ProcessErrors(null, data, status, null, {
-                    hdr: 'Error!',
-                    msg: 'Failed to get instance groups info. GET returned status: ' + status
-                });
-            });
-        }]
-    }
-};

awx/ui/client/src/instance-groups/instances/instance-modal.block.less

@@ -0,0 +1,24 @@
+.Modal-backdrop {
+ position: fixed;
+ top: 0px;
+ left: 0px;
+ height:100%;
+ width:100%;
+ background: #000;
+ z-index: 2;
+ opacity: 0.5;
+}
+
+.Modal-holder {
+  position: fixed;
+  top: 1;
+  left: 0px;
+  right: 0px;
+  top: 0px;
+  bottom: 0px;
+  z-index: 3;
+
+  .modal-dialog {
+    padding-top: 100px;
+  }
+}

awx/ui/client/src/instance-groups/instances/instance-modal.controller.js

@@ -0,0 +1,79 @@
+function InstanceModalController ($scope, $state, models, strings, ProcessErrors) {
+    const { instance, instanceGroup } = models;
+    const vm = this || {};
+
+    vm.setInstances = () => {
+        vm.instances = instance.get('results').map(instance => {
+            instance.isSelected = false;
+            return instance;
+        });
+    };
+
+    vm.setRelatedInstances = () => {
+        vm.instanceGroupName = instanceGroup.get('name');
+        vm.relatedInstances = instanceGroup.get('related.instances.results');
+        vm.relatedInstanceIds = vm.relatedInstances.map(instance => instance.id);
+        vm.instances = instance.get('results').map(instance => {
+            instance.isSelected = vm.relatedInstanceIds.includes(instance.id);
+            return instance;
+        });
+    };
+
+    init();
+
+    function init() {
+        vm.strings = strings;
+        vm.panelTitle = strings.get('instance.PANEL_TITLE');
+        vm.instanceGroupId = instanceGroup.get('id');
+
+        if (vm.instanceGroupId === undefined) {
+            vm.setInstances();
+        } else {
+            vm.setRelatedInstances();
+        }
+    }
+
+    $scope.$watch('vm.instances', function() {
+        vm.selectedRows = _.filter(vm.instances, 'isSelected');
+        vm.deselectedRows = _.filter(vm.instances, 'isSelected', false);
+     }, true);
+
+    vm.submit = () => {
+        const associate = vm.selectedRows
+            .map(instance => ({id: instance.id}));
+        const disassociate = vm.deselectedRows
+            .map(instance => ({id: instance.id, disassociate: true}));
+
+        const all = associate.concat(disassociate);
+        const defers = all.map((data) => {
+            const config = {
+                url: `${vm.instanceGroupId}/instances/`,
+                data: data
+            };
+            return instanceGroup.http.post(config);
+        });
+
+        Promise.all(defers)
+            .then(vm.onSaveSuccess)
+            .catch(({data, status}) => {
+                ProcessErrors($scope, data, status, null, {
+                    hdr: 'Error!',
+                    msg: 'Call failed. Return status: ' + status
+                });
+            });
+    };
+
+    vm.onSaveSuccess = () => {
+        $state.go('instanceGroups.instances', {}, {reload: 'instanceGroups.instances'});
+    };
+}
+
+InstanceModalController.$inject = [
+    '$scope',
+    '$state',
+    'resolvedModels',
+    'InstanceGroupsStrings',
+    'ProcessErrors'
+];
+
+export default InstanceModalController;

awx/ui/client/src/instance-groups/instances/instance-modal.partial.html

@@ -0,0 +1,55 @@
+<div id="instance-modal" class="modal-dialog">
+    <at-panel on-dismiss="instanceGroups.instances">
+        <at-panel-heading>
+            {{ vm.panelTitle }} | {{ vm.instanceGroupName}}
+        </at-panel-heading>
+        <multi-select-preview selected-rows='vm.selectedRows' available-rows='vm.instances'></multi-select-preview>
+        <at-panel-body>
+            <div class="at-List-toolbar">
+                <smart-search
+                    class="at-List-search"
+                    django-model="instances"
+                    base-path="instances"
+                    iterator="instance"
+                    list="list"
+                    dataset="vm.instances"
+                    collection="collection"
+                    search-tags="searchTags">
+                </smart-search>
+            </div>
+            <at-list results='vm.instances'>
+                <at-row ng-repeat="instance in vm.instances"
+                    ng-class="{'at-Row--active': (instance.id === vm.activeId)}">
+
+                    <input type="checkbox"
+                        class="at-Row-checkbox"
+                        ng-class="{ 'at-Input--rejected': state.rejected }"
+                        ng-model="instance.isSelected"
+                        ng-checked="instance.isSelected"
+                        ng-attr-tabindex="{{ tab || undefined }}"
+                        ng-disabled="state._disabled" />
+
+                    <div class="at-Row-items" style="flex: 1">
+                        <at-row-item
+                            header-value="{{ instance.hostname }}"
+                            header-tag="{{ vm.instanceTypes[instance.type] }}"></at-row-item>
+                        </at-row-item>
+                    </div>
+                </at-row>
+
+            </at-list>
+            <div class="at-ActionGroup">
+                <div class="pull-right">
+                    <button class="btn at-ButtonHollow--default"
+                        ng-click="$state.go('instanceGroups.instances')">
+                        {{:: vm.strings.get('CANCEL') }}
+                    </button>
+                    <button class="btn at-Button--success"
+                        ng-click="vm.submit()">
+                        {{:: vm.strings.get('SAVE') }}
+                    </button>
+                </div>
+            </div>
+        </at-panel-body>
+    </at-panel>
+</div>

awx/ui/client/src/instance-groups/instances/instances-list.partial.html

@@ -1,44 +1,73 @@
-<div class="instances-list">
+<at-panel>
-    <smart-search django-model="instances" base-path="instances" iterator="instance" dataset="instance_dataset"
+    <at-panel-heading>
-        list="list" collection="instances" search-tags="searchTags">
+        {{ vm.panelTitle }}
-    </smart-search>
+    </at-panel-heading>
 
-    <div class="List-noItems ng-hide" ng-show="instances.length === 0 &amp;&amp; (searchTags | isEmpty)" translate>PLEASE ADD ITEMS TO THIS LIST</div>
+    <at-tab-group>
-    <div class="list-table-container" ng-show="instances.length > 0">
+        <at-tab state="vm.tab.details">{{:: vm.strings.get('tab.DETAILS') }}</at-tab>
-        <table id="instances_table" class="List-table" is-extended="false">
+        <at-tab state="vm.tab.instances">{{:: vm.strings.get('tab.INSTANCES') }}</at-tab>
-            <thead>
+        <at-tab state="vm.tab.jobs" ng-hide="$state.current.name === 'instanceGroups.add'">{{:: vm.strings.get('tab.JOBS') }}</at-tab>
-                <tr class="List-tableHeaderRow">
+    </at-tab-group>
-                    <th id="instance-hostname-header" class="List-tableHeader list-header col-md-5 col-sm-5 col-xs-5" ng-click="columnNoSort !== 'true' &amp;&amp; toggleColumnOrderBy()"
+
-                        ng-class="{'list-header-noSort' : columnNoSort === 'true'}" base-path="instances" collection="instances"
+    <at-panel-body>
-                        dataset="instance_dataset" column-sort="" column-field="hostname" column-iterator="instance" column-no-sort="undefined"
+        <div class="at-List-toolbar">
-                        column-label="Name" column-custom-class="col-md-5 col-sm-5 col-xs-5" query-set="instance_queryset">
+            <smart-search
-                        "{{'Name' | translate}}"
+                class="at-List-search"
-                        <i ng-if="columnNoSort !== 'true'" class="fa columnSortIcon fa-sort-up" ng-class="orderByIcon()"></i>
+                django-model="instances"
-                    </th>
+                base-path="instances"
-                    <th id="instance-jobs_running-header" class="List-tableHeader list-header  list-header-noSort" translate>
+                iterator="instance"
-                        Running Jobs
+                list="list"
-                    </th>
+                dataset="vm.instances"
-                    <th id="instance-consumed_capacity-header" class="List-tableHeader list-header  list-header-noSort" translate>
+                collection="collection"
-                        Used Capacity
+                search-tags="searchTags">
-                    </th>
+            </smart-search>
-                </tr>
+
-            </thead>
+            <div class="at-List-toolbarAction">
-            <tbody>
+                <button
-                <!-- ngRepeat: instance in instances -->
+                    type="button"
-                <tr ng-class="{isActive: isActive(instance.id)}" id="instance.id" class="List-tableRow instance_class ng-scope" ng-repeat="instance in instances">
+                    ng-click="$state.go('instanceGroups.instances.modal.add')"
-                    <td class="List-tableCell hostname-column col-md-5 col-sm-5 col-xs-5">
+                    class="at-Button--add"
-                        <a ui-sref="instanceGroups.instances.list.job.list({instance_id: instance.id})" class="ng-binding">{{ instance.hostname }}</a>
+                    aria-expanded="false">
-                    </td>
+                </button>
-                    <td class="List-tableCell jobs_running-column ng-binding">
+                <div ui-view="modal"></div>
-                        <a ui-sref="instanceGroups.instances.jobs({instance_group_id: $stateParams.instance_group_id})">
+            </div>
-                            {{ instance.jobs_running }}
+        </div>
-                        </a>
+        <at-list results='vm.instances'>
-                    </td>
+            <at-row ng-repeat="instance in vm.instances"
-                    <td class="List-tableCell List-tableCell--capacityColumn ng-binding">
+                ng-class="{'at-Row--active': (instance.id === vm.activeId)}">
-                        <capacity-bar capacity="instance.consumed_capacity" total-capacity="instance.capacity">
+                <div class="at-Row-firstColumn">
-                    </td>
+                    <div class="ScheduleToggle"
-                </tr>
+                        ng-class="{'is-on': instance.enabled}">
-            </tbody>
+                        <button ng-show="instance.enabled"
-        </table>
+                            class="ScheduleToggle-switch is-on ng-hide"
-    </div>
+                            ng-click="vm.toggle(instance)">
-</div>
+                            {{:: vm.strings.get('ON') }}
+                        </button>
+                        <button ng-show="!instance.enabled"
+                            class="ScheduleToggle-switch"
+                            ng-click="vm.toggle(instance)">
+                            {{:: vm.strings.get('OFF') }}
+                        </button>
+                    </div>
+                </div>
+
+                <div class="at-Row-items">
+                    <at-row-item header-value="{{ instance.hostname }}"></at-row-item>
+                    <div class="at-Row--rowLayout">
+                        <at-row-item
+                            label-value="Running Jobs"
+                            label-state="instanceGroups.instanceJobs({instance_group_id: {{vm.instance_group_id}}, instance_id: {{instance.id}}})"
+                            value="{{ instance.jobs_running }}"
+                            badge="true">
+                        </at-row-item>
+                    </div>
+                </div>
+
+                <div class="at-Row-actions">
+                    <capacity-adjuster state="instance"></capacity-adjuster>
+                    <capacity-bar label-value="Used Capacity" capacity="instance.consumed_capacity" total-capacity="instance.capacity"></capacity-bar>
+                </div>
+            </at-row>
+        </at-list>
+    </at-panel-body>
+</at-panel>

awx/ui/client/src/instance-groups/instances/instances-list.route.js

@@ -1,35 +0,0 @@
-import {templateUrl} from '../../shared/template-url/template-url.factory';
-import { N_ } from '../../i18n';
-
-export default {
-    name: 'instanceGroups.instances.list',
-    url: '/instances',
-    searchPrefix: 'instance',
-    ncyBreadcrumb: {
-        parent: 'instanceGroups',
-        label: N_('{{breadcrumb.instance_group_name}}')
-    },
-    params: {
-        instance_search: {
-            value: {
-                page_size: '20',
-                order_by: 'hostname'
-            },
-            dynamic: true
-        }
-    },
-    views: {
-        'list@instanceGroups.instances': {
-            templateUrl: templateUrl('./instance-groups/instances/instances-list'),
-            controller: 'InstanceListController'
-        }
-    },
-    resolve: {
-        Dataset: ['InstanceList', 'QuerySet', '$stateParams', 'GetBasePath',
-            function(list, qs, $stateParams, GetBasePath) {
-                let path = `${GetBasePath('instance_groups')}${$stateParams.instance_group_id}/instances`;
-                return qs.search(path, $stateParams[`${list.iterator}_search`]);
-            }
-        ]
-    }
-};

awx/ui/client/src/instance-groups/instances/instances.controller.js

@@ -1,20 +1,86 @@
-export default ['$scope', 'InstanceList', 'GetBasePath', 'Rest', 'Dataset','Find', '$state', '$q',
+function InstancesController ($scope, $state, $http, models, Instance, strings, Dataset, ProcessErrors) {
-    function($scope, InstanceList, GetBasePath, Rest, Dataset, Find, $state, $q) {
+    const { instanceGroup } = models;
-        let list = InstanceList;
+    const vm = this || {};
+    vm.strings = strings;
+    vm.panelTitle = instanceGroup.get('name');
+    vm.instances = instanceGroup.get('related.instances.results');
+    vm.instance_group_id = instanceGroup.get('id');
 
-        init();
+    init();
 
-        function init(){
+    function init() {
-            $scope.optionsDefer = $q.defer();
+        $scope.list = {
-            $scope.list = list;
+            iterator: 'instance',
-            $scope[`${list.iterator}_dataset`] = Dataset.data;
+            name: 'instances'
-            $scope[list.name] = $scope[`${list.iterator}_dataset`].results;
+        };
+        $scope.collection = {
+            basepath: 'instances',
+            iterator: 'instance'
+        };
+        $scope[`${$scope.list.iterator}_dataset`] = Dataset.data;
+        $scope[$scope.list.name] = $scope[`${$scope.list.iterator}_dataset`].results;
+    }
+
+    vm.tab = {
+        details: {
+            _go: 'instanceGroups.edit',
+            _params: { instance_group_id: vm.instance_group_id }
+        },
+        instances: {
+            _active: true,
+            _go: 'instanceGroups.instances',
+            _params: { instance_group_id: vm.instance_group_id }
+        },
+        jobs: {
+            _go: 'instanceGroups.jobs',
+            _params: { instance_group_id: vm.instance_group_id }
         }
+    };
 
-        $scope.isActive = function(id) {
+    vm.toggle = (toggled) => {
-            let selected = parseInt($state.params.instance_id);
+        const instance = _.find(vm.instances, 'id', toggled.id);
-            return id === selected;
+        instance.enabled = !instance.enabled;
+
+        const data = {
+            "enabled": instance.enabled
         };
 
-    }
+        const req = {
+            method: 'PUT',
+            url: instance.url,
+            data
+        };
+
+        $http(req).then(vm.onSaveSuccess)
+            .catch(({data, status}) => {
+                ProcessErrors($scope, data, status, null, {
+                    hdr: 'Error!',
+                    msg: 'Call failed. Return status: ' + status
+                });
+            });
+    };
+
+    vm.onSaveSuccess = () => {
+        $state.transitionTo($state.current, $state.params, {
+            reload: true, location: true, inherit: false, notify: true
+        });
+    };
+
+    $scope.isActive = function(id) {
+        let selected = parseInt($state.params.instance_id);
+        return id === selected;
+    };
+}
+
+InstancesController.$inject = [
+    '$scope',
+    '$state',
+    '$http',
+    'resolvedModels',
+    'InstanceModel',
+    'InstanceGroupsStrings',
+    'Dataset',
+    'ProcessErrors'
 ];
+
+export default InstancesController;

awx/ui/client/src/instance-groups/instances/instances.list.js

@@ -1,29 +0,0 @@
-export default ['i18n', function(i18n) {
-    return {
-        name:  'instances' ,
-        iterator: 'instance',
-        listTitle: false,
-        index: false,
-        hover: false,
-        tabs: true,
-        well: true,
-
-        fields: {
-            hostname: {
-                key: true,
-                label: i18n._('Name'),
-                columnClass: 'col-md-3 col-sm-9 col-xs-9',
-                modalColumnClass: 'col-md-8',
-                uiSref: 'instanceGroups.instances.list.job({instance_id: instance.id})'
-            },
-            consumed_capacity: {
-                label: i18n._('Capacity'),
-                nosort: true,
-            },
-            jobs_running: {
-                label: i18n._('Running Jobs'),
-                nosort: true,
-            },
-        }
-    };
-}];

awx/ui/client/src/instance-groups/instances/instances.route.js

@@ -1,35 +0,0 @@
-import {templateUrl} from '../../shared/template-url/template-url.factory';
-
-export default {
-    name: 'instanceGroups.instances',
-    url: '/:instance_group_id',
-    abstract: true,
-    views: {
-        'instances@instanceGroups': {
-            templateUrl: templateUrl('./instance-groups/instance-group'),
-            controller: function($scope, $rootScope, instanceGroup) {
-                $scope.instanceGroupName = instanceGroup.name;
-                $scope.instanceGroupCapacity = instanceGroup.consumed_capacity;
-                $scope.instanceGroupTotalCapacity = instanceGroup.capacity;
-                $scope.instanceGroupJobsRunning = instanceGroup.jobs_running;
-                $rootScope.breadcrumb.instance_group_name = instanceGroup.name;
-            }
-        }
-    },
-    resolve: {
-        instanceGroup: ['GetBasePath', 'Rest', 'ProcessErrors', '$stateParams', function(GetBasePath, Rest, ProcessErrors, $stateParams) {
-            let url = GetBasePath('instance_groups') + $stateParams.instance_group_id;
-            Rest.setUrl(url);
-            return Rest.get()
-                .then(({data}) => {
-                    return data;
-                })
-                .catch(({data, status}) => {
-                    ProcessErrors(null, data, status, null, {
-                        hdr: 'Error!',
-                        msg: 'Failed to get instance groups info. GET returned status: ' + status
-                    });
-                });
-        }]
-    }
-};

awx/ui/client/src/instance-groups/jobs/jobs-list.partial.html

@@ -0,0 +1,89 @@
+<at-panel>
+    <at-panel-heading>
+        {{ vm.panelTitle }}
+    </at-panel-heading>
+    <at-tab-group>
+        <at-tab state="vm.tab.details">{{:: vm.strings.get('tab.DETAILS') }}</at-tab>
+        <at-tab state="vm.tab.instances">{{:: vm.strings.get('tab.INSTANCES') }}</at-tab>
+        <at-tab state="vm.tab.jobs">{{:: vm.strings.get('tab.JOBS') }}</at-tab>
+    </at-tab-group>
+
+    <at-panel-body>
+        <div class="at-List-toolbar">
+            <smart-search
+                class="at-List-search"
+                django-model="jobs"
+                base-path="unified_jobs"
+                iterator="job"
+                list="list"
+                dataset="job_dataset"
+                collection="collection"
+                search-tags="searchTags"
+                query-set="querySet">
+            </smart-search>
+        </div>
+        <at-list results="vm.jobs">
+            <at-row ng-repeat="job in vm.jobs"
+                ng-class="{'at-Row--active': (job.id === vm.activeId)}"
+                job-id="{{ job.id }}">
+                <div class="at-RowStatus">
+                    <a href="{{ job.detailsUrl }}" ng-if="isSuccessful(job.status)" aw-tool-tip="Job successful. Click for details." aw-tip-placement="right">
+                        <i class="fa DashboardList-status DashboardList-status--success icon-job-successful"></i>
+                    </a>
+                    <a href="{{ job.detailsUrl }}" ng-if="!isSuccessful(job.status)" aw-tool-tip="Job failed. Click for details." aw-tip-placement="right">
+                        <i class="fa DashboardList-status DashboardList-status--failed icon-job-failed"></i>
+                    </a>
+                </div>
+                <div class="at-Row-items">
+                    <at-row-item
+                        class="at-RowItem--isHeaderLink"
+                        header-value="{{ job.name }}"
+                        header-tag="{{ job.type }}"
+                        ng-click="vm.viewjobResults(job)">
+                    </at-row-item>
+                    <at-row-item
+                        label-value="{{:: vm.jobStrings.get('list.ROW_ITEM_LABEL_STARTED') }}"
+                        value="{{ vm.getTime(job.started) }}">
+                    </at-row-item>
+                    <at-row-item
+                        label-value="{{:: vm.jobStrings.get('list.ROW_ITEM_LABEL_FINISHED') }}"
+                        value="{{ vm.getTime(job.finished) }}">
+                    </at-row-item>
+                    <at-row-item
+                        header-value="{{ job.name }}"
+                        header-link="/#/jobs/workflow_job/{{ job.id }}"
+                        header-tag="{{ vm.jobTypes[job.type] }}"
+                        ng-if="job.type === 'workflow_job_job'">
+                    </at-row-item>
+                    <at-row-item
+                        label-value="{{:: vm.jobStrings.get('list.ROW_ITEM_LABEL_TEMPLATE') }}"
+                        value="{{ job.summary_fields.job_template.name }}"
+                        value-link="/#/templates/job_template/{{ job.summary_fields.job_template.id }}">
+                    </at-row-item>
+                    <at-row-item
+                        label-value="{{:: vm.jobStrings.get('list.ROW_ITEM_LABEL_INVENTORY') }}"
+                        value="{{ job.summary_fields.inventory.name }}"
+                        value-link="/#/inventories/inventory/{{ job.summary_fields.inventory.id }}">
+                    </at-row-item>
+                    <at-row-item
+                        label-value="{{:: vm.jobStrings.get('list.ROW_ITEM_LABEL_PROJECT') }}"
+                        value="{{ job.summary_fields.project.name }}"
+                        value-link="/#/projects/{{ job.summary_fields.project.id }}">
+                    </at-row-item>
+                    <at-row-item
+                        label-value="{{:: vm.jobStrings.get('list.ROW_ITEM_LABEL_CREDENTIALS') }}"
+                        tag-values="job.summary_fields.credentials"
+                        tags-are-creds="true">
+                    </at-row-item>
+                </div>
+            </at-row>
+        </at-list>
+        <paginate
+            collection="vm.jobs"
+            dataset="vm.dataset"
+            iterator="job"
+            base-path="unified_jobs"
+            query-set="vm.queryset">
+        </paginate>
+    </at-panel-body>
+</at-panel>

awx/ui/client/src/instance-groups/jobs/jobs-list.route.js

@@ -1,41 +0,0 @@
-import { N_ } from '../../i18n';
-
-export default {
-    name: 'instanceGroups.instances.jobs',
-    url: '/jobs',
-    searchPrefix: 'job',
-    ncyBreadcrumb: {
-        parent: 'instanceGroups.instances.list',
-        label: N_('JOBS')
-    },
-    params: {
-        job_search: {
-            value: {
-                page_size: '20',
-                order_by: '-finished',
-                not__launch_type: 'sync'
-            },
-            dynamic: true
-        },
-        instance_group_id: null
-    },
-    views: {
-        'list@instanceGroups.instances': {
-            templateProvider: function(JobsList, generateList) {
-                let html = generateList.build({
-                    list: JobsList
-                });
-                return html;
-            },
-            controller: 'JobsListController'
-        }
-    },
-    resolve: {
-        Dataset: ['JobsList', 'QuerySet', '$stateParams', 'GetBasePath',
-            function(list, qs, $stateParams, GetBasePath) {
-                let path = `${GetBasePath('instance_groups')}${$stateParams.instance_group_id}/jobs`;
-                return qs.search(path, $stateParams[`${list.iterator}_search`]);
-            }
-        ]
-    }
-};

awx/ui/client/src/instance-groups/jobs/jobs.controller.js

@@ -1,82 +1,88 @@
-export default ['$scope','JobsList', 'GetBasePath', 'Rest', 'Dataset','Find', '$state', '$q',
-    function($scope, JobsList, GetBasePath, Rest, Dataset, Find, $state, $q) {
 
-        let list = JobsList;
+function InstanceGroupJobsController ($scope, $filter, $state, model, strings, jobStrings) {
+    const vm = this || {};
+    const { instanceGroup } = model;
 
-        init();
+    init();
 
-        function init(){
+    function init(){
-            $scope.optionsDefer = $q.defer();
+        const instance_group_id = instanceGroup.get('id');
-            $scope.list = list;
+        vm.strings = strings;
-            $scope[`${list.iterator}_dataset`] = Dataset.data;
+        vm.jobStrings = jobStrings;
-            $scope[list.name] = $scope[`${list.iterator}_dataset`].results;
+        vm.queryset = { page_size: '10', order_by: '-finished', instance_group_id: instance_group_id };
-        }
+        vm.jobs = instanceGroup.get('related.jobs.results');
+        vm.dataset = instanceGroup.get('related.jobs');
+        vm.count = instanceGroup.get('related.jobs.count');
+        vm.panelTitle = instanceGroup.get('name');
 
-        $scope.$on(`${list.iterator}_options`, function(event, data){
+        vm.tab = {
-            $scope.options = data.data.actions.GET;
+            details: {
-            optionsRequestDataProcessing();
+                _go: 'instanceGroups.edit',
-        });
+                _params: { instance_group_id },
-
+                _label: strings.get('tab.DETAILS')
-        // iterate over the list and add fields like type label, after the
+            },
-        // OPTIONS request returns, or the list is sorted/paginated/searched
+            instances: {
-        function optionsRequestDataProcessing(){
+                _go: 'instanceGroups.instances',
-
+                _params: { instance_group_id },
-            if($scope[list.name] && $scope[list.name].length > 0) {
+                _label: strings.get('tab.INSTANCES')
-                $scope[list.name].forEach(function(item, item_idx) {
+            },
-                    var itm = $scope[list.name][item_idx];
+            jobs: {
-                    if(item.summary_fields && item.summary_fields.source_workflow_job &&
+                _active: true,
-                        item.summary_fields.source_workflow_job.id){
+                _label: strings.get('tab.JOBS')
-                            item.workflow_result_link = `/#/workflows/${item.summary_fields.source_workflow_job.id}`;
-                    }
-
-                    // Set the item type label
-                    if (list.fields.type && $scope.options &&
-                            $scope.options.hasOwnProperty('type')) {
-                                $scope.options.type.choices.forEach(function(choice) {
-                                    if (choice[0] === item.type) {
-                                    itm.type_label = choice[1];
-                                }
-                            });
-                        }
-                        buildTooltips(itm);
-                });
             }
-        }
-
-        function buildTooltips(job) {
-            job.status_tip = 'Job ' + job.status + ". Click for details.";
-        }
-
-        $scope.viewjobResults = function(job) {
-            var goTojobResults = function(state) {
-                $state.go(state, { id: job.id }, { reload: true });
-            };
-            switch (job.type) {
-                case 'job':
-                    goTojobResults('jobResult');
-                    break;
-                case 'ad_hoc_command':
-                    goTojobResults('adHocJobStdout');
-                    break;
-                case 'system_job':
-                    goTojobResults('managementJobStdout');
-                    break;
-                case 'project_update':
-                    goTojobResults('scmUpdateStdout');
-                    break;
-                case 'inventory_update':
-                    goTojobResults('inventorySyncStdout');
-                    break;
-                case 'workflow_job':
-                    goTojobResults('workflowResults');
-                    break;
-            }
-
         };
-
-        $scope.$watchCollection(`${$scope.list.name}`, function() {
-                optionsRequestDataProcessing();
-            }
-        );
     }
+
+    vm.getTime = function(time) {
+        let val = "";
+        if (time) {
+            val += $filter('longDate')(time);
+        }
+        if (val === "") {
+            val = undefined;
+        }
+        return val;
+    };
+
+    $scope.isSuccessful = function (status) {
+        return (status === "successful");
+    };
+
+    vm.viewjobResults = function(job) {
+        var goTojobResults = function(state) {
+            $state.go(state, { id: job.id }, { reload: true });
+        };
+        switch (job.type) {
+            case 'job':
+                goTojobResults('jobResult');
+                break;
+            case 'ad_hoc_command':
+                goTojobResults('adHocJobStdout');
+                break;
+            case 'system_job':
+                goTojobResults('managementJobStdout');
+                break;
+            case 'project_update':
+                goTojobResults('scmUpdateStdout');
+                break;
+            case 'inventory_update':
+                goTojobResults('inventorySyncStdout');
+                break;
+            case 'workflow_job':
+                goTojobResults('workflowResults');
+                break;
+        }
+
+    };
+}
+
+InstanceGroupJobsController.$inject = [
+    '$scope',
+    '$filter',
+    '$state',
+    'resolvedModels',
+    'InstanceGroupsStrings',
+    'JobStrings'
 ];
+
+export default InstanceGroupJobsController;

awx/ui/client/src/instance-groups/jobs/jobs.strings.js

@@ -0,0 +1,30 @@
+function JobStrings (BaseString) {
+    BaseString.call(this, 'jobs');
+
+    const { t } = this;
+    const ns = this.jobs;
+
+    ns.state = {
+        LIST_BREADCRUMB_LABEL: t.s('JOBS')
+    };
+
+    ns.list = {
+        PANEL_TITLE: t.s('JOBS'),
+        ADD_BUTTON_LABEL: t.s('ADD'),
+        ADD_DD_JT_LABEL: t.s('Job Template'),
+        ADD_DD_WF_LABEL: t.s('Workflow Template'),
+        ROW_ITEM_LABEL_ACTIVITY: t.s('Activity'),
+        ROW_ITEM_LABEL_INVENTORY: t.s('Inventory'),
+        ROW_ITEM_LABEL_PROJECT: t.s('Project'),
+        ROW_ITEM_LABEL_TEMPLATE: t.s('Template'),
+        ROW_ITEM_LABEL_CREDENTIALS: t.s('Credentials'),
+        ROW_ITEM_LABEL_MODIFIED: t.s('Last Modified'),
+        ROW_ITEM_LABEL_RAN: t.s('Last Ran'),
+        ROW_ITEM_LABEL_STARTED: t.s('Started'),
+        ROW_ITEM_LABEL_FINISHED: t.s('Finished')
+    };
+}
+
+JobStrings.$inject = ['BaseStringService'];
+
+export default JobStrings;

awx/ui/client/src/instance-groups/list/instance-groups-list.controller.js

@@ -1,6 +1,10 @@
-export default ['$scope', 'InstanceGroupList', 'GetBasePath', 'Rest', 'Dataset','Find', '$state',
+export default ['$scope', 'InstanceGroupList', 'resolvedModels', 'Dataset', '$state', 'ComponentsStrings', 'ProcessErrors',
-    function($scope, InstanceGroupList, GetBasePath, Rest, Dataset, Find, $state) {
+    function($scope, InstanceGroupList, resolvedModels, Dataset, $state, strings, ProcessErrors) {
-        let list = InstanceGroupList;
+        const list = InstanceGroupList;
+        const vm = this;
+        const { instanceGroup } = resolvedModels;
+
+        vm.strings = strings;
 
         init();
 
@@ -11,9 +15,36 @@ export default ['$scope', 'InstanceGroupList', 'GetBasePath', 'Rest', 'Dataset',
             $scope.instanceGroupCount = Dataset.data.count;
         }
 
-        $scope.isActive = function(id) {
+        $scope.selection = {};
-            let selected = parseInt($state.params.instance_group_id);
+
-            return id === selected;
+        $scope.$watch('$state.params.instance_group_id', () => {
+            vm.activeId = parseInt($state.params.instance_group_id);
+        });
+
+        vm.delete = () => {
+            let deletables = $scope.selection;
+            deletables = Object.keys(deletables).filter((n) => deletables[n]);
+
+            deletables.forEach((data) => {
+                let promise = instanceGroup.http.delete({resource: data});
+                Promise.resolve(promise).then(vm.onSaveSuccess)
+                    .catch(({data, status}) => {
+                        ProcessErrors($scope, data, status, null, {
+                            hdr: 'Error!',
+                            msg: 'Call failed. Return status: ' + status
+                        });
+                    });
+            });
+        };
+
+        vm.onSaveSuccess = () => {
+            $state.transitionTo($state.current, $state.params, {
+                reload: true, location: true, inherit: false, notify: true
+            });
+        };
+
+        $scope.createInstanceGroup = () => {
+            $state.go('instanceGroups.add');
         };
     }
 ];

awx/ui/client/src/instance-groups/list/instance-groups-list.partial.html

@@ -1,63 +1,82 @@
-<div class="List-header">
+<at-panel>
-    <div class="List-title">
+    <at-panel-heading hide-dismiss="true">
-        <div class="List-titleText" translate>
+        {{ vm.strings.get('layout.INSTANCE_GROUPS') }}
-            INSTANCE GROUPS
-        </div>
         <span class="badge List-titleBadge">
             {{ instanceGroupCount }}
         </span>
-    </div>
+    </at-panel-heading>
-</div>
 
-<smart-search django-model="instance_groups" base-path="instance_groups" iterator="instance_group" list="list" dataset="instance_group_dataset"
+    <at-panel-body>
-    collection="instance_groups" search-tags="searchTags">
+        <div class="at-List-toolbar">
-</smart-search>
+            <smart-search
+                class="at-List-search"
+                django-model="instance_groups"
+                base-path="instance_groups"
+                iterator="instance_group"
+                list="list"
+                dataset="instance_group_dataset"
+                collection="collection"
+                search-tags="searchTags">
+            </smart-search>
+            <div class="at-List-toolbarAction">
+                <div ng-click="vm.delete()"
+                    class="at-RowAction at-RowAction--danger">
+                    <i class="fa fa-trash"></i>
+                </div>
+                <button
+                    type="button"
+                    ui-sref="instanceGroups.add"
+                    class="at-Button--add"
+                    aria-haspopup="true"
+                    aria-expanded="false">
+                </button>
+            </div>
+        </div>
 
-<div class="List-noItems" ng-show="instance_groups.length < 1" translate>PLEASE ADD ITEMS TO THIS LIST</div>
+        <at-list>
+            <at-row ng-repeat="instance_group in instance_groups"
+                ng-class="{'at-Row--active': (instance_group.id === vm.activeId)}">
 
-<div class="list-table-container" ng-show="instance_groups.length > 0">
+                <input type="checkbox"
-    <table id="instance_groups_table" class="List-table" is-extended="false">
+                class="at-Row-checkbox"
-        <thead>
+                ng-model="selection[instance_group.id]"/>
-            <tr class="List-tableHeaderRow">
+
-                <th id="instance_group-name-header" class="List-tableHeader list-header col-md-5 col-sm-5 col-xs-5" ng-click="columnNoSort !== 'true' &amp;&amp; toggleColumnOrderBy()"
+                <div class="at-Row-items">
-                    ng-class="{'list-header-noSort' : columnNoSort === 'true'}" base-path="instance_groups" collection="instance_groups"
+                    <at-row-item
-                    dataset="instance_group_dataset" column-sort="" column-field="name" column-iterator="instance_group" column-no-sort="undefined"
+                        header-value="{{ instance_group.name }}"
-                    column-label="Name" column-custom-class="col-md-5 col-sm-5 col-xs-5" query-set="instance_group_queryset">
+                        header-link="/#/instance_groups/{{ instance_group.id }}">
-                    "{{'Name' | translate}}"
+                    </at-row-item>
-                    <i ng-if="columnNoSort !== 'true'" class="fa columnSortIcon fa-sort-up" ng-class="orderByIcon()"></i>
+
-                </th>
+                    <div class="at-Row--rowLayout">
-                <th id="instance_group-jobs_running-header" class="List-tableHeader list-header list-header-noSort" translate>
+                        <at-row-item
-                    Running Jobs
+                            label-value="Instances"
-                </th>
+                            label-link="/#/instance_groups/{{ instance_group.id }}/instances"
-                <th id="instance_group-consumed_capacity-header" class="List-tableHeader list-header list-header-noSort" translate>
+                            value="{{ instance_group.instances }}"
-                    Used Capacity
+                            badge="true">
-                </th>
+                        </at-row-item>
-            </tr>
+
-        </thead>
+                        <at-row-item
-        <tbody>
+                            label-value="Running Jobs"
-            <!-- ngRepeat: instance_group in instance_groups -->
+                            label-link="/#/instance_groups/{{ instance_group.id }}/jobs"
-            <tr ng-class="{isActive: isActive(instance_group.id)}" id="instance_group.id" class="List-tableRow instance_group_class ng-scope" ng-repeat="instance_group in instance_groups">
+                            value="{{ instance_group.jobs_running }}"
-                <td class="List-tableCell name-column col-md-5 col-sm-5 col-xs-5">
+                            badge="true">
-                    <a ui-sref="instanceGroups.instances.list({instance_group_id: instance_group.id})" class="ng-binding" >{{ instance_group.name }}</a>
+                        </at-row-item>
-                    <span class="badge List-titleBadge">{{ instance_group.instances }}</span>
+                    </div>
-                </td>
+
-                <td class="List-tableCell jobs_running-column ng-binding">
+                </div>
-                    <a ui-sref="instanceGroups.instances.jobs({instance_group_id: instance_group.id})">
+
-                        {{ instance_group.jobs_running }}
+                <div class="at-Row-actions">
-                    </a>
+                    <capacity-bar label-value="Used Capacity" capacity="instance_group.consumed_capacity" total-capacity="instance_group.capacity"></capacity-bar>
-                </td>
+                </div>
-                <td class="List-tableCell List-tableCell--capacityColumn ng-binding">
+            </at-row>
-                    <capacity-bar capacity="instance_group.consumed_capacity" total-capacity="instance_group.capacity"></capacity-bar>
+        </at-list>
-                </td>
+    </at-panel-body>
-            </tr>
+    <paginate
-        </tbody>
+        base-path="instance_groups"
-    </table>
+        iterator="instance_group"
-</div>
+        dataset="instance_group_dataset"
+        collection="instance_groups"
+        query-set="instance_group_queryset">
+    </paginate>
+</at-panel>
 
-<paginate
-    base-path="instance_groups"
-    iterator="instance_group"
-    dataset="instance_group_dataset"
-    collection="instance_groups"
-    query-set="instance_group_queryset">
-</paginate>

awx/ui/client/src/instance-groups/main.js

@@ -1,58 +1,335 @@
-import InstanceGroupsList from './list/instance-groups-list.controller';
+import { templateUrl } from '../shared/template-url/template-url.factory';
+import CapacityAdjuster from './capacity-adjuster/capacity-adjuster.directive';
+import CapacityBar from './capacity-bar/capacity-bar.directive';
 import instanceGroupsMultiselect from '../shared/instance-groups-multiselect/instance-groups.directive';
 import instanceGroupsModal from '../shared/instance-groups-multiselect/instance-groups-modal/instance-groups-modal.directive';
-import instanceGroupsRoute from './instance-groups.route';
+
-import instancesListRoute from './instances/instances-list.route';
+import AddEditTemplate from './add-edit/add-edit-instance-groups.view.html';
-import JobsList from './jobs/jobs.list';
+import AddInstanceGroupController from './add-edit/add-instance-group.controller';
-import jobsListRoute from './jobs/jobs-list.route';
+import EditInstanceGroupController from './add-edit/edit-instance-group.controller';
-import JobsListController from './jobs/jobs.controller';
+import InstanceListPolicy from './add-edit/instance-list-policy.directive.js';
-import InstanceList from './instances/instances.list';
+
-import instancesRoute from './instances/instances.route';
+import InstanceGroupsTemplate from './list/instance-groups-list.partial.html';
+import InstanceGroupsListController from './list/instance-groups-list.controller';
+
+import InstancesTemplate from './instances/instances-list.partial.html';
 import InstanceListController from './instances/instances.controller';
-import InstanceJobsList from './instances/instance-jobs/instance-jobs.list';
+
-import instanceJobsRoute from './instances/instance-jobs/instance-jobs.route';
+import JobsTemplate from './jobs/jobs-list.partial.html';
-import instanceJobsListRoute from './instances/instance-jobs/instance-jobs-list.route';
+import InstanceGroupJobsListController from './jobs/jobs.controller';
-import InstanceJobsController from './instances/instance-jobs/instance-jobs.controller';
+import InstanceJobsListController from './instances/instance-jobs/instance-jobs.controller';
-import CapacityBar from './capacity-bar/main';
+
+import InstanceModalTemplate from './instances/instance-modal.partial.html';
+import InstanceModalController from './instances/instance-modal.controller.js';
+
 import list from './instance-groups.list';
 import service from './instance-groups.service';
 
-export default
+import InstanceGroupsStrings from './instance-groups.strings';
-angular.module('instanceGroups', [CapacityBar.name])
+import JobStrings from './jobs/jobs.strings';
+
+const MODULE_NAME = 'instanceGroups';
+
+function InstanceGroupsResolve ($q, $stateParams, InstanceGroup, Instance) {
+    const instanceGroupId = $stateParams.instance_group_id;
+    const instanceId = $stateParams.instance_id;
+    let promises = {};
+
+    if (!instanceGroupId && !instanceId) {
+        promises.instanceGroup = new InstanceGroup(['get', 'options']);
+        promises.instance = new Instance(['get', 'options']);
+
+        return $q.all(promises);
+    }
+
+    if (instanceGroupId && instanceId) {
+        promises.instance = new Instance(['get', 'options'], [instanceId, instanceId])
+            .then((instance) => instance.extend('get', 'jobs', {params: {page_size: "10", order_by: "-finished"}}));
+
+        return $q.all(promises);
+    }
+
+    promises.instanceGroup = new InstanceGroup(['get', 'options'], [instanceGroupId, instanceGroupId])
+            .then((instanceGroup) =>  instanceGroup.extend('get', 'jobs', {params: {page_size: "10", order_by: "-finished"}}))
+            .then((instanceGroup) =>  instanceGroup.extend('get', 'instances'));
+    promises.instance = new Instance('get');
+
+
+    return $q.all(promises)
+        .then(models => models);
+}
+
+InstanceGroupsResolve.$inject = [
+    '$q',
+    '$stateParams',
+    'InstanceGroupModel',
+    'InstanceModel'
+];
+
+function InstanceGroupsRun ($stateExtender, strings, ComponentsStrings) {
+    $stateExtender.addState({
+        name: 'instanceGroups',
+        url: '/instance_groups',
+        searchPrefix: 'instance_group',
+        ncyBreadcrumb: {
+            label: ComponentsStrings.get('layout.INSTANCE_GROUPS')
+        },
+        params: {
+            instance_group_search: {
+                value: {
+                    page_size: '10',
+                    order_by: 'name'
+                },
+                dynamic: true
+            }
+        },
+        data: {
+            alwaysShowRefreshButton: true,
+        },
+        views: {
+            '@': {
+                templateUrl: templateUrl('./instance-groups/instance-groups'),
+            },
+            'list@instanceGroups': {
+                templateUrl: InstanceGroupsTemplate,
+                controller: 'InstanceGroupsListController',
+                controllerAs: 'vm'
+            }
+        },
+        resolve: {
+            resolvedModels: InstanceGroupsResolve,
+            Dataset: ['InstanceGroupList', 'QuerySet', '$stateParams', 'GetBasePath',
+                function(list, qs, $stateParams, GetBasePath) {
+                    let path = GetBasePath(list.basePath) || GetBasePath(list.name);
+                    return qs.search(path, $stateParams[`${list.iterator}_search`]);
+                }
+            ]
+        }
+    });
+
+    $stateExtender.addState({
+        name: 'instanceGroups.add',
+        url: '/add',
+        ncyBreadcrumb: {
+            label: strings.get('state.ADD_BREADCRUMB_LABEL')
+        },
+        views: {
+            'add@instanceGroups': {
+                templateUrl: AddEditTemplate,
+                controller: AddInstanceGroupController,
+                controllerAs: 'vm'
+            }
+        },
+        resolve: {
+            resolvedModels: InstanceGroupsResolve
+        }
+    });
+
+    $stateExtender.addState({
+        name: 'instanceGroups.add.modal',
+        abstract: true,
+        ncyBreadcrumb: {
+            skip: true,
+        },
+        views: {
+            "modal": {
+                template: `<div class="Modal-backdrop"></div>
+                <div class="Modal-holder" ui-view="modal" autoscroll="false"></div>`,
+            }
+        }
+    });
+
+    $stateExtender.addState({
+        name: 'instanceGroups.add.modal.instances',
+        ncyBreadcrumb: {
+            skip: true,
+        },
+        views: {
+            "modal": {
+                template: '<instance-list-policy></instance-list-policy>',
+            }
+        },
+        resolvedModels: InstanceGroupsResolve
+    });
+
+    $stateExtender.addState({
+        name: 'instanceGroups.edit',
+        route: '/:instance_group_id',
+        ncyBreadcrumb: {
+            label: strings.get('state.EDIT_BREADCRUMB_LABEL')
+        },
+        views: {
+            'edit@instanceGroups': {
+                templateUrl: AddEditTemplate,
+                controller: EditInstanceGroupController,
+                controllerAs: 'vm'
+            }
+        },
+        resolve: {
+            resolvedModels: InstanceGroupsResolve
+        }
+    });
+
+
+    $stateExtender.addState({
+        name: 'instanceGroups.edit.modal',
+        abstract: true,
+        ncyBreadcrumb: {
+            skip: true,
+        },
+        views: {
+            "modal": {
+                template: `<div class="Modal-backdrop"></div>
+                <div class="Modal-holder" ui-view="modal" autoscroll="false"></div>`,
+            }
+        }
+    });
+
+    $stateExtender.addState({
+        name: 'instanceGroups.edit.modal.instances',
+        ncyBreadcrumb: {
+            skip: true,
+        },
+        views: {
+            "modal": {
+                template: '<instance-list-policy></instance-list-policy>',
+            }
+        },
+        resolvedModels: InstanceGroupsResolve
+    });
+
+    $stateExtender.addState({
+        name: 'instanceGroups.instances',
+        url: '/:instance_group_id/instances',
+        ncyBreadcrumb: {
+            parent: 'instanceGroups.edit',
+            label: ComponentsStrings.get('layout.INSTANCES')
+        },
+        params: {
+            instance_search: {
+                value: {
+                    page_size: '10',
+                    order_by: 'hostname'
+                },
+                dynamic: true
+            }
+        },
+        views: {
+            'instances@instanceGroups': {
+                templateUrl: InstancesTemplate,
+                controller: 'InstanceListController',
+                controllerAs: 'vm'
+            }
+        },
+        resolve: {
+            resolvedModels: InstanceGroupsResolve
+        }
+    });
+
+    $stateExtender.addState({
+        name: 'instanceGroups.instances.modal',
+        abstract: true,
+        ncyBreadcrumb: {
+            skip: true,
+        },
+        views: {
+            "modal": {
+                template: `<div class="Modal-backdrop"></div>
+                <div class="Modal-holder" ui-view="modal" autoscroll="false"></div>`,
+            }
+        }
+    });
+
+    $stateExtender.addState({
+        name: 'instanceGroups.instances.modal.add',
+        ncyBreadcrumb: {
+            skip: true,
+        },
+        views: {
+            "modal": {
+                templateUrl: InstanceModalTemplate,
+                controller: InstanceModalController,
+                controllerAs: 'vm'
+            }
+        },
+        resolvedModels: InstanceGroupsResolve
+    });
+
+    $stateExtender.addState({
+        name: 'instanceGroups.instanceJobs',
+        url: '/:instance_group_id/instances/:instance_id/jobs',
+        ncyBreadcrumb: {
+            parent: 'instanceGroups.instances',
+            label: ComponentsStrings.get('layout.JOBS')
+        },
+        views: {
+            'instanceJobs@instanceGroups': {
+                templateUrl: JobsTemplate,
+                controller: 'InstanceJobsListController',
+                controllerAs: 'vm'
+            },
+        },
+        params: {
+            job_search: {
+                value: {
+                    page_size: '10',
+                    order_by: '-finished'
+                },
+                dynamic: true
+            },
+        },
+        resolvedModels: InstanceGroupsResolve
+    });
+
+    $stateExtender.addState({
+        name: 'instanceGroups.jobs',
+        url: '/:instance_group_id/jobs',
+        ncyBreadcrumb: {
+            parent: 'instanceGroups.edit',
+            label: ComponentsStrings.get('layout.JOBS')
+        },
+        params: {
+            job_search: {
+                value: {
+                    page_size: '10',
+                    order_by: '-finished'
+                },
+                dynamic: true
+            }
+        },
+        views: {
+            'jobs@instanceGroups': {
+                templateUrl: JobsTemplate,
+                controller: 'InstanceGroupJobsListController',
+                controllerAs: 'vm'
+            },
+        },
+        resolve: {
+            resolvedModels: InstanceGroupsResolve
+        }
+    });
+}
+
+InstanceGroupsRun.$inject = [
+    '$stateExtender',
+    'InstanceGroupsStrings',
+    'ComponentsStrings'
+];
+
+angular.module(MODULE_NAME, [])
     .service('InstanceGroupsService', service)
     .factory('InstanceGroupList', list)
-    .factory('JobsList', JobsList)
+    .controller('InstanceGroupsListController', InstanceGroupsListController)
-    .factory('InstanceList', InstanceList)
+    .controller('InstanceGroupJobsListController', InstanceGroupJobsListController)
-    .factory('InstanceJobsList', InstanceJobsList)
-    .controller('InstanceGroupsList', InstanceGroupsList)
-    .controller('JobsListController', JobsListController)
     .controller('InstanceListController', InstanceListController)
-    .controller('InstanceJobsController', InstanceJobsController)
+    .controller('InstanceJobsListController', InstanceJobsListController)
+    .directive('instanceListPolicy', InstanceListPolicy)
     .directive('instanceGroupsMultiselect', instanceGroupsMultiselect)
     .directive('instanceGroupsModal', instanceGroupsModal)
-    .config(['$stateProvider', 'stateDefinitionsProvider', '$stateExtenderProvider',
+    .directive('capacityAdjuster', CapacityAdjuster)
-        function($stateProvider, stateDefinitionsProvider, $stateExtenderProvider) {
+    .directive('capacityBar', CapacityBar)
-            let stateExtender = $stateExtenderProvider.$get();
+    .service('InstanceGroupsStrings', InstanceGroupsStrings)
+    .service('JobStrings', JobStrings)
+    .run(InstanceGroupsRun);
 
-
+export default MODULE_NAME;
-            function generateInstanceGroupsStates() {
-                return new Promise((resolve) => {
-                    resolve({
-                        states: [
-                            stateExtender.buildDefinition(instanceGroupsRoute),
-                            stateExtender.buildDefinition(instancesRoute),
-                            stateExtender.buildDefinition(instancesListRoute),
-                            stateExtender.buildDefinition(jobsListRoute),
-                            stateExtender.buildDefinition(instanceJobsRoute),
-                            stateExtender.buildDefinition(instanceJobsListRoute)
-                        ]
-                    });
-                });
-            }
-
-            $stateProvider.state({
-                name: 'instanceGroups.**',
-                url: '/instance_groups',
-                lazyLoad: () => generateInstanceGroupsStates()
-            });
-        }]);

awx/ui/client/src/shared/directives.js

@@ -38,7 +38,7 @@ angular.module('AWDirectives', ['RestServices', 'Utilities'])
     };
 })
 
-// caplitalize  Add to any input field where the first letter of each
+// capitalize  Add to any input field where the first letter of each
 //              word should be capitalized. Use in place of css test-transform.
 //              For some reason "text-transform: capitalize" in breadcrumbs
 //              causes a break at each blank space. And of course,
@@ -65,6 +65,26 @@ angular.module('AWDirectives', ['RestServices', 'Utilities'])
     };
 })
 
+// stringToNumber
+//
+// If your model does not contain actual numbers then this directive
+// will do the conversion in the ngModel $formatters and $parsers pipeline.
+//
+.directive('stringToNumber', function() {
+    return {
+        require: 'ngModel',
+        restrict: 'A',
+        link: function(scope, element, attrs, ngModel) {
+            ngModel.$parsers.push(function(value) {
+                return '' + value;
+            });
+            ngModel.$formatters.push(function(value) {
+                return parseFloat(value);
+            });
+        }
+    };
+})
+
 // imageUpload
 //
 // Accepts image and returns base64 information with basic validation

awx/ui/client/src/shared/multi-select-preview/multi-select-preview.partial.html

@@ -11,6 +11,7 @@
                     </div>
                     <div class="MultiSelectPreview-previewTag MultiSelectPreview-previewTag--deletable">
                         <span>{{selectedRow.name}}</span>
+                        <span ng-if="selectedRow.hostname">{{selectedRow.hostname}}</span>
                     </div>
                 </div>
             </div>

docs/clustering.md

@@ -28,6 +28,8 @@ It's important to point out a few existing things:
 * Existing old-style HA deployments will be transitioned automatically to the new HA system during the upgrade process to 3.1.
 * Manual projects will need to be synced to all instances by the customer
 
+Ansible Tower 3.3 adds support for container-based clusters using Openshift or Kubernetes
+
 ## Important Changes
 
 * There is no concept of primary/secondary in the new Tower system. *All* systems are primary.
@@ -226,6 +228,47 @@ show up in api endpoints and stats monitoring. These groups can be removed with
 $ awx-manage unregister_queue --queuename=<name>
 ```
 
+### Configuring Instances and Instance Groups from the API
+
+Instance Groups can be created by posting to `/api/v2/instance_groups` as a System Admin.
+
+Once created, `Instances` can be associated with an Instance Group with:
+
+```
+HTTP POST /api/v2/instance_groups/x/instances/ {'id': y}`
+```
+
+An `Instance` that is added to an `InstanceGroup` will automatically reconfigure itself to listen on the group's work queue. See the following
+section `Instance Group Policies` for more details.
+
+### Instance Group Policies
+
+Tower `Instances` can be configured to automatically join `Instance Groups` when they come online by defining a policy. These policies are evaluated for
+every new Instance that comes online.
+
+Instance Group Policies are controlled by 3 optional fields on an `Instance Group`:
+
+* `policy_instance_percentage`: This is a number between 0 - 100. It gaurantees that this percentage of active Tower instances will be added
+  to this `Instance Group`. As new instances come online, if the number of Instances in this group relative to the total number of instances
+  is less than the given percentage then new ones will be added until the percentage condition is satisfied.
+* `policy_instance_minimum`: This policy attempts to keep at least this many `Instances` in the `Instance Group`. If the number of
+  available instances is lower than this minimum then all `Instances` will be placed in this `Instance Group`.
+* `policy_instance_list`: This is a fixed list of `Instance` names. These `Instances` will *always* be added to this `Instance Group`.
+  Further, by adding Instances to this list you are declaring that you will manually manage those Instances and they will not be eligible under any other
+  policy. This means they will not be automatically added to any other `Instance Group` even if the policy would cause them to be matched.
+
+> NOTES
+
+* `Instances` that are assigned directly to `Instance Groups` by posting to `/api/v2/instance_groups/x/instances` or
+  `/api/v2/instances/x/instance_groups` are automatically added to the `policy_instance_list`. This means they are subject to the
+  normal caveats for `policy_instance_list` and must be manually managed.
+* `policy_instance_percentage` and `policy_instance_minimum` work together. For example, if you have a `policy_instance_percentage` of
+  50% and a `policy_instance_minimum` of 2 and you start 6 `Instances`. 3 of them would be assigned to the `Instance Group`. If you reduce the number
+  of `Instances` to 2 then both of them would be assigned to the `Instance Group` to satisfy `policy_instance_minimum`. In this way, you can set a lower
+  bound on the amount of available resources.
+* Policies don't actively prevent `Instances` from being associated with multiple `Instance Groups` but this can effectively be achieved by making the percentages
+  sum to 100. If you have 4 `Instance Groups` assign each a percentage value of 25 and the `Instances` will be distributed among them with no overlap.
+
 ### Status and Monitoring
 
 Tower itself reports as much status as it can via the api at `/api/v2/ping` in order to provide validation of the health

installer/image_build/files/launch_awx_task.sh

@@ -4,7 +4,12 @@ if [ `id -u` -ge 500 ]; then
     cat /tmp/passwd > /etc/passwd
     rm /tmp/passwd
 fi
+
+ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=$DATABASE_HOST port=$DATABASE_PORT" all
+ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=localhost port=11211" all
+ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=localhost port=5672" all
 ANSIBLE_REMOTE_TEMP=/tmp ANSIBLE_LOCAL_TEMP=/tmp ansible -i "127.0.0.1," -c local -v -m postgresql_db -U $DATABASE_USER -a "name=$DATABASE_NAME owner=$DATABASE_USER login_user=$DATABASE_USER login_host=$DATABASE_HOST login_password=$DATABASE_PASSWORD port=$DATABASE_PORT" all
+
 awx-manage migrate --noinput --fake-initial
 if [ ! -z "$AWX_ADMIN_USER" ]&&[ ! -z "$AWX_ADMIN_PASSWORD" ]; then
     echo "from django.contrib.auth.models import User; User.objects.create_superuser('$AWX_ADMIN_USER', 'root@localhost', '$AWX_ADMIN_PASSWORD')" | awx-manage shell
@@ -14,5 +19,5 @@ else
     awx-manage create_preload_data
 fi
 awx-manage provision_instance --hostname=$(hostname)
-awx-manage register_queue --queuename=tower --hostnames=$(hostname)
+awx-manage register_queue --queuename=tower --instance_percent=100
 supervisord -c /supervisor_task.conf

installer/image_build/files/settings.py

@@ -31,9 +31,6 @@ AWX_PROOT_ENABLED = False
 
 CLUSTER_HOST_ID = "awx"
 SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'
-CELERY_TASK_QUEUES += (Queue(CLUSTER_HOST_ID, Exchange(CLUSTER_HOST_ID), routing_key=CLUSTER_HOST_ID),)
-CELERY_TASK_ROUTES['awx.main.tasks.cluster_node_heartbeat'] = {'queue': CLUSTER_HOST_ID, 'routing_key': CLUSTER_HOST_ID}
-CELERY_TASK_ROUTES['awx.main.tasks.purge_old_stdout_files'] = {'queue': CLUSTER_HOST_ID, 'routing_key': CLUSTER_HOST_ID}
 
 
 ###############################################################################

installer/image_build/files/supervisor.conf

@@ -41,6 +41,15 @@ priority=5
 
 # TODO: Exit Handler
 
+[eventlistener:awx-config-watcher]
+command=/usr/bin/config-watcher
+stderr_logfile=/dev/stdout
+stderr_logfile_maxbytes=0
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+events=TICK_60
+priority=0
+
 [unix_http_server]
 file=/tmp/supervisor.sock
 

installer/image_build/files/supervisor_task.conf

@@ -3,8 +3,7 @@ nodaemon = True
 umask = 022
 
 [program:celery]
-# TODO: Needs to be reworked to dynamically use instance group queues
+command = /var/lib/awx/venv/awx/bin/celery worker -A awx -B -l debug --autoscale=4 -Ofair -s /var/lib/awx/beat.db -Q tower_broadcast_all -n celery@%(ENV_HOSTNAME)s
-command = /var/lib/awx/venv/awx/bin/celery worker -A awx -l debug --autoscale=4 -Ofair -Q tower_scheduler,tower_broadcast_all,tower,%(host_node_name)s -n celery@localhost
 directory = /var/lib/awx
 environment = LANGUAGE="en_US.UTF-8",LANG="en_US.UTF-8",LC_ALL="en_US.UTF-8",LC_CTYPE="en_US.UTF-8"
 #user = {{ aw_user }}
@@ -16,18 +15,6 @@ stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
-[program:awx-celeryd-beat]
-command = /var/lib/awx/venv/awx/bin/celery beat -A awx -l debug --pidfile= -s /var/lib/awx/beat.db
-directory = /var/lib/awx
-autostart = true
-autorestart = true
-stopwaitsecs = 5
-redirect_stderr=true
-stdout_logfile = /dev/stdout
-stdout_logfile_maxbytes = 0
-stderr_logfile = /dev/stderr
-stderr_logfile_maxbytes = 0
-
 [program:callback-receiver]
 command = awx-manage run_callback_receiver
 directory = /var/lib/awx
@@ -56,6 +43,15 @@ priority=5
 
 # TODO: Exit Handler
 
+[eventlistener:awx-config-watcher]
+command=/usr/bin/config-watcher
+stderr_logfile=/dev/stdout
+stderr_logfile_maxbytes=0
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+events=TICK_60
+priority=0
+
 [unix_http_server]
 file=/tmp/supervisor.sock
 

installer/image_build/tasks/main.yml

@@ -163,6 +163,12 @@
     dest: "{{ docker_base_path }}/requirements"
   delegate_to: localhost
 
+- name: Stage config watcher
+  copy:
+    src: ../tools/scripts/config-watcher
+    dest: "{{ docker_base_path }}/config-watcher"
+  delegate_to: localhost
+
 - name: Stage Makefile
   copy:
     src: ../Makefile

installer/image_build/templates/Dockerfile.j2

@@ -22,6 +22,7 @@ ADD requirements/requirements_ansible.txt \
     requirements/requirements_git.txt \
     /tmp/requirements/
 ADD ansible.repo /etc/yum.repos.d/ansible.repo
+ADD config-watcher /usr/bin/config-watcher
 ADD RPM-GPG-KEY-ansible-release /etc/pki/rpm-gpg/RPM-GPG-KEY-ansible-release
 # OS Dependencies
 WORKDIR /tmp
@@ -50,7 +51,7 @@ ADD supervisor.conf /supervisor.conf
 ADD supervisor_task.conf /supervisor_task.conf
 ADD launch_awx.sh /usr/bin/launch_awx.sh
 ADD launch_awx_task.sh /usr/bin/launch_awx_task.sh
-RUN chmod +rx /usr/bin/launch_awx.sh && chmod +rx /usr/bin/launch_awx_task.sh
+RUN chmod +rx /usr/bin/launch_awx.sh && chmod +rx /usr/bin/launch_awx_task.sh && chmod +rx /usr/bin/config-watcher
 ADD settings.py /etc/tower/settings.py
 RUN chmod g+w /etc/passwd
 RUN chmod -R 777 /var/log/nginx && chmod -R 777 /var/lib/nginx

installer/kubernetes/tasks/main.yml

@@ -96,6 +96,12 @@
     path: "{{ kubernetes_base_path }}"
     state: directory
 
+- name: Template Kubernetes AWX etcd2
+  template:
+    src: etcd.yml.j2
+    dest: "{{ kubernetes_base_path }}/etcd.yml"
+    mode: '0600'
+
 - name: Template Kubernetes AWX Config
   template:
     src: configmap.yml.j2
@@ -108,6 +114,9 @@
     dest: "{{ kubernetes_base_path }}/deployment.yml"
     mode: '0600'
 
+- name: Apply etcd deployment
+  shell: "kubectl apply -f {{ kubernetes_base_path }}/etcd.yml"
+  
 - name: Apply Configmap
   shell: "kubectl apply -f {{ kubernetes_base_path }}/configmap.yml"
 

installer/kubernetes/templates/configmap.yml.j2

@@ -13,6 +13,8 @@ data:
     # Container environments don't like chroots
     AWX_PROOT_ENABLED = False
     
+    AWX_AUTO_DEPROVISION_INSTANCES = True
+
     #Autoprovisioning should replace this
     CLUSTER_HOST_ID = socket.gethostname()
     SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'

installer/kubernetes/templates/deployment.yml.j2

@@ -41,18 +41,42 @@ spec:
             - name: AWX_ADMIN_PASSWORD
               value: {{ default_admin_password|default('password') }}
         - name: awx-rabbit
-          image: rabbitmq:3
+          image: ansible/awx_rabbitmq:latest
+          imagePullPolicy: Always
           env:
+            # For consupmption by rabbitmq-env.conf
+            - name: MY_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: RABBITMQ_USE_LONGNAME
+              value: "true"
+            - name: ERLANG_COOKIE
+              value: "test"
             - name: RABBITMQ_ERLANG_COOKIE
-              value: secretb
+              value: "secretb"
             - name: RABBITMQ_NODENAME
-              value: rabbitmq
+              value: "rabbit@$(MY_POD_IP)"
+            - name: AUTOCLUSTER_TYPE
+              value: "etcd"
+            - name: AUTOCLUSTER_DELAY
+              value: "60"
+            - name: ETCD_HOST
+              value: "etcd"
+            - name: AUTOCLUSTER_CLEANUP
+              value: "true"
+            - name: CLEANUP_WARN_ONLY
+              value: "false"
+            - name: CLEANUP_INTERVAL
+              value: "30"
             - name: RABBITMQ_DEFAULT_USER
               value: awx
             - name: RABBITMQ_DEFAULT_PASS
               value: abcdefg
             - name: RABBITMQ_DEFAULT_VHOST
               value: awx
+            - name: RABBITMQ_CONFIG_FILE
+              value: /etc/rabbitmq/rabbitmq
         - name: awx-memcached
           image: memcached
       volumes:

installer/kubernetes/templates/etcd.yml.j2

@@ -0,0 +1,44 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: etcd
+  namespace: {{ awx_kubernetes_project }}
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        name: awx-etcd2
+        service: etcd
+    spec:
+      containers:
+        - name: etcd
+          image: elcolio/etcd:latest
+          ports:
+            - containerPort: 4001
+          volumeMounts:
+            - mountPath: /data
+              name: datadir
+      volumes:
+        - name: datadir
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+  labels:
+    name: awx-etcd
+  name: etcd
+  namespace: {{ awx_kubernetes_project }}
+spec:
+  ports:
+    - name: etcd
+      port: 4001
+      protocol: TCP
+      targetPort: 4001
+  selector:
+    name: awx-etcd2
+  sessionAffinity: None
+  type: ClusterIP

installer/openshift/tasks/main.yml

@@ -121,6 +121,15 @@
     dest: "{{ openshift_base_path }}/deployment.yml"
     mode: '0600'
 
+- name: Template Openshift AWX etcd2
+  template:
+    src: etcd.yml.j2
+    dest: "{{ openshift_base_path }}/etcd.yml"
+    mode: '0600'
+
+- name: Apply etcd deployment
+  shell: "oc apply -f {{ openshift_base_path }}/etcd.yml"
+
 - name: Apply Configmap
   shell: "oc apply -f {{ openshift_base_path }}/configmap.yml"
 

installer/openshift/templates/configmap.yml.j2

@@ -13,6 +13,9 @@ data:
     # Container environments don't like chroots
     AWX_PROOT_ENABLED = False
 
+    # Automatically deprovision pods that go offline
+    AWX_AUTO_DEPROVISION_INSTANCES = True
+
     #Autoprovisioning should replace this
     CLUSTER_HOST_ID = socket.gethostname()
     SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'

installer/openshift/templates/deployment.yml.j2

@@ -41,18 +41,42 @@ spec:
             - name: AWX_ADMIN_PASSWORD
               value: {{ default_admin_password|default('password') }}
         - name: awx-rabbit
-          image: rabbitmq:3
+          image: ansible/awx_rabbitmq:latest
+          imagePullPolicy: Always
           env:
+            # For consupmption by rabbitmq-env.conf
+            - name: MY_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: RABBITMQ_USE_LONGNAME
+              value: "true"
+            - name: ERLANG_COOKIE
+              value: "test"
             - name: RABBITMQ_ERLANG_COOKIE
-              value: secretb
+              value: "secretb"
             - name: RABBITMQ_NODENAME
-              value: rabbitmq
+              value: "rabbit@$(MY_POD_IP)"
+            - name: AUTOCLUSTER_TYPE
+              value: "etcd"
+            - name: AUTOCLUSTER_DELAY
+              value: "60"
+            - name: ETCD_HOST
+              value: "etcd"
+            - name: AUTOCLUSTER_CLEANUP
+              value: "true"
+            - name: CLEANUP_WARN_ONLY
+              value: "false"
+            - name: CLEANUP_INTERVAL
+              value: "30"
             - name: RABBITMQ_DEFAULT_USER
               value: awx
             - name: RABBITMQ_DEFAULT_PASS
               value: abcdefg
             - name: RABBITMQ_DEFAULT_VHOST
               value: awx
+            - name: RABBITMQ_CONFIG_FILE
+              value: /etc/rabbitmq/rabbitmq
         - name: awx-memcached
           image: memcached
       volumes:
@@ -80,6 +104,23 @@ spec:
   selector:
     name: awx-web-deploy
 ---
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: awx-rmq-mgmt
+  namespace: {{ awx_openshift_project }}
+  labels:
+    name: awx-rmq-mgmt
+spec:
+  type: ClusterIP
+  ports:
+    - name: rmqmgmt
+      port: 15672
+      targetPort: 15672
+  selector:
+    name: awx-web-deploy
+---
 apiVersion: v1
 kind: Route
 metadata:

installer/openshift/templates/etcd.yml.j2

@@ -0,0 +1,44 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: etcd
+  namespace: {{ awx_openshift_project }}
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        name: awx-etcd2
+        service: etcd
+    spec:
+      containers:
+        - name: etcd
+          image: elcolio/etcd:latest
+          ports:
+            - containerPort: 4001
+          volumeMounts:
+            - mountPath: /data
+              name: datadir
+      volumes:
+        - name: datadir
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+  labels:
+    name: awx-etcd
+  name: etcd
+  namespace: {{ awx_openshift_project }}
+spec:
+  ports:
+    - name: etcd
+      port: 4001
+      protocol: TCP
+      targetPort: 4001
+  selector:
+    name: awx-etcd2
+  sessionAffinity: None
+  type: ClusterIP

tools/docker-compose/supervisor.conf

@@ -4,7 +4,7 @@ minfds = 4096
 nodaemon=true
 
 [program:celeryd]
-command = celery worker -A awx -l DEBUG -B -Ofair --autoscale=100,4 --schedule=/celerybeat-schedule -Q tower_scheduler,tower_broadcast_all,%(ENV_AWX_GROUP_QUEUES)s,%(ENV_HOSTNAME)s -n celery@%(ENV_HOSTNAME)s
+command = celery worker -A awx -l DEBUG -B --autoscale=20,3 -Ofair -s /var/lib/awx/beat.db -Q tower_broadcast_all -n celery@%(ENV_HOSTNAME)s
 autostart = true
 autorestart = true
 redirect_stderr=true