From 56abfa732efa38c1bfa707cb110bbca7c4fab817 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Tue, 14 Nov 2017 13:49:06 -0500 Subject: [PATCH] Adding initial instance group policies and policy evaluation planner --- awx/api/serializers.py | 3 +- .../management/commands/register_queue.py | 10 ++++- .../0013_v330_instancegroup_policies.py | 30 ++++++++++++++ awx/main/models/ha.py | 14 +++++++ awx/main/tasks.py | 40 ++++++++++++++++++- 5 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 awx/main/migrations/0013_v330_instancegroup_policies.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index a0822cea0f..adb17dd65b 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -4011,7 +4011,8 @@ class InstanceGroupSerializer(BaseSerializer): model = InstanceGroup fields = ("id", "type", "url", "related", "name", "created", "modified", "capacity", "committed_capacity", "consumed_capacity", - "percent_capacity_remaining", "jobs_running", "instances", "controller") + "percent_capacity_remaining", "jobs_running", "instances", "controller", + "policy_instance_percentage", "policy_instance_minimum") def get_related(self, obj): res = super(InstanceGroupSerializer, self).get_related(obj) diff --git a/awx/main/management/commands/register_queue.py b/awx/main/management/commands/register_queue.py index 548e305bcc..1e7912836d 100644 --- a/awx/main/management/commands/register_queue.py +++ b/awx/main/management/commands/register_queue.py @@ -17,6 +17,10 @@ class Command(BaseCommand): help='Comma-Delimited Hosts to add to the Queue') parser.add_argument('--controller', dest='controller', type=str, default='', help='The controlling group (makes this an isolated group)') + parser.add_argument('--instance_percent', dest='instance_percent', type=int, default=0, + help='The percentage of active instances that will be assigned to this group'), + parser.add_argument('--instance_minimum', dest='instance_minimum', type=int, default=0, + help='The minimum number of instance that will be retained for this group from available instances') def handle(self, **options): queuename = options.get('queuename') @@ -38,7 +42,9 @@ class Command(BaseCommand): changed = True else: print("Creating instance group {}".format(queuename)) - ig = InstanceGroup(name=queuename) + ig = InstanceGroup(name=queuename, + policy_instance_percentage=options.get('instance_percent'), + policy_instance_minimum=options.get('instance_minimum')) if control_ig: ig.controller = control_ig ig.save() @@ -60,5 +66,7 @@ class Command(BaseCommand): sys.exit(1) else: print("Instance already registered {}".format(instance[0].hostname)) + ig.policy_instance_list = instance_list + ig.save() if changed: print('(changed: True)') diff --git a/awx/main/migrations/0013_v330_instancegroup_policies.py b/awx/main/migrations/0013_v330_instancegroup_policies.py new file mode 100644 index 0000000000..b8fa658fb8 --- /dev/null +++ b/awx/main/migrations/0013_v330_instancegroup_policies.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import awx.main.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0008_v320_drop_v1_credential_fields'), + ] + + operations = [ + migrations.AddField( + model_name='instancegroup', + name='policy_instance_list', + field=awx.main.fields.JSONField(default=[], help_text='List of exact-match Instances that will always be automatically assigned to this group', blank=True), + ), + migrations.AddField( + model_name='instancegroup', + name='policy_instance_minimum', + field=models.IntegerField(default=0, help_text='Static minimum number of Instances to automatically assign to this group'), + ), + migrations.AddField( + model_name='instancegroup', + name='policy_instance_percentage', + field=models.IntegerField(default=0, help_text='Percentage of Instances to automatically assign to this group'), + ), + ] diff --git a/awx/main/models/ha.py b/awx/main/models/ha.py index cb63beb126..eac0bec22f 100644 --- a/awx/main/models/ha.py +++ b/awx/main/models/ha.py @@ -12,6 +12,7 @@ from solo.models import SingletonModel from awx.api.versioning import reverse from awx.main.managers import InstanceManager, InstanceGroupManager +from awx.main.fields import JSONField from awx.main.models.inventory import InventoryUpdate from awx.main.models.jobs import Job from awx.main.models.projects import ProjectUpdate @@ -88,6 +89,19 @@ class InstanceGroup(models.Model): default=None, null=True ) + policy_instance_percentage = models.IntegerField( + default=0, + help_text=_("Percentage of Instances to automatically assign to this group") + ) + policy_instance_minimum = models.IntegerField( + default=0, + help_text=_("Static minimum number of Instances to automatically assign to this group") + ) + policy_instance_list = JSONField( + default=[], + blank=True, + help_text=_("List of exact-match Instances that will always be automatically assigned to this group") + ) def get_absolute_url(self, request=None): return reverse('api:instance_group_detail', kwargs={'pk': self.pk}, request=request) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 0e9c8bcea5..109b92a771 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2,7 +2,8 @@ # All Rights Reserved. # Python -from collections import OrderedDict +import codecs +from collections import OrderedDict, namedtuple import ConfigParser import cStringIO import functools @@ -131,6 +132,43 @@ def inform_cluster_of_shutdown(*args, **kwargs): logger.exception('Encountered problem with normal shutdown signal.') +@shared_task(bind=True, queue='tower', base=LogErrorsTask) +def apply_cluster_membership_policies(self): + considered_instances = Instance.objects.all().order_by('id').only('id') + total_instances = considered_instances.count() + actual_groups = [] + actual_instances = [] + Group = namedtuple('Group', ['obj', 'instances']) + Instance = namedtuple('Instance', ['obj', 'groups']) + # Process policy instance list first, these will represent manually managed instances + # that will not go through automatic policy determination + for ig in InstanceGroup.objects.all(): + group_actual = Group(obj=ig, instances=[]) + for i in ig.policy_instance_list: + group_actual.instances.append(i) + if i in considered_instances: + considered_instances.remove(i) + actual_groups.append(group_actual) + # Process Instance minimum policies next, since it represents a concrete lower bound to the + # number of instances to make available to instance groups + for i in considered_instances: + instance_actual = Instance(obj=i, groups=[]) + for g in sorted(actual_groups, cmp=lambda x,y: len(x.instances) - len(y.instances)): + if len(g.instances) < g.obj.policy_instance_minimum: + g.instances.append(instance_actual.obj.id) + instance_actual.groups.append(g.obj.id) + break + actual_instances.append(instance_actual) + # Finally process instance policy percentages + for i in sorted(actual_instances, cmp=lambda x,y: len(x.groups) - len(y.groups)): + for g in sorted(actual_groups, cmp=lambda x,y: len(x.instances) - len(y.instances)): + if 100 * float(len(g.instances)) / total_instances < g.obj.policy_instance_percentage: + g.instances.append(i.obj.id) + i.groups.append(g.obj.id) + break + # Next step + + @shared_task(queue='tower_broadcast_all', bind=True, base=LogErrorsTask) def handle_setting_changes(self, setting_keys): orig_len = len(setting_keys)