From 4e7827829fad486fbe7208f2d76e2939c801ddb7 Mon Sep 17 00:00:00 2001 From: Chris Church Date: Fri, 22 Mar 2013 18:55:10 -0400 Subject: [PATCH] Work on executing launch job via celery to run ansible playbook. --- lib/main/admin.py | 11 ++- lib/main/management/__init__.py | 0 lib/main/management/commands/__init__.py | 0 .../management/commands/acom_inventory.py | 90 +++++++++++++++++++ lib/main/models/__init__.py | 24 ++++- lib/main/tasks.py | 11 +++ lib/settings/defaults.py | 4 + 7 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 lib/main/management/__init__.py create mode 100644 lib/main/management/commands/__init__.py create mode 100755 lib/main/management/commands/acom_inventory.py diff --git a/lib/main/admin.py b/lib/main/admin.py index ec0b1299e0..d7a093088b 100644 --- a/lib/main/admin.py +++ b/lib/main/admin.py @@ -33,7 +33,10 @@ class OrganizationAdmin(admin.ModelAdmin): class InventoryAdmin(admin.ModelAdmin): - list_display = ('name', 'description', 'active') + fields = ('name', 'organization', 'description', 'active', 'tags', + 'created_by', 'audit_trail') + list_display = ('name', 'organization', 'description', 'active') + list_filter = ('organization', 'active') filter_horizontal = ('tags',) class TagAdmin(admin.ModelAdmin): @@ -47,8 +50,12 @@ class AuditTrailAdmin(admin.ModelAdmin): class HostAdmin(admin.ModelAdmin): - list_display = ('name', 'description', 'active') + fields = ('name', 'inventory', 'description', 'active', 'tags', + 'created_by', 'audit_trail') + list_display = ('name', 'inventory', 'description', 'active') + list_filter = ('inventory', 'active') filter_horizontal = ('tags',) + # FIXME: Edit reverse of many to many for groups. class GroupAdmin(admin.ModelAdmin): diff --git a/lib/main/management/__init__.py b/lib/main/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/main/management/commands/__init__.py b/lib/main/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/main/management/commands/acom_inventory.py b/lib/main/management/commands/acom_inventory.py new file mode 100755 index 0000000000..bbec213aaf --- /dev/null +++ b/lib/main/management/commands/acom_inventory.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python + +import json +from optparse import make_option +import os +import sys +from django.core.management.base import NoArgsCommand, CommandError + +class Command(NoArgsCommand): + + help = 'Ansible Commander Inventory script' + + option_list = NoArgsCommand.option_list + ( + make_option('-i', '--inventory', dest='inventory', type='int', default=0, + help='Inventory ID (can also be specified using ' + 'ACOM_INVENTORY environment variable)'), + make_option('--list', action='store_true', dest='list', default=False, + help='Return JSON hash of host groups.'), + make_option('--host', dest='host', default='', + help='Return JSON hash of host vars.'), + make_option('--indent', dest='indent', type='int', default=None, + help='Indentation level for pretty printing output'), + ) + + def get_list(self, inventory, indent=None): + groups = {} + for group in inventory.groups.all(): + # FIXME: Check if group is active? + group_info = { + 'hosts': list(group.hosts.values_list('name', flat=True)), + # FIXME: Include host vars here? + 'vars': dict(group.variable_data.values_list('name', 'data')), + 'children': list(group.children.values_list('name', flat=True)), + } + group_info = dict(filter(lambda x: bool(x[1]), group_info.items())) + if group_info.keys() in ([], ['hosts']): + groups[group.name] = group_info.get('hosts', []) + else: + groups[group.name] = group_info + self.stdout.write(json.dumps(groups, indent=indent)) + + def get_host(self, inventory, hostname, indent=None): + from lib.main.models import Host + hostvars = {} + try: + # FIXME: Check if active? + host = inventory.hosts.get(name=hostname) + except Host.DoesNotExist: + raise CommandError('Host %s not found in the given inventory' % hostname) + hostvars = dict(host.variable_data.values_list('name', 'data')) + # FIXME: Do we also need to include variables defined for groups of which + # this host is a member? + self.stdout.write(json.dumps(hostvars, indent=indent)) + + def handle_noargs(self, **options): + from lib.main.models import Inventory + try: + inventory_id = int(os.getenv('ACOM_INVENTORY', options.get('inventory', 0))) + except ValueError: + raise CommandError('Inventory ID must be an integer') + if not inventory_id: + raise CommandError('No inventory ID specified') + try: + inventory = Inventory.objects.get(id=inventory_id) + except Inventory.DoesNotExist: + raise CommandError('Inventory with ID %d not found' % inventory_id) + list_ = options.get('list', False) + host = options.get('host', '') + indent = options.get('indent', None) + if list_ and host: + raise CommandError('Only one of --list or --host can be specified') + elif list_: + self.get_list(inventory, indent=indent) + elif host: + self.get_host(inventory, host, indent=indent) + else: + self.stderr.write('Either --list or --host must be specified') + self.print_help() + +if __name__ == '__main__': + # FIXME: This environment variable *should* already be set if this script + # is called from a celery task. Probably won't work otherwise. + try: + import lib.settings + except ImportError: + sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..', '..'))) + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'lib.settings') + from django.core.management import execute_from_command_line + argv = [sys.argv[0], 'acom_inventory'] + sys.argv[1:] + execute_from_command_line(argv) diff --git a/lib/main/models/__init__.py b/lib/main/models/__init__.py index ea3451b17b..51e247a53a 100644 --- a/lib/main/models/__init__.py +++ b/lib/main/models/__init__.py @@ -125,6 +125,9 @@ class Organization(CommonModel): def can_user_delete(cls, user, obj): return cls.can_user_administrate(user, obj) + def __unicode__(self): + return self.name + class Inventory(CommonModel): ''' an inventory source contains lists and hosts. @@ -136,6 +139,12 @@ class Inventory(CommonModel): organization = models.ForeignKey(Organization, null=True, on_delete=SET_NULL, related_name='inventories') + def __unicode__(self): + if self.organization: + return u'%s (%s)' % (self.name, self.organization) + else: + return self.name + class Host(CommonModel): ''' A managed node @@ -146,6 +155,9 @@ class Host(CommonModel): inventory = models.ForeignKey('Inventory', null=True, on_delete=SET_NULL, related_name='hosts') + def __unicode__(self): + return self.name + class Group(CommonModel): ''' A group of managed nodes. May belong to multiple groups @@ -155,9 +167,12 @@ class Group(CommonModel): app_label = 'main' inventory = models.ForeignKey('Inventory', null=True, on_delete=SET_NULL, related_name='groups') - parents = models.ManyToManyField('self', related_name='children', blank=True) + parents = models.ManyToManyField('self', symmetrical=False, related_name='children', blank=True) hosts = models.ManyToManyField('Host', related_name='groups', blank=True) + def __unicode__(self): + return self.name + # FIXME: audit nullables # FIXME: audit cascades @@ -174,6 +189,9 @@ class VariableData(CommonModel): group = models.ForeignKey('Group', null=True, default=None, blank=True, on_delete=CASCADE, related_name='variable_data') data = models.TextField() # FIXME: JsonField + def __unicode__(self): + return '%s = %s' % (self.name, self.data) + class Credential(CommonModel): ''' A credential contains information about how to talk to a remote set of hosts @@ -269,6 +287,10 @@ class LaunchJob(CommonModel): user = models.ForeignKey('auth.User', on_delete=SET_NULL, null=True, default=None, blank=True, related_name='launch_jobs') job_type = models.CharField(max_length=64, choices=JOB_TYPE_CHOICES) + def start(self): + from lib.main.tasks import run_launch_job + return run_launch_job.delay(self.pk) + # project has one default playbook but really should have a list of playbooks and flags ... diff --git a/lib/main/tasks.py b/lib/main/tasks.py index e1118061e7..9af962b1df 100644 --- a/lib/main/tasks.py +++ b/lib/main/tasks.py @@ -1,7 +1,18 @@ +import os +import subprocess from celery import task from lib.main.models import * @task(name='run_launch_job') def run_launch_job(launch_job_pk): launch_job = LaunchJob.objects.get(pk=launch_job_pk) + os.environ['ACOM_INVENTORY'] = str(launch_job.inventory.pk) + inventory_script = os.path.abspath(os.path.join(os.path.dirname(__file__), + 'management', 'commands', 'acom_inventory.py')) + playbook = launch_job.project.default_playbook + cmd = ['ansible-playbook', '-i', inventory_script, '-v'] + if False: # local mode + cmd.extend(['-c', 'local']) + cmd.append(playbook) + subprocess.check_call(cmd) # FIXME: Do stuff here! diff --git a/lib/settings/defaults.py b/lib/settings/defaults.py index 887a600211..63c7bbae4f 100644 --- a/lib/settings/defaults.py +++ b/lib/settings/defaults.py @@ -157,6 +157,10 @@ if 'djcelery' in INSTALLED_APPS: djcelery.setup_loader() BROKER_URL = 'django://' +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' CELERY_TRACK_STARTED = True +CELERYD_TASK_TIME_LIMIT = 600 +CELERYD_TASK_SOFT_TIME_LIMIT = 540 CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler' CELERYBEAT_MAX_LOOP_INTERVAL = 60