From 1b93886be268a69808c90eeb66cb1aa93bc8c38c Mon Sep 17 00:00:00 2001 From: Chris Church Date: Thu, 4 Apr 2013 13:59:32 -0400 Subject: [PATCH] Updated callback module to delegate to acom_callback_event management command. --- lib/main/admin.py | 2 +- .../commands/acom_callback_event.py | 56 +++++-- .../management/commands/acom_inventory.py | 1 - lib/main/tasks.py | 4 + lib/main/tests/__init__.py | 2 +- lib/main/tests/commands.py | 156 +++++++++++++++++- lib/main/tests/tasks.py | 10 +- lib/plugins/callback/acom_callback.py | 45 +---- 8 files changed, 216 insertions(+), 60 deletions(-) mode change 100644 => 100755 lib/main/management/commands/acom_callback_event.py diff --git a/lib/main/admin.py b/lib/main/admin.py index ec1561e0f7..1347aa22dc 100644 --- a/lib/main/admin.py +++ b/lib/main/admin.py @@ -132,7 +132,7 @@ class CredentialAdmin(admin.ModelAdmin): class TeamAdmin(admin.ModelAdmin): list_display = ('name', 'description', 'active') - filter_horizontal = ('projects', 'users', 'organizations', 'tags') + filter_horizontal = ('projects', 'users', 'tags') class ProjectAdmin(admin.ModelAdmin): diff --git a/lib/main/management/commands/acom_callback_event.py b/lib/main/management/commands/acom_callback_event.py old mode 100644 new mode 100755 index 1c7ef33dcd..abc1eaae29 --- a/lib/main/management/commands/acom_callback_event.py +++ b/lib/main/management/commands/acom_callback_event.py @@ -22,23 +22,40 @@ import json from optparse import make_option import os import sys -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import NoArgsCommand, CommandError -class Command(BaseCommmand): +class Command(NoArgsCommand): + ''' + Management command to log callback events from ansible-playbook. + ''' help = 'Ansible Commander Callback Event Capture' - option_list = BaseCommmand.option_list + ( + option_list = NoArgsCommand.option_list + ( make_option('-i', '--launch-job-status', dest='launch_job_status_id', type='int', default=0, - help='Inventory ID (can also be specified using ' - 'ACOM_INVENTORY_ID environment variable)'), - #make_option('--indent', dest='indent', type='int', default=None, - # help='Indentation level for pretty printing output'), + help='Launch job status ID (can also be specified using ' + 'ACOM_LAUNCH_JOB_STATUS_ID environment variable)'), + make_option('-e', '--event', dest='event_type', default=None, + help='Event type'), + make_option('-f', '--file', dest='event_data_file', default=None, + help='JSON-formatted data file containing callback event ' + 'data (specify "-" to read from stdin)'), + make_option('-d', '--data', dest='event_data_json', default=None, + help='JSON-formatted callback event data'), ) - def handle(self, *args, **options): - from lib.main.models import LaunchJobStatus + def handle_noargs(self, **options): + from lib.main.models import LaunchJobStatus, LaunchJobStatusEvent + event_type = options.get('event_type', None) + if not event_type: + raise CommandError('No event specified') + if event_type not in [x[0] for x in LaunchJobStatusEvent.EVENT_TYPES]: + raise CommandError('Unsupported event') + event_data_file = options.get('event_data_file', None) + event_data_json = options.get('event_data_json', None) + if event_data_file is None and event_data_json is None: + raise CommandError('Either --file or --data must be specified') try: launch_job_status_id = int(os.getenv('ACOM_LAUNCH_JOB_STATUS_ID', options.get('launch_job_status_id', 0))) @@ -48,9 +65,26 @@ class Command(BaseCommmand): raise CommandError('No launch job status ID specified') try: launch_job_status = LaunchJobStatus.objects.get(id=launch_job_status_id) - except Inventory.DoesNotExist: + except LaunchJobStatus.DoesNotExist: raise CommandError('Launch job status with ID %d not found' % launch_job_status_id) - # FIXME: Do stuff here. + if launch_job_status.status != 'running': + raise CommandError('Unable to add event except when launch job is running') + try: + if event_data_json is None: + try: + if event_data_file == '-': + event_data_fileobj = sys.stdin + else: + event_data_fileobj = file(event_data_file, 'rb') + event_data = json.load(event_data_fileobj) + except IOError, e: + raise CommandError('Error %r reading from %s' % (e, event_data_file)) + else: + event_data = json.loads(event_data_json) + except ValueError: + raise CommandError('Error parsing JSON data') + launch_job_status.launch_job_status_events.create(event=event_type, + event_data=event_data) if __name__ == '__main__': from __init__ import run_command_as_script diff --git a/lib/main/management/commands/acom_inventory.py b/lib/main/management/commands/acom_inventory.py index 248bc33405..fa441a5c53 100755 --- a/lib/main/management/commands/acom_inventory.py +++ b/lib/main/management/commands/acom_inventory.py @@ -21,7 +21,6 @@ import json from optparse import make_option import os -import sys from django.core.management.base import NoArgsCommand, CommandError class Command(NoArgsCommand): diff --git a/lib/main/tasks.py b/lib/main/tasks.py index 9953905284..7424029f55 100644 --- a/lib/main/tasks.py +++ b/lib/main/tasks.py @@ -34,10 +34,14 @@ def run_launch_job(launch_job_status_pk): inventory_script = os.path.abspath(os.path.join(os.path.dirname(__file__), 'management', 'commands', 'acom_inventory.py')) + callback_script = os.path.abspath(os.path.join(os.path.dirname(__file__), + 'management', 'commands', + 'acom_callback_event.py')) env = dict(os.environ.items()) env['ACOM_LAUNCH_JOB_STATUS_ID'] = str(launch_job_status.pk) env['ACOM_INVENTORY_ID'] = str(launch_job.inventory.pk) env['ANSIBLE_CALLBACK_PLUGINS'] = plugin_dir + env['ACOM_CALLBACK_EVENT_SCRIPT'] = callback_script if hasattr(settings, 'ANSIBLE_TRANSPORT'): env['ANSIBLE_TRANSPORT'] = getattr(settings, 'ANSIBLE_TRANSPORT') diff --git a/lib/main/tests/__init__.py b/lib/main/tests/__init__.py index 98d3fdbad5..1f94b18f7a 100644 --- a/lib/main/tests/__init__.py +++ b/lib/main/tests/__init__.py @@ -20,5 +20,5 @@ from lib.main.tests.organizations import OrganizationsTest from lib.main.tests.users import UsersTest from lib.main.tests.inventory import InventoryTest from lib.main.tests.projects import ProjectsTest -from lib.main.tests.commands import AcomInventoryTest +from lib.main.tests.commands import * from lib.main.tests.tasks import RunLaunchJobTest diff --git a/lib/main/tests/commands.py b/lib/main/tests/commands.py index 72f2c566fd..696182d8aa 100644 --- a/lib/main/tests/commands.py +++ b/lib/main/tests/commands.py @@ -20,11 +20,15 @@ import json import os import StringIO import sys +import tempfile from django.core.management import call_command from django.core.management.base import CommandError +from django.utils.timezone import now from lib.main.models import * from lib.main.tests.base import BaseTest +__all__ = ['AcomInventoryTest', 'AcomCallbackEventTest'] + class BaseCommandTest(BaseTest): ''' Base class for tests that run management commands. @@ -33,6 +37,7 @@ class BaseCommandTest(BaseTest): def setUp(self): super(BaseCommandTest, self).setUp() self._environ = dict(os.environ.items()) + self._temp_files = [] def tearDown(self): super(BaseCommandTest, self).tearDown() @@ -42,16 +47,23 @@ class BaseCommandTest(BaseTest): for k,v in os.environ.items(): if k not in self._environ.keys(): del os.environ[k] + for tf in self._temp_files: + if os.path.exists(tf): + os.remove(tf) def run_command(self, name, *args, **options): ''' Run a management command and capture its stdout/stderr along with any exceptions. ''' + stdin_fileobj = options.pop('stdin_fileobj', None) options.setdefault('verbosity', 1) options.setdefault('interactive', False) + original_stdin = sys.stdin original_stdout = sys.stdout original_stderr = sys.stderr + if stdin_fileobj: + sys.stdin = stdin_fileobj sys.stdout = StringIO.StringIO() sys.stderr = StringIO.StringIO() result = None @@ -64,6 +76,7 @@ class BaseCommandTest(BaseTest): finally: captured_stdout = sys.stdout.getvalue() captured_stderr = sys.stderr.getvalue() + sys.stdin = original_stdin sys.stdout = original_stdout sys.stderr = original_stderr return result, captured_stdout, captured_stderr @@ -244,4 +257,145 @@ class AcomInventoryTest(BaseCommandTest): host='blah') self.assertTrue(isinstance(result, CommandError)) self.assertEqual(json.loads(stdout), {}) - \ No newline at end of file + +class AcomCallbackEventTest(BaseCommandTest): + ''' + Test cases for acom_callback_event management command. + ''' + + def setUp(self): + super(AcomCallbackEventTest, self).setUp() + self.setup_users() + self.organization = self.make_organizations(self.super_django_user, 1)[0] + self.project = self.make_projects(self.normal_django_user, 1)[0] + self.organization.projects.add(self.project) + self.inventory = Inventory.objects.create(name='test-inventory', + organization=self.organization) + self.host = self.inventory.hosts.create(name='host.example.com', + inventory=self.inventory) + self.group = self.inventory.groups.create(name='test-group', + inventory=self.inventory) + self.group.hosts.add(self.host) + self.launch_job = LaunchJob.objects.create(name='test-launch-job', + inventory=self.inventory, + project=self.project) + self.launch_job_status = self.launch_job.launch_job_statuses.create( + name='launch-job-status-%s' % now().isoformat()) + self.valid_kwargs = { + 'launch_job_status_id': self.launch_job_status.id, + 'event_type': 'playbook_on_start', + 'event_data_json': json.dumps({'test_event_data': [2,4,6]}), + } + + def test_with_launch_job_status_not_running(self): + # Events can only be added when the launch job is running. + self.assertEqual(self.launch_job_status.status, 'pending') + result, stdout, stderr = self.run_command('acom_callback_event', + **self.valid_kwargs) + self.assertTrue(isinstance(result, CommandError)) + self.assertTrue('unable to add event ' in str(result).lower()) + self.launch_job_status.status = 'successful' + self.launch_job_status.save() + result, stdout, stderr = self.run_command('acom_callback_event', + **self.valid_kwargs) + self.assertTrue(isinstance(result, CommandError)) + self.assertTrue('unable to add event ' in str(result).lower()) + self.launch_job_status.status = 'failed' + self.launch_job_status.save() + result, stdout, stderr = self.run_command('acom_callback_event', + **self.valid_kwargs) + self.assertTrue(isinstance(result, CommandError)) + self.assertTrue('unable to add event ' in str(result).lower()) + + def test_with_invalid_args(self): + self.launch_job_status.status = 'running' + self.launch_job_status.save() + # Event type not given. + kwargs = dict(self.valid_kwargs.items()) + kwargs.pop('event_type') + result, stdout, stderr = self.run_command('acom_callback_event', **kwargs) + self.assertTrue(isinstance(result, CommandError)) + self.assertTrue('no event specified' in str(result).lower()) + # Invalid event type. + kwargs = dict(self.valid_kwargs.items()) + kwargs['event_type'] = 'invalid_event_type' + result, stdout, stderr = self.run_command('acom_callback_event', **kwargs) + self.assertTrue(isinstance(result, CommandError)) + self.assertTrue('unsupported event' in str(result).lower()) + # Neither file or data specified. + kwargs = dict(self.valid_kwargs.items()) + kwargs.pop('event_data_json') + result, stdout, stderr = self.run_command('acom_callback_event', **kwargs) + self.assertTrue(isinstance(result, CommandError)) + self.assertTrue('either --file or --data' in str(result).lower()) + # Non-integer launch job status ID. + kwargs = dict(self.valid_kwargs.items()) + kwargs['launch_job_status_id'] = 'foo' + result, stdout, stderr = self.run_command('acom_callback_event', **kwargs) + self.assertTrue(isinstance(result, CommandError)) + self.assertTrue('id must be an integer' in str(result).lower()) + # No launch job status ID. + kwargs = dict(self.valid_kwargs.items()) + kwargs.pop('launch_job_status_id') + result, stdout, stderr = self.run_command('acom_callback_event', **kwargs) + self.assertTrue(isinstance(result, CommandError)) + self.assertTrue('no launch job status id' in str(result).lower()) + # Invalid launch job status ID. + kwargs = dict(self.valid_kwargs.items()) + kwargs['launch_job_status_id'] = 9999 + result, stdout, stderr = self.run_command('acom_callback_event', **kwargs) + self.assertTrue(isinstance(result, CommandError)) + self.assertTrue('not found' in str(result).lower()) + # Invalid inline JSON data. + kwargs = dict(self.valid_kwargs.items()) + kwargs['event_data_json'] = 'invalid json' + result, stdout, stderr = self.run_command('acom_callback_event', **kwargs) + self.assertTrue(isinstance(result, CommandError)) + self.assertTrue('error parsing json' in str(result).lower()) + # Invalid file specified. + kwargs = dict(self.valid_kwargs.items()) + kwargs.pop('event_data_json') + h, tf = tempfile.mkstemp() + os.close(h) + os.remove(tf) + kwargs['event_data_file'] = '%s.json' % tf + result, stdout, stderr = self.run_command('acom_callback_event', **kwargs) + self.assertTrue(isinstance(result, CommandError)) + self.assertTrue('reading from' in str(result).lower()) + + def test_with_valid_args(self): + self.launch_job_status.status = 'running' + self.launch_job_status.save() + # Default valid args. + kwargs = dict(self.valid_kwargs.items()) + result, stdout, stderr = self.run_command('acom_callback_event', **kwargs) + self.assertEqual(result, None) + self.assertEqual(self.launch_job_status.launch_job_status_events.count(), 1) + # Pass launch job status in environment instead. + kwargs = dict(self.valid_kwargs.items()) + kwargs.pop('launch_job_status_id') + os.environ['ACOM_LAUNCH_JOB_STATUS_ID'] = str(self.launch_job_status.id) + result, stdout, stderr = self.run_command('acom_callback_event', **kwargs) + self.assertEqual(result, None) + self.assertEqual(self.launch_job_status.launch_job_status_events.count(), 2) + os.environ.pop('ACOM_LAUNCH_JOB_STATUS_ID', None) + # Test with JSON data in a file instead. + kwargs = dict(self.valid_kwargs.items()) + kwargs.pop('event_data_json') + h, tf = tempfile.mkstemp(suffix='.json') + self._temp_files.append(tf) + f = os.fdopen(h, 'w') + json.dump({'some_event_data': [1, 2, 3]}, f) + f.close() + kwargs['event_data_file'] = tf + result, stdout, stderr = self.run_command('acom_callback_event', **kwargs) + self.assertEqual(result, None) + self.assertEqual(self.launch_job_status.launch_job_status_events.count(), 3) + # Test with JSON data from stdin. + kwargs = dict(self.valid_kwargs.items()) + kwargs.pop('event_data_json') + kwargs['event_data_file'] = '-' + kwargs['stdin_fileobj'] = StringIO.StringIO(json.dumps({'blah': 'bleep'})) + result, stdout, stderr = self.run_command('acom_callback_event', **kwargs) + self.assertEqual(result, None) + self.assertEqual(self.launch_job_status.launch_job_status_events.count(), 4) diff --git a/lib/main/tests/tasks.py b/lib/main/tests/tasks.py index d62e0c8d7d..9491b341ef 100644 --- a/lib/main/tests/tasks.py +++ b/lib/main/tests/tasks.py @@ -88,12 +88,10 @@ class RunLaunchJobTest(BaseCeleryTest): launch_job_status = self.launch_job.start() self.assertEqual(launch_job_status.status, 'pending') launch_job_status = LaunchJobStatus.objects.get(pk=launch_job_status.pk) - print 'stdout:', launch_job_status.result_stdout - print 'stderr:', launch_job_status.result_stderr - print launch_job_status.status - - print settings.DATABASES - + #print 'stdout:', launch_job_status.result_stdout + #print 'stderr:', launch_job_status.result_stderr + #print launch_job_status.status + #print settings.DATABASES self.assertEqual(launch_job_status.status, 'successful') self.assertTrue(launch_job_status.result_stdout) launch_job_status_events = launch_job_status.launch_job_status_events.all() diff --git a/lib/plugins/callback/acom_callback.py b/lib/plugins/callback/acom_callback.py index 02ad259223..3c560f6238 100644 --- a/lib/plugins/callback/acom_callback.py +++ b/lib/plugins/callback/acom_callback.py @@ -16,7 +16,9 @@ # along with this program. If not, see . +import json import os +import subprocess import sys class CallbackModule(object): @@ -25,47 +27,12 @@ class CallbackModule(object): ''' def __init__(self): - # the DJANGO_SETTINGS_MODULE environment variable *should* already - # be set if this callback is called when executing a playbook via a - # celery task, otherwise just bail out. - settings_module_name = os.environ.get('DJANGO_SETTINGS_MODULE', None) - if not settings_module_name: - return - # FIXME: Not particularly fond of this sys.path hack, but it is needed - # when a celery task calls ansible-playbook and needs to execute this - # script directly. - try: - settings_parent_module = __import__(settings_module_name) - except ImportError: - top_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..') - sys.path.insert(0, os.path.abspath(top_dir)) - settings_parent_module = __import__(settings_module_name) - settings_module = getattr(settings_parent_module, settings_module_name.split('.')[-1]) - # Use the ACOM_TEST_DATABASE_NAME environment variable to specify the test - # database name when called from unit tests. - if os.environ.get('ACOM_TEST_DATABASE_NAME', None): - settings_module.DATABASES['default']['NAME'] = os.environ['ACOM_TEST_DATABASE_NAME'] - # Try to get the launch job status ID from the environment, otherwise - # just bail out now. - try: - launch_job_status_pk = int(os.environ.get('ACOM_LAUNCH_JOB_STATUS_ID', '')) - except ValueError: - return - from lib.main.models import LaunchJobStatus - try: - self.launch_job_status = LaunchJobStatus.objects.get(pk=launch_job_status_pk) - except LaunchJobStatus.DoesNotExist: - pass + self.acom_callback_event_script = os.getenv('ACOM_CALLBACK_EVENT_SCRIPT') def _log_event(self, event, **event_data): - #print '====', event, args, kwargs - # self.playbook.inventory - if hasattr(self, 'launch_job_status'): - kwargs = { - 'event': event, - 'event_data': event_data, - } - self.launch_job_status.launch_job_status_events.create(**kwargs) + event_data_json = json.dumps(event_data) + cmdline = [self.acom_callback_event_script, '-e', event, '-d', event_data_json] + subprocess.check_call(cmdline) def on_any(self, *args, **kwargs): pass