Updated callback module to delegate to acom_callback_event management command.

lib/main/admin.py

@@ -132,7 +132,7 @@ class CredentialAdmin(admin.ModelAdmin):
 class TeamAdmin(admin.ModelAdmin):
 
     list_display = ('name', 'description', 'active')
-    filter_horizontal = ('projects', 'users', 'organizations', 'tags')
+    filter_horizontal = ('projects', 'users', 'tags')
 
 class ProjectAdmin(admin.ModelAdmin):
 

lib/main/management/commands/acom_callback_event.py

@@ -22,23 +22,40 @@ import json
 from optparse import make_option
 import os
 import sys
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import NoArgsCommand, CommandError
 
-class Command(BaseCommmand):
+class Command(NoArgsCommand):
+    '''
+    Management command to log callback events from ansible-playbook.
+    '''
 
     help = 'Ansible Commander Callback Event Capture'
 
-    option_list = BaseCommmand.option_list + (
+    option_list = NoArgsCommand.option_list + (
         make_option('-i', '--launch-job-status', dest='launch_job_status_id',
                     type='int', default=0,
-                    help='Inventory ID (can also be specified using '
+                    help='Launch job status ID (can also be specified using '
-                         'ACOM_INVENTORY_ID environment variable)'),
+                         'ACOM_LAUNCH_JOB_STATUS_ID environment variable)'),
-        #make_option('--indent', dest='indent', type='int', default=None,
+        make_option('-e', '--event', dest='event_type', default=None,
-        #            help='Indentation level for pretty printing output'),
+                 help='Event type'),
+        make_option('-f', '--file', dest='event_data_file', default=None,
+                    help='JSON-formatted data file containing callback event '
+                         'data (specify "-" to read from stdin)'),
+        make_option('-d', '--data', dest='event_data_json', default=None,
+                    help='JSON-formatted callback event data'),
     )
 
-    def handle(self, *args, **options):
+    def handle_noargs(self, **options):
-        from lib.main.models import LaunchJobStatus
+        from lib.main.models import LaunchJobStatus, LaunchJobStatusEvent
+        event_type = options.get('event_type', None)
+        if not event_type:
+            raise CommandError('No event specified')
+        if event_type not in [x[0] for x in LaunchJobStatusEvent.EVENT_TYPES]:
+            raise CommandError('Unsupported event')
+        event_data_file = options.get('event_data_file', None)
+        event_data_json = options.get('event_data_json', None)
+        if event_data_file is None and event_data_json is None:
+            raise CommandError('Either --file or --data must be specified')
         try:
             launch_job_status_id = int(os.getenv('ACOM_LAUNCH_JOB_STATUS_ID',
                                    options.get('launch_job_status_id', 0)))
@@ -48,9 +65,26 @@ class Command(BaseCommmand):
             raise CommandError('No launch job status ID specified')
         try:
             launch_job_status = LaunchJobStatus.objects.get(id=launch_job_status_id)
-        except Inventory.DoesNotExist:
+        except LaunchJobStatus.DoesNotExist:
             raise CommandError('Launch job status with ID %d not found' % launch_job_status_id)
-        # FIXME: Do stuff here.
+        if launch_job_status.status != 'running':
+            raise CommandError('Unable to add event except when launch job is running')
+        try:
+            if event_data_json is None:
+                try:
+                    if event_data_file == '-':
+                        event_data_fileobj = sys.stdin
+                    else:
+                        event_data_fileobj = file(event_data_file, 'rb')
+                    event_data = json.load(event_data_fileobj)
+                except IOError, e:
+                    raise CommandError('Error %r reading from %s' % (e, event_data_file))
+            else:
+                event_data = json.loads(event_data_json)
+        except ValueError:
+            raise CommandError('Error parsing JSON data')
+        launch_job_status.launch_job_status_events.create(event=event_type,
+                                                          event_data=event_data)
 
 if __name__ == '__main__':
     from __init__ import run_command_as_script

lib/main/management/commands/acom_inventory.py

@@ -21,7 +21,6 @@
 import json
 from optparse import make_option
 import os
-import sys
 from django.core.management.base import NoArgsCommand, CommandError
 
 class Command(NoArgsCommand):

lib/main/tasks.py

@@ -34,10 +34,14 @@ def run_launch_job(launch_job_status_pk):
     inventory_script = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                     'management', 'commands',
                                                     'acom_inventory.py'))
+    callback_script = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                                    'management', 'commands',
+                                                    'acom_callback_event.py'))
     env = dict(os.environ.items())
     env['ACOM_LAUNCH_JOB_STATUS_ID'] = str(launch_job_status.pk)
     env['ACOM_INVENTORY_ID'] = str(launch_job.inventory.pk)
     env['ANSIBLE_CALLBACK_PLUGINS'] = plugin_dir
+    env['ACOM_CALLBACK_EVENT_SCRIPT'] = callback_script
  
     if hasattr(settings, 'ANSIBLE_TRANSPORT'):
         env['ANSIBLE_TRANSPORT'] = getattr(settings, 'ANSIBLE_TRANSPORT')

lib/main/tests/__init__.py

@@ -20,5 +20,5 @@ from lib.main.tests.organizations import OrganizationsTest
 from lib.main.tests.users import UsersTest
 from lib.main.tests.inventory import InventoryTest
 from lib.main.tests.projects import ProjectsTest
-from lib.main.tests.commands import AcomInventoryTest
+from lib.main.tests.commands import *
 from lib.main.tests.tasks import RunLaunchJobTest

lib/main/tests/commands.py

@@ -20,11 +20,15 @@ import json
 import os
 import StringIO
 import sys
+import tempfile
 from django.core.management import call_command
 from django.core.management.base import CommandError
+from django.utils.timezone import now
 from lib.main.models import *
 from lib.main.tests.base import BaseTest
 
+__all__ = ['AcomInventoryTest', 'AcomCallbackEventTest']
+
 class BaseCommandTest(BaseTest):
     '''
     Base class for tests that run management commands.
@@ -33,6 +37,7 @@ class BaseCommandTest(BaseTest):
     def setUp(self):
         super(BaseCommandTest, self).setUp()
         self._environ = dict(os.environ.items())
+        self._temp_files = []
 
     def tearDown(self):
         super(BaseCommandTest, self).tearDown()
@@ -42,16 +47,23 @@ class BaseCommandTest(BaseTest):
         for k,v in os.environ.items():
             if k not in self._environ.keys():
                 del os.environ[k]
+        for tf in self._temp_files:
+            if os.path.exists(tf):
+                os.remove(tf)
 
     def run_command(self, name, *args, **options):
         '''
         Run a management command and capture its stdout/stderr along with any
         exceptions.
         '''
+        stdin_fileobj = options.pop('stdin_fileobj', None)
         options.setdefault('verbosity', 1)
         options.setdefault('interactive', False)
+        original_stdin = sys.stdin
         original_stdout = sys.stdout
         original_stderr = sys.stderr
+        if stdin_fileobj:
+            sys.stdin = stdin_fileobj
         sys.stdout = StringIO.StringIO()
         sys.stderr = StringIO.StringIO()
         result = None
@@ -64,6 +76,7 @@ class BaseCommandTest(BaseTest):
         finally:
             captured_stdout = sys.stdout.getvalue()
             captured_stderr = sys.stderr.getvalue()
+            sys.stdin = original_stdin
             sys.stdout = original_stdout
             sys.stderr = original_stderr
         return result, captured_stdout, captured_stderr
@@ -244,4 +257,145 @@ class AcomInventoryTest(BaseCommandTest):
                                                   host='blah')
         self.assertTrue(isinstance(result, CommandError))
         self.assertEqual(json.loads(stdout), {})
-    
+
+class AcomCallbackEventTest(BaseCommandTest):
+    '''
+    Test cases for acom_callback_event management command.
+    '''
+
+    def setUp(self):
+        super(AcomCallbackEventTest, self).setUp()
+        self.setup_users()
+        self.organization = self.make_organizations(self.super_django_user, 1)[0]
+        self.project = self.make_projects(self.normal_django_user, 1)[0]
+        self.organization.projects.add(self.project)
+        self.inventory = Inventory.objects.create(name='test-inventory',
+                                                  organization=self.organization)
+        self.host = self.inventory.hosts.create(name='host.example.com',
+                                                inventory=self.inventory)
+        self.group = self.inventory.groups.create(name='test-group',
+                                                  inventory=self.inventory)
+        self.group.hosts.add(self.host)
+        self.launch_job = LaunchJob.objects.create(name='test-launch-job',
+                                                   inventory=self.inventory,
+                                                   project=self.project)
+        self.launch_job_status = self.launch_job.launch_job_statuses.create(
+            name='launch-job-status-%s' % now().isoformat())
+        self.valid_kwargs = {
+            'launch_job_status_id': self.launch_job_status.id,
+            'event_type': 'playbook_on_start',
+            'event_data_json': json.dumps({'test_event_data': [2,4,6]}),
+        }
+
+    def test_with_launch_job_status_not_running(self):
+        # Events can only be added when the launch job is running.
+        self.assertEqual(self.launch_job_status.status, 'pending')
+        result, stdout, stderr = self.run_command('acom_callback_event',
+                                                  **self.valid_kwargs)
+        self.assertTrue(isinstance(result, CommandError))
+        self.assertTrue('unable to add event ' in str(result).lower())
+        self.launch_job_status.status = 'successful'
+        self.launch_job_status.save()
+        result, stdout, stderr = self.run_command('acom_callback_event',
+                                                  **self.valid_kwargs)
+        self.assertTrue(isinstance(result, CommandError))
+        self.assertTrue('unable to add event ' in str(result).lower())
+        self.launch_job_status.status = 'failed'
+        self.launch_job_status.save()
+        result, stdout, stderr = self.run_command('acom_callback_event',
+                                                  **self.valid_kwargs)
+        self.assertTrue(isinstance(result, CommandError))
+        self.assertTrue('unable to add event ' in str(result).lower())
+
+    def test_with_invalid_args(self):
+        self.launch_job_status.status = 'running'
+        self.launch_job_status.save()
+        # Event type not given.
+        kwargs = dict(self.valid_kwargs.items())
+        kwargs.pop('event_type')
+        result, stdout, stderr = self.run_command('acom_callback_event', **kwargs)
+        self.assertTrue(isinstance(result, CommandError))
+        self.assertTrue('no event specified' in str(result).lower())
+        # Invalid event type.
+        kwargs = dict(self.valid_kwargs.items())
+        kwargs['event_type'] = 'invalid_event_type'
+        result, stdout, stderr = self.run_command('acom_callback_event', **kwargs)
+        self.assertTrue(isinstance(result, CommandError))
+        self.assertTrue('unsupported event' in str(result).lower())
+        # Neither file or data specified.
+        kwargs = dict(self.valid_kwargs.items())
+        kwargs.pop('event_data_json')
+        result, stdout, stderr = self.run_command('acom_callback_event', **kwargs)
+        self.assertTrue(isinstance(result, CommandError))
+        self.assertTrue('either --file or --data' in str(result).lower())
+        # Non-integer launch job status ID.
+        kwargs = dict(self.valid_kwargs.items())
+        kwargs['launch_job_status_id'] = 'foo'
+        result, stdout, stderr = self.run_command('acom_callback_event', **kwargs)
+        self.assertTrue(isinstance(result, CommandError))
+        self.assertTrue('id must be an integer' in str(result).lower())
+        # No launch job status ID.
+        kwargs = dict(self.valid_kwargs.items())
+        kwargs.pop('launch_job_status_id')
+        result, stdout, stderr = self.run_command('acom_callback_event', **kwargs)
+        self.assertTrue(isinstance(result, CommandError))
+        self.assertTrue('no launch job status id' in str(result).lower())
+        # Invalid launch job status ID.
+        kwargs = dict(self.valid_kwargs.items())
+        kwargs['launch_job_status_id'] = 9999
+        result, stdout, stderr = self.run_command('acom_callback_event', **kwargs)
+        self.assertTrue(isinstance(result, CommandError))
+        self.assertTrue('not found' in str(result).lower())
+        # Invalid inline JSON data.
+        kwargs = dict(self.valid_kwargs.items())
+        kwargs['event_data_json'] = 'invalid json'
+        result, stdout, stderr = self.run_command('acom_callback_event', **kwargs)
+        self.assertTrue(isinstance(result, CommandError))
+        self.assertTrue('error parsing json' in str(result).lower())
+        # Invalid file specified.
+        kwargs = dict(self.valid_kwargs.items())
+        kwargs.pop('event_data_json')
+        h, tf = tempfile.mkstemp()
+        os.close(h)
+        os.remove(tf)
+        kwargs['event_data_file'] = '%s.json' % tf
+        result, stdout, stderr = self.run_command('acom_callback_event', **kwargs)
+        self.assertTrue(isinstance(result, CommandError))
+        self.assertTrue('reading from' in str(result).lower())
+
+    def test_with_valid_args(self):
+        self.launch_job_status.status = 'running'
+        self.launch_job_status.save()
+        # Default valid args.
+        kwargs = dict(self.valid_kwargs.items())
+        result, stdout, stderr = self.run_command('acom_callback_event', **kwargs)
+        self.assertEqual(result, None)
+        self.assertEqual(self.launch_job_status.launch_job_status_events.count(), 1)
+        # Pass launch job status in environment instead.
+        kwargs = dict(self.valid_kwargs.items())
+        kwargs.pop('launch_job_status_id')
+        os.environ['ACOM_LAUNCH_JOB_STATUS_ID'] = str(self.launch_job_status.id)
+        result, stdout, stderr = self.run_command('acom_callback_event', **kwargs)
+        self.assertEqual(result, None)
+        self.assertEqual(self.launch_job_status.launch_job_status_events.count(), 2)
+        os.environ.pop('ACOM_LAUNCH_JOB_STATUS_ID', None)
+        # Test with JSON data in a file instead.
+        kwargs = dict(self.valid_kwargs.items())
+        kwargs.pop('event_data_json')
+        h, tf = tempfile.mkstemp(suffix='.json')
+        self._temp_files.append(tf)
+        f = os.fdopen(h, 'w')
+        json.dump({'some_event_data': [1, 2, 3]}, f)
+        f.close()
+        kwargs['event_data_file'] = tf
+        result, stdout, stderr = self.run_command('acom_callback_event', **kwargs)
+        self.assertEqual(result, None)
+        self.assertEqual(self.launch_job_status.launch_job_status_events.count(), 3)
+        # Test with JSON data from stdin.
+        kwargs = dict(self.valid_kwargs.items())
+        kwargs.pop('event_data_json')
+        kwargs['event_data_file'] = '-'
+        kwargs['stdin_fileobj'] = StringIO.StringIO(json.dumps({'blah': 'bleep'}))
+        result, stdout, stderr = self.run_command('acom_callback_event', **kwargs)
+        self.assertEqual(result, None)
+        self.assertEqual(self.launch_job_status.launch_job_status_events.count(), 4)

lib/main/tests/tasks.py

@@ -88,12 +88,10 @@ class RunLaunchJobTest(BaseCeleryTest):
         launch_job_status = self.launch_job.start()
         self.assertEqual(launch_job_status.status, 'pending')
         launch_job_status = LaunchJobStatus.objects.get(pk=launch_job_status.pk)
-        print 'stdout:', launch_job_status.result_stdout
+        #print 'stdout:', launch_job_status.result_stdout
-        print 'stderr:', launch_job_status.result_stderr
+        #print 'stderr:', launch_job_status.result_stderr
-        print launch_job_status.status
+        #print launch_job_status.status
-        
+        #print settings.DATABASES
-        print settings.DATABASES
-        
         self.assertEqual(launch_job_status.status, 'successful')
         self.assertTrue(launch_job_status.result_stdout)
         launch_job_status_events = launch_job_status.launch_job_status_events.all()

lib/plugins/callback/acom_callback.py

@@ -16,7 +16,9 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 
+import json
 import os
+import subprocess
 import sys
 
 class CallbackModule(object):
@@ -25,47 +27,12 @@ class CallbackModule(object):
     '''
 
     def __init__(self):
-        # the DJANGO_SETTINGS_MODULE environment variable *should* already
+        self.acom_callback_event_script = os.getenv('ACOM_CALLBACK_EVENT_SCRIPT')
-        # be set if this callback is called when executing a playbook via a
-        # celery task, otherwise just bail out.
-        settings_module_name = os.environ.get('DJANGO_SETTINGS_MODULE', None)
-        if not settings_module_name:
-            return
-        # FIXME: Not particularly fond of this sys.path hack, but it is needed
-        # when a celery task calls ansible-playbook and needs to execute this
-        # script directly.
-        try:
-            settings_parent_module = __import__(settings_module_name)
-        except ImportError:
-            top_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..')
-            sys.path.insert(0, os.path.abspath(top_dir))
-            settings_parent_module = __import__(settings_module_name)
-        settings_module = getattr(settings_parent_module, settings_module_name.split('.')[-1])
-        # Use the ACOM_TEST_DATABASE_NAME environment variable to specify the test
-        # database name when called from unit tests.
-        if os.environ.get('ACOM_TEST_DATABASE_NAME', None):
-            settings_module.DATABASES['default']['NAME'] = os.environ['ACOM_TEST_DATABASE_NAME']
-        # Try to get the launch job status ID from the environment, otherwise
-        # just bail out now.
-        try:
-            launch_job_status_pk = int(os.environ.get('ACOM_LAUNCH_JOB_STATUS_ID', ''))
-        except ValueError:
-            return
-        from lib.main.models import LaunchJobStatus
-        try:
-            self.launch_job_status = LaunchJobStatus.objects.get(pk=launch_job_status_pk)
-        except LaunchJobStatus.DoesNotExist:
-            pass
 
     def _log_event(self, event, **event_data):
-        #print '====', event, args, kwargs
+        event_data_json = json.dumps(event_data)
-        # self.playbook.inventory
+        cmdline = [self.acom_callback_event_script, '-e', event, '-d', event_data_json]
-        if hasattr(self, 'launch_job_status'):
+        subprocess.check_call(cmdline)
-            kwargs = {
-                'event': event,
-                'event_data': event_data,
-            }
-            self.launch_job_status.launch_job_status_events.create(**kwargs)
 
     def on_any(self, *args, **kwargs):
         pass