Initial task status to file implementation

awx/api/serializers.py

@@ -7,6 +7,7 @@ import re
 import socket
 import urlparse
 import logging
+import os.path
 
 # PyYAML
 import yaml
@@ -188,6 +189,21 @@ class BaseSerializer(serializers.ModelSerializer):
         else:
             return obj.active
 
+class BaseTaskSerializer(BaseSerializer):
+
+    result_stdout = serializers.SerializerMethodField('get_result_stdout')
+
+    def get_result_stdout(self, obj):
+        if obj is None:
+            return ''
+        if obj.result_stdout_file != "":
+            if not os.path.exists(obj.result_stdout_file):
+                return "stdout capture is missing"
+            stdout_fd = open(obj.result_stdout_file, "r")
+            output = stdout_fd.read()
+            stdout_fd.close()
+            return output
+        return obj.result_stdout
 
 class UserSerializer(BaseSerializer):
 
@@ -361,7 +377,7 @@ class ProjectPlaybooksSerializer(ProjectSerializer):
         return ret.get('playbooks', [])
 
 
-class ProjectUpdateSerializer(BaseSerializer):
+class ProjectUpdateSerializer(BaseTaskSerializer):
 
     class Meta:
         model = ProjectUpdate
@@ -685,7 +701,7 @@ class InventorySourceSerializer(BaseSerializer):
         return metadata
 
 
-class InventoryUpdateSerializer(BaseSerializer):
+class InventoryUpdateSerializer(BaseTaskSerializer):
 
     class Meta:
         model = InventoryUpdate
@@ -839,7 +855,7 @@ class JobTemplateSerializer(BaseSerializer):
         return attrs
 
 
-class JobSerializer(BaseSerializer):
+class JobSerializer(BaseTaskSerializer):
 
     passwords_needed_to_start = serializers.Field(source='passwords_needed_to_start')
 
@@ -849,9 +865,8 @@ class JobSerializer(BaseSerializer):
                   'modified', 'job_template', 'job_type', 'inventory',
                   'project', 'playbook', 'credential', 'cloud_credential',
                   'forks', 'limit', 'verbosity', 'extra_vars',
-                  'job_tags', 'launch_type', 'status', 'failed',
+                  'job_tags', 'launch_type', 'status', 'failed', 'result_stdout',
-                  'result_stdout', 'result_traceback',
+                  'result_traceback', 'passwords_needed_to_start', 'job_args',
-                  'passwords_needed_to_start', 'job_args',
                   'job_cwd', 'job_env')
 
     def get_related(self, obj):

awx/main/models/base.py

@@ -273,6 +273,11 @@ class CommonTask(PrimordialModel):
         default='',
         editable=False,
     )
+    result_stdout_file = models.TextField(
+        blank=True,
+        default='',
+        editable=False,
+    )
     result_traceback = models.TextField(
         blank=True,
         default='',

awx/main/tasks.py

@@ -17,6 +17,7 @@ import tempfile
 import time
 import traceback
 import urlparse
+import uuid
 
 # Pexpect
 import pexpect
@@ -68,6 +69,9 @@ class BaseTask(Task):
             transaction.commit()
         return instance
 
+    def get_model(self, pk):
+        return self.model.objects.get(pk=pk)
+
     def get_path_to(self, *args):
         '''
         Return absolute path relative to this file.
@@ -155,7 +159,7 @@ class BaseTask(Task):
         '''
         return SortedDict()
 
-    def run_pexpect(self, instance, args, cwd, env, passwords,
+    def run_pexpect(self, instance, args, cwd, env, passwords, task_stdout_handle,
                     output_replacements=None):
         '''
         Run the given command using pexpect to capture output and provide
@@ -176,18 +180,20 @@ class BaseTask(Task):
             expect_list.append(item[0])
             expect_passwords[n] = passwords.get(item[1], '') or ''
         expect_list.extend([pexpect.TIMEOUT, pexpect.EOF])
+        self.update_model(instance.pk, status='running', output_replacements=output_replacements)
         while child.isalive():
             result_id = child.expect(expect_list, timeout=pexpect_timeout)
             #print 'pexpect result_id', result_id, expect_list[result_id], expect_passwords.get(result_id, None)
             if result_id in expect_passwords:
                 child.sendline(expect_passwords[result_id])
-            updates = {'status': 'running',
-                       'output_replacements': output_replacements}
             if logfile_pos != logfile.tell():
+                old_logfile_pos = logfile_pos
                 logfile_pos = logfile.tell()
-                updates['result_stdout'] = logfile.getvalue()
+                #updates['result_stdout'] = logfile.getvalue()
+                task_stdout_handle.write(logfile.getvalue()[old_logfile_pos:logfile_pos])
+                task_stdout_handle.flush()
                 last_stdout_update = time.time()
-            instance = self.update_model(instance.pk, **updates)
+            instance = self.get_model(instance.pk)
             # Commit transaction needed when running unit tests. FIXME: Is it
             # needed or breaks anything for normal operation?
             transaction.commit()
@@ -213,6 +219,7 @@ class BaseTask(Task):
         '''
         if instance.status != 'pending':
             return False
+        # TODO: Check that we can write to the stdout data directory
         return True
 
     def post_run_hook(self, instance, **kwargs):
@@ -264,10 +271,11 @@ class BaseTask(Task):
             cwd = self.build_cwd(instance, **kwargs)
             env = self.build_env(instance, **kwargs)
             safe_env = self.build_safe_env(instance, **kwargs)
+            stdout_filename = os.path.join(settings.JOBOUTPUT_ROOT, str(uuid.uuid1()) + ".out")
+            stdout_handle = open(stdout_filename, 'w')
             instance = self.update_model(pk, job_args=json.dumps(safe_args),
-                                         job_cwd=cwd, job_env=safe_env)
+                                         job_cwd=cwd, job_env=safe_env, result_stdout_file=stdout_filename)
-            status, stdout = self.run_pexpect(instance, args, cwd, env,
+            status, stdout = self.run_pexpect(instance, args, cwd, env, kwargs['passwords'], stdout_handle)
-                                              kwargs['passwords'])
         except Exception:
             tb = traceback.format_exc()
         finally:
@@ -276,7 +284,11 @@ class BaseTask(Task):
                     os.remove(kwargs['private_data_file'])
                 except IOError:
                     pass
-        instance = self.update_model(pk, status=status, result_stdout=stdout,
+                try:
+                    stdout_handle.close()
+                except Exception:
+                    pass
+        instance = self.update_model(pk, status=status,
                                      result_traceback=tb,
                                      output_replacements=output_replacements)
         self.post_run_hook(instance, **kwargs)

awx/settings/defaults.py

@@ -83,6 +83,10 @@ MEDIA_URL = '/media/'
 # This directory should not be web-accessible.
 PROJECTS_ROOT = os.path.join(BASE_DIR, 'projects')
 
+# Absolute filesystem path to the directory for job status stdout
+# This directory should not be web-accessible
+JOBOUTPUT_ROOT = os.path.join(BASE_DIR, 'job_status')
+
 SITE_ID = 1
 
 # Make this unique, and don't share it with anybody.

awx/settings/local_settings.py.example

@@ -47,6 +47,10 @@ if len(sys.argv) >= 2 and sys.argv[1] == 'test':
 # This directory should NOT be web-accessible.
 PROJECTS_ROOT = os.path.join(BASE_DIR, 'projects')
 
+# Absolute filesystem path to the directory for job status stdout
+# This directory should not be web-accessible
+JOBOUTPUT_ROOT = os.path.join(BASE_DIR, 'job_status')
+
 # Local time zone for this installation. Choices can be found here:
 # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
 # although not all choices may be available on all operating systems.

config/deb/settings.py

@@ -34,6 +34,8 @@ STATIC_ROOT = '/var/lib/awx/public/static'
 
 PROJECTS_ROOT = '/var/lib/awx/projects'
 
+JOBOUTPUT_ROOT = '/var/lib/awx/job_status'
+
 SECRET_KEY = file('/etc/awx/SECRET_KEY', 'rb').read().strip()
 
 ALLOWED_HOSTS = ['*']

config/rpm/settings.py

@@ -34,6 +34,8 @@ STATIC_ROOT = '/var/lib/awx/public/static'
 
 PROJECTS_ROOT = '/var/lib/awx/projects'
 
+JOBOUTPUT_ROOT = '/var/lib/awx/job_status'
+
 SECRET_KEY = file('/etc/awx/SECRET_KEY', 'rb').read().strip()
 
 ALLOWED_HOSTS = ['*']