Merge pull request #320 from matburt/stdout_file_download

Improve stdout behavior by implementing a downloadable stdout.

awx/api/generics.py

@@ -202,6 +202,7 @@ class GenericAPIView(generics.GenericAPIView, APIView):
                     'model_verbose_name_plural': unicode(self.model._meta.verbose_name_plural),
                 })
             d.update({'serializer_fields': self.get_serializer().metadata()})
+        d['settings'] = settings
         return d
 
     def metadata(self, request):

awx/api/renderers.py

@@ -45,6 +45,9 @@ class PlainTextRenderer(renderers.BaseRenderer):
             data = unicode(data)
         return data.encode(self.charset)
 
+class DownloadTextRenderer(PlainTextRenderer):
+    format = "txt_download"
+
 class AnsiTextRenderer(PlainTextRenderer):
 
     media_type = 'text/plain'

awx/api/serializers.py

@@ -464,7 +464,8 @@ class UnifiedJobTemplateSerializer(BaseSerializer):
 
 class UnifiedJobSerializer(BaseSerializer):
 
-    result_stdout = serializers.CharField(source='result_stdout', label='result stdout', read_only=True)
+    #result_stdout = serializers.CharField(source='result_stdout', label='result stdout', read_only=True)
+    result_stdout = serializers.SerializerMethodField('get_result_stdout')
     unified_job_template = serializers.Field(source='unified_job_template_id', label='unified job template')
     job_env = serializers.CharField(source='job_env', label='job env', read_only=True)
 
@@ -475,6 +476,13 @@ class UnifiedJobSerializer(BaseSerializer):
                   'job_cwd', 'job_env', 'job_explanation', 'result_stdout',
                   'result_traceback')
 
+
+    def get_result_stdout(self, obj):
+        obj_size = obj.result_stdout_size
+        if obj_size > settings.STDOUT_MAX_BYTES_DISPLAY:
+            return "Standard Output too large to display (%d bytes), only download supported for sizes over %d bytes" % (obj_size, settings.STDOUT_MAX_BYTES_DISPLAY)
+        return obj.result_stdout
+
     def get_types(self):
         if type(self) is UnifiedJobSerializer:
             return ['project_update', 'inventory_update', 'job', 'ad_hoc_command', 'system_job']

awx/api/templates/api/unified_job_stdout.md

@@ -12,6 +12,7 @@ Use the `format` query string parameter to specify the output format.
 * Plain Text: `?format=txt`
 * Plain Text with ANSI color codes: `?format=ansi`
 * JSON structure: `?format=json`
+* Downloaded Plain Text: `?format=txt_download`
 
 (_New in Ansible Tower 2.0.0_) When using the Browsable API, HTML and JSON
 formats, the `start_line` and `end_line` query string parameters can be used
@@ -20,4 +21,7 @@ to specify a range of line numbers to retrieve.
 Use `dark=1` or `dark=0` as a query string parameter to force or disable a
 dark background.
 
+Files over {{ settings.STDOUT_MAX_BYTES_DISPLAY|filesizeformat }} (configurable) will not display in the browser. Use the `txt_download`
+format to download the file directly to view it.
+
 {% include "api/_new_in_awx.md" %}

awx/api/views.py

@@ -25,6 +25,8 @@ from django.utils.safestring import mark_safe
 from django.utils.timezone import now
 from django.views.decorators.csrf import csrf_exempt
 from django.template.loader import render_to_string
+from django.core.servers.basehttp import FileWrapper
+from django.http import HttpResponse
 
 # Django REST Framework
 from rest_framework.exceptions import PermissionDenied, ParseError
@@ -2109,11 +2111,20 @@ class JobList(ListCreateAPIView):
     model = Job
     serializer_class = JobListSerializer
 
+    def get_queryset(self):
+        qs = self.request.user.get_queryset(self.model).defer('result_stdout_text')
+        return qs
+
 class JobDetail(RetrieveUpdateDestroyAPIView):
 
     model = Job
     serializer_class = JobSerializer
 
+    
+    def get_queryset(self):
+        qs = super(JobDetail, self).get_queryset().defer('result_stdout_text')
+        return qs
+
     def update(self, request, *args, **kwargs):
         obj = self.get_object()
         # Only allow changes (PUT/PATCH) when job status is "new".
@@ -2783,18 +2794,32 @@ class UnifiedJobList(ListAPIView):
     model = UnifiedJob
     serializer_class = UnifiedJobListSerializer
     new_in_148 = True
+    
+    def get_queryset(self):
+        qs = self.request.user.get_queryset(self.model).defer('result_stdout_text')
+        return qs
+
 
 class UnifiedJobStdout(RetrieveAPIView):
 
     serializer_class = UnifiedJobStdoutSerializer
     renderer_classes = [BrowsableAPIRenderer, renderers.StaticHTMLRenderer,
-                        PlainTextRenderer, AnsiTextRenderer,
+                        PlainTextRenderer, AnsiTextRenderer, DownloadTextRenderer,
                         renderers.JSONRenderer]
     filter_backends = ()
     new_in_148 = True
 
+    def get_queryset(self):
+        qs = super(UnifiedJobStdout, self).get_queryset().defer('result_stdout_text')
+        return qs    
+
     def retrieve(self, request, *args, **kwargs):
         unified_job = self.get_object()
+        obj_size = unified_job.result_stdout_size
+        if request.accepted_renderer.format != 'txt_download' and obj_size > settings.STDOUT_MAX_BYTES_DISPLAY:
+            return Response("Standard Output too large to display (%d bytes), "
+                            "only download supported for sizes over %d bytes" % (obj_size, settings.STDOUT_MAX_BYTES_DISPLAY))
+
         if request.accepted_renderer.format in ('html', 'api', 'json'):
             start_line = request.QUERY_PARAMS.get('start_line', 0)
             end_line = request.QUERY_PARAMS.get('end_line', None)
@@ -2820,6 +2845,14 @@ class UnifiedJobStdout(RetrieveAPIView):
             return Response(data)
         elif request.accepted_renderer.format == 'ansi':
             return Response(unified_job.result_stdout_raw)
+        elif request.accepted_renderer.format == 'txt_download':
+            try:
+                content_fd = open(unified_job.dump_result_stdout(), 'r')
+                response = HttpResponse(FileWrapper(content_fd), content_type='text/plain')
+                response["Content-Disposition"] = 'attachment; filename="job_%s.txt"' % str(unified_job.id)
+                return response
+            except Exception, e:
+                return Response({"error": "Error generating stdout download file: %s" % str(e)}, status=status.HTTP_400_BAD_REQUEST)
         else:
             return super(UnifiedJobStdout, self).retrieve(request, *args, **kwargs)
 

awx/main/models/unified_jobs.py

@@ -5,6 +5,7 @@
 import codecs
 import json
 import logging
+import uuid
 import re
 import os
 import os.path
@@ -12,7 +13,7 @@ from StringIO import StringIO
 
 # Django
 from django.conf import settings
-from django.db import models
+from django.db import models, connection
 from django.core.exceptions import NON_FIELD_ERRORS
 from django.utils.datastructures import SortedDict
 from django.utils.translation import ugettext_lazy as _
@@ -657,6 +658,22 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
     def result_stdout(self):
         return self._result_stdout_raw(escape_ascii=True)
 
+    @property
+    def result_stdout_size(self):
+        cursor = connection.cursor()
+        cursor.execute("select length(result_stdout_text) from main_unifiedjob where id = %d" % self.pk)
+        record_size = cursor.fetchone()[0]
+        return record_size
+
+    def dump_result_stdout(self):
+        tower_file = "towerjob-%s" % str(uuid.uuid1())[:8]
+        out_path = os.path.join(settings.STDOUT_TEMP_DIR, tower_file)
+        tower_fd = open(out_path, 'w')
+        cursor = connection.cursor()
+        cursor.copy_expert("copy (select result_stdout_text from main_unifiedjob where id = %d) to stdout" % (self.pk), tower_fd)
+        tower_fd.close()
+        return out_path
+
     def _result_stdout_raw_limited(self, start_line=0, end_line=None, redact_sensitive=True, escape_ascii=False):
         return_buffer = u""
         if end_line is not None:

awx/main/tasks.py

@@ -114,6 +114,17 @@ def tower_periodic_scheduler(self):
             new_unified_job.socketio_emit_status("failed")
         emit_websocket_notification('/socket.io/schedules', 'schedule_changed', dict(id=schedule.id))
 
+@task(bind=True)
+def clean_stdout_tempfiles(self):
+    nowtime = time.time()
+    removed = 0
+    for this_file in os.listdir(settings.STDOUT_TEMP_DIR):
+        this_file = os.path.join(settings.STDOUT_TEMP_DIR, this_file)
+        if "towerjob" in this_file and os.stat(this_file).st_mtime < nowtime - 86400:
+            os.remove(this_file)
+            removed += 1
+    print("Removed %d files" % removed)
+
 @task()
 def notify_task_runner(metadata_dict):
     """Add the given task into the Tower task manager's queue, to be consumed

awx/settings/defaults.py

@@ -303,6 +303,10 @@ CELERYBEAT_SCHEDULE = {
         'task': 'awx.main.tasks.tower_periodic_scheduler',
         'schedule': timedelta(seconds=30)
     },
+    'job_stdout_cleanup': {
+        'task': 'awx.main.tasks.clean_stdout_tempfiles',
+        'schedule': timedelta(hours=3)
+    },
 }
 
 # Any ANSIBLE_* settings will be passed to the subprocess environment by the
@@ -562,6 +566,10 @@ FACT_CACHE_PORT = 6564
 
 ORG_ADMINS_CAN_SEE_ALL_USERS = True
 
+# Control when we display stdout and where to store the temporary files for downloading
+STDOUT_MAX_BYTES_DISPLAY = 1048576
+STDOUT_TEMP_DIR = "/var/lib/awx/"
+
 # Logging configuration.
 LOGGING = {
     'version': 1,

awx/settings/development.py

@@ -64,6 +64,8 @@ PASSWORD_HASHERS = (
 # Configure a default UUID for development only.
 SYSTEM_UUID = '00000000-0000-0000-0000-000000000000'
 
+STDOUT_TEMP_DIR = "/tmp"
+
 # If there is an `/etc/tower/settings.py`, include it.
 # If there is a `/etc/tower/conf.d/*.py`, include them.
 include(optional('/etc/tower/settings.py'), scope=locals())