change stdout composition to generate from job events on the fly

this approach totally removes the process of reading and writing stdout
files on the local file system at settings.JOBOUTPUT_ROOT when jobs are
run; now stdout content is only written on-demand as it's fetched for
the deprecated `stdout` endpoint

see: https://github.com/ansible/awx/issues/200

awx/api/serializers.py

@@ -614,14 +614,12 @@ class UnifiedJobTemplateSerializer(BaseSerializer):
 class UnifiedJobSerializer(BaseSerializer):
     show_capabilities = ['start', 'delete']
 
-    result_stdout = serializers.SerializerMethodField()
-
     class Meta:
         model = UnifiedJob
         fields = ('*', 'unified_job_template', 'launch_type', 'status',
                   'failed', 'started', 'finished', 'elapsed', 'job_args',
-                  'job_cwd', 'job_env', 'job_explanation', 'result_stdout',
-                  'execution_node', 'result_traceback')
+                  'job_cwd', 'job_env', 'job_explanation', 'execution_node',
+                  'result_traceback')
         extra_kwargs = {
             'unified_job_template': {
                 'source': 'unified_job_template_id',
@@ -702,25 +700,17 @@ class UnifiedJobSerializer(BaseSerializer):
 
         return ret
 
-    def get_result_stdout(self, obj):
-        obj_size = obj.result_stdout_size
-        if obj_size > settings.STDOUT_MAX_BYTES_DISPLAY:
-            return _("Standard Output too large to display (%(text_size)d bytes), "
-                     "only download supported for sizes over %(supported_size)d bytes") % {
-                'text_size': obj_size, 'supported_size': settings.STDOUT_MAX_BYTES_DISPLAY}
-        return obj.result_stdout
-
 
 class UnifiedJobListSerializer(UnifiedJobSerializer):
 
     class Meta:
-        fields = ('*', '-job_args', '-job_cwd', '-job_env', '-result_traceback', '-result_stdout')
+        fields = ('*', '-job_args', '-job_cwd', '-job_env', '-result_traceback')
 
     def get_field_names(self, declared_fields, info):
         field_names = super(UnifiedJobListSerializer, self).get_field_names(declared_fields, info)
         # Meta multiple inheritance and -field_name options don't seem to be
         # taking effect above, so remove the undesired fields here.
-        return tuple(x for x in field_names if x not in ('job_args', 'job_cwd', 'job_env', 'result_traceback', 'result_stdout'))
+        return tuple(x for x in field_names if x not in ('job_args', 'job_cwd', 'job_env', 'result_traceback'))
 
     def get_types(self):
         if type(self) is UnifiedJobListSerializer:
@@ -760,14 +750,6 @@ class UnifiedJobStdoutSerializer(UnifiedJobSerializer):
     class Meta:
         fields = ('result_stdout',)
 
-    def get_result_stdout(self, obj):
-        obj_size = obj.result_stdout_size
-        if obj_size > settings.STDOUT_MAX_BYTES_DISPLAY:
-            return _("Standard Output too large to display (%(text_size)d bytes), "
-                     "only download supported for sizes over %(supported_size)d bytes") % {
-                'text_size': obj_size, 'supported_size': settings.STDOUT_MAX_BYTES_DISPLAY}
-        return obj.result_stdout
-
     def get_types(self):
         if type(self) is UnifiedJobStdoutSerializer:
             return ['project_update', 'inventory_update', 'job', 'ad_hoc_command', 'system_job']
@@ -2966,9 +2948,11 @@ class SystemJobTemplateSerializer(UnifiedJobTemplateSerializer):
 
 class SystemJobSerializer(UnifiedJobSerializer):
 
+    result_stdout = serializers.SerializerMethodField()
+
     class Meta:
         model = SystemJob
-        fields = ('*', 'system_job_template', 'job_type', 'extra_vars')
+        fields = ('*', 'system_job_template', 'job_type', 'extra_vars', 'result_stdout')
 
     def get_related(self, obj):
         res = super(SystemJobSerializer, self).get_related(obj)
@@ -2980,6 +2964,9 @@ class SystemJobSerializer(UnifiedJobSerializer):
             res['cancel'] = self.reverse('api:system_job_cancel', kwargs={'pk': obj.pk})
         return res
 
+    def get_result_stdout(self, obj):
+        return obj.result_stdout
+
 
 class SystemJobCancelSerializer(SystemJobSerializer):
 

awx/api/views.py

@@ -2,13 +2,11 @@
 # All Rights Reserved.
 
 # Python
-import os
 import re
 import cgi
 import dateutil
 import time
 import socket
-import subprocess
 import sys
 import logging
 import requests
@@ -20,7 +18,7 @@ import six
 from django.conf import settings
 from django.core.exceptions import FieldError
 from django.db.models import Q, Count, F
-from django.db import IntegrityError, transaction, connection
+from django.db import IntegrityError, transaction
 from django.shortcuts import get_object_or_404
 from django.utils.encoding import smart_text, force_text
 from django.utils.safestring import mark_safe
@@ -4498,7 +4496,7 @@ class StdoutANSIFilter(object):
     def __init__(self, fileobj):
         self.fileobj = fileobj
         self.extra_data = ''
-        if hasattr(fileobj,'close'):
+        if hasattr(fileobj, 'close'):
             self.close = fileobj.close
 
     def read(self, size=-1):
@@ -4529,93 +4527,73 @@ class UnifiedJobStdout(RetrieveAPIView):
                         renderers.JSONRenderer, DownloadTextRenderer, AnsiDownloadRenderer]
     filter_backends = ()
     new_in_148 = True
+    deprecated = True
 
     def retrieve(self, request, *args, **kwargs):
         unified_job = self.get_object()
-        obj_size = unified_job.result_stdout_size
-        if request.accepted_renderer.format not in {'txt_download', 'ansi_download'} and obj_size > settings.STDOUT_MAX_BYTES_DISPLAY:
-            response_message = _("Standard Output too large to display (%(text_size)d bytes), "
-                                 "only download supported for sizes over %(supported_size)d bytes") % {
-                'text_size': obj_size, 'supported_size': settings.STDOUT_MAX_BYTES_DISPLAY}
+        try:
+            target_format = request.accepted_renderer.format
+            if target_format in ('html', 'api', 'json'):
+                content_format = request.query_params.get('content_format', 'html')
+                content_encoding = request.query_params.get('content_encoding', None)
+                start_line = request.query_params.get('start_line', 0)
+                end_line = request.query_params.get('end_line', None)
+                dark_val = request.query_params.get('dark', '')
+                dark = bool(dark_val and dark_val[0].lower() in ('1', 't', 'y'))
+                content_only = bool(target_format in ('api', 'json'))
+                dark_bg = (content_only and dark) or (not content_only and (dark or not dark_val))
+                content, start, end, absolute_end = unified_job.result_stdout_raw_limited(start_line, end_line)
+
+                # Remove any ANSI escape sequences containing job event data.
+                content = re.sub(r'\x1b\[K(?:[A-Za-z0-9+/=]+\x1b\[\d+D)+\x1b\[K', '', content)
+
+                body = ansiconv.to_html(cgi.escape(content))
+
+                context = {
+                    'title': get_view_name(self.__class__),
+                    'body': mark_safe(body),
+                    'dark': dark_bg,
+                    'content_only': content_only,
+                }
+                data = render_to_string('api/stdout.html', context).strip()
+
+                if target_format == 'api':
+                    return Response(mark_safe(data))
+                if target_format == 'json':
+                    if content_encoding == 'base64' and content_format == 'ansi':
+                        return Response({'range': {'start': start, 'end': end, 'absolute_end': absolute_end}, 'content': b64encode(content)})
+                    elif content_format == 'html':
+                        return Response({'range': {'start': start, 'end': end, 'absolute_end': absolute_end}, 'content': body})
+                return Response(data)
+            elif target_format == 'txt':
+                return Response(unified_job.result_stdout)
+            elif target_format == 'ansi':
+                return Response(unified_job.result_stdout_raw)
+            elif target_format in {'txt_download', 'ansi_download'}:
+                filename = '{type}_{pk}{suffix}.txt'.format(
+                    type=camelcase_to_underscore(unified_job.__class__.__name__),
+                    pk=unified_job.id,
+                    suffix='.ansi' if target_format == 'ansi_download' else ''
+                )
+                content_fd = unified_job.result_stdout_raw_handle(enforce_max_bytes=False)
+                if target_format == 'txt_download':
+                    content_fd = StdoutANSIFilter(content_fd)
+                response = HttpResponse(FileWrapper(content_fd), content_type='text/plain')
+                response["Content-Disposition"] = 'attachment; filename="{}"'.format(filename)
+                return response
+            else:
+                return super(UnifiedJobStdout, self).retrieve(request, *args, **kwargs)
+        except StdoutMaxBytesExceeded as e:
+            response_message = _(
+                "Standard Output too large to display {text_size} bytes), "
+                "only download supported for sizes over {supported_size} bytes").format(
+                    text_size=e.total, supported_size=e.supported
+            )
             if request.accepted_renderer.format == 'json':
                 return Response({'range': {'start': 0, 'end': 1, 'absolute_end': 1}, 'content': response_message})
             else:
                 return Response(response_message)
 
-        if request.accepted_renderer.format in ('html', 'api', 'json'):
-            content_format = request.query_params.get('content_format', 'html')
-            content_encoding = request.query_params.get('content_encoding', None)
-            start_line = request.query_params.get('start_line', 0)
-            end_line = request.query_params.get('end_line', None)
-            dark_val = request.query_params.get('dark', '')
-            dark = bool(dark_val and dark_val[0].lower() in ('1', 't', 'y'))
-            content_only = bool(request.accepted_renderer.format in ('api', 'json'))
-            dark_bg = (content_only and dark) or (not content_only and (dark or not dark_val))
-            content, start, end, absolute_end = unified_job.result_stdout_raw_limited(start_line, end_line)
-
-            # Remove any ANSI escape sequences containing job event data.
-            content = re.sub(r'\x1b\[K(?:[A-Za-z0-9+/=]+\x1b\[\d+D)+\x1b\[K', '', content)
-
-            body = ansiconv.to_html(cgi.escape(content))
-
-            context = {
-                'title': get_view_name(self.__class__),
-                'body': mark_safe(body),
-                'dark': dark_bg,
-                'content_only': content_only,
-            }
-            data = render_to_string('api/stdout.html', context).strip()
-
-            if request.accepted_renderer.format == 'api':
-                return Response(mark_safe(data))
-            if request.accepted_renderer.format == 'json':
-                if content_encoding == 'base64' and content_format == 'ansi':
-                    return Response({'range': {'start': start, 'end': end, 'absolute_end': absolute_end}, 'content': b64encode(content)})
-                elif content_format == 'html':
-                    return Response({'range': {'start': start, 'end': end, 'absolute_end': absolute_end}, 'content': body})
-            return Response(data)
-        elif request.accepted_renderer.format == 'txt':
-            return Response(unified_job.result_stdout)
-        elif request.accepted_renderer.format == 'ansi':
-            return Response(unified_job.result_stdout_raw)
-        elif request.accepted_renderer.format in {'txt_download', 'ansi_download'}:
-            if not os.path.exists(unified_job.result_stdout_file):
-                write_fd = open(unified_job.result_stdout_file, 'w')
-                with connection.cursor() as cursor:
-                    try:
-                        tablename, related_name = {
-                            Job: ('main_jobevent', 'job_id'),
-                            AdHocCommand: ('main_adhoccommandevent', 'ad_hoc_command_id'),
-                        }.get(unified_job.__class__, ('main_genericcommandevent', 'unified_job_id'))
-                        cursor.copy_expert(
-                            "copy (select stdout from {} where {}={} order by start_line) to stdout".format(
-                                tablename,
-                                related_name,
-                                unified_job.id
-                            ),
-                            write_fd
-                        )
-                        write_fd.close()
-                        subprocess.Popen("sed -i 's/\\\\r\\\\n/\\n/g' {}".format(unified_job.result_stdout_file),
-                                         shell=True).wait()
-                    except Exception as e:
-                        return Response({"error": _("Error generating stdout download file: {}".format(e))})
-            try:
-                content_fd = open(unified_job.result_stdout_file, 'r')
-                if request.accepted_renderer.format == 'txt_download':
-                    # For txt downloads, filter out ANSI escape sequences.
-                    content_fd = StdoutANSIFilter(content_fd)
-                    suffix = ''
-                else:
-                    suffix = '_ansi'
-                response = HttpResponse(FileWrapper(content_fd), content_type='text/plain')
-                response["Content-Disposition"] = 'attachment; filename="job_%s%s.txt"' % (str(unified_job.id), suffix)
-                return response
-            except Exception as e:
-                return Response({"error": _("Error generating stdout download file: %s") % str(e)}, status=status.HTTP_400_BAD_REQUEST)
-        else:
-            return super(UnifiedJobStdout, self).retrieve(request, *args, **kwargs)
-
 
 class ProjectUpdateStdout(UnifiedJobStdout):