properly compose stdout downloads that contain unicode

awx/main/models/unified_jobs.py

@@ -2,7 +2,7 @@
 # All Rights Reserved.
 
 # Python
-from cStringIO import StringIO
+from StringIO import StringIO
 import json
 import logging
 import os
@@ -1013,7 +1013,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
         return content
 
     def _result_stdout_raw(self, redact_sensitive=False, escape_ascii=False):
-        content = self.result_stdout_raw_handle().read()
+        content = self.result_stdout_raw_handle().read().decode('utf-8')
         if redact_sensitive:
             content = UriCleaner.remove_sensitive(content)
         if escape_ascii:
@@ -1029,13 +1029,13 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
         return self._result_stdout_raw(escape_ascii=True)
 
     def _result_stdout_raw_limited(self, start_line=0, end_line=None, redact_sensitive=True, escape_ascii=False):
-        return_buffer = u""
+        return_buffer = StringIO()
         if end_line is not None:
             end_line = int(end_line)
         stdout_lines = self.result_stdout_raw_handle().readlines()
         absolute_end = len(stdout_lines)
         for line in stdout_lines[int(start_line):end_line]:
-            return_buffer += line
+            return_buffer.write(line)
         if int(start_line) < 0:
             start_actual = len(stdout_lines) + int(start_line)
             end_actual = len(stdout_lines)
@@ -1046,6 +1046,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
             else:
                 end_actual = len(stdout_lines)
 
+        return_buffer = return_buffer.getvalue().decode('utf-8')
         if redact_sensitive:
             return_buffer = UriCleaner.remove_sensitive(return_buffer)
         if escape_ascii:

awx/main/tests/functional/api/test_unified_jobs_stdout.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import re
 import shutil
 import tempfile
@@ -40,7 +42,7 @@ def sqlite_copy_expert(request):
                     InventoryUpdateEvent, SystemJobEvent):
             if cls._meta.db_table == tablename:
                 for event in cls.objects.order_by('start_line').all():
-                    fd.write(event.stdout)
+                    fd.write(event.stdout.encode('utf-8'))
 
     setattr(SQLiteCursorWrapper, 'copy_expert', write_stdout)
     request.addfinalizer(lambda: shutil.rmtree(path))
@@ -229,3 +231,23 @@ def test_legacy_result_stdout_with_max_bytes(Cls, view, fmt, get, admin):
 
     response = get(url + '?format={}'.format(fmt + '_download'), user=admin, expect=200)
     assert response.content == large_stdout
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize('Parent, Child, relation, view', [
+    [Job, JobEvent, 'job', 'api:job_stdout'],
+    [AdHocCommand, AdHocCommandEvent, 'ad_hoc_command', 'api:ad_hoc_command_stdout'],
+    [_mk_project_update, ProjectUpdateEvent, 'project_update', 'api:project_update_stdout'],
+    [_mk_inventory_update, InventoryUpdateEvent, 'inventory_update', 'api:inventory_update_stdout'],
+])
+@pytest.mark.parametrize('fmt', ['txt', 'ansi', 'txt_download', 'ansi_download'])
+def test_text_with_unicode_stdout(sqlite_copy_expert, Parent, Child, relation,
+                                  view, get, admin, fmt):
+    job = Parent()
+    job.save()
+    for i in range(3):
+        Child(**{relation: job, 'stdout': u'オ{}\n'.format(i), 'start_line': i}).save()
+    url = reverse(view, kwargs={'pk': job.pk}) + '?format=' + fmt
+
+    response = get(url, user=admin, expect=200)
+    assert response.content.splitlines() == ['オ%d' % i for i in range(3)]