Merge pull request #946 from ryanpetrello/fix-7846

fix a handful of issues for playbooks that contain unicode

awx/main/expect/isolated_manager.py

@@ -1,5 +1,4 @@
 import base64
-import cStringIO
 import codecs
 import StringIO
 import json
@@ -143,7 +142,7 @@ class IsolatedManager(object):
 
         # if an ssh private key fifo exists, read its contents and delete it
         if self.ssh_key_path:
-            buff = cStringIO.StringIO()
+            buff = StringIO.StringIO()
             with open(self.ssh_key_path, 'r') as fifo:
                 for line in fifo:
                     buff.write(line)
@@ -183,7 +182,7 @@ class IsolatedManager(object):
             job_timeout=settings.AWX_ISOLATED_LAUNCH_TIMEOUT,
             pexpect_timeout=5
         )
-        output = buff.getvalue()
+        output = buff.getvalue().encode('utf-8')
         playbook_logger.info('Isolated job {} dispatch:\n{}'.format(self.instance.id, output))
         if status != 'successful':
             self.stdout_handle.write(output)
@@ -283,7 +282,7 @@ class IsolatedManager(object):
         status = 'failed'
         output = ''
         rc = None
-        buff = cStringIO.StringIO()
+        buff = StringIO.StringIO()
         last_check = time.time()
         seek = 0
         job_timeout = remaining = self.job_timeout
@@ -304,7 +303,7 @@ class IsolatedManager(object):
                 time.sleep(1)
                 continue
 
-            buff = cStringIO.StringIO()
+            buff = StringIO.StringIO()
             logger.debug('Checking on isolated job {} with `check_isolated.yml`.'.format(self.instance.id))
             status, rc = IsolatedManager.run_pexpect(
                 args, self.awx_playbook_path(), self.management_env, buff,
@@ -314,7 +313,7 @@ class IsolatedManager(object):
                 pexpect_timeout=5,
                 proot_cmd=self.proot_cmd
             )
-            output = buff.getvalue()
+            output = buff.getvalue().encode('utf-8')
             playbook_logger.info('Isolated job {} check:\n{}'.format(self.instance.id, output))
 
             path = self.path_to('artifacts', 'stdout')
@@ -356,14 +355,14 @@ class IsolatedManager(object):
         }
         args = self._build_args('clean_isolated.yml', '%s,' % self.host, extra_vars)
         logger.debug('Cleaning up job {} on isolated host with `clean_isolated.yml` playbook.'.format(self.instance.id))
-        buff = cStringIO.StringIO()
+        buff = StringIO.StringIO()
         timeout = max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT)
         status, rc = IsolatedManager.run_pexpect(
             args, self.awx_playbook_path(), self.management_env, buff,
             idle_timeout=timeout, job_timeout=timeout,
             pexpect_timeout=5
         )
-        output = buff.getvalue()
+        output = buff.getvalue().encode('utf-8')
         playbook_logger.info('Isolated job {} cleanup:\n{}'.format(self.instance.id, output))
 
         if status != 'successful':
@@ -406,14 +405,14 @@ class IsolatedManager(object):
         env = cls._base_management_env()
         env['ANSIBLE_STDOUT_CALLBACK'] = 'json'
 
-        buff = cStringIO.StringIO()
+        buff = StringIO.StringIO()
         timeout = max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT)
         status, rc = IsolatedManager.run_pexpect(
             args, cls.awx_playbook_path(), env, buff,
             idle_timeout=timeout, job_timeout=timeout,
             pexpect_timeout=5
         )
-        output = buff.getvalue()
+        output = buff.getvalue().encode('utf-8')
         buff.close()
 
         try:

awx/main/models/unified_jobs.py

@@ -2,7 +2,7 @@
 # All Rights Reserved.
 
 # Python
-from cStringIO import StringIO
+from StringIO import StringIO
 import json
 import logging
 import os
@@ -1013,7 +1013,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
         return content
 
     def _result_stdout_raw(self, redact_sensitive=False, escape_ascii=False):
-        content = self.result_stdout_raw_handle().read()
+        content = self.result_stdout_raw_handle().read().decode('utf-8')
         if redact_sensitive:
             content = UriCleaner.remove_sensitive(content)
         if escape_ascii:
@@ -1029,13 +1029,13 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
         return self._result_stdout_raw(escape_ascii=True)
 
     def _result_stdout_raw_limited(self, start_line=0, end_line=None, redact_sensitive=True, escape_ascii=False):
-        return_buffer = u""
+        return_buffer = StringIO()
         if end_line is not None:
             end_line = int(end_line)
         stdout_lines = self.result_stdout_raw_handle().readlines()
         absolute_end = len(stdout_lines)
         for line in stdout_lines[int(start_line):end_line]:
-            return_buffer += line
+            return_buffer.write(line)
         if int(start_line) < 0:
             start_actual = len(stdout_lines) + int(start_line)
             end_actual = len(stdout_lines)
@@ -1046,6 +1046,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
             else:
                 end_actual = len(stdout_lines)
 
+        return_buffer = return_buffer.getvalue().decode('utf-8')
         if redact_sensitive:
             return_buffer = UriCleaner.remove_sensitive(return_buffer)
         if escape_ascii:

awx/main/tests/functional/api/test_unified_jobs_stdout.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import re
 import shutil
 import tempfile
@@ -40,7 +42,7 @@ def sqlite_copy_expert(request):
                     InventoryUpdateEvent, SystemJobEvent):
             if cls._meta.db_table == tablename:
                 for event in cls.objects.order_by('start_line').all():
-                    fd.write(event.stdout)
+                    fd.write(event.stdout.encode('utf-8'))
 
     setattr(SQLiteCursorWrapper, 'copy_expert', write_stdout)
     request.addfinalizer(lambda: shutil.rmtree(path))
@@ -229,3 +231,23 @@ def test_legacy_result_stdout_with_max_bytes(Cls, view, fmt, get, admin):
 
     response = get(url + '?format={}'.format(fmt + '_download'), user=admin, expect=200)
     assert response.content == large_stdout
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize('Parent, Child, relation, view', [
+    [Job, JobEvent, 'job', 'api:job_stdout'],
+    [AdHocCommand, AdHocCommandEvent, 'ad_hoc_command', 'api:ad_hoc_command_stdout'],
+    [_mk_project_update, ProjectUpdateEvent, 'project_update', 'api:project_update_stdout'],
+    [_mk_inventory_update, InventoryUpdateEvent, 'inventory_update', 'api:inventory_update_stdout'],
+])
+@pytest.mark.parametrize('fmt', ['txt', 'ansi', 'txt_download', 'ansi_download'])
+def test_text_with_unicode_stdout(sqlite_copy_expert, Parent, Child, relation,
+                                  view, get, admin, fmt):
+    job = Parent()
+    job.save()
+    for i in range(3):
+        Child(**{relation: job, 'stdout': u'オ{}\n'.format(i), 'start_line': i}).save()
+    url = reverse(view, kwargs={'pk': job.pk}) + '?format=' + fmt
+
+    response = get(url, user=admin, expect=200)
+    assert response.content.splitlines() == ['オ%d' % i for i in range(3)]