diff --git a/awx/main/expect/isolated_manager.py b/awx/main/expect/isolated_manager.py index c4bd95efa7..8ed03e9e44 100644 --- a/awx/main/expect/isolated_manager.py +++ b/awx/main/expect/isolated_manager.py @@ -1,5 +1,4 @@ import base64 -import cStringIO import codecs import StringIO import json @@ -143,7 +142,7 @@ class IsolatedManager(object): # if an ssh private key fifo exists, read its contents and delete it if self.ssh_key_path: - buff = cStringIO.StringIO() + buff = StringIO.StringIO() with open(self.ssh_key_path, 'r') as fifo: for line in fifo: buff.write(line) @@ -183,7 +182,7 @@ class IsolatedManager(object): job_timeout=settings.AWX_ISOLATED_LAUNCH_TIMEOUT, pexpect_timeout=5 ) - output = buff.getvalue() + output = buff.getvalue().encode('utf-8') playbook_logger.info('Isolated job {} dispatch:\n{}'.format(self.instance.id, output)) if status != 'successful': self.stdout_handle.write(output) @@ -283,7 +282,7 @@ class IsolatedManager(object): status = 'failed' output = '' rc = None - buff = cStringIO.StringIO() + buff = StringIO.StringIO() last_check = time.time() seek = 0 job_timeout = remaining = self.job_timeout @@ -304,7 +303,7 @@ class IsolatedManager(object): time.sleep(1) continue - buff = cStringIO.StringIO() + buff = StringIO.StringIO() logger.debug('Checking on isolated job {} with `check_isolated.yml`.'.format(self.instance.id)) status, rc = IsolatedManager.run_pexpect( args, self.awx_playbook_path(), self.management_env, buff, @@ -314,7 +313,7 @@ class IsolatedManager(object): pexpect_timeout=5, proot_cmd=self.proot_cmd ) - output = buff.getvalue() + output = buff.getvalue().encode('utf-8') playbook_logger.info('Isolated job {} check:\n{}'.format(self.instance.id, output)) path = self.path_to('artifacts', 'stdout') @@ -356,14 +355,14 @@ class IsolatedManager(object): } args = self._build_args('clean_isolated.yml', '%s,' % self.host, extra_vars) logger.debug('Cleaning up job {} on isolated host with `clean_isolated.yml` playbook.'.format(self.instance.id)) - buff = cStringIO.StringIO() + buff = StringIO.StringIO() timeout = max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT) status, rc = IsolatedManager.run_pexpect( args, self.awx_playbook_path(), self.management_env, buff, idle_timeout=timeout, job_timeout=timeout, pexpect_timeout=5 ) - output = buff.getvalue() + output = buff.getvalue().encode('utf-8') playbook_logger.info('Isolated job {} cleanup:\n{}'.format(self.instance.id, output)) if status != 'successful': @@ -406,14 +405,14 @@ class IsolatedManager(object): env = cls._base_management_env() env['ANSIBLE_STDOUT_CALLBACK'] = 'json' - buff = cStringIO.StringIO() + buff = StringIO.StringIO() timeout = max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT) status, rc = IsolatedManager.run_pexpect( args, cls.awx_playbook_path(), env, buff, idle_timeout=timeout, job_timeout=timeout, pexpect_timeout=5 ) - output = buff.getvalue() + output = buff.getvalue().encode('utf-8') buff.close() try: diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index a7625f5529..dd9deff77f 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -2,7 +2,7 @@ # All Rights Reserved. # Python -from cStringIO import StringIO +from StringIO import StringIO import json import logging import os @@ -1013,7 +1013,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique return content def _result_stdout_raw(self, redact_sensitive=False, escape_ascii=False): - content = self.result_stdout_raw_handle().read() + content = self.result_stdout_raw_handle().read().decode('utf-8') if redact_sensitive: content = UriCleaner.remove_sensitive(content) if escape_ascii: @@ -1029,13 +1029,13 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique return self._result_stdout_raw(escape_ascii=True) def _result_stdout_raw_limited(self, start_line=0, end_line=None, redact_sensitive=True, escape_ascii=False): - return_buffer = u"" + return_buffer = StringIO() if end_line is not None: end_line = int(end_line) stdout_lines = self.result_stdout_raw_handle().readlines() absolute_end = len(stdout_lines) for line in stdout_lines[int(start_line):end_line]: - return_buffer += line + return_buffer.write(line) if int(start_line) < 0: start_actual = len(stdout_lines) + int(start_line) end_actual = len(stdout_lines) @@ -1046,6 +1046,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique else: end_actual = len(stdout_lines) + return_buffer = return_buffer.getvalue().decode('utf-8') if redact_sensitive: return_buffer = UriCleaner.remove_sensitive(return_buffer) if escape_ascii: diff --git a/awx/main/tests/functional/api/test_unified_jobs_stdout.py b/awx/main/tests/functional/api/test_unified_jobs_stdout.py index f3bb8bbbaf..c4a1897b87 100644 --- a/awx/main/tests/functional/api/test_unified_jobs_stdout.py +++ b/awx/main/tests/functional/api/test_unified_jobs_stdout.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import re import shutil import tempfile @@ -40,7 +42,7 @@ def sqlite_copy_expert(request): InventoryUpdateEvent, SystemJobEvent): if cls._meta.db_table == tablename: for event in cls.objects.order_by('start_line').all(): - fd.write(event.stdout) + fd.write(event.stdout.encode('utf-8')) setattr(SQLiteCursorWrapper, 'copy_expert', write_stdout) request.addfinalizer(lambda: shutil.rmtree(path)) @@ -229,3 +231,23 @@ def test_legacy_result_stdout_with_max_bytes(Cls, view, fmt, get, admin): response = get(url + '?format={}'.format(fmt + '_download'), user=admin, expect=200) assert response.content == large_stdout + + +@pytest.mark.django_db +@pytest.mark.parametrize('Parent, Child, relation, view', [ + [Job, JobEvent, 'job', 'api:job_stdout'], + [AdHocCommand, AdHocCommandEvent, 'ad_hoc_command', 'api:ad_hoc_command_stdout'], + [_mk_project_update, ProjectUpdateEvent, 'project_update', 'api:project_update_stdout'], + [_mk_inventory_update, InventoryUpdateEvent, 'inventory_update', 'api:inventory_update_stdout'], +]) +@pytest.mark.parametrize('fmt', ['txt', 'ansi', 'txt_download', 'ansi_download']) +def test_text_with_unicode_stdout(sqlite_copy_expert, Parent, Child, relation, + view, get, admin, fmt): + job = Parent() + job.save() + for i in range(3): + Child(**{relation: job, 'stdout': u'オ{}\n'.format(i), 'start_line': i}).save() + url = reverse(view, kwargs={'pk': job.pk}) + '?format=' + fmt + + response = get(url, user=admin, expect=200) + assert response.content.splitlines() == ['オ%d' % i for i in range(3)]