fix a few UTF-8 bugs on Ubuntu related to stdout text downloads

This commit is contained in:
Ryan Petrello
2019-01-14 17:14:36 -05:00
parent 39d119534c
commit 2016798e0f

View File

@@ -3,6 +3,7 @@
# Python # Python
from io import StringIO from io import StringIO
import codecs
import json import json
import logging import logging
import os import os
@@ -995,7 +996,8 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
mode='w', mode='w',
prefix='{}-{}-'.format(self.model_to_str(), self.pk), prefix='{}-{}-'.format(self.model_to_str(), self.pk),
suffix='.out', suffix='.out',
dir=settings.JOBOUTPUT_ROOT dir=settings.JOBOUTPUT_ROOT,
encoding='utf-8'
) )
# Before the addition of event-based stdout, older versions of # Before the addition of event-based stdout, older versions of
@@ -1010,7 +1012,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
fd.write(legacy_stdout_text) fd.write(legacy_stdout_text)
if hasattr(fd, 'name'): if hasattr(fd, 'name'):
fd.flush() fd.flush()
return open(fd.name, 'r') return codecs.open(fd.name, 'r', encoding='utf-8')
else: else:
# we just wrote to this StringIO, so rewind it # we just wrote to this StringIO, so rewind it
fd.seek(0) fd.seek(0)
@@ -1056,7 +1058,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
# up escaped line sequences # up escaped line sequences
fd.flush() fd.flush()
subprocess.Popen("sed -i 's/\\\\r\\\\n/\\n/g' {}".format(fd.name), shell=True).wait() subprocess.Popen("sed -i 's/\\\\r\\\\n/\\n/g' {}".format(fd.name), shell=True).wait()
return open(fd.name, 'r') return codecs.open(fd.name, 'r', encoding='utf-8')
else: else:
# If we're dealing with an in-memory string buffer, use # If we're dealing with an in-memory string buffer, use
# string.replace() # string.replace()