From a803cedd7ce3d5534ae9be524e7b098678f17dab Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 10 Oct 2019 16:07:08 -0400 Subject: [PATCH 1/4] Break out a new reusable truncate_stdout utility function --- awx/api/serializers.py | 42 ++++++++++++-------------------------- awx/main/utils/common.py | 44 +++++++++++++++++++++++++++++----------- 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 14d8944dc1..a5bdad1c39 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -45,7 +45,6 @@ from polymorphic.models import PolymorphicModel from awx.main.access import get_user_capabilities from awx.main.constants import ( SCHEDULEABLE_PROVIDERS, - ANSI_SGR_PATTERN, ACTIVE_STATES, CENSOR_VALUE, ) @@ -70,7 +69,8 @@ from awx.main.utils import ( get_type_for_model, get_model_for_type, camelcase_to_underscore, getattrd, parse_yaml_or_json, has_model_field_prefetched, extract_ansible_vars, encrypt_dict, - prefetch_page_capabilities, get_external_account) + prefetch_page_capabilities, get_external_account, truncate_stdout, +) from awx.main.utils.filters import SmartFilter from awx.main.redact import UriCleaner, REPLACE_STR @@ -3851,25 +3851,17 @@ class JobEventSerializer(BaseSerializer): return d def to_representation(self, obj): - ret = super(JobEventSerializer, self).to_representation(obj) + data = super(JobEventSerializer, self).to_representation(obj) # Show full stdout for event detail view, truncate only for list view. if hasattr(self.context.get('view', None), 'retrieve'): - return ret + return data # Show full stdout for playbook_on_* events. if obj and obj.event.startswith('playbook_on'): - return ret + return data max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY - if max_bytes > 0 and 'stdout' in ret and len(ret['stdout']) >= max_bytes: - ret['stdout'] = ret['stdout'][:(max_bytes - 1)] + u'\u2026' - set_count = 0 - reset_count = 0 - for m in ANSI_SGR_PATTERN.finditer(ret['stdout']): - if m.string[m.start():m.end()] == u'\u001b[0m': - reset_count += 1 - else: - set_count += 1 - ret['stdout'] += u'\u001b[0m' * (set_count - reset_count) - return ret + if 'stdout' in data: + data['stdout'] = truncate_stdout(data['stdout'], max_bytes) + return data class JobEventWebSocketSerializer(JobEventSerializer): @@ -3964,22 +3956,14 @@ class AdHocCommandEventSerializer(BaseSerializer): return res def to_representation(self, obj): - ret = super(AdHocCommandEventSerializer, self).to_representation(obj) + data = super(AdHocCommandEventSerializer, self).to_representation(obj) # Show full stdout for event detail view, truncate only for list view. if hasattr(self.context.get('view', None), 'retrieve'): - return ret + return data max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY - if max_bytes > 0 and 'stdout' in ret and len(ret['stdout']) >= max_bytes: - ret['stdout'] = ret['stdout'][:(max_bytes - 1)] + u'\u2026' - set_count = 0 - reset_count = 0 - for m in ANSI_SGR_PATTERN.finditer(ret['stdout']): - if m.string[m.start():m.end()] == u'\u001b[0m': - reset_count += 1 - else: - set_count += 1 - ret['stdout'] += u'\u001b[0m' * (set_count - reset_count) - return ret + if 'stdout' in data: + data['stdout'] = truncate_stdout(data['stdout'], max_bytes) + return data class AdHocCommandEventWebSocketSerializer(AdHocCommandEventSerializer): diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index d36dfa272b..11e4722f5f 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -38,18 +38,22 @@ from django.apps import apps logger = logging.getLogger('awx.main.utils') -__all__ = ['get_object_or_400', 'camelcase_to_underscore', 'underscore_to_camelcase', 'memoize', 'memoize_delete', - 'get_ansible_version', 'get_ssh_version', 'get_licenser', 'get_awx_version', 'update_scm_url', - 'get_type_for_model', 'get_model_for_type', 'copy_model_by_class', 'region_sorting', - 'copy_m2m_relationships', 'prefetch_page_capabilities', 'to_python_boolean', - 'ignore_inventory_computed_fields', 'ignore_inventory_group_removal', - '_inventory_updates', 'get_pk_from_dict', 'getattrd', 'getattr_dne', 'NoDefaultProvided', - 'get_current_apps', 'set_current_apps', - 'extract_ansible_vars', 'get_search_fields', 'get_system_task_capacity', 'get_cpu_capacity', 'get_mem_capacity', - 'wrap_args_with_proot', 'build_proot_temp_dir', 'check_proot_installed', 'model_to_dict', - 'NullablePromptPseudoField', 'model_instance_diff', 'parse_yaml_or_json', 'RequireDebugTrueOrTest', - 'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError', 'get_custom_venv_choices', 'get_external_account', - 'task_manager_bulk_reschedule', 'schedule_task_manager', 'classproperty', 'create_temporary_fifo'] +__all__ = [ + 'get_object_or_400', 'camelcase_to_underscore', 'underscore_to_camelcase', 'memoize', + 'memoize_delete', 'get_ansible_version', 'get_ssh_version', 'get_licenser', + 'get_awx_version', 'update_scm_url', 'get_type_for_model', 'get_model_for_type', + 'copy_model_by_class', 'region_sorting', 'copy_m2m_relationships', + 'prefetch_page_capabilities', 'to_python_boolean', 'ignore_inventory_computed_fields', + 'ignore_inventory_group_removal', '_inventory_updates', 'get_pk_from_dict', 'getattrd', + 'getattr_dne', 'NoDefaultProvided', 'get_current_apps', 'set_current_apps', + 'extract_ansible_vars', 'get_search_fields', 'get_system_task_capacity', + 'get_cpu_capacity', 'get_mem_capacity', 'wrap_args_with_proot', 'build_proot_temp_dir', + 'check_proot_installed', 'model_to_dict', 'NullablePromptPseudoField', + 'model_instance_diff', 'parse_yaml_or_json', 'RequireDebugTrueOrTest', + 'has_model_field_prefetched', 'set_environ', 'IllegalArgumentError', + 'get_custom_venv_choices', 'get_external_account', 'task_manager_bulk_reschedule', + 'schedule_task_manager', 'classproperty', 'create_temporary_fifo', 'truncate_stdout', +] def get_object_or_400(klass, *args, **kwargs): @@ -1088,3 +1092,19 @@ def create_temporary_fifo(data): ).start() return path + +def truncate_stdout(stdout, size): + from awx.main.constants import ANSI_SGR_PATTERN + + if size <= 0 or len(stdout) <= size: + return stdout + + stdout = stdout[:(size - 1)] + u'\u2026' + set_count, reset_count = 0, 0 + for m in ANSI_SGR_PATTERN.finditer(stdout): + if m.group() == u'\u001b[0m': + reset_count += 1 + else: + set_count += 1 + + return stdout + u'\u001b[0m' * (set_count - reset_count) From 9efa7b84dfbb0e4654f440f6b8cdc48b79789343 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 10 Oct 2019 16:08:17 -0400 Subject: [PATCH 2/4] Depend on a serializer context variable `no_truncate` to decide whether to turn off the ANSI control sequence-aware truncation, instead of needing inappropriate awareness of the details of the view that invoked the serializer. This will also allow us to have views that can more flexibly turn off the truncation under other circumstances. --- awx/api/serializers.py | 10 +++++----- awx/api/views/__init__.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index a5bdad1c39..71525ccf0c 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -3852,12 +3852,12 @@ class JobEventSerializer(BaseSerializer): def to_representation(self, obj): data = super(JobEventSerializer, self).to_representation(obj) - # Show full stdout for event detail view, truncate only for list view. - if hasattr(self.context.get('view', None), 'retrieve'): - return data # Show full stdout for playbook_on_* events. if obj and obj.event.startswith('playbook_on'): return data + # If the view logic says to not trunctate (request was to the detail view or a param was used) + if self.context.get('no_truncate', False): + return data max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY if 'stdout' in data: data['stdout'] = truncate_stdout(data['stdout'], max_bytes) @@ -3957,8 +3957,8 @@ class AdHocCommandEventSerializer(BaseSerializer): def to_representation(self, obj): data = super(AdHocCommandEventSerializer, self).to_representation(obj) - # Show full stdout for event detail view, truncate only for list view. - if hasattr(self.context.get('view', None), 'retrieve'): + # If the view logic says to not trunctate (request was to the detail view or a param was used) + if self.context.get('no_truncate', False): return data max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY if 'stdout' in data: diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 383c7aeee9..646ff2c746 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -3774,6 +3774,11 @@ class JobEventDetail(RetrieveAPIView): model = models.JobEvent serializer_class = serializers.JobEventSerializer + def get_serializer_context(self): + context = super().get_serializer_context() + context.update(no_truncate=True) + return context + class JobEventChildrenList(SubListAPIView): @@ -4008,6 +4013,11 @@ class AdHocCommandEventDetail(RetrieveAPIView): model = models.AdHocCommandEvent serializer_class = serializers.AdHocCommandEventSerializer + def get_serializer_context(self): + context = super().get_serializer_context() + context.update(no_truncate=True) + return context + class BaseAdHocCommandEventsList(SubListAPIView): From e672e68a02d03090db7868fd7ca60613950f9b7e Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 10 Oct 2019 16:21:53 -0400 Subject: [PATCH 3/4] Allow the job event list views to take a no_truncate GET param --- awx/api/filters.py | 2 +- awx/api/views/__init__.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/awx/api/filters.py b/awx/api/filters.py index 327303dd2e..ea9d011562 100644 --- a/awx/api/filters.py +++ b/awx/api/filters.py @@ -126,7 +126,7 @@ class FieldLookupBackend(BaseFilterBackend): ''' RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by', - 'search', 'type', 'host_filter', 'count_disabled',) + 'search', 'type', 'host_filter', 'count_disabled', 'no_truncate') SUPPORTED_LOOKUPS = ('exact', 'iexact', 'contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 646ff2c746..f337345df9 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -3768,6 +3768,12 @@ class JobEventList(ListAPIView): serializer_class = serializers.JobEventSerializer search_fields = ('stdout',) + def get_serializer_context(self): + context = super().get_serializer_context() + if self.request.query_params.get('no_truncate'): + context.update(no_truncate=True) + return context + class JobEventDetail(RetrieveAPIView): @@ -4007,6 +4013,12 @@ class AdHocCommandEventList(ListAPIView): serializer_class = serializers.AdHocCommandEventSerializer search_fields = ('stdout',) + def get_serializer_context(self): + context = super().get_serializer_context() + if self.request.query_params.get('no_truncate'): + context.update(no_truncate=True) + return context + class AdHocCommandEventDetail(RetrieveAPIView): From cf89108edffeea2b2e594150159193eb5eb0be88 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 10 Oct 2019 16:57:39 -0400 Subject: [PATCH 4/4] Force the CLI to use no_truncate for the monitor calls --- awxkit/awxkit/cli/stdout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awxkit/awxkit/cli/stdout.py b/awxkit/awxkit/cli/stdout.py index 47ca7f79f6..1cf18168e1 100644 --- a/awxkit/awxkit/cli/stdout.py +++ b/awxkit/awxkit/cli/stdout.py @@ -73,7 +73,7 @@ def monitor_workflow(response, session, print_stdout=True, timeout=None, def monitor(response, session, print_stdout=True, timeout=None, interval=.25): get = response.url.get - payload = {'order_by': 'start_line'} + payload = {'order_by': 'start_line', 'no_truncate': True} if response.type == 'job': events = response.related.job_events.get else: