From 86599cf1e302c2451481ec4f427fa7aa6e8d71cb Mon Sep 17 00:00:00 2001 From: Chris Church Date: Tue, 1 Apr 2014 20:15:36 -0400 Subject: [PATCH] Add API resource for unified job stdout with HTML output of ANSI color codes. --- awx/api/renderers.py | 15 + awx/api/serializers.py | 25 +- awx/api/templates/api/unified_job_stdout.md | 28 ++ awx/api/urls.py | 3 + awx/api/views.py | 54 ++ awx/lib/site-packages/README | 1 + awx/lib/site-packages/ansi2html/__init__.py | 2 + awx/lib/site-packages/ansi2html/converter.py | 492 +++++++++++++++++++ awx/lib/site-packages/ansi2html/style.py | 113 +++++ awx/lib/site-packages/ansi2html/util.py | 2 + awx/main/models/unified_jobs.py | 17 +- awx/main/tasks.py | 1 - awx/templates/rest_framework/base.html | 8 +- requirements/ansi2html-1.0.6.tar.gz | Bin 0 -> 51701 bytes requirements/dev.txt | 1 + requirements/dev_local.txt | 1 + requirements/prod.txt | 1 + requirements/prod_local.txt | 1 + 18 files changed, 747 insertions(+), 18 deletions(-) create mode 100644 awx/api/templates/api/unified_job_stdout.md create mode 100644 awx/lib/site-packages/ansi2html/__init__.py create mode 100644 awx/lib/site-packages/ansi2html/converter.py create mode 100644 awx/lib/site-packages/ansi2html/style.py create mode 100644 awx/lib/site-packages/ansi2html/util.py create mode 100644 requirements/ansi2html-1.0.6.tar.gz diff --git a/awx/api/renderers.py b/awx/api/renderers.py index 0f106dd25f..125c3f6cfc 100644 --- a/awx/api/renderers.py +++ b/awx/api/renderers.py @@ -16,3 +16,18 @@ class BrowsableAPIRenderer(renderers.BrowsableAPIRenderer): return if method in ('DELETE', 'OPTIONS'): return True # Don't actually need to return a form + +class PlainTextRenderer(renderers.BaseRenderer): + + media_type = 'text/plain' + format = 'txt' + + def render(self, data, media_type=None, renderer_context=None): + if not isinstance(data, basestring): + data = unicode(data) + return data.encode(self.charset) + +class AnsiTextRenderer(PlainTextRenderer): + + media_type = 'text/plain' + format = 'ansi' diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 18ede5caad..0d2cd22829 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -393,6 +393,12 @@ class UnifiedJobSerializer(BaseSerializer): res['unified_job_template'] = obj.unified_job_template.get_absolute_url() if obj.schedule and obj.schedule.active: res['schedule'] = obj.schedule.get_absolute_url() + if isinstance(obj, ProjectUpdate): + res['stdout'] = reverse('api:project_update_stdout', args=(obj.pk,)) + elif isinstance(obj, InventoryUpdate): + res['stdout'] = reverse('api:inventory_update_stdout', args=(obj.pk,)) + elif isinstance(obj, Job): + res['stdout'] = reverse('api:job_stdout', args=(obj.pk,)) return res def to_native(self, obj): @@ -445,15 +451,20 @@ class UnifiedJobListSerializer(UnifiedJobSerializer): return ret -class BaseTaskSerializer(BaseSerializer): +class UnifiedJobStdoutSerializer(UnifiedJobSerializer): - job_env = serializers.SerializerMethodField('get_job_env') + class Meta: + fields = ('result_stdout',) - def get_job_env(self, obj): - job_env_d = obj.job_env - if 'BROKER_URL' in job_env_d: - job_env_d.pop('BROKER_URL') - return job_env_d + def get_types(self): + if type(self) is UnifiedJobSerializer: + return ['project_update', 'inventory_update', 'job'] + else: + return super(UnifiedJobSerializer, self).get_types() + + def to_native(self, obj): + ret = super(UnifiedJobStdoutSerializer, self).to_native(obj) + return ret.get('result_stdout', '') class UserSerializer(BaseSerializer): diff --git a/awx/api/templates/api/unified_job_stdout.md b/awx/api/templates/api/unified_job_stdout.md new file mode 100644 index 0000000000..13f827fc7d --- /dev/null +++ b/awx/api/templates/api/unified_job_stdout.md @@ -0,0 +1,28 @@ +# Retrieve {{ model_verbose_name|title }} Stdout: + +Make GET request to this resource to retrieve the stdout from running this +{{ model_verbose_name }}. + +## Format + +Use the `format` query string parameter to specify the output format. + +* Browsable API: `?format=api` +* HTML: `?format=html` +* Plain Text: `?format=txt` +* Plain Text with ANSI color codes: `?format=ansi` + +When using the HTML or API formats, use the `scheme` query string parameter to +change the output colors. The value must be one of the following (default is +`ansi2html`): + +* `ansi2html` +* `osx` +* `xterm` +* `xterm-bright` +* `solarized` + +Use `dark=1` or `dark=0` as a query string parameter to force or disable a +dark background. + +{% include "api/_new_in_awx.md" %} diff --git a/awx/api/urls.py b/awx/api/urls.py index 6aa7e9d3cb..dd83eb98d8 100644 --- a/awx/api/urls.py +++ b/awx/api/urls.py @@ -47,6 +47,7 @@ project_urls = patterns('awx.api.views', project_update_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/$', 'project_update_detail'), url(r'^(?P[0-9]+)/cancel/$', 'project_update_cancel'), + url(r'^(?P[0-9]+)/stdout/$', 'project_update_stdout'), ) team_urls = patterns('awx.api.views', @@ -112,6 +113,7 @@ inventory_source_urls = patterns('awx.api.views', inventory_update_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/$', 'inventory_update_detail'), url(r'^(?P[0-9]+)/cancel/$', 'inventory_update_cancel'), + url(r'^(?P[0-9]+)/stdout/$', 'inventory_update_stdout'), ) credential_urls = patterns('awx.api.views', @@ -142,6 +144,7 @@ job_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/job_host_summaries/$', 'job_job_host_summaries_list'), url(r'^(?P[0-9]+)/job_events/$', 'job_job_events_list'), url(r'^(?P[0-9]+)/activity_stream/$', 'job_activity_stream_list'), + url(r'^(?P[0-9]+)/stdout/$', 'job_stdout'), ) job_host_summary_urls = patterns('awx.api.views', diff --git a/awx/api/views.py b/awx/api/views.py index 40abba8067..214e2ce5b4 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -29,6 +29,10 @@ from rest_framework.settings import api_settings from rest_framework.views import exception_handler from rest_framework import status +# Ansi2HTML +from ansi2html import Ansi2HTMLConverter +from ansi2html.style import SCHEME + # AWX from awx.main.licenses import LicenseReader from awx.main.models import * @@ -37,8 +41,10 @@ from awx.main.access import get_user_queryset from awx.main.signals import ignore_inventory_computed_fields, ignore_inventory_group_removal from awx.api.authentication import JobTaskAuthentication from awx.api.permissions import * +from awx.api.renderers import * from awx.api.serializers import * from awx.api.generics import * +from awx.api.generics import get_view_name def api_exception_handler(exc): @@ -1407,6 +1413,54 @@ class UnifiedJobList(ListAPIView): serializer_class = UnifiedJobListSerializer new_in_148 = True +class UnifiedJobStdout(RetrieveAPIView): + + serializer_class = UnifiedJobStdoutSerializer + renderer_classes = [BrowsableAPIRenderer, renderers.StaticHTMLRenderer, + PlainTextRenderer, AnsiTextRenderer] + filter_backends = () + new_in_148 = True + + def retrieve(self, request, *args, **kwargs): + unified_job = self.get_object() + if request.accepted_renderer.format in ('html', 'api'): + scheme = request.QUERY_PARAMS.get('scheme', None) + if scheme not in SCHEME: + scheme = 'ansi2html' + dark_val = request.QUERY_PARAMS.get('dark', '') + dark = bool(dark_val and dark_val[0].lower() in ('1', 't', 'y')) + content_only = bool(request.accepted_renderer.format == 'api') + dark_bg = (content_only and dark) or (not content_only and (dark or not dark_val)) + conv = Ansi2HTMLConverter(scheme=scheme, dark_bg=dark_bg, + title=get_view_name(self.__class__)) + if content_only: + headers = conv.produce_headers() + body = conv.convert(unified_job.result_stdout_raw, full=False) + data = '\n'.join([headers, body]) + data = '
%s
' % data + else: + data = conv.convert(unified_job.result_stdout_raw) + # Fix ugly grey background used by default. + data = data.replace('.body_background { background-color: #AAAAAA; }', + '.body_background { background-color: #f5f5f5; }') + return Response(data) + elif request.accepted_renderer.format == 'ansi': + return Response(unified_job.result_stdout_raw) + else: + return super(UnifiedJobStdout, self).retrieve(request, *args, **kwargs) + +class ProjectUpdateStdout(UnifiedJobStdout): + + model = ProjectUpdate + +class InventoryUpdateStdout(UnifiedJobStdout): + + model = InventoryUpdate + +class JobStdout(UnifiedJobStdout): + + model = Job + class ActivityStreamList(SimpleListAPIView): model = ActivityStream diff --git a/awx/lib/site-packages/README b/awx/lib/site-packages/README index d3f904b1b4..817940e954 100644 --- a/awx/lib/site-packages/README +++ b/awx/lib/site-packages/README @@ -2,6 +2,7 @@ Local versions of third-party packages required by Tower. Package names and versions are listed below, along with notes on which files are included. amqp==1.4.4 (amqp/*) +ansi2html==1.0.6 (ansi2html/*) anyjson==0.3.3 (anyjson/*) argparse==1.2.1 (argparse.py, needed for Python 2.6 support) Babel==1.3 (babel/*, excluded bin/pybabel) diff --git a/awx/lib/site-packages/ansi2html/__init__.py b/awx/lib/site-packages/ansi2html/__init__.py new file mode 100644 index 0000000000..58250b8196 --- /dev/null +++ b/awx/lib/site-packages/ansi2html/__init__.py @@ -0,0 +1,2 @@ +from ansi2html.converter import Ansi2HTMLConverter +__all__ = ['Ansi2HTMLConverter'] diff --git a/awx/lib/site-packages/ansi2html/converter.py b/awx/lib/site-packages/ansi2html/converter.py new file mode 100644 index 0000000000..04cf89cd87 --- /dev/null +++ b/awx/lib/site-packages/ansi2html/converter.py @@ -0,0 +1,492 @@ +# This file is part of ansi2html +# Convert ANSI (terminal) colours and attributes to HTML +# Copyright (C) 2012 Ralph Bean +# Copyright (C) 2013 Sebastian Pipping +# +# Inspired by and developed off of the work by pixelbeat and blackjack. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# . + +import re +import sys +import optparse +import pkg_resources + +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict + +from ansi2html.style import get_styles, SCHEME +import six +from six.moves import map +from six.moves import zip + + +ANSI_FULL_RESET = 0 +ANSI_INTENSITY_INCREASED = 1 +ANSI_INTENSITY_REDUCED = 2 +ANSI_INTENSITY_NORMAL = 22 +ANSI_STYLE_ITALIC = 3 +ANSI_STYLE_NORMAL = 23 +ANSI_BLINK_SLOW = 5 +ANSI_BLINK_FAST = 6 +ANSI_BLINK_OFF = 25 +ANSI_UNDERLINE_ON = 4 +ANSI_UNDERLINE_OFF = 24 +ANSI_CROSSED_OUT_ON = 9 +ANSI_CROSSED_OUT_OFF = 29 +ANSI_VISIBILITY_ON = 28 +ANSI_VISIBILITY_OFF = 8 +ANSI_FOREGROUND_CUSTOM_MIN = 30 +ANSI_FOREGROUND_CUSTOM_MAX = 37 +ANSI_FOREGROUND_256 = 38 +ANSI_FOREGROUND_DEFAULT = 39 +ANSI_BACKGROUND_CUSTOM_MIN = 40 +ANSI_BACKGROUND_CUSTOM_MAX = 47 +ANSI_BACKGROUND_256 = 48 +ANSI_BACKGROUND_DEFAULT = 49 +ANSI_NEGATIVE_ON = 7 +ANSI_NEGATIVE_OFF = 27 + + +_template = six.u(""" + + + +%(title)s + + + +
+%(content)s
+
+ + + +""") + + +class _State(object): + def __init__(self): + self.reset() + + def reset(self): + self.intensity = ANSI_INTENSITY_NORMAL + self.style = ANSI_STYLE_NORMAL + self.blink = ANSI_BLINK_OFF + self.underline = ANSI_UNDERLINE_OFF + self.crossedout = ANSI_CROSSED_OUT_OFF + self.visibility = ANSI_VISIBILITY_ON + self.foreground = (ANSI_FOREGROUND_DEFAULT, None) + self.background = (ANSI_BACKGROUND_DEFAULT, None) + self.negative = ANSI_NEGATIVE_OFF + + def adjust(self, ansi_code, parameter=None): + if ansi_code in (ANSI_INTENSITY_INCREASED, ANSI_INTENSITY_REDUCED, ANSI_INTENSITY_NORMAL): + self.intensity = ansi_code + elif ansi_code in (ANSI_STYLE_ITALIC, ANSI_STYLE_NORMAL): + self.style = ansi_code + elif ansi_code in (ANSI_BLINK_SLOW, ANSI_BLINK_FAST, ANSI_BLINK_OFF): + self.blink = ansi_code + elif ansi_code in (ANSI_UNDERLINE_ON, ANSI_UNDERLINE_OFF): + self.underline = ansi_code + elif ansi_code in (ANSI_CROSSED_OUT_ON, ANSI_CROSSED_OUT_OFF): + self.crossedout = ansi_code + elif ansi_code in (ANSI_VISIBILITY_ON, ANSI_VISIBILITY_OFF): + self.visibility = ansi_code + elif ANSI_FOREGROUND_CUSTOM_MIN <= ansi_code <= ANSI_FOREGROUND_CUSTOM_MAX: + self.foreground = (ansi_code, None) + elif ansi_code == ANSI_FOREGROUND_256: + self.foreground = (ansi_code, parameter) + elif ansi_code == ANSI_FOREGROUND_DEFAULT: + self.foreground = (ansi_code, None) + elif ANSI_BACKGROUND_CUSTOM_MIN <= ansi_code <= ANSI_BACKGROUND_CUSTOM_MAX: + self.background = (ansi_code, None) + elif ansi_code == ANSI_BACKGROUND_256: + self.background = (ansi_code, parameter) + elif ansi_code == ANSI_BACKGROUND_DEFAULT: + self.background = (ansi_code, None) + elif ansi_code in (ANSI_NEGATIVE_ON, ANSI_NEGATIVE_OFF): + self.negative = ansi_code + + def to_css_classes(self): + css_classes = [] + + def append_unless_default(output, value, default): + if value != default: + css_class = 'ansi%d' % value + output.append(css_class) + + def append_color_unless_default(output, color, default, negative, neg_css_class): + value, parameter = color + if value != default: + prefix = 'inv' if negative else 'ansi' + css_class_index = str(value) \ + if (parameter is None) \ + else '%d-%d' % (value, parameter) + output.append(prefix + css_class_index) + elif negative: + output.append(neg_css_class) + + append_unless_default(css_classes, self.intensity, ANSI_INTENSITY_NORMAL) + append_unless_default(css_classes, self.style, ANSI_STYLE_NORMAL) + append_unless_default(css_classes, self.blink, ANSI_BLINK_OFF) + append_unless_default(css_classes, self.underline, ANSI_UNDERLINE_OFF) + append_unless_default(css_classes, self.crossedout, ANSI_CROSSED_OUT_OFF) + append_unless_default(css_classes, self.visibility, ANSI_VISIBILITY_ON) + + flip_fore_and_background = (self.negative == ANSI_NEGATIVE_ON) + append_color_unless_default(css_classes, self.foreground, ANSI_FOREGROUND_DEFAULT, flip_fore_and_background, 'inv_background') + append_color_unless_default(css_classes, self.background, ANSI_BACKGROUND_DEFAULT, flip_fore_and_background, 'inv_foreground') + + return css_classes + + +def linkify(line): + for match in re.findall(r'https?:\/\/\S+', line): + line = line.replace(match, '%s' % (match, match)) + + return line + + +def _needs_extra_newline(text): + if not text or text.endswith('\n'): + return False + return True + + +class CursorMoveUp(object): + pass + + +class Ansi2HTMLConverter(object): + """ Convert Ansi color codes to CSS+HTML + + Example: + >>> conv = Ansi2HTMLConverter() + >>> ansi = " ".join(sys.stdin.readlines()) + >>> html = conv.convert(ansi) + """ + + def __init__(self, + inline=False, + dark_bg=True, + font_size='normal', + linkify=False, + escaped=True, + markup_lines=False, + output_encoding='utf-8', + scheme='ansi2html', + title='' + ): + + self.inline = inline + self.dark_bg = dark_bg + self.font_size = font_size + self.linkify = linkify + self.escaped = escaped + self.markup_lines = markup_lines + self.output_encoding = output_encoding + self.scheme = scheme + self.title = title + self._attrs = None + + if inline: + self.styles = dict([(item.klass.strip('.'), item) for item in get_styles(self.dark_bg, self.scheme)]) + + self.ansi_codes_prog = re.compile('\033\\[' '([\\d;]*)' '([a-zA-z])') + + def apply_regex(self, ansi): + parts = self._apply_regex(ansi) + parts = self._collapse_cursor(parts) + parts = list(parts) + + if self.linkify: + parts = [linkify(part) for part in parts] + + combined = "".join(parts) + + if self.markup_lines: + combined = "\n".join([ + """%s""" % (i, line) + for i, line in enumerate(combined.split('\n')) + ]) + + return combined + + def _apply_regex(self, ansi): + if self.escaped: + specials = OrderedDict([ + ('&', '&'), + ('<', '<'), + ('>', '>'), + ]) + for pattern, special in specials.items(): + ansi = ansi.replace(pattern, special) + + state = _State() + inside_span = False + last_end = 0 # the index of the last end of a code we've seen + for match in self.ansi_codes_prog.finditer(ansi): + yield ansi[last_end:match.start()] + last_end = match.end() + + params, command = match.groups() + + if command not in 'mMA': + continue + + # Special cursor-moving code. The only supported one. + if command == 'A': + yield CursorMoveUp + continue + + try: + params = list(map(int, params.split(';'))) + except ValueError: + params = [ANSI_FULL_RESET] + + # Find latest reset marker + last_null_index = None + skip_after_index = -1 + for i, v in enumerate(params): + if i <= skip_after_index: + continue + + if v == ANSI_FULL_RESET: + last_null_index = i + elif v in (ANSI_FOREGROUND_256, ANSI_BACKGROUND_256): + skip_after_index = i + 2 + + # Process reset marker, drop everything before + if last_null_index is not None: + params = params[last_null_index + 1:] + if inside_span: + inside_span = False + yield '' + state.reset() + + if not params: + continue + + # Turn codes into CSS classes + skip_after_index = -1 + for i, v in enumerate(params): + if i <= skip_after_index: + continue + + if v in (ANSI_FOREGROUND_256, ANSI_BACKGROUND_256): + try: + parameter = params[i + 2] + except IndexError: + continue + skip_after_index = i + 2 + else: + parameter = None + state.adjust(v, parameter=parameter) + + if inside_span: + yield '' + inside_span = False + + css_classes = state.to_css_classes() + if not css_classes: + continue + + if self.inline: + style = [self.styles[klass].kw for klass in css_classes if + klass in self.styles] + yield '' % "; ".join(style) + else: + yield '' % " ".join(css_classes) + inside_span = True + + yield ansi[last_end:] + if inside_span: + yield '' + inside_span = False + + def _collapse_cursor(self, parts): + """ Act on any CursorMoveUp commands by deleting preceding tokens """ + + final_parts = [] + for part in parts: + + # Throw out empty string tokens ("") + if not part: + continue + + # Go back, deleting every token in the last 'line' + if part == CursorMoveUp: + final_parts.pop() + while '\n' not in final_parts[-1]: + final_parts.pop() + + continue + + # Otherwise, just pass this token forward + final_parts.append(part) + + return final_parts + + def prepare(self, ansi='', ensure_trailing_newline=False): + """ Load the contents of 'ansi' into this object """ + + body = self.apply_regex(ansi) + + if ensure_trailing_newline and _needs_extra_newline(body): + body += '\n' + + self._attrs = { + 'dark_bg': self.dark_bg, + 'font_size': self.font_size, + 'body': body, + } + + return self._attrs + + def attrs(self): + """ Prepare attributes for the template """ + if not self._attrs: + raise Exception("Method .prepare not yet called.") + return self._attrs + + def convert(self, ansi, full=True, ensure_trailing_newline=False): + attrs = self.prepare(ansi, ensure_trailing_newline=ensure_trailing_newline) + if not full: + return attrs["body"] + else: + return _template % { + 'style' : "\n".join(map(str, get_styles(self.dark_bg, self.scheme))), + 'title' : self.title, + 'font_size' : self.font_size, + 'content' : attrs["body"], + 'output_encoding' : self.output_encoding, + } + + def produce_headers(self): + return '\n' % { + 'style' : "\n".join(map(str, get_styles(self.dark_bg, self.scheme))) + } + + +def main(): + """ + $ ls --color=always | ansi2html > directories.html + $ sudo tail /var/log/messages | ccze -A | ansi2html > logs.html + $ task burndown | ansi2html > burndown.html + """ + + scheme_names = sorted(six.iterkeys(SCHEME)) + version_str = pkg_resources.get_distribution('ansi2html').version + parser = optparse.OptionParser( + usage=main.__doc__, + version="%%prog %s" % version_str) + parser.add_option( + "-p", "--partial", dest="partial", + default=False, action="store_true", + help="Process lines as them come in. No headers are produced.") + parser.add_option( + "-i", "--inline", dest="inline", + default=False, action="store_true", + help="Inline style without headers or template.") + parser.add_option( + "-H", "--headers", dest="headers", + default=False, action="store_true", + help="Just produce the