Merge pull request #4626 from chrismeyersfsu/enhancement-graph_jobs_3_8_0

terminal graph of job status changes

awx/main/management/commands/graph_jobs.py

@@ -0,0 +1,117 @@
+# Python
+import asciichartpy as chart
+import collections
+import time
+import sys
+
+# Django
+from django.db.models import Count
+from django.core.management.base import BaseCommand
+
+# AWX
+from awx.main.models import (
+    Job,
+    Instance
+)
+
+
+DEFAULT_WIDTH = 100
+DEFAULT_HEIGHT = 30
+
+
+def chart_color_lookup(color_str):
+    return getattr(chart, color_str)
+
+
+def clear_screen():
+    print(chr(27) + "[2J")
+
+
+class JobStatus():
+    def __init__(self, status, color, width):
+        self.status = status
+        self.color = color
+        self.color_code = chart_color_lookup(color)
+        self.x = collections.deque(maxlen=width)
+        self.y = collections.deque(maxlen=width)
+
+    def tick(self, x, y):
+        self.x.append(x)
+        self.y.append(y)
+
+
+class JobStatusController:
+    RESET = chart_color_lookup('reset')
+
+    def __init__(self, width):
+        self.plots = [
+            JobStatus('pending', 'red', width),
+            JobStatus('waiting', 'blue', width),
+            JobStatus('running', 'green', width)
+        ]
+        self.ts_start = int(time.time())
+
+    def tick(self):
+        ts = int(time.time()) - self.ts_start
+        q = Job.objects.filter(status__in=['pending','waiting','running']).values_list('status').order_by().annotate(Count('status'))
+        status_count = dict(pending=0, waiting=0, running=0)
+        for status, count in q:
+            status_count[status] = count
+
+        for p in self.plots:
+            p.tick(ts, status_count[p.status])
+
+    def series(self):
+        return [list(p.y) for p in self.plots]
+
+    def generate_status(self):
+        line = ""
+        lines = []
+        for p in self.plots:
+            lines.append(f'{p.color_code}{p.status} {p.y[-1]}{self.RESET}')
+
+        line += ", ".join(lines) + '\n'
+
+        width = 5
+        time_running = int(time.time()) - self.ts_start
+        instances = Instance.objects.all().order_by('hostname')
+        line += "Capacity:  " + ", ".join([f"{instance.capacity:{width}}" for instance in instances]) + '\n'
+        line += "Remaining: " + ", ".join([f"{instance.remaining_capacity:{width}}" for instance in instances]) + '\n'
+        line += f"Seconds running: {time_running}" + '\n'
+
+        return line
+
+
+class Command(BaseCommand):
+    help = "Plot pending, waiting, running jobs over time on the terminal"
+
+    def add_arguments(self, parser):
+        parser.add_argument('--refresh', dest='refresh', type=float, default=1.0,
+                            help='Time between refreshes of the graph and data in seconds (defaults to 1.0)')
+        parser.add_argument('--width', dest='width', type=int, default=DEFAULT_WIDTH,
+                            help=f'Width of the graph (defaults to {DEFAULT_WIDTH})')
+        parser.add_argument('--height', dest='height', type=int, default=DEFAULT_HEIGHT,
+                            help=f'Height of the graph (defaults to {DEFAULT_HEIGHT})')
+
+    def handle(self, *args, **options):
+        refresh_seconds = options['refresh']
+        width = options['width']
+        height = options['height']
+
+        jctl = JobStatusController(width)
+
+        conf = {
+            'colors': [chart_color_lookup(p.color) for p in jctl.plots],
+            'height': height,
+        }
+
+        while True:
+            jctl.tick()
+
+            draw = chart.plot(jctl.series(), conf)
+            status_line = jctl.generate_status()
+            clear_screen()
+            print(draw)
+            sys.stdout.write(status_line)
+            time.sleep(refresh_seconds)
+

docs/debugging.md

@@ -98,3 +98,29 @@ sdb-listen
 
 This will open a Python process that listens for new debugger sessions and
 automatically connects to them for you.
+
+Graph Jobs
+----------
+The `awx-manage graph_jobs` can be used to visualize how Jobs progress from
+pending to waiting to running.
+
+```
+awx-manage graph_jobs --help
+usage: awx-manage graph_jobs [-h] [--refresh REFRESH] [--width WIDTH]
+                             [--height HEIGHT] [--version] [-v {0,1,2,3}]
+                             [--settings SETTINGS] [--pythonpath PYTHONPATH]
+                             [--traceback] [--no-color] [--force-color]
+
+Plot pending, waiting, running jobs over time on the terminal
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --refresh REFRESH     Time between refreshes of the graph and data in
+                        seconds (defaults to 1.0)
+  --width WIDTH         Width of the graph (defaults to 100)
+  --height HEIGHT       Height of the graph (defaults to 30)
+```
+
+Below is an example run with 200 Jobs flowing through the system.
+
+[![asciicast](https://asciinema.org/a/xnfzMQ30xWPdhwORiISz0wcEw.svg)](https://asciinema.org/a/xnfzMQ30xWPdhwORiISz0wcEw)

docs/licenses/asciichartpy.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright © 2016 Igor Kroitor
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

requirements/requirements.in

@@ -1,6 +1,7 @@
 aiohttp
 ansible-runner>=1.4.6
 ansiconv==1.0.0  # UPGRADE BLOCKER: from 2013, consider replacing instead of upgrading
+asciichartpy
 azure-keyvault==1.1.0  # see UPGRADE BLOCKERs
 channels
 channels-redis>=3.1.0  # https://github.com/django/channels_redis/issues/212

requirements/requirements.txt

@@ -3,6 +3,7 @@ aiohttp==3.6.2            # via -r /awx_devel/requirements/requirements.in
 aioredis==1.3.1           # via channels-redis
 ansible-runner==1.4.6     # via -r /awx_devel/requirements/requirements.in
 ansiconv==1.0.0           # via -r /awx_devel/requirements/requirements.in
+asciichartpy==1.5.25      # via -r /awx_devel/requirements/requirements.in
 asgiref==3.2.5            # via channels, channels-redis, daphne
 async-timeout==3.0.1      # via aiohttp, aioredis
 attrs==19.3.0             # via aiohttp, automat, jsonschema, service-identity, twisted
@@ -130,4 +131,4 @@ zope.interface==5.0.0     # via twisted
 
 # The following packages are considered to be unsafe in a requirements file:
 pip==19.3.1               # via -r /awx_devel/requirements/requirements.in
-setuptools==41.6.0        # via -r /awx_devel/requirements/requirements.in, google-auth, jsonschema, kubernetes, markdown, python-daemon, zope.interface
+setuptools==41.6.0        # via -r /awx_devel/requirements/requirements.in, asciichartpy, google-auth, jsonschema, kubernetes, markdown, python-daemon, zope.interface