From d743b773532ceeb732ee283b348fff770123b5cd Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Mon, 26 Feb 2018 18:27:35 -0500 Subject: [PATCH] replace our rdb tooling w/ the sdb PyPI package --- Makefile | 3 - awx/settings/development.py | 5 + docs/debugging.md | 10 +- requirements/requirements_dev.txt | 1 + tools/docker-compose-cluster.yml | 6 +- tools/docker-compose.yml | 2 +- tools/rdb.py | 242 ------------------------------ 7 files changed, 15 insertions(+), 254 deletions(-) delete mode 100644 tools/rdb.py diff --git a/Makefile b/Makefile index fe221fffe4..1daca68516 100644 --- a/Makefile +++ b/Makefile @@ -336,9 +336,6 @@ receiver: nginx: nginx -g "daemon off;" -rdb: - $(PYTHON) tools/rdb.py - jupyter: @if [ "$(VENV_BASE)" ]; then \ . $(VENV_BASE)/awx/bin/activate; \ diff --git a/awx/settings/development.py b/awx/settings/development.py index 36fc290d6d..eec3689ee7 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -152,3 +152,8 @@ SERVICE_NAME_DICT = { # Used for sending commands in automatic restart UWSGI_FIFO_LOCATION = '/awxfifo' +try: + socket.gethostbyname('docker.for.mac.internal') + os.environ['SDB_NOTIFY_HOST'] = 'docker.for.mac.internal' +except Exception: + os.environ['SDB_NOTIFY_HOST'] = os.popen('ip route').read().split(' ')[2] diff --git a/docs/debugging.md b/docs/debugging.md index ee0405af3b..1257a8cf11 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -9,7 +9,7 @@ Python processes in Tower's development environment are kept running in the background via supervisord. As such, interacting with them via Python's standard `pdb.set_trace()` isn't possible. -Bundled in our container environment is a remote debugging tool, `rdb`. You +Bundled in our container environment is a remote debugging tool, `sdb`. You can use it to set remote breakpoints in Tower code and debug interactively over a telnet session: @@ -28,8 +28,8 @@ a telnet session: # You can access it from your host machine using telnet: # # $ telnet localhost - import rdb - rdb.set_trace() + import sdb + sdb.set_trace() ``` Keep in mind that when you interactively debug in this way, any process @@ -43,8 +43,8 @@ remote debugging sessions and automatically connect to them. From your *host* machine (i.e., _outside_ of the development container), you can run: ``` -make rdb +sdb-listen ``` This will open a Python process that listens for new debugger sessions and -automatically connects to them for you. \ No newline at end of file +automatically connects to them for you. diff --git a/requirements/requirements_dev.txt b/requirements/requirements_dev.txt index 367bf85567..5c5f8e30f5 100644 --- a/requirements/requirements_dev.txt +++ b/requirements/requirements_dev.txt @@ -19,3 +19,4 @@ jupyter matplotlib backports.tempfile # support in unit tests for py32+ tempfile.TemporaryDirectory mockldap +sdb diff --git a/tools/docker-compose-cluster.yml b/tools/docker-compose-cluster.yml index 0c90d036b5..141a62da7c 100644 --- a/tools/docker-compose-cluster.yml +++ b/tools/docker-compose-cluster.yml @@ -23,7 +23,7 @@ services: RABBITMQ_USER: guest RABBITMQ_PASS: guest RABBITMQ_VHOST: / - CELERY_RDB_HOST: 0.0.0.0 + SDB_HOST: 0.0.0.0 AWX_GROUP_QUEUES: alpha,tower volumes: - "../:/awx_devel" @@ -37,7 +37,7 @@ services: RABBITMQ_USER: guest RABBITMQ_PASS: guest RABBITMQ_VHOST: / - CELERY_RDB_HOST: 0.0.0.0 + SDB_HOST: 0.0.0.0 AWX_GROUP_QUEUES: bravo,tower volumes: - "../:/awx_devel" @@ -50,7 +50,7 @@ services: RABBITMQ_USER: guest RABBITMQ_PASS: guest RABBITMQ_VHOST: / - CELERY_RDB_HOST: 0.0.0.0 + SDB_HOST: 0.0.0.0 AWX_GROUP_QUEUES: charlie,tower volumes: - "../:/awx_devel" diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml index ef9c4cad79..ee6234ec8c 100644 --- a/tools/docker-compose.yml +++ b/tools/docker-compose.yml @@ -9,7 +9,7 @@ services: RABBITMQ_USER: guest RABBITMQ_PASS: guest RABBITMQ_VHOST: / - CELERY_RDB_HOST: 0.0.0.0 + SDB_HOST: 0.0.0.0 AWX_GROUP_QUEUES: tower ports: - "8888:8888" diff --git a/tools/rdb.py b/tools/rdb.py deleted file mode 100644 index d184922d2d..0000000000 --- a/tools/rdb.py +++ /dev/null @@ -1,242 +0,0 @@ -import rlcompleter -try: - import readline -except ImportError: - print("Module readline not available.") -else: - if 'libedit' in readline.__doc__: - readline.parse_and_bind("bind ^I rl_complete") - else: - readline.parse_and_bind("tab: complete") - -import sys -from celery.contrib.rdb import Rdb - -import cmd -import contextlib -import logging -import os -import pprint -import re -import select -import socket -import threading -from cStringIO import StringIO -from Queue import Queue, Empty - -from pygments import highlight -from pygments.lexers import PythonLexer -from pygments.formatters import Terminal256Formatter - -logger = logging.getLogger('awx') - - -@contextlib.contextmanager -def style(im_self, filepart=None, lexer=None): - - lexer = PythonLexer - old_stdout = im_self.stdout - buff = StringIO() - im_self.stdout = buff - yield - - value = buff.getvalue() - context = len(value.splitlines()) - file_cache = {} - - if filepart: - filepath, lineno = filepart - if filepath not in file_cache: - with open(filepath, 'r') as source: - file_cache[filepath] = source.readlines() - value = ''.join(file_cache[filepath][:lineno - 1]) + value - - formatter = Terminal256Formatter(style='friendly') - value = highlight(value, lexer(), formatter) - - # Properly format line numbers when they show up in multi-line strings - strcolor, _ = formatter.style_string['Token.Literal.String'] - intcolor, _ = formatter.style_string['Token.Literal.Number.Integer'] - value = re.sub( - r'%s([0-9]+)' % re.escape(strcolor), - lambda match: intcolor + match.group(1) + strcolor, - value, - ) - - # Highlight the "current" line in yellow for visibility - lineno = im_self.curframe.f_lineno - - value = re.sub( - '(?]+>[^\[]+\[39m([^\x1b]+)[^m]+m([^\n]+)' % (re.escape(intcolor), lineno), - lambda match: ''.join([ - str(lineno), - ' ->', - '\x1b[93m', - match.group(1), - re.sub('\x1b[^m]+m', '', match.group(2)), - '\x1b[0m' - ]), - value - ) - - if filepart: - _, first = filepart - value = '\n'.join(value.splitlines()[-context:]) + '\n' - - if value.strip(): - old_stdout.write(value) - im_self.stdout = old_stdout - - -class CustomPdb(Rdb): - - def cmdloop(self): - self.do_list(tuple()) - return cmd.Cmd.cmdloop(self) - - def do_list(self, args): - lines = 60 - context = (lines - 2) / 2 - if not args: - first = max(1, self.curframe.f_lineno - context) - last = first + context * 2 - 1 - args = "(%s, %s)" % (first, last) - self.lineno = None - with style(self, ( - self.curframe.f_code.co_filename, self.curframe.f_lineno - context) - ): - return Rdb.do_list(self, args) - do_l = do_list - - def format_stack_entry(self, *args, **kwargs): - entry = Rdb.format_stack_entry(self, *args, **kwargs) - return '\n'.join( - filter(lambda x: not x.startswith('->'), entry.splitlines()) - ) - - def print_stack_entry(self, *args, **kwargs): - with style(self): - return Rdb.print_stack_entry(self, *args, **kwargs) - - def set_next(self, curframe): - os.system('clear') - Rdb.set_next(self, curframe) - - def set_return(self, arg): - os.system('clear') - Rdb.set_return(self, arg) - - def set_step(self): - os.system('clear') - Rdb.set_step(self) - - def default(self, line): - with style(self): - return Rdb.default(self, line) - - def parseline(self, line): - line = line.strip() - match = re.search('^([0-9]+)([a-zA-Z]+)', line) - if match: - times, command = match.group(1), match.group(2) - line = command - self.cmdqueue.extend(list(command * (int(times) - 1))) - if line == '?': - line = 'dir()' - elif line.endswith('??'): - line = "import inspect; print ''.join(inspect.getsourcelines(%s)[0][:25])" % line[:-2] - elif line.endswith('?'): - line = 'dir(%s)' % line[:-1] - return cmd.Cmd.parseline(self, line) - - def displayhook(self, obj): - if obj is not None and not isinstance(obj, list): - return pprint.pprint(obj) - return Rdb.displayhook(self, obj) - - def get_avail_port(self, *args, **kwargs): - try: - socket.gethostbyname('docker.for.mac.localhost') - host = 'docker.for.mac.localhost' - except Exception: - host = os.popen('ip route').read().split(' ')[2] - sock, port = Rdb.get_avail_port(self, *args, **kwargs) - socket.socket(socket.AF_INET, socket.SOCK_DGRAM).sendto( - str(port), (host, 6899) - ) - return (sock, port) - - def say(self, m): - logger.warning(m) - - -CustomPdb.complete = rlcompleter.Completer(locals()).complete - - -def set_trace(): - return CustomPdb().set_trace(sys._getframe().f_back) - - -def listen(): - queue = Queue() - - def _consume(queue): - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.bind(('0.0.0.0', 6899)) - print('listening for rdb notifications on :6899...') - while True: - r, w, x = select.select([sock], [], []) - for i in r: - data = i.recv(1024) - queue.put(data) - worker = threading.Thread(target=_consume, args=(queue,)) - worker.setDaemon(True) - worker.start() - - try: - while True: - try: - port = queue.get(timeout=1) - queue.task_done() - if port == 'q': - break - port = int(port) - print('opening telnet session at localhost:%d...' % port) - telnet(port) - print('listening for rdb notifications on :6899...') - except Empty: - pass - except KeyboardInterrupt: - print('got Ctrl-C') - queue.put('q') - - -def telnet(port): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(2) - - try: - s.connect(('0.0.0.0', port)) - except Exception: - print('unable to connect') - return - print('connected to 0.0.0.0:%d' % port) - - while True: - socket_list = [sys.stdin, s] - r, w, e = select.select(socket_list, [], []) - for sock in r: - if sock == s: - data = sock.recv(4096) - if not data: - print('connection closed') - return - else: - sys.stdout.write(data) - else: - msg = sys.stdin.readline() - s.send(msg) - - -if __name__ == '__main__': - listen()