External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-02-24 06:26:00 -03:30
Code Issues Packages Projects Releases Wiki Activity

Fix a bug in the callback receiver where we weren't properly

Browse Source 
initializing the parent signal handler
This commit is contained in:
Matthew Jones
2014-11-17 13:27:48 -05:00
parent c7079cf338
commit e99466efc2
1 changed files with 23 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
awx/main/management/commands/run_callback_receiver.py
View File

@@ -41,10 +41,13 @@ class CallbackReceiver(object):
def run_subscriber(self, use_workers=True): def run_subscriber(self, use_workers=True):
def shutdown_handler(active_workers): def shutdown_handler(active_workers):
def _handler(signum, frame): def _handler(signum, frame):
for active_worker in active_workers: try:
active_worker.terminate() for active_worker in active_workers:
signal.signal(signum, signal.SIG_DFL) active_worker.terminate()
os.kill(os.getpid(), signum) # Rethrow signal, this time without catching it signal.signal(signum, signal.SIG_DFL)
os.kill(os.getpid(), signum) # Rethrow signal, this time without catching it
except Exception, e:
os.system('echo "exception %s" > /tmp/foob' % str(e))
return _handler return _handler
def check_pre_handle(data): def check_pre_handle(data):
event = data.get('event', '') event = data.get('event', '')
@@ -60,9 +63,6 @@ class CallbackReceiver(object):
queue_actual = Queue() queue_actual = Queue()
w = Process(target=self.callback_worker, args=(queue_actual,)) w = Process(target=self.callback_worker, args=(queue_actual,))
w.start() w.start()
signal.signal(signal.SIGINT, shutdown_handler([w]))
signal.signal(signal.SIGTERM, shutdown_handler([w]))
if settings.DEBUG: if settings.DEBUG:
self.print_log('Started worker %s' % str(idx)) self.print_log('Started worker %s' % str(idx))
worker_queues.append([0, queue_actual, w]) worker_queues.append([0, queue_actual, w])
@@ -76,12 +76,13 @@ class CallbackReceiver(object):
main_process.daemon = True main_process.daemon = True
main_process.start() main_process.start()
signal.signal(signal.SIGINT, shutdown_handler([main_process])) signal.signal(signal.SIGINT, shutdown_handler([p[2] for p in worker_queues] + [main_process]))
signal.signal(signal.SIGTERM, shutdown_handler([main_process])) signal.signal(signal.SIGTERM, shutdown_handler([p[2] for p in worker_queues] + [main_process]))
while True: while True:
workers_changed = False
for queue_worker in worker_queues: for queue_worker in worker_queues:
if not queue_worker[2].is_alive(): if not queue_worker[2].is_alive():
workers_changed = True
queue_worker[2].join() queue_worker[2].join()
w = Process(target=self.callback_worker, args=(queue_worker[1],)) w = Process(target=self.callback_worker, args=(queue_worker[1],))
w.daemon = True w.daemon = True
@@ -89,6 +90,9 @@ class CallbackReceiver(object):
signal.signal(signal.SIGINT, shutdown_handler([w])) signal.signal(signal.SIGINT, shutdown_handler([w]))
signal.signal(signal.SIGTERM, shutdown_handler([w])) signal.signal(signal.SIGTERM, shutdown_handler([w]))
queue_worker[2] = w queue_worker[2] = w
if workers_changed:
signal.signal(signal.SIGINT, shutdown_handler([p[2] for p in worker_queues] + [main_process]))
signal.signal(signal.SIGTERM, shutdown_handler([p[2] for p in worker_queues] + [main_process]))
if not main_process.is_alive(): if not main_process.is_alive():
sys.exit(1) sys.exit(1)
time.sleep(0.1) time.sleep(0.1)
@@ -134,7 +138,11 @@ class CallbackReceiver(object):
job_parent_events = {} job_parent_events = {}
queue_actual_worker = worker_queues[total_messages % WORKERS] queue_actual_worker = worker_queues[total_messages % WORKERS]
queue_actual_worker[0] += 1 queue_actual_worker[0] += 1
queue_actual_worker[1].put(message) try:
queue_actual_worker[1].put(message, block=True, timeout=2)
except Exception:
print("Queue workers went away, continuing...")
continue
if queue_actual_worker[0] >= MAX_REQUESTS: if queue_actual_worker[0] >= MAX_REQUESTS:
queue_actual_worker[0] = 0 queue_actual_worker[0] = 0
# print("Recycling worker process") # print("Recycling worker process")
@@ -221,7 +229,10 @@ class CallbackReceiver(object):
def callback_worker(self, queue_actual): def callback_worker(self, queue_actual):
messages_processed = 0 messages_processed = 0
while True: while True:
message = queue_actual.get() try:
message = queue_actual.get(block=True, timeout=1)
except Exception:
continue
self.process_job_event(message) self.process_job_event(message)
messages_processed += 1 messages_processed += 1
if messages_processed >= MAX_REQUESTS: if messages_processed >= MAX_REQUESTS: