move code linting to a stricter pep8-esque auto-formatting tool, black

This commit is contained in:
Ryan Petrello
2021-03-19 12:44:51 -04:00
parent 9b702e46fe
commit c2ef0a6500
671 changed files with 20538 additions and 21924 deletions

View File

@@ -30,13 +30,12 @@ else:
class NoOpResultQueue(object):
def put(self, item):
pass
class PoolWorker(object):
'''
"""
Used to track a worker child process and its pending and finished messages.
This class makes use of two distinct multiprocessing.Queues to track state:
@@ -62,7 +61,7 @@ class PoolWorker(object):
A worker is "busy" when it has at least one message in self.managed_tasks.
It is "idle" when self.managed_tasks is empty.
'''
"""
track_managed_tasks = False
@@ -91,10 +90,10 @@ class PoolWorker(object):
self.calculate_managed_tasks()
def quit(self):
'''
"""
Send a special control message to the worker that tells it to exit
gracefully.
'''
"""
self.queue.put('QUIT')
@property
@@ -112,9 +111,7 @@ class PoolWorker(object):
@property
def mb(self):
if self.alive:
return '{:0.3f}'.format(
psutil.Process(self.pid).memory_info().rss / 1024.0 / 1024.0
)
return '{:0.3f}'.format(psutil.Process(self.pid).memory_info().rss / 1024.0 / 1024.0)
return '0'
@property
@@ -179,11 +176,7 @@ class PoolWorker(object):
except QueueEmpty:
break # qsize is not always _totally_ up to date
if len(orphaned):
logger.error(
'requeuing {} messages from gone worker pid:{}'.format(
len(orphaned), self.pid
)
)
logger.error('requeuing {} messages from gone worker pid:{}'.format(len(orphaned), self.pid))
return orphaned
@property
@@ -202,7 +195,7 @@ class StatefulPoolWorker(PoolWorker):
class WorkerPool(object):
'''
"""
Creates a pool of forked PoolWorkers.
As WorkerPool.write(...) is called (generally, by a kombu consumer
@@ -220,7 +213,7 @@ class WorkerPool(object):
0, # preferred worker 0
'Hello, World!'
)
'''
"""
pool_cls = PoolWorker
debug_meta = ''
@@ -284,13 +277,10 @@ class WorkerPool(object):
'{% endfor %}'
)
now = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')
return tmpl.render(
pool=self, workers=self.workers, meta=self.debug_meta,
dt=now
)
return tmpl.render(pool=self, workers=self.workers, meta=self.debug_meta, dt=now)
def write(self, preferred_queue, body):
queue_order = sorted(range(len(self.workers)), key=lambda x: -1 if x==preferred_queue else x)
queue_order = sorted(range(len(self.workers)), key=lambda x: -1 if x == preferred_queue else x)
write_attempt_order = []
for queue_actual in queue_order:
try:
@@ -315,10 +305,10 @@ class WorkerPool(object):
class AutoscalePool(WorkerPool):
'''
"""
An extended pool implementation that automatically scales workers up and
down based on demand
'''
"""
pool_cls = StatefulPoolWorker
@@ -333,7 +323,7 @@ class AutoscalePool(WorkerPool):
else:
total_memory_gb = (psutil.virtual_memory().total >> 30) + 1 # noqa: round up
# 5 workers per GB of total memory
self.max_workers = (total_memory_gb * 5)
self.max_workers = total_memory_gb * 5
# max workers can't be less than min_workers
self.max_workers = max(self.min_workers, self.max_workers)
@@ -410,15 +400,11 @@ class AutoscalePool(WorkerPool):
if current_task and isinstance(current_task, dict):
if current_task.get('task', '').endswith('tasks.run_task_manager'):
if 'started' not in current_task:
w.managed_tasks[
current_task['uuid']
]['started'] = time.time()
w.managed_tasks[current_task['uuid']]['started'] = time.time()
age = time.time() - current_task['started']
w.managed_tasks[current_task['uuid']]['age'] = age
if age > (60 * 5):
logger.error(
f'run_task_manager has held the advisory lock for >5m, sending SIGTERM to {w.pid}'
) # noqa
logger.error(f'run_task_manager has held the advisory lock for >5m, sending SIGTERM to {w.pid}') # noqa
os.kill(w.pid, signal.SIGTERM)
for m in orphaned: