Register system again if deleted by another pod

Avoid cases where missing instance
  would throw error on startup
  this gives time for heartbeat to register it
This commit is contained in:
Alan Rominger
2022-07-08 12:06:21 -04:00
parent 2c9a0444e6
commit 585d3f4e2a
4 changed files with 19 additions and 5 deletions

View File

@@ -15,7 +15,11 @@ def startup_reaping():
If this particular instance is starting, then we know that any running jobs are invalid
so we will reap those jobs as a special action here
"""
me = Instance.objects.me()
try:
me = Instance.objects.me()
except RuntimeError as e:
logger.warning(f'Local instance is not registered, not running startup reaper: {e}')
return
jobs = UnifiedJob.objects.filter(status='running', controller_node=me.hostname)
job_ids = []
for j in jobs: