Files
awx/awx/main/tests/functional/tasks/test_tasks_jobs.py
T
Hui SongandClaude Sonnet 4.6 a5b2179a3e AAP-89607: enable controller adoption after restart via receptor work unit reconnection
Add ability for a restarted controller to reconnect to in-progress work units
on the execution environment and replay their events, eliminating the need
to re-run jobs after controller restart in hybrid/mesh deployments.

Key changes:

1. Extract _process_phase() so it can be called standalone without re-transmitting
   artifacts. Unified per-job loops consolidate startup/heartbeat processing
   with explicit adoption/error branches for each job.

2. Add reattach_to_work_unit() to reconnect and stream events from a completed
   receptor work unit. Uses hybrid memory-safe dedup (safe_threshold + collision_zone)
   to avoid re-sending events already in the database to the callback receiver.

3. Unify job finalization: extract _finalize_job_run() shared between normal
   execution and adoption paths. Extract _configure_runner_callback() as the
   canonical callback factory used by both paths.

4. Move _receptor_release_work() after terminal status is committed to DB,
   eliminating the race where a kill between release and commit left jobs
   stuck in 'running' without adoptable work units.

5. Add adopt_job_async @task to run adoption in background, enabling real-time
   event streaming from the EE during controller restart recovery.

Cluster-verified across 9 scenarios with all event stream data checked.
Unit and functional tests cover all adoption paths and edge cases.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
2026-09-23 14:27:21 -04:00

145 lines
4.5 KiB
Python

import pytest
from awx.main.tasks.jobs import RunJob
from awx.main.models import Job
@pytest.mark.django_db
def test_does_not_run_reaped_job(mocker, mock_me):
job = Job.objects.create(status='failed', job_explanation='This job has been reaped.')
mock_run = mocker.patch('awx.main.tasks.jobs.ansible_runner.interface.run')
try:
RunJob().run(job.id)
except Exception:
pass
job.refresh_from_db()
assert job.status == 'failed'
mock_run.assert_not_called()
@pytest.mark.django_db
def test_cancel_flag_on_start(jt_linked, caplog):
job = jt_linked.create_unified_job()
job.status = 'waiting'
job.cancel_flag = True
job.save()
task = RunJob()
task.run(job.id)
job = Job.objects.get(id=job.id)
assert job.status == 'canceled'
@pytest.mark.django_db
def test_runjob_run_can_accept_waiting_status(jt_linked, mocker):
"""Test that RunJob.run() can accept a job in 'waiting' status and transition it to 'running'
before the pre_run_hook is called"""
job = jt_linked.create_unified_job()
job.status = 'waiting'
job.save()
status_at_pre_run = None
def capture_status(instance, private_data_dir):
nonlocal status_at_pre_run
instance.refresh_from_db()
status_at_pre_run = instance.status
mock_pre_run = mocker.patch.object(RunJob, 'pre_run_hook', side_effect=capture_status)
task = RunJob()
try:
task.run(job.id)
except Exception:
pass
mock_pre_run.assert_called_once()
assert status_at_pre_run == 'running'
@pytest.mark.django_db
def test_finalize_job_run_with_real_job_instance(jt_linked, mocker):
"""Test that _finalize_job_run executes the real finalization path with actual Job instance."""
from awx.main.tasks.jobs import _finalize_job_run
from awx.main.tasks.callback import RunnerCallback
job = jt_linked.create_unified_job()
job.status = 'running'
job.save()
# Create a real callback
callback = RunnerCallback(model=Job)
callback.event_ct = 5
callback.host_status_counts = {'ok': 1, 'failed': 0} # Non-None triggers events_processed_hook
# Mock the hooks that would be called
mocker.patch('awx.main.tasks.jobs.events_processed_hook')
mocker.patch('awx.main.tasks.jobs.ScheduleTaskManager')
mocker.patch('awx.main.tasks.jobs.ScheduleWorkflowManager')
# Call the real finalization function
result = _finalize_job_run(Job, job.id, callback, 'successful')
# Verify the job was updated with successful status
job.refresh_from_db()
assert job.status == 'successful'
assert result.id == job.id
@pytest.mark.django_db
def test_finalize_job_run_with_blocked_jobs(jt_linked, mocker):
"""Test that _finalize_job_run calls ScheduleTaskManager for jobs with blocked dependencies."""
from awx.main.tasks.jobs import _finalize_job_run
from awx.main.tasks.callback import RunnerCallback
job = jt_linked.create_unified_job()
job.status = 'running'
job.save()
# Create another job that depends on this one
blocked_job = jt_linked.create_unified_job()
blocked_job.status = 'blocked'
blocked_job.save()
job.unifiedjob_blocked_jobs.add(blocked_job)
callback = RunnerCallback(model=Job)
callback.event_ct = 0
# Mock the hooks
mock_schedule_task = mocker.patch('awx.main.tasks.jobs.ScheduleTaskManager')
mocker.patch('awx.main.tasks.jobs.events_processed_hook')
mocker.patch('awx.main.tasks.jobs.ScheduleWorkflowManager')
# Call finalization
_finalize_job_run(Job, job.id, callback, 'successful')
# Verify ScheduleTaskManager was called
mock_schedule_task.return_value.schedule.assert_called_once()
@pytest.mark.django_db
def test_finalize_job_run_calls_websocket_emit(jt_linked, mocker):
"""Test that websocket status is emitted after finalization."""
from awx.main.tasks.jobs import _finalize_job_run
from awx.main.tasks.callback import RunnerCallback
job = jt_linked.create_unified_job()
job.status = 'running'
job.save()
callback = RunnerCallback(model=Job)
callback.event_ct = 0
# Mock websocket_emit_status
mocker.patch('awx.main.tasks.jobs.events_processed_hook')
mocker.patch('awx.main.tasks.jobs.ScheduleTaskManager')
mocker.patch('awx.main.tasks.jobs.ScheduleWorkflowManager')
mock_websocket = mocker.patch.object(type(job), 'websocket_emit_status')
# Call finalization with failed status
_finalize_job_run(Job, job.id, callback, 'failed')
# Verify websocket was called with correct status
mock_websocket.assert_called_with('failed')