diff --git a/awx/main/management/commands/cleanup_jobs.py b/awx/main/management/commands/cleanup_jobs.py index 3b23cf7dee..391f2ce2b4 100644 --- a/awx/main/management/commands/cleanup_jobs.py +++ b/awx/main/management/commands/cleanup_jobs.py @@ -49,6 +49,41 @@ def dt_to_partition_name(tbl_name, dt): return f"{tbl_name}_{dt.strftime('%Y%m%d_%H')}" +JHS_CHUNK_SIZE = 1000 + + +def _pre_delete_job_host_summaries(job_pks, logger=None): + """Pre-delete JobHostSummary rows and clear Host FK references in batches. + + Django's cascade collector materializes all JHS IDs into a single + UPDATE ... IN (...) to SET_NULL on Host.last_job_host_summary. + With many jobs x hosts this exceeds PostgreSQL's 1GB alloc limit. + Doing it in chunks with raw SQL avoids that. + """ + if not job_pks: + return + + # ANY(%s) is PostgreSQL-specific; AWX only supports PostgreSQL + with connection.cursor() as cursor: + for i in range(0, len(job_pks), JHS_CHUNK_SIZE): + chunk = list(job_pks[i : i + JHS_CHUNK_SIZE]) + + cursor.execute( + "UPDATE main_host SET last_job_host_summary_id = NULL" + " WHERE last_job_host_summary_id IN" + " (SELECT id FROM main_jobhostsummary WHERE job_id = ANY(%s))", + [chunk], + ) + + cursor.execute( + "DELETE FROM main_jobhostsummary WHERE job_id = ANY(%s)", + [chunk], + ) + + if logger: + logger.debug("Pre-deleted JobHostSummary chunk %d-%d of %d job PKs", i, i + len(chunk), len(job_pks)) + + class DeleteMeta: def __init__(self, logger, job_class, cutoff, dry_run): self.logger = logger @@ -91,6 +126,8 @@ class DeleteMeta: def delete_jobs(self): if not self.dry_run: + if self.job_class is Job: + _pre_delete_job_host_summaries(self.jobs_pk_list, self.logger) self.job_class.objects.filter(pk__in=self.jobs_pk_list).delete() def find_partitions_to_drop(self): @@ -265,8 +302,9 @@ class Command(BaseCommand): if info['min'] is not None: for start in range(info['min'], info['max'] + 1, self.batch_size): qs_batch = qs.filter(id__gte=start, id__lte=start + self.batch_size) - pk_list = qs_batch.values_list('id', flat=True) + pk_list = list(qs_batch.values_list('id', flat=True)) + _pre_delete_job_host_summaries(pk_list, self.logger) _, results = qs_batch.delete() deleted += results['main.Job'] # Avoid dropping the job event table in case we have interacted with it already diff --git a/awx/main/tests/unit/commands/test_cleanup_jobs.py b/awx/main/tests/unit/commands/test_cleanup_jobs.py new file mode 100644 index 0000000000..0e36807583 --- /dev/null +++ b/awx/main/tests/unit/commands/test_cleanup_jobs.py @@ -0,0 +1,137 @@ +from unittest import mock + +from awx.main.management.commands.cleanup_jobs import _pre_delete_job_host_summaries, JHS_CHUNK_SIZE + + +class TestPreDeleteJobHostSummaries: + def test_empty_list_is_noop(self): + with mock.patch('awx.main.management.commands.cleanup_jobs.connection') as mock_conn: + _pre_delete_job_host_summaries([]) + mock_conn.cursor.assert_not_called() + + def test_single_chunk(self): + job_pks = [1, 2, 3] + with mock.patch('awx.main.management.commands.cleanup_jobs.connection') as mock_conn: + mock_cursor = mock.MagicMock() + mock_conn.cursor.return_value.__enter__ = mock.Mock(return_value=mock_cursor) + mock_conn.cursor.return_value.__exit__ = mock.Mock(return_value=False) + + _pre_delete_job_host_summaries(job_pks) + + assert mock_cursor.execute.call_count == 2 + update_call = mock_cursor.execute.call_args_list[0] + assert 'UPDATE main_host SET last_job_host_summary_id = NULL' in update_call[0][0] + assert 'ANY(%s)' in update_call[0][0] + assert update_call[0][1] == [[1, 2, 3]] + + delete_call = mock_cursor.execute.call_args_list[1] + assert 'DELETE FROM main_jobhostsummary' in delete_call[0][0] + assert 'ANY(%s)' in delete_call[0][0] + assert delete_call[0][1] == [[1, 2, 3]] + + def test_multiple_chunks(self): + job_pks = list(range(1, JHS_CHUNK_SIZE + 500)) + with mock.patch('awx.main.management.commands.cleanup_jobs.connection') as mock_conn: + mock_cursor = mock.MagicMock() + mock_conn.cursor.return_value.__enter__ = mock.Mock(return_value=mock_cursor) + mock_conn.cursor.return_value.__exit__ = mock.Mock(return_value=False) + + _pre_delete_job_host_summaries(job_pks) + + # 2 chunks x 2 SQL statements each = 4 execute calls + assert mock_cursor.execute.call_count == 4 + + # First chunk should have JHS_CHUNK_SIZE items + first_update = mock_cursor.execute.call_args_list[0] + assert len(first_update[0][1][0]) == JHS_CHUNK_SIZE + + # Second chunk should have the remainder + second_update = mock_cursor.execute.call_args_list[2] + assert len(second_update[0][1][0]) == 499 + + def test_sql_is_fully_static(self): + """SQL strings contain no interpolated values — only ANY(%s) placeholders.""" + job_pks = [100, 200] + with mock.patch('awx.main.management.commands.cleanup_jobs.connection') as mock_conn: + mock_cursor = mock.MagicMock() + mock_conn.cursor.return_value.__enter__ = mock.Mock(return_value=mock_cursor) + mock_conn.cursor.return_value.__exit__ = mock.Mock(return_value=False) + + _pre_delete_job_host_summaries(job_pks) + + for call in mock_cursor.execute.call_args_list: + sql = call[0][0] + assert 'ANY(%s)' in sql + assert '100' not in sql + assert '200' not in sql + + def test_logger_called_per_chunk(self): + job_pks = [1, 2, 3] + logger = mock.MagicMock() + with mock.patch('awx.main.management.commands.cleanup_jobs.connection') as mock_conn: + mock_cursor = mock.MagicMock() + mock_conn.cursor.return_value.__enter__ = mock.Mock(return_value=mock_cursor) + mock_conn.cursor.return_value.__exit__ = mock.Mock(return_value=False) + + _pre_delete_job_host_summaries(job_pks, logger=logger) + + logger.debug.assert_called_once() + + def test_update_runs_before_delete(self): + """Host FK must be NULLed before JHS rows are deleted.""" + job_pks = [1] + with mock.patch('awx.main.management.commands.cleanup_jobs.connection') as mock_conn: + mock_cursor = mock.MagicMock() + mock_conn.cursor.return_value.__enter__ = mock.Mock(return_value=mock_cursor) + mock_conn.cursor.return_value.__exit__ = mock.Mock(return_value=False) + + _pre_delete_job_host_summaries(job_pks) + + first_sql = mock_cursor.execute.call_args_list[0][0][0] + second_sql = mock_cursor.execute.call_args_list[1][0][0] + assert 'UPDATE' in first_sql + assert 'DELETE' in second_sql + + +class TestDeleteMetaPreDelete: + """Verify DeleteMeta.delete_jobs() calls _pre_delete_job_host_summaries correctly.""" + + @mock.patch('awx.main.management.commands.cleanup_jobs._pre_delete_job_host_summaries') + def test_called_for_job_class(self, mock_pre_delete): + from awx.main.management.commands.cleanup_jobs import DeleteMeta + from awx.main.models import Job + + dm = DeleteMeta(logger=mock.MagicMock(), job_class=Job, cutoff=mock.MagicMock(), dry_run=False) + dm.jobs_pk_list = [10, 20, 30] + + with mock.patch.object(Job.objects, 'filter') as mock_filter: + mock_filter.return_value.delete.return_value = (3, {}) + dm.delete_jobs() + + mock_pre_delete.assert_called_once_with([10, 20, 30], dm.logger) + + @mock.patch('awx.main.management.commands.cleanup_jobs._pre_delete_job_host_summaries') + def test_skipped_for_non_job_class(self, mock_pre_delete): + from awx.main.management.commands.cleanup_jobs import DeleteMeta + from awx.main.models import ProjectUpdate + + dm = DeleteMeta(logger=mock.MagicMock(), job_class=ProjectUpdate, cutoff=mock.MagicMock(), dry_run=False) + dm.jobs_pk_list = [10, 20] + + with mock.patch.object(ProjectUpdate.objects, 'filter') as mock_filter: + mock_filter.return_value.delete.return_value = (2, {}) + dm.delete_jobs() + + mock_pre_delete.assert_not_called() + + @mock.patch('awx.main.management.commands.cleanup_jobs._pre_delete_job_host_summaries') + def test_skipped_for_dry_run(self, mock_pre_delete): + from awx.main.management.commands.cleanup_jobs import DeleteMeta + from awx.main.models import Job + + dm = DeleteMeta(logger=mock.MagicMock(), job_class=Job, cutoff=mock.MagicMock(), dry_run=True) + dm.jobs_pk_list = [10, 20] + + dm.delete_jobs() + + mock_pre_delete.assert_not_called()