From e371de38edf2bc308f431f0b95f83fd3b6938360 Mon Sep 17 00:00:00 2001 From: Jim Ladd Date: Thu, 13 May 2021 14:28:24 -0700 Subject: [PATCH] update job cleanup tests for sqlite-based execution --- awx/main/tests/functional/__init__.py | 59 +++++++++++++++++++ .../functional/commands/test_cleanup_jobs.py | 17 ++++-- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/awx/main/tests/functional/__init__.py b/awx/main/tests/functional/__init__.py index 6b9ac00c34..07d89e9739 100644 --- a/awx/main/tests/functional/__init__.py +++ b/awx/main/tests/functional/__init__.py @@ -16,6 +16,65 @@ def app_post_migration(sender, app_config, **kwargs): if 'result_stdout_text' not in cols: cur.execute('ALTER TABLE main_unifiedjob ADD COLUMN result_stdout_text TEXT') + # we also need to make sure that the `_unpartitioned_` tables are present. + # these tables represent old job event tables that were renamed / preserved during a + # migration which introduces partitioned event tables + # https://github.com/ansible/awx/issues/9039 + for tblname in ('main_jobevent', 'main_inventoryupdateevent', 'main_projectupdateevent', 'main_adhoccommandevent', 'main_systemjobevent'): + table_entries = cur.execute(f'SELECT count(*) from sqlite_master WHERE tbl_name="_unpartitioned_{tblname}";').fetchone()[0] + if table_entries > 0: + continue + if tblname == 'main_adhoccommandevent': + unique_columns = """host_name character varying(1024) NOT NULL, + event character varying(100) NOT NULL, + failed boolean NOT NULL, + changed boolean NOT NULL, + host_id integer, + ad_hoc_command_id integer NOT NULL + """ + elif tblname == 'main_inventoryupdateevent': + unique_columns = "inventory_update_id integer NOT NULL" + elif tblname == 'main_jobevent': + unique_columns = """event character varying(100) NOT NULL, + failed boolean NOT NULL, + changed boolean NOT NULL, + host_name character varying(1024) NOT NULL, + play character varying(1024) NOT NULL, + role character varying(1024) NOT NULL, + task character varying(1024) NOT NULL, + host_id integer, + job_id integer NOT NULL, + playbook character varying(1024) NOT NULL + """ + elif tblname == 'main_projectupdateevent': + unique_columns = """event character varying(100) NOT NULL, + failed boolean NOT NULL, + changed boolean NOT NULL, + playbook character varying(1024) NOT NULL, + play character varying(1024) NOT NULL, + role character varying(1024) NOT NULL, + task character varying(1024) NOT NULL, + project_update_id integer NOT NULL + """ + elif tblname == 'main_systemjobevent': + unique_columns = "system_job_id integer NOT NULL" + + cur.execute( + f"""CREATE TABLE _unpartitioned_{tblname} ( + id bigint NOT NULL, + created timestamp with time zone NOT NULL, + modified timestamp with time zone NOT NULL, + event_data text NOT NULL, + counter integer NOT NULL, + end_line integer NOT NULL, + start_line integer NOT NULL, + stdout text NOT NULL, + uuid character varying(1024) NOT NULL, + verbosity integer NOT NULL, + {unique_columns}); + """ + ) + if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': post_migrate.connect(app_post_migration, sender=apps.get_app_config('main')) diff --git a/awx/main/tests/functional/commands/test_cleanup_jobs.py b/awx/main/tests/functional/commands/test_cleanup_jobs.py index 0b934a43ed..c78c85077a 100644 --- a/awx/main/tests/functional/commands/test_cleanup_jobs.py +++ b/awx/main/tests/functional/commands/test_cleanup_jobs.py @@ -2,10 +2,12 @@ import pytest from datetime import datetime, timedelta from pytz import timezone from collections import OrderedDict +from unittest import mock from django.db.models.deletion import Collector, SET_NULL, CASCADE from django.core.management import call_command +from awx.main.management.commands import cleanup_jobs from awx.main.utils.deletion import AWXCollector from awx.main.models import JobTemplate, User, Job, JobEvent, Notification, WorkflowJobNode, JobHostSummary @@ -32,19 +34,20 @@ def setup_environment(inventory, project, machine_credential, host, notification notification.save() for i in range(3): + # create jobs with current time job1 = jt.create_job() job1.created = datetime.now(tz=timezone('UTC')) job1.save() - # create jobs with current time - JobEvent.create_from_data(job_id=job1.pk, uuid='abc123', event='runner_on_start', stdout='a' * 1025).save() + # sqlite does not support partitioning so we cannot test partition-based jobevent cleanup + # JobEvent.create_from_data(job_id=job1.pk, uuid='abc123', event='runner_on_start', stdout='a' * 1025).save() new_jobs.append(job1) - job2 = jt.create_job() # create jobs 10 days ago + job2 = jt.create_job() job2.created = datetime.now(tz=timezone('UTC')) - timedelta(days=days) job2.save() job2.dependent_jobs.add(job1) - JobEvent.create_from_data(job_id=job2.pk, uuid='abc123', event='runner_on_start', stdout='a' * 1025).save() + # JobEvent.create_from_data(job_id=job2.pk, uuid='abc123', event='runner_on_start', stdout='a' * 1025).save() old_jobs.append(job2) jt.last_job = job2 @@ -62,7 +65,13 @@ def setup_environment(inventory, project, machine_credential, host, notification return (old_jobs, new_jobs, days_str) +# sqlite does not support table partitioning so we mock out the methods responsible for pruning +# job event partitions during the job cleanup task +# https://github.com/ansible/awx/issues/9039 @pytest.mark.django_db +@mock.patch.object(cleanup_jobs.DeleteMeta, 'identify_excluded_partitions', mock.MagicMock()) +@mock.patch.object(cleanup_jobs.DeleteMeta, 'find_partitions_to_drop', mock.MagicMock()) +@mock.patch.object(cleanup_jobs.DeleteMeta, 'drop_partitions', mock.MagicMock()) def test_cleanup_jobs(setup_environment): (old_jobs, new_jobs, days_str) = setup_environment