AAP-37989 Tests for exclude list with multiple jobs (#15722)

* Tests for exclude list with multiple jobs

awx/main/tests/conftest.py

@@ -216,6 +216,16 @@ def mock_get_event_queryset_no_job_created():
 
 @pytest.fixture
 def mock_me():
+    "Allows Instance.objects.me() to work without touching the database"
     me_mock = mock.MagicMock(return_value=Instance(id=1, hostname=settings.CLUSTER_HOST_ID, uuid='00000000-0000-0000-0000-000000000000'))
     with mock.patch.object(Instance.objects, 'me', me_mock):
         yield
+
+
+@pytest.fixture
+def me_inst():
+    "Inserts an instance to the database for Instance.objects.me(), and goes ahead and mocks it in"
+    inst = Instance.objects.create(hostname='local_node', uuid='00000000-0000-0000-0000-000000000000')
+    me_mock = mock.MagicMock(return_value=inst)
+    with mock.patch.object(Instance.objects, 'me', me_mock):
+        yield inst

awx/main/tests/functional/test_tasks.py

@@ -1,5 +1,4 @@
 import pytest
-from unittest import mock
 import os
 import tempfile
 import shutil
@@ -27,16 +26,24 @@ def test_no_worker_info_on_AWX_nodes(node_type):
 
 
 @pytest.fixture
-def mock_job_folder(request):
+def job_folder_factory(request):
-    pdd_path = tempfile.mkdtemp(prefix='awx_123_')
+    def _rf(job_id='1234'):
+        pdd_path = tempfile.mkdtemp(prefix=f'awx_{job_id}_')
 
-    def test_folder_cleanup():
+        def test_folder_cleanup():
-        if os.path.exists(pdd_path):
+            if os.path.exists(pdd_path):
-            shutil.rmtree(pdd_path)
+                shutil.rmtree(pdd_path)
 
-    request.addfinalizer(test_folder_cleanup)
+        request.addfinalizer(test_folder_cleanup)
 
-    return pdd_path
+        return pdd_path
+
+    return _rf
+
+
+@pytest.fixture
+def mock_job_folder(job_folder_factory):
+    return job_folder_factory()
 
 
 @pytest.mark.django_db
@@ -49,17 +56,31 @@ def test_folder_cleanup_stale_file(mock_job_folder, mock_me):
 
 
 @pytest.mark.django_db
-def test_folder_cleanup_running_job(mock_job_folder, mock_me):
+def test_folder_cleanup_running_job(mock_job_folder, me_inst):
-    me_inst = Instance.objects.create(hostname='local_node', uuid='00000000-0000-0000-0000-000000000000')
+    job = Job.objects.create(id=1234, controller_node=me_inst.hostname, status='running')
-    with mock.patch.object(Instance.objects, 'me', return_value=me_inst):
+    _cleanup_images_and_files(grace_period=0)
-        job = Job.objects.create(id=123, controller_node=me_inst.hostname, status='running')
+    assert os.path.exists(mock_job_folder)  # running job should prevent folder from getting deleted
-        _cleanup_images_and_files(grace_period=0)
-        assert os.path.exists(mock_job_folder)  # running job should prevent folder from getting deleted
 
-        job.status = 'failed'
+    job.status = 'failed'
-        job.save(update_fields=['status'])
+    job.save(update_fields=['status'])
-        _cleanup_images_and_files(grace_period=0)
+    _cleanup_images_and_files(grace_period=0)
-        assert not os.path.exists(mock_job_folder)  # job is finished and no grace period, should delete
+    assert not os.path.exists(mock_job_folder)  # job is finished and no grace period, should delete
+
+
+@pytest.mark.django_db
+def test_folder_cleanup_multiple_running_jobs(job_folder_factory, me_inst):
+    jobs = []
+    dirs = []
+    num_jobs = 3
+
+    for i in range(num_jobs):
+        job = Job.objects.create(controller_node=me_inst.hostname, status='running')
+        dirs.append(job_folder_factory(job.id))
+        jobs.append(job)
+
+    _cleanup_images_and_files(grace_period=0)
+
+    assert [os.path.exists(d) for d in dirs] == [True for i in range(num_jobs)]
 
 
 @pytest.mark.django_db

awx/main/tests/live/tests/test_cleanup_task.py

@@ -0,0 +1,39 @@
+import os
+import tempfile
+import subprocess
+
+from awx.main.tasks.receptor import _convert_args_to_cli
+from awx.main.models import Instance, JobTemplate
+
+
+def test_folder_cleanup_multiple_running_jobs_execution_node(request):
+    demo_jt = JobTemplate.objects.get(name='Demo Job Template')
+
+    jobs = [demo_jt.create_unified_job(_eager_fields={'status': 'running'}) for i in range(3)]
+
+    def delete_jobs():
+        for job in jobs:
+            job.delete()
+
+    request.addfinalizer(delete_jobs)
+
+    job_dirs = []
+    job_patterns = []
+    for job in jobs:
+        job_pattern = f'awx_{job.id}_1234'
+        job_dir = os.path.join(tempfile.gettempdir(), job_pattern)
+        job_patterns.append(job_pattern)
+        job_dirs.append(job_dir)
+        os.mkdir(job_dir)
+
+    inst = Instance.objects.me()
+    runner_cleanup_kwargs = inst.get_cleanup_task_kwargs(exclude_strings=job_patterns, grace_period=0)
+
+    # We can not call worker_cleanup directly because execution and control nodes are not fungible
+    args = _convert_args_to_cli(runner_cleanup_kwargs)
+    remote_command = ' '.join(args)
+
+    subprocess.call('ansible-runner worker ' + remote_command, shell=True)
+    print('ansible-runner worker ' + remote_command)
+
+    assert [os.path.exists(job_dir) for job_dir in job_dirs] == [True for i in range(3)]