diff --git a/awx/main/tasks/jobs.py b/awx/main/tasks/jobs.py index 9d8a9dfef8..ee0e12f04e 100644 --- a/awx/main/tasks/jobs.py +++ b/awx/main/tasks/jobs.py @@ -9,6 +9,7 @@ import os from pathlib import Path import shutil import stat +import tarfile import yaml import tempfile import traceback @@ -1616,18 +1617,97 @@ class RunProjectUpdate(BaseTask): shutil.copytree(awx_playbooks, os.path.join(private_data_dir, 'project')) @staticmethod - def clear_project_cache(cache_dir, keep_value): + def clear_project_cache(cache_dir, keep_value, max_reqcache=10): if os.path.isdir(cache_dir): + # Read the current requirements hash from stage so we don't evict it + current_reqcache = None + stage_hash_file = os.path.join(cache_dir, 'stage', '.requirements_hash') + try: + if os.path.exists(stage_hash_file): + with open(stage_hash_file) as f: + req_hash = f.read().strip() + if req_hash: + current_reqcache = f'reqcache_{req_hash}' + except OSError: + pass + + reqcache_entries = [] for entry in os.listdir(cache_dir): old_path = os.path.join(cache_dir, entry) - if entry not in (keep_value, 'stage'): - # invalidate, then delete + if entry.startswith('reqcache_'): + reqcache_entries.append(old_path) + elif entry not in (keep_value, 'stage'): new_path = os.path.join(cache_dir, '.~~delete~~' + entry) try: os.rename(old_path, new_path) shutil.rmtree(new_path) except OSError: logger.warning(f"Could not remove cache directory {old_path}") + if len(reqcache_entries) > max_reqcache: + reqcache_entries.sort(key=lambda p: os.path.getmtime(p)) + for stale in reqcache_entries[: len(reqcache_entries) - max_reqcache]: + if current_reqcache and os.path.basename(stale) == current_reqcache: + continue + try: + shutil.rmtree(stale) + logger.debug(f'Evicted stale requirements cache {stale}') + except OSError: + logger.warning(f"Could not remove stale requirements cache {stale}") + + @staticmethod + def _sync_requirements_cache(base_path, cache_path): + """After a successful install, sync the requirements hash cache. + + If galaxy install ran (collections/roles exist in cache_path), save them + to a hash-keyed reqcache directory for reuse across commits. + If galaxy install was skipped (cache hit), the reqcache already has the content. + """ + hash_file = os.path.join(cache_path, '.requirements_hash') + if not os.path.exists(hash_file): + return + + try: + with open(hash_file) as f: + req_hash = f.read().strip() + except OSError: + return + + if not req_hash: + return + + reqcache_path = os.path.join(base_path, f'reqcache_{req_hash}') + + for subfolder in ('requirements_collections', 'requirements_roles'): + src = os.path.join(cache_path, subfolder) + dst = os.path.join(reqcache_path, subfolder) + if os.path.exists(src) and not os.path.exists(dst): + tmp_path = tempfile.mkdtemp(dir=base_path, prefix=f'.reqcache_{req_hash}_tmp_') + try: + tmp_dst = os.path.join(tmp_path, subfolder) + shutil.copytree(src, tmp_dst, symlinks=True) + os.makedirs(reqcache_path, exist_ok=True) + os.rename(tmp_dst, dst) + logger.debug(f'Saved {subfolder} to requirements cache {reqcache_path}') + except Exception: + logger.warning(f'Failed to save {subfolder} to requirements cache, cleaning up') + raise + finally: + shutil.rmtree(tmp_path, ignore_errors=True) + + # Create a cached tarball for faster per-job copies (AAP-96176) + tar_path = os.path.join(reqcache_path, f'{subfolder}.tar') + if os.path.exists(dst) and not os.path.exists(tar_path): + tmp_tar = tar_path + '.tmp' + try: + with tarfile.open(tmp_tar, 'w') as tar: + tar.add(dst, arcname=subfolder) + os.rename(tmp_tar, tar_path) + logger.debug(f'Created tarball cache {tar_path}') + except Exception: + logger.warning(f'Failed to create tarball cache {tar_path}') + if os.path.exists(tmp_tar): + os.remove(tmp_tar) + # Non-fatal: copytree fallback still works @staticmethod def make_local_copy(project, job_private_data_dir): @@ -1649,8 +1729,34 @@ class RunProjectUpdate(BaseTask): subfolders.append('requirements_roles') for subfolder in subfolders: cache_subpath = os.path.join(cache_path, subfolder) + if not os.path.exists(cache_subpath): + # Fall back to requirements-hash-keyed cache (reqcache) + hash_file = os.path.join(cache_path, '.requirements_hash') + if os.path.exists(hash_file): + try: + with open(hash_file) as f: + req_hash = f.read().strip() + if req_hash: + reqcache_subpath = os.path.join(project.get_cache_path(), f'reqcache_{req_hash}', subfolder) + if os.path.exists(reqcache_subpath): + cache_subpath = reqcache_subpath + logger.debug(f'Using requirements cache for {subfolder} (hash {req_hash[:12]})') + except OSError: + pass if os.path.exists(cache_subpath): dest_subpath = os.path.join(job_private_data_dir, subfolder) + # Prefer cached tarball for fewer file operations on SAN storage + tar_path = cache_subpath + '.tar' + if os.path.exists(tar_path): + try: + with tarfile.open(tar_path, 'r') as tar: + tar.extractall(path=job_private_data_dir, filter='data') + logger.debug('{0} {1} prepared {2} from tarball'.format(type(project).__name__, project.pk, dest_subpath)) + continue + except Exception: + logger.warning(f'Failed to extract tarball {tar_path}, falling back to copytree') + if os.path.exists(dest_subpath): + shutil.rmtree(dest_subpath) shutil.copytree(cache_subpath, dest_subpath, symlinks=True) logger.debug('{0} {1} prepared {2} from cache'.format(type(project).__name__, project.pk, dest_subpath)) @@ -1685,6 +1791,8 @@ class RunProjectUpdate(BaseTask): shutil.rmtree(cache_path) os.rename(stage_path, cache_path) logger.debug('{0} wrote to cache at {1}'.format(instance.log_format, cache_path)) + + self._sync_requirements_cache(base_path, cache_path) elif os.path.exists(stage_path): shutil.rmtree(stage_path) # cannot trust content update produced diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 278f7c3437..251f7ffaec 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import json import os +import tarfile from pathlib import Path import fcntl @@ -1767,6 +1768,442 @@ def test_sync_and_copy_get_sync_needs_called_under_shared_lock(get_sync_needs, s assert call_order[1] == 'get_sync_needs' +class TestClearProjectCachePreservesReqcache: + def test_reqcache_entries_preserved(self, tmp_path): + cache_dir = tmp_path / 'cache' + cache_dir.mkdir() + (cache_dir / 'abc123').mkdir() + (cache_dir / 'reqcache_deadbeef').mkdir() + (cache_dir / 'reqcache_cafebabe').mkdir() + (cache_dir / 'old_revision').mkdir() + (cache_dir / 'stage').mkdir() + + jobs.RunProjectUpdate.clear_project_cache(str(cache_dir), 'abc123') + + remaining = set(os.listdir(cache_dir)) + assert 'abc123' in remaining + assert 'reqcache_deadbeef' in remaining + assert 'reqcache_cafebabe' in remaining + assert 'stage' in remaining + assert 'old_revision' not in remaining + + def test_no_reqcache_entries(self, tmp_path): + cache_dir = tmp_path / 'cache' + cache_dir.mkdir() + (cache_dir / 'keep_me').mkdir() + (cache_dir / 'delete_me').mkdir() + + jobs.RunProjectUpdate.clear_project_cache(str(cache_dir), 'keep_me') + + remaining = set(os.listdir(cache_dir)) + assert 'keep_me' in remaining + assert 'delete_me' not in remaining + + def test_evicts_oldest_reqcache_when_over_limit(self, tmp_path): + cache_dir = tmp_path / 'cache' + cache_dir.mkdir() + (cache_dir / 'current').mkdir() + + for i in range(12): + entry = cache_dir / f'reqcache_hash{i:02d}' + entry.mkdir() + os.utime(str(entry), (1000 + i, 1000 + i)) + + jobs.RunProjectUpdate.clear_project_cache(str(cache_dir), 'current', max_reqcache=10) + + remaining = set(os.listdir(cache_dir)) + assert 'reqcache_hash00' not in remaining + assert 'reqcache_hash01' not in remaining + assert 'reqcache_hash02' in remaining + assert 'reqcache_hash11' in remaining + assert len([e for e in remaining if e.startswith('reqcache_')]) == 10 + + def test_eviction_protects_current_reqcache(self, tmp_path): + cache_dir = tmp_path / 'cache' + cache_dir.mkdir() + (cache_dir / 'current').mkdir() + + stage_dir = cache_dir / 'stage' + stage_dir.mkdir() + (stage_dir / '.requirements_hash').write_text('hash00') + + for i in range(12): + entry = cache_dir / f'reqcache_hash{i:02d}' + entry.mkdir() + os.utime(str(entry), (1000 + i, 1000 + i)) + + jobs.RunProjectUpdate.clear_project_cache(str(cache_dir), 'current', max_reqcache=10) + + remaining = set(os.listdir(cache_dir)) + # hash00 is the oldest but should be protected (current update's hash) + assert 'reqcache_hash00' in remaining + # hash01 is the next oldest and should be evicted + assert 'reqcache_hash01' not in remaining + # hash02+ should survive (only 1 non-protected entry evicted) + assert 'reqcache_hash02' in remaining + assert 'reqcache_hash11' in remaining + + +class TestSyncRequirementsCache: + def test_saves_collections_to_reqcache(self, tmp_path): + base_path = tmp_path / 'cache' + base_path.mkdir() + cache_path = base_path / 'abc123' + cache_path.mkdir() + + # Simulate a galaxy install that produced collections + collections_dir = cache_path / 'requirements_collections' + collections_dir.mkdir() + (collections_dir / 'ansible_collections').mkdir() + (collections_dir / 'ansible_collections' / 'test.txt').write_text('test') + + # Write the hash marker + (cache_path / '.requirements_hash').write_text('deadbeef1234') + + jobs.RunProjectUpdate._sync_requirements_cache(str(base_path), str(cache_path)) + + reqcache = base_path / 'reqcache_deadbeef1234' / 'requirements_collections' + assert reqcache.exists() + assert (reqcache / 'ansible_collections' / 'test.txt').read_text() == 'test' + + def test_skips_when_no_hash_file(self, tmp_path): + base_path = tmp_path / 'cache' + base_path.mkdir() + cache_path = base_path / 'abc123' + cache_path.mkdir() + + jobs.RunProjectUpdate._sync_requirements_cache(str(base_path), str(cache_path)) + + entries = os.listdir(base_path) + assert not any(e.startswith('reqcache_') for e in entries) + + def test_no_partial_cache_on_copytree_failure(self, tmp_path): + base_path = tmp_path / 'cache' + base_path.mkdir() + cache_path = base_path / 'abc123' + cache_path.mkdir() + + collections_dir = cache_path / 'requirements_collections' + collections_dir.mkdir() + (collections_dir / 'test.txt').write_text('test') + (cache_path / '.requirements_hash').write_text('deadbeef1234') + + with mock.patch('shutil.copytree', side_effect=OSError('disk full')): + try: + jobs.RunProjectUpdate._sync_requirements_cache(str(base_path), str(cache_path)) + except OSError: + pass + + reqcache = base_path / 'reqcache_deadbeef1234' + assert not reqcache.exists() + tmp_dirs = [e for e in os.listdir(base_path) if e.startswith('.reqcache_')] + assert len(tmp_dirs) == 0 + + def test_creates_tarball_after_sync(self, tmp_path): + base_path = tmp_path / 'cache' + base_path.mkdir() + cache_path = base_path / 'abc123' + cache_path.mkdir() + + collections_dir = cache_path / 'requirements_collections' + collections_dir.mkdir() + (collections_dir / 'ansible_collections').mkdir() + (collections_dir / 'ansible_collections' / 'test.txt').write_text('test') + (cache_path / '.requirements_hash').write_text('deadbeef1234') + + jobs.RunProjectUpdate._sync_requirements_cache(str(base_path), str(cache_path)) + + tar_path = base_path / 'reqcache_deadbeef1234' / 'requirements_collections.tar' + assert tar_path.exists() + with tarfile.open(str(tar_path), 'r') as tar: + names = tar.getnames() + assert 'requirements_collections/ansible_collections/test.txt' in names + + def test_skips_when_reqcache_already_exists(self, tmp_path): + base_path = tmp_path / 'cache' + base_path.mkdir() + cache_path = base_path / 'abc123' + cache_path.mkdir() + + # Pre-existing reqcache + reqcache = base_path / 'reqcache_deadbeef1234' / 'requirements_collections' + reqcache.mkdir(parents=True) + (reqcache / 'existing.txt').write_text('original') + + # New cache with same hash but different content + collections_dir = cache_path / 'requirements_collections' + collections_dir.mkdir() + (collections_dir / 'new.txt').write_text('new') + (cache_path / '.requirements_hash').write_text('deadbeef1234') + + jobs.RunProjectUpdate._sync_requirements_cache(str(base_path), str(cache_path)) + + # Should not overwrite existing reqcache + assert (reqcache / 'existing.txt').read_text() == 'original' + assert not (reqcache / 'new.txt').exists() + + +class TestMakeLocalCopyReqcacheFallback: + def test_falls_back_to_reqcache(self, tmp_path): + cache_base = tmp_path / 'cache' + cache_base.mkdir() + + # Create cache_id directory with hash pointer but no collections + cache_id_dir = cache_base / 'abc123' + cache_id_dir.mkdir() + (cache_id_dir / '.requirements_hash').write_text('deadbeef1234') + + # Create reqcache with collections + reqcache_collections = cache_base / 'reqcache_deadbeef1234' / 'requirements_collections' + reqcache_collections.mkdir(parents=True) + (reqcache_collections / 'community').mkdir() + (reqcache_collections / 'community' / 'general.txt').write_text('collection content') + + # Create project path + project_path = tmp_path / 'project' + project_path.mkdir() + (project_path / 'playbook.yml').write_text('- hosts: all') + + # Create job private data dir + job_dir = tmp_path / 'job' + job_dir.mkdir() + + project = mock.Mock() + project.get_project_path.return_value = str(project_path) + project.get_cache_path.return_value = str(cache_base) + project.cache_id = 'abc123' + project.pk = 1 + + with mock.patch('awx.main.tasks.jobs.settings') as mock_settings: + mock_settings.AWX_COLLECTIONS_ENABLED = True + mock_settings.AWX_ROLES_ENABLED = False + jobs.RunProjectUpdate.make_local_copy(project, str(job_dir)) + + # Collections should be copied from reqcache + dest = job_dir / 'requirements_collections' / 'community' / 'general.txt' + assert dest.exists() + assert dest.read_text() == 'collection content' + + def test_uses_tarball_from_reqcache(self, tmp_path): + cache_base = tmp_path / 'cache' + cache_base.mkdir() + + cache_id_dir = cache_base / 'abc123' + cache_id_dir.mkdir() + (cache_id_dir / '.requirements_hash').write_text('deadbeef1234') + + # Create reqcache with collections directory AND tarball + reqcache = cache_base / 'reqcache_deadbeef1234' + reqcache.mkdir() + reqcache_collections = reqcache / 'requirements_collections' + reqcache_collections.mkdir() + (reqcache_collections / 'community').mkdir() + (reqcache_collections / 'community' / 'general.txt').write_text('collection content') + + tar_path = reqcache / 'requirements_collections.tar' + with tarfile.open(str(tar_path), 'w') as tar: + tar.add(str(reqcache_collections), arcname='requirements_collections') + + project_path = tmp_path / 'project' + project_path.mkdir() + (project_path / 'playbook.yml').write_text('- hosts: all') + + job_dir = tmp_path / 'job' + job_dir.mkdir() + + project = mock.Mock() + project.get_project_path.return_value = str(project_path) + project.get_cache_path.return_value = str(cache_base) + project.cache_id = 'abc123' + project.pk = 1 + + with mock.patch('awx.main.tasks.jobs.settings') as mock_settings: + mock_settings.AWX_COLLECTIONS_ENABLED = True + mock_settings.AWX_ROLES_ENABLED = False + jobs.RunProjectUpdate.make_local_copy(project, str(job_dir)) + + dest = job_dir / 'requirements_collections' / 'community' / 'general.txt' + assert dest.exists() + assert dest.read_text() == 'collection content' + + def test_direct_cache_ignores_reqcache_tarball(self, tmp_path): + """Direct cache hit must use copytree, not a reqcache tarball that could have stale content.""" + cache_base = tmp_path / 'cache' + cache_base.mkdir() + + cache_id_dir = cache_base / 'abc123' + cache_id_dir.mkdir() + direct_collections = cache_id_dir / 'requirements_collections' + direct_collections.mkdir() + (direct_collections / 'direct.txt').write_text('direct content') + (cache_id_dir / '.requirements_hash').write_text('deadbeef1234') + + # Reqcache tarball with DIFFERENT content — must not be used + reqcache = cache_base / 'reqcache_deadbeef1234' + reqcache.mkdir() + stale_dir = tmp_path / '_stale' + stale_dir.mkdir() + stale_collections = stale_dir / 'requirements_collections' + stale_collections.mkdir() + (stale_collections / 'direct.txt').write_text('stale tarball content') + tar_path = reqcache / 'requirements_collections.tar' + with tarfile.open(str(tar_path), 'w') as tar: + tar.add(str(stale_collections), arcname='requirements_collections') + + project_path = tmp_path / 'project' + project_path.mkdir() + (project_path / 'playbook.yml').write_text('- hosts: all') + + job_dir = tmp_path / 'job' + job_dir.mkdir() + + project = mock.Mock() + project.get_project_path.return_value = str(project_path) + project.get_cache_path.return_value = str(cache_base) + project.cache_id = 'abc123' + project.pk = 1 + + with mock.patch('awx.main.tasks.jobs.settings') as mock_settings: + mock_settings.AWX_COLLECTIONS_ENABLED = True + mock_settings.AWX_ROLES_ENABLED = False + jobs.RunProjectUpdate.make_local_copy(project, str(job_dir)) + + dest = job_dir / 'requirements_collections' / 'direct.txt' + assert dest.exists() + assert dest.read_text() == 'direct content' + + def test_tarball_partial_extract_falls_back_to_copytree(self, tmp_path): + """If tarball extraction partially writes then fails, copytree fallback must still work.""" + cache_base = tmp_path / 'cache' + cache_base.mkdir() + + cache_id_dir = cache_base / 'abc123' + cache_id_dir.mkdir() + (cache_id_dir / '.requirements_hash').write_text('deadbeef1234') + + reqcache = cache_base / 'reqcache_deadbeef1234' + reqcache.mkdir() + reqcache_collections = reqcache / 'requirements_collections' + reqcache_collections.mkdir() + (reqcache_collections / 'community').mkdir() + (reqcache_collections / 'community' / 'general.txt').write_text('collection content') + + # Create a valid tarball that we'll make fail during extraction + tar_path = reqcache / 'requirements_collections.tar' + with tarfile.open(str(tar_path), 'w') as tar: + tar.add(str(reqcache_collections), arcname='requirements_collections') + + project_path = tmp_path / 'project' + project_path.mkdir() + (project_path / 'playbook.yml').write_text('- hosts: all') + + job_dir = tmp_path / 'job' + job_dir.mkdir() + + # Simulate partial extraction: create dest dir to mimic what a failed extractall leaves behind + partial_dest = job_dir / 'requirements_collections' + partial_dest.mkdir() + + project = mock.Mock() + project.get_project_path.return_value = str(project_path) + project.get_cache_path.return_value = str(cache_base) + project.cache_id = 'abc123' + project.pk = 1 + + original_open = tarfile.open + + def failing_tarfile_open(*args, **kwargs): + tar = original_open(*args, **kwargs) + + def raise_after_partial(*a, **kw): + raise OSError("Simulated extraction failure") + + tar.extractall = raise_after_partial + return tar + + with mock.patch('awx.main.tasks.jobs.settings') as mock_settings: + mock_settings.AWX_COLLECTIONS_ENABLED = True + mock_settings.AWX_ROLES_ENABLED = False + with mock.patch('awx.main.tasks.jobs.tarfile.open', side_effect=failing_tarfile_open): + jobs.RunProjectUpdate.make_local_copy(project, str(job_dir)) + + dest = job_dir / 'requirements_collections' / 'community' / 'general.txt' + assert dest.exists() + assert dest.read_text() == 'collection content' + + def test_falls_back_to_copytree_when_no_tarball(self, tmp_path): + cache_base = tmp_path / 'cache' + cache_base.mkdir() + + cache_id_dir = cache_base / 'abc123' + cache_id_dir.mkdir() + (cache_id_dir / '.requirements_hash').write_text('deadbeef1234') + + # Reqcache with directory but NO tarball + reqcache_collections = cache_base / 'reqcache_deadbeef1234' / 'requirements_collections' + reqcache_collections.mkdir(parents=True) + (reqcache_collections / 'community').mkdir() + (reqcache_collections / 'community' / 'general.txt').write_text('collection content') + + project_path = tmp_path / 'project' + project_path.mkdir() + (project_path / 'playbook.yml').write_text('- hosts: all') + + job_dir = tmp_path / 'job' + job_dir.mkdir() + + project = mock.Mock() + project.get_project_path.return_value = str(project_path) + project.get_cache_path.return_value = str(cache_base) + project.cache_id = 'abc123' + project.pk = 1 + + with mock.patch('awx.main.tasks.jobs.settings') as mock_settings: + mock_settings.AWX_COLLECTIONS_ENABLED = True + mock_settings.AWX_ROLES_ENABLED = False + jobs.RunProjectUpdate.make_local_copy(project, str(job_dir)) + + # Should still work via copytree fallback + dest = job_dir / 'requirements_collections' / 'community' / 'general.txt' + assert dest.exists() + assert dest.read_text() == 'collection content' + + def test_uses_direct_cache_when_available(self, tmp_path): + cache_base = tmp_path / 'cache' + cache_base.mkdir() + + # Create cache_id directory with collections directly present + cache_id_dir = cache_base / 'abc123' + cache_id_dir.mkdir() + direct_collections = cache_id_dir / 'requirements_collections' + direct_collections.mkdir() + (direct_collections / 'direct.txt').write_text('direct content') + + # Create project path + project_path = tmp_path / 'project' + project_path.mkdir() + (project_path / 'playbook.yml').write_text('- hosts: all') + + job_dir = tmp_path / 'job' + job_dir.mkdir() + + project = mock.Mock() + project.get_project_path.return_value = str(project_path) + project.get_cache_path.return_value = str(cache_base) + project.cache_id = 'abc123' + project.pk = 1 + + with mock.patch('awx.main.tasks.jobs.settings') as mock_settings: + mock_settings.AWX_COLLECTIONS_ENABLED = True + mock_settings.AWX_ROLES_ENABLED = False + jobs.RunProjectUpdate.make_local_copy(project, str(job_dir)) + + # Should use direct cache, not reqcache + dest = job_dir / 'requirements_collections' / 'direct.txt' + assert dest.exists() + assert dest.read_text() == 'direct content' + + @pytest.mark.parametrize('injector_cls', [cls for cls in ManagedCredentialType.registry.values() if cls.injectors]) def test_managed_injector_redaction(injector_cls): """See awx.main.models.inventory.PluginFileInjector._get_shared_env diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml index 40354f2d6a..36b34c2c3f 100644 --- a/awx/playbooks/project_update.yml +++ b/awx/playbooks/project_update.yml @@ -198,14 +198,15 @@ name: Install content with ansible-galaxy command if necessary vars: galaxy_task_env: # configured in settings + awx_cache_path: "{{ projects_root }}/.__awx_cache/{{ local_path }}" # additional_galaxy_env contains environment variables are used for installing roles and collections and will take precedence over items in galaxy_task_env additional_galaxy_env: # These paths control where ansible-galaxy installs collections and roles on top the filesystem - ANSIBLE_COLLECTIONS_PATH: "{{ projects_root }}/.__awx_cache/{{ local_path }}/stage/requirements_collections" - ANSIBLE_ROLES_PATH: "{{ projects_root }}/.__awx_cache/{{ local_path }}/stage/requirements_roles" + ANSIBLE_COLLECTIONS_PATH: "{{ awx_cache_path }}/stage/requirements_collections" + ANSIBLE_ROLES_PATH: "{{ awx_cache_path }}/stage/requirements_roles" # Put the local tmp directory in same volume as collection destination # otherwise, files cannot be moved accross volumes and will cause error - ANSIBLE_LOCAL_TEMP: "{{ projects_root }}/.__awx_cache/{{ local_path }}/stage/tmp" + ANSIBLE_LOCAL_TEMP: "{{ awx_cache_path }}/stage/tmp" tasks: - name: Check content sync settings when: not roles_enabled | bool and not collections_enabled | bool @@ -221,6 +222,49 @@ - name: End play due to disabled content sync ansible.builtin.meta: end_play + - name: Compute requirements hash for cache reuse + tags: + - install_roles + - install_collections + block: + - name: Find all requirements files + ansible.builtin.stat: + path: "{{ item }}" + checksum_algorithm: sha256 + get_attributes: false + get_mime: false + register: req_files_stat + loop: + - "{{ project_path | quote }}/collections/requirements.yml" + - "{{ project_path | quote }}/collections/requirements.yaml" + - "{{ project_path | quote }}/roles/requirements.yml" + - "{{ project_path | quote }}/roles/requirements.yaml" + - "{{ project_path | quote }}/requirements.yml" + - "{{ project_path | quote }}/requirements.yaml" + + - name: Build combined requirements hash + ansible.builtin.set_fact: + requirements_hash: "{{ _existing | map(attribute='item') | zip(_existing | map(attribute='stat.checksum')) | list | to_json | hash('sha256') }}" + has_requirements: "{{ _existing | length > 0 }}" + vars: + _existing: "{{ req_files_stat.results | selectattr('stat.exists', 'defined') | selectattr('stat.exists') | list }}" + + - name: Check requirements hash cache + ansible.builtin.stat: + path: "{{ awx_cache_path }}/reqcache_{{ requirements_hash }}" + register: reqcache_stat + when: has_requirements | bool + + - name: Write requirements hash marker for post-run cache management + ansible.builtin.copy: + content: "{{ requirements_hash }}" + dest: "{{ awx_cache_path }}/stage/.requirements_hash" + when: has_requirements | bool + + - name: Set cache hit fact + ansible.builtin.set_fact: + requirements_cache_hit: "{{ has_requirements | bool and (reqcache_stat.stat.exists | default(false)) }}" + - block: - name: Fetch galaxy roles from roles/requirements.(yml/yaml) ansible.builtin.command: @@ -237,6 +281,7 @@ when: - roles_enabled | bool - req_file | length > 0 + - not (requirements_cache_hit | default(false) | bool) tags: - install_roles @@ -255,6 +300,7 @@ when: - collections_enabled | bool - req_file | length > 0 + - not (requirements_cache_hit | default(false) | bool) tags: - install_collections @@ -275,6 +321,7 @@ - collections_enabled | bool - roles_enabled | bool - req_file | length > 0 + - not (requirements_cache_hit | default(false) | bool) tags: - install_collections - install_roles