Integrate content caching with existing task logic

Revert the --force flags

use the update id as metric for role caching

Shift the movement of cache to job folder from rsync task to python

Only install roles and collections if needed

Deal with roles and collections for jobs without sync
Skip local copy if roles or collections turned off

update docs for content caching

Design pivot - use empty cache dir to indicate lack of content

Do not cache content if we did not install content

Test changes to allay concerns about reliability of local_path

Do not blow away cache for SCM inventory updates

awx/playbooks/project_update.yml

@@ -1,7 +1,9 @@
 ---
 # The following variables will be set by the runner of this playbook:
-# project_path: PROJECTS_DIR/_local_path_
-# project_cache: CACHE_DIR/.__awx_cache/_local_path_
+# projects_root: Global location for caching project checkouts and roles and collections
+#     should not have trailing slash on end
+# local_path: Path within projects_root to use for this project
+# project_path: projects_root/local_path
 # scm_url: https://server/repo
 # insights_url: Insights service URL (from configuration)
 # scm_branch: branch/tag/revision (HEAD if unset)
@@ -30,13 +32,6 @@
       tags:
         - delete
 
-    - name: delete project cache directory before update
-      file:
-        path: "{{project_cache|quote}}"
-        state: absent
-      tags:
-        - delete
-
     - block:
         - name: update project using git
           git:
@@ -117,12 +112,11 @@
         - update_svn
         - update_insights
 
-    - name: Set content cache location
-      set_fact:
-        cache_dir: "{{ project_cache }}/{{ scm_version|default(scm_branch) }}"
-      tags:
-        - install_collections
-        - install_roles
+- hosts: localhost
+  gather_facts: false
+  connection: local
+  name: Install content with ansible-galaxy command if necessary
+  tasks:
 
     - block:
         - name: detect requirements.yml
@@ -131,7 +125,10 @@
           register: doesRequirementsExist
 
         - name: fetch galaxy roles from requirements.yml
-          command: ansible-galaxy role install {{ '--force' if roles_destination is not defined else '' }} -r roles/requirements.yml -p {{ cache_dir }}/requirements_roles {{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
+          command: >
+            ansible-galaxy role install -r roles/requirements.yml
+            --roles-path {{projects_root}}/.__awx_cache/{{local_path}}/stage/requirements_roles
+            {{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
           args:
             chdir: "{{project_path|quote}}"
           register: galaxy_result
@@ -141,12 +138,6 @@
             ANSIBLE_FORCE_COLOR: false
             GIT_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no"
 
-        - name: populate job directory with needed roles
-          synchronize:
-            src: "{{ cache_dir }}/requirements_roles/"
-            dest: "{{ roles_destination }}"
-          when: roles_destination is defined and doesRequirementsExist.stat.exists
-
       when: roles_enabled|bool
       tags:
         - install_roles
@@ -158,7 +149,10 @@
           register: doesCollectionRequirementsExist
 
         - name: fetch galaxy collections from collections/requirements.yml
-          command: ansible-galaxy collection install {{ '--force' if collections_destination is not defined else '' }} -r collections/requirements.yml -p {{ cache_dir }}/requirements_collections {{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
+          command: >
+            ansible-galaxy collection install -r collections/requirements.yml
+            --collections-path {{projects_root}}/.__awx_cache/{{local_path}}/stage/requirements_collections
+            {{ ' -' + 'v' * ansible_verbosity if ansible_verbosity else '' }}
           args:
             chdir: "{{project_path|quote}}"
           register: galaxy_collection_result
@@ -166,15 +160,9 @@
           changed_when: "'Installing ' in galaxy_collection_result.stdout"
           environment:
             ANSIBLE_FORCE_COLOR: false
-            ANSIBLE_COLLECTIONS_PATHS: "{{ cache_dir }}/requirements_collections"
+            ANSIBLE_COLLECTIONS_PATHS: "{{projects_root}}/.__awx_cache/{{local_path}}/stage/requirements_collections"
             GIT_SSH_COMMAND: "ssh -o StrictHostKeyChecking=no"
 
-        - name: populate job directory with needed collections
-          synchronize:
-            src: "{{ cache_dir }}/requirements_collections/"
-            dest: "{{ collections_destination }}"
-          when: collections_destination is defined and doesCollectionRequirementsExist.stat.exists
-
       when:
         - "ansible_version.full is version_compare('2.8', '>=')"
         - collections_enabled|bool