Use tags to reduce project update output

Handle folder deletion as tag

remove -v use by default

Change meaning of roles_enabled playbook var to
 value of AWX global setting

awx/api/serializers.py

@@ -1472,7 +1472,7 @@ class ProjectUpdateSerializer(UnifiedJobSerializer, ProjectOptionsSerializer):
 
     class Meta:
         model = ProjectUpdate
-        fields = ('*', 'project', 'job_type', '-controller_node')
+        fields = ('*', 'project', 'job_type', 'job_tags', '-controller_node')
 
     def get_related(self, obj):
         res = super(ProjectUpdateSerializer, self).get_related(obj)

awx/main/migrations/0099_v370_projectupdate_job_tags.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.4 on 2019-11-01 18:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0098_v360_rename_cyberark_aim_credential_type'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='projectupdate',
+            name='job_tags',
+            field=models.CharField(blank=True, default='', help_text='Parts of the project update playbook that will be run.', max_length=1024),
+        ),
+    ]

awx/main/models/projects.py

@@ -483,6 +483,12 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
         choices=PROJECT_UPDATE_JOB_TYPE_CHOICES,
         default='check',
     )
+    job_tags = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        help_text=_('Parts of the project update playbook that will be run.'),
+    )
     scm_revision = models.CharField(
         max_length=1024,
         blank=True,
@@ -587,3 +593,24 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
         if not selected_groups:
             return self.global_instance_groups
         return selected_groups
+
+    def save(self, *args, **kwargs):
+        added_update_fields = []
+        if not self.job_tags:
+            job_tags = ['update_{}'.format(self.scm_type)]
+            if self.job_type == 'run':
+                job_tags.append('install_roles')
+                job_tags.append('install_collections')
+            self.job_tags = ','.join(job_tags)
+            added_update_fields.append('job_tags')
+        if self.scm_delete_on_update and 'delete' not in self.job_tags and self.job_type == 'check':
+            self.job_tags = ','.join([self.job_tags, 'delete'])
+            added_update_fields.append('job_tags')
+        elif (not self.scm_delete_on_update) and 'delete' in self.job_tags:
+            job_tags = self.job_tags.split(',')
+            job_tags.remove('delete')
+            self.job_tags = ','.join(job_tags)
+            added_update_fields.append('job_tags')
+        if 'update_fields' in kwargs:
+            kwargs['update_fields'].extend(added_update_fields)
+        return super(ProjectUpdate, self).save(*args, **kwargs)

awx/main/tasks.py

@@ -71,7 +71,7 @@ from awx.main.utils import (get_ssh_version, update_scm_url,
                             ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager,
                             get_awx_version)
 from awx.main.utils.ansible import read_ansible_config
-from awx.main.utils.common import get_ansible_version, _get_ansible_version, get_custom_venv_choices
+from awx.main.utils.common import _get_ansible_version, get_custom_venv_choices
 from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja
 from awx.main.utils.reload import stop_local_services
 from awx.main.utils.pglock import advisory_lock
@@ -1734,14 +1734,16 @@ class RunJob(BaseTask):
 
         project_path = job.project.get_project_path(check_if_exists=False)
         job_revision = job.project.scm_revision
-        needs_sync = True
+        sync_needs = []
+        all_sync_needs = ['update_{}'.format(job.project.scm_type), 'install_roles', 'install_collections']
         if not job.project.scm_type:
-            # manual projects are not synced, user has responsibility for that
-            needs_sync = False
+            pass # manual projects are not synced, user has responsibility for that
         elif not os.path.exists(project_path):
             logger.debug('Performing fresh clone of {} on this instance.'.format(job.project))
+            sync_needs = all_sync_needs
         elif not job.project.scm_revision:
             logger.debug('Revision not known for {}, will sync with remote'.format(job.project))
+            sync_needs = all_sync_needs
         elif job.project.scm_type == 'git':
             git_repo = git.Repo(project_path)
             try:
@@ -1752,23 +1754,27 @@ class RunJob(BaseTask):
                 if desired_revision == current_revision:
                     job_revision = desired_revision
                     logger.info('Skipping project sync for {} because commit is locally available'.format(job.log_format))
-                    needs_sync = False
+                else:
+                    sync_needs = all_sync_needs
             except (ValueError, BadGitName):
                 logger.debug('Needed commit for {} not in local source tree, will sync with remote'.format(job.log_format))
+                sync_needs = all_sync_needs
+        else:
+            sync_needs = all_sync_needs
         # Galaxy requirements are not supported for manual projects
-        if not needs_sync and job.project.scm_type:
+        if not sync_needs and job.project.scm_type:
             # see if we need a sync because of presence of roles
             galaxy_req_path = os.path.join(project_path, 'roles', 'requirements.yml')
             if os.path.exists(galaxy_req_path):
                 logger.debug('Running project sync for {} because of galaxy role requirements.'.format(job.log_format))
-                needs_sync = True
+                sync_needs.append('install_roles')
 
             galaxy_collections_req_path = os.path.join(project_path, 'collections', 'requirements.yml')
             if os.path.exists(galaxy_collections_req_path):
                 logger.debug('Running project sync for {} because of galaxy collections requirements.'.format(job.log_format))
-                needs_sync = True
+                sync_needs.append('install_collections')
 
-        if needs_sync:
+        if sync_needs:
             pu_ig = job.instance_group
             pu_en = job.execution_node
             if job.is_isolated() is True:
@@ -1778,6 +1784,7 @@ class RunJob(BaseTask):
             sync_metafields = dict(
                 launch_type="sync",
                 job_type='run',
+                job_tags=','.join(sync_needs),
                 status='running',
                 instance_group = pu_ig,
                 execution_node=pu_en,
@@ -1785,6 +1792,8 @@ class RunJob(BaseTask):
             )
             if job.scm_branch and job.scm_branch != job.project.scm_branch:
                 sync_metafields['scm_branch'] = job.scm_branch
+            if 'update_' not in sync_metafields['job_tags']:
+                sync_metafields['scm_revision'] = job_revision
             local_project_sync = job.project.create_project_update(_eager_fields=sync_metafields)
             # save the associated job before calling run() so that a
             # cancel() call on the job can cancel the project update
@@ -2008,8 +2017,8 @@ class RunProjectUpdate(BaseTask):
         args = []
         if getattr(settings, 'PROJECT_UPDATE_VVV', False):
             args.append('-vvv')
-        else:
-            args.append('-v')
+        if project_update.job_tags:
+            args.extend(['-t', project_update.job_tags])
         return args
 
     def build_extra_vars_file(self, project_update, private_data_dir):
@@ -2023,28 +2032,16 @@ class RunProjectUpdate(BaseTask):
             scm_branch = project_update.project.scm_revision
         elif not scm_branch:
             scm_branch = {'hg': 'tip'}.get(project_update.scm_type, 'HEAD')
-        if project_update.job_type == 'check':
-            roles_enabled = False
-            collections_enabled = False
-        else:
-            roles_enabled = getattr(settings, 'AWX_ROLES_ENABLED', True)
-            collections_enabled = getattr(settings, 'AWX_COLLECTIONS_ENABLED', True)
-            # collections were introduced in Ansible version 2.8
-            if Version(get_ansible_version()) <= Version('2.8'):
-                collections_enabled = False
         extra_vars.update({
             'project_path': project_update.get_project_path(check_if_exists=False),
             'insights_url': settings.INSIGHTS_URL_BASE,
             'awx_license_type': get_license(show_key=False).get('license_type', 'UNLICENSED'),
             'awx_version': get_awx_version(),
-            'scm_type': project_update.scm_type,
             'scm_url': scm_url,
             'scm_branch': scm_branch,
             'scm_clean': project_update.scm_clean,
-            'scm_delete_on_update': project_update.scm_delete_on_update if project_update.job_type == 'check' else False,
-            'scm_full_checkout': True if project_update.job_type == 'run' else False,
-            'roles_enabled': roles_enabled,
-            'collections_enabled': collections_enabled,
+            'roles_enabled': settings.AWX_ROLES_ENABLED,
+            'collections_enabled': settings.AWX_COLLECTIONS_ENABLED,
         })
         if project_update.job_type != 'check' and self.job_private_data_dir:
             extra_vars['collections_destination'] = os.path.join(self.job_private_data_dir, 'requirements_collections')
@@ -2217,12 +2214,15 @@ class RunProjectUpdate(BaseTask):
             copy_tree(project_path, destination_folder)
 
     def post_run_hook(self, instance, status):
+        if self.playbook_new_revision:
+            instance.scm_revision = self.playbook_new_revision
+            instance.save(update_fields=['scm_revision'])
         if self.job_private_data_dir:
             # copy project folder before resetting to default branch
             # because some git-tree-specific resources (like submodules) might matter
             self.make_local_copy(
                 instance.get_project_path(check_if_exists=False), os.path.join(self.job_private_data_dir, 'project'),
-                instance.scm_type, self.playbook_new_revision
+                instance.scm_type, instance.scm_revision
             )
             if self.original_branch:
                 # for git project syncs, non-default branches can be problems
@@ -2234,9 +2234,6 @@ class RunProjectUpdate(BaseTask):
                     logger.exception('Failed to restore project repo to prior state after {}'.format(instance.log_format))
         self.release_lock(instance)
         p = instance.project
-        if self.playbook_new_revision:
-            instance.scm_revision = self.playbook_new_revision
-            instance.save(update_fields=['scm_revision'])
         if instance.job_type == 'check' and status not in ('failed', 'canceled',):
             if self.playbook_new_revision:
                 p.scm_revision = self.playbook_new_revision
@@ -2497,6 +2494,7 @@ class RunInventoryUpdate(BaseTask):
                 _eager_fields=dict(
                     launch_type="sync",
                     job_type='run',
+                    job_tags='update_{},install_collections'.format(source_project.scm_type),  # roles are never valid for inventory
                     status='running',
                     execution_node=inventory_update.execution_node,
                     instance_group = inventory_update.instance_group,

awx/playbooks/project_update.yml

@@ -1,32 +1,33 @@
 ---
 # The following variables will be set by the runner of this playbook:
 # project_path: PROJECTS_DIR/_local_path_
-# scm_type: git|hg|svn|insights
 # scm_url: https://server/repo
 # insights_url: Insights service URL (from configuration)
 # scm_branch: branch/tag/revision (HEAD if unset)
 # scm_clean: true/false
-# scm_delete_on_update: true/false
-# scm_full_checkout: true (if for a job template run), false (if retrieving revision)
 # scm_username: username (only for svn/insights)
 # scm_password: password (only for svn/insights)
 # scm_accept_hostkey: true/false (only for git, set automatically)
 # scm_refspec: a refspec to fetch in addition to obtaining version
-# roles_enabled: Allow us to pull roles from a requirements.yml file
+# roles_enabled: Value of the global setting to enable roles downloading
+# collections_enabled: Value of the global setting to enable collections downloading
 # roles_destination: Path to save roles from galaxy to
+# collections_destination: Path to save collections from galaxy to
 # awx_version: Current running version of the awx or tower as a string
 # awx_license_type: "open" for AWX; else presume Tower
 
-- hosts: all
+- hosts: localhost
   gather_facts: false
+  connection: local
+  name: Update source tree if necessary
   tasks:
 
     - name: delete project directory before update
       file:
         path: "{{project_path|quote}}"
         state: absent
-      when: scm_delete_on_update|default('')
-      delegate_to: localhost
+      tags:
+        - delete
 
     - block:
       - name: update project using git
@@ -43,8 +44,8 @@
         set_fact:
           scm_version: "{{ git_result['after'] }}"
         when: "'after' in git_result"
-      when: scm_type == 'git'
-      delegate_to: localhost
+      tags:
+        - update_git
 
     - block:
       - name: update project using hg
@@ -63,8 +64,8 @@
       - name: parse hg version string properly
         set_fact:
           scm_version: "{{scm_version|regex_replace('^([A-Za-z0-9]+).*$', '\\1')}}"
-      when: scm_type == 'hg'
-      delegate_to: localhost
+      tags:
+        - update_hg
 
     - block:
       - name: update project using svn
@@ -87,8 +88,8 @@
       - name: parse subversion version string properly
         set_fact:
           scm_version: "{{scm_version|regex_replace('^.*Revision: ([0-9]+).*$', '\\1')}}"
-      when: scm_type == 'svn'
-      delegate_to: localhost
+      tags:
+        - update_svn
 
     - block:
       - name: Ensure the project directory is present
@@ -110,16 +111,21 @@
         set_fact:
           scm_version: "{{results.version}}"
         when: results is defined
-      when: scm_type == 'insights'
-      delegate_to: localhost
-
+      tags:
+        - update_insights
 
     - name: Repository Version
       debug: msg="Repository Version {{ scm_version }}"
-      when: scm_version is defined
+      tags:
+        - update_git
+        - update_hg
+        - update_svn
+        - update_insights
 
-- hosts: all
+- hosts: localhost
   gather_facts: false
+  connection: local
+  name: Install content with ansible-galaxy command if necessary
   tasks:
 
     - block:
@@ -138,7 +144,8 @@
           ANSIBLE_FORCE_COLOR: False
 
       when: roles_enabled|bool
-      delegate_to: localhost
+      tags:
+        - install_roles
 
     - block:
       - name: detect collections/requirements.yml
@@ -156,5 +163,8 @@
           ANSIBLE_FORCE_COLOR: False
           ANSIBLE_COLLECTIONS_PATHS: "{{ collections_destination }}"
 
-      when: collections_enabled|bool
-      delegate_to: localhost
+      when:
+        - "ansible_version.full is version_compare('2.8', '>=')"
+        - collections_enabled|bool
+      tags:
+        - install_collections