From f64d0dde5a9afb7c8bd0d9d388b337adee6dd771 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Fri, 1 Nov 2019 15:07:56 -0400 Subject: [PATCH] Use tags to reduce project update output Handle folder deletion as tag remove -v use by default Change meaning of roles_enabled playbook var to value of AWX global setting --- awx/api/serializers.py | 2 +- .../0099_v370_projectupdate_job_tags.py | 18 ++++++ awx/main/models/projects.py | 27 +++++++++ awx/main/tasks.py | 56 +++++++++---------- awx/playbooks/project_update.yml | 52 ++++++++++------- 5 files changed, 104 insertions(+), 51 deletions(-) create mode 100644 awx/main/migrations/0099_v370_projectupdate_job_tags.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index a3d8d43306..7bb09c4acf 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1472,7 +1472,7 @@ class ProjectUpdateSerializer(UnifiedJobSerializer, ProjectOptionsSerializer): class Meta: model = ProjectUpdate - fields = ('*', 'project', 'job_type', '-controller_node') + fields = ('*', 'project', 'job_type', 'job_tags', '-controller_node') def get_related(self, obj): res = super(ProjectUpdateSerializer, self).get_related(obj) diff --git a/awx/main/migrations/0099_v370_projectupdate_job_tags.py b/awx/main/migrations/0099_v370_projectupdate_job_tags.py new file mode 100644 index 0000000000..92c550a49e --- /dev/null +++ b/awx/main/migrations/0099_v370_projectupdate_job_tags.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2019-11-01 18:46 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0098_v360_rename_cyberark_aim_credential_type'), + ] + + operations = [ + migrations.AddField( + model_name='projectupdate', + name='job_tags', + field=models.CharField(blank=True, default='', help_text='Parts of the project update playbook that will be run.', max_length=1024), + ), + ] diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index b7b52dcf6b..cc31842d4d 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -483,6 +483,12 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage choices=PROJECT_UPDATE_JOB_TYPE_CHOICES, default='check', ) + job_tags = models.CharField( + max_length=1024, + blank=True, + default='', + help_text=_('Parts of the project update playbook that will be run.'), + ) scm_revision = models.CharField( max_length=1024, blank=True, @@ -587,3 +593,24 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage if not selected_groups: return self.global_instance_groups return selected_groups + + def save(self, *args, **kwargs): + added_update_fields = [] + if not self.job_tags: + job_tags = ['update_{}'.format(self.scm_type)] + if self.job_type == 'run': + job_tags.append('install_roles') + job_tags.append('install_collections') + self.job_tags = ','.join(job_tags) + added_update_fields.append('job_tags') + if self.scm_delete_on_update and 'delete' not in self.job_tags and self.job_type == 'check': + self.job_tags = ','.join([self.job_tags, 'delete']) + added_update_fields.append('job_tags') + elif (not self.scm_delete_on_update) and 'delete' in self.job_tags: + job_tags = self.job_tags.split(',') + job_tags.remove('delete') + self.job_tags = ','.join(job_tags) + added_update_fields.append('job_tags') + if 'update_fields' in kwargs: + kwargs['update_fields'].extend(added_update_fields) + return super(ProjectUpdate, self).save(*args, **kwargs) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index a904fd5b45..ad7d43e4a3 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -71,7 +71,7 @@ from awx.main.utils import (get_ssh_version, update_scm_url, ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager, get_awx_version) from awx.main.utils.ansible import read_ansible_config -from awx.main.utils.common import get_ansible_version, _get_ansible_version, get_custom_venv_choices +from awx.main.utils.common import _get_ansible_version, get_custom_venv_choices from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja from awx.main.utils.reload import stop_local_services from awx.main.utils.pglock import advisory_lock @@ -1734,14 +1734,16 @@ class RunJob(BaseTask): project_path = job.project.get_project_path(check_if_exists=False) job_revision = job.project.scm_revision - needs_sync = True + sync_needs = [] + all_sync_needs = ['update_{}'.format(job.project.scm_type), 'install_roles', 'install_collections'] if not job.project.scm_type: - # manual projects are not synced, user has responsibility for that - needs_sync = False + pass # manual projects are not synced, user has responsibility for that elif not os.path.exists(project_path): logger.debug('Performing fresh clone of {} on this instance.'.format(job.project)) + sync_needs = all_sync_needs elif not job.project.scm_revision: logger.debug('Revision not known for {}, will sync with remote'.format(job.project)) + sync_needs = all_sync_needs elif job.project.scm_type == 'git': git_repo = git.Repo(project_path) try: @@ -1752,23 +1754,27 @@ class RunJob(BaseTask): if desired_revision == current_revision: job_revision = desired_revision logger.info('Skipping project sync for {} because commit is locally available'.format(job.log_format)) - needs_sync = False + else: + sync_needs = all_sync_needs except (ValueError, BadGitName): logger.debug('Needed commit for {} not in local source tree, will sync with remote'.format(job.log_format)) + sync_needs = all_sync_needs + else: + sync_needs = all_sync_needs # Galaxy requirements are not supported for manual projects - if not needs_sync and job.project.scm_type: + if not sync_needs and job.project.scm_type: # see if we need a sync because of presence of roles galaxy_req_path = os.path.join(project_path, 'roles', 'requirements.yml') if os.path.exists(galaxy_req_path): logger.debug('Running project sync for {} because of galaxy role requirements.'.format(job.log_format)) - needs_sync = True + sync_needs.append('install_roles') galaxy_collections_req_path = os.path.join(project_path, 'collections', 'requirements.yml') if os.path.exists(galaxy_collections_req_path): logger.debug('Running project sync for {} because of galaxy collections requirements.'.format(job.log_format)) - needs_sync = True + sync_needs.append('install_collections') - if needs_sync: + if sync_needs: pu_ig = job.instance_group pu_en = job.execution_node if job.is_isolated() is True: @@ -1778,6 +1784,7 @@ class RunJob(BaseTask): sync_metafields = dict( launch_type="sync", job_type='run', + job_tags=','.join(sync_needs), status='running', instance_group = pu_ig, execution_node=pu_en, @@ -1785,6 +1792,8 @@ class RunJob(BaseTask): ) if job.scm_branch and job.scm_branch != job.project.scm_branch: sync_metafields['scm_branch'] = job.scm_branch + if 'update_' not in sync_metafields['job_tags']: + sync_metafields['scm_revision'] = job_revision local_project_sync = job.project.create_project_update(_eager_fields=sync_metafields) # save the associated job before calling run() so that a # cancel() call on the job can cancel the project update @@ -2008,8 +2017,8 @@ class RunProjectUpdate(BaseTask): args = [] if getattr(settings, 'PROJECT_UPDATE_VVV', False): args.append('-vvv') - else: - args.append('-v') + if project_update.job_tags: + args.extend(['-t', project_update.job_tags]) return args def build_extra_vars_file(self, project_update, private_data_dir): @@ -2023,28 +2032,16 @@ class RunProjectUpdate(BaseTask): scm_branch = project_update.project.scm_revision elif not scm_branch: scm_branch = {'hg': 'tip'}.get(project_update.scm_type, 'HEAD') - if project_update.job_type == 'check': - roles_enabled = False - collections_enabled = False - else: - roles_enabled = getattr(settings, 'AWX_ROLES_ENABLED', True) - collections_enabled = getattr(settings, 'AWX_COLLECTIONS_ENABLED', True) - # collections were introduced in Ansible version 2.8 - if Version(get_ansible_version()) <= Version('2.8'): - collections_enabled = False extra_vars.update({ 'project_path': project_update.get_project_path(check_if_exists=False), 'insights_url': settings.INSIGHTS_URL_BASE, 'awx_license_type': get_license(show_key=False).get('license_type', 'UNLICENSED'), 'awx_version': get_awx_version(), - 'scm_type': project_update.scm_type, 'scm_url': scm_url, 'scm_branch': scm_branch, 'scm_clean': project_update.scm_clean, - 'scm_delete_on_update': project_update.scm_delete_on_update if project_update.job_type == 'check' else False, - 'scm_full_checkout': True if project_update.job_type == 'run' else False, - 'roles_enabled': roles_enabled, - 'collections_enabled': collections_enabled, + 'roles_enabled': settings.AWX_ROLES_ENABLED, + 'collections_enabled': settings.AWX_COLLECTIONS_ENABLED, }) if project_update.job_type != 'check' and self.job_private_data_dir: extra_vars['collections_destination'] = os.path.join(self.job_private_data_dir, 'requirements_collections') @@ -2217,12 +2214,15 @@ class RunProjectUpdate(BaseTask): copy_tree(project_path, destination_folder) def post_run_hook(self, instance, status): + if self.playbook_new_revision: + instance.scm_revision = self.playbook_new_revision + instance.save(update_fields=['scm_revision']) if self.job_private_data_dir: # copy project folder before resetting to default branch # because some git-tree-specific resources (like submodules) might matter self.make_local_copy( instance.get_project_path(check_if_exists=False), os.path.join(self.job_private_data_dir, 'project'), - instance.scm_type, self.playbook_new_revision + instance.scm_type, instance.scm_revision ) if self.original_branch: # for git project syncs, non-default branches can be problems @@ -2234,9 +2234,6 @@ class RunProjectUpdate(BaseTask): logger.exception('Failed to restore project repo to prior state after {}'.format(instance.log_format)) self.release_lock(instance) p = instance.project - if self.playbook_new_revision: - instance.scm_revision = self.playbook_new_revision - instance.save(update_fields=['scm_revision']) if instance.job_type == 'check' and status not in ('failed', 'canceled',): if self.playbook_new_revision: p.scm_revision = self.playbook_new_revision @@ -2497,6 +2494,7 @@ class RunInventoryUpdate(BaseTask): _eager_fields=dict( launch_type="sync", job_type='run', + job_tags='update_{},install_collections'.format(source_project.scm_type), # roles are never valid for inventory status='running', execution_node=inventory_update.execution_node, instance_group = inventory_update.instance_group, diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml index 87d00b7fe4..c64cedade0 100644 --- a/awx/playbooks/project_update.yml +++ b/awx/playbooks/project_update.yml @@ -1,32 +1,33 @@ --- # The following variables will be set by the runner of this playbook: # project_path: PROJECTS_DIR/_local_path_ -# scm_type: git|hg|svn|insights # scm_url: https://server/repo # insights_url: Insights service URL (from configuration) # scm_branch: branch/tag/revision (HEAD if unset) # scm_clean: true/false -# scm_delete_on_update: true/false -# scm_full_checkout: true (if for a job template run), false (if retrieving revision) # scm_username: username (only for svn/insights) # scm_password: password (only for svn/insights) # scm_accept_hostkey: true/false (only for git, set automatically) # scm_refspec: a refspec to fetch in addition to obtaining version -# roles_enabled: Allow us to pull roles from a requirements.yml file +# roles_enabled: Value of the global setting to enable roles downloading +# collections_enabled: Value of the global setting to enable collections downloading # roles_destination: Path to save roles from galaxy to +# collections_destination: Path to save collections from galaxy to # awx_version: Current running version of the awx or tower as a string # awx_license_type: "open" for AWX; else presume Tower -- hosts: all +- hosts: localhost gather_facts: false + connection: local + name: Update source tree if necessary tasks: - name: delete project directory before update file: path: "{{project_path|quote}}" state: absent - when: scm_delete_on_update|default('') - delegate_to: localhost + tags: + - delete - block: - name: update project using git @@ -43,8 +44,8 @@ set_fact: scm_version: "{{ git_result['after'] }}" when: "'after' in git_result" - when: scm_type == 'git' - delegate_to: localhost + tags: + - update_git - block: - name: update project using hg @@ -63,8 +64,8 @@ - name: parse hg version string properly set_fact: scm_version: "{{scm_version|regex_replace('^([A-Za-z0-9]+).*$', '\\1')}}" - when: scm_type == 'hg' - delegate_to: localhost + tags: + - update_hg - block: - name: update project using svn @@ -87,8 +88,8 @@ - name: parse subversion version string properly set_fact: scm_version: "{{scm_version|regex_replace('^.*Revision: ([0-9]+).*$', '\\1')}}" - when: scm_type == 'svn' - delegate_to: localhost + tags: + - update_svn - block: - name: Ensure the project directory is present @@ -110,16 +111,21 @@ set_fact: scm_version: "{{results.version}}" when: results is defined - when: scm_type == 'insights' - delegate_to: localhost - + tags: + - update_insights - name: Repository Version debug: msg="Repository Version {{ scm_version }}" - when: scm_version is defined + tags: + - update_git + - update_hg + - update_svn + - update_insights -- hosts: all +- hosts: localhost gather_facts: false + connection: local + name: Install content with ansible-galaxy command if necessary tasks: - block: @@ -138,7 +144,8 @@ ANSIBLE_FORCE_COLOR: False when: roles_enabled|bool - delegate_to: localhost + tags: + - install_roles - block: - name: detect collections/requirements.yml @@ -156,5 +163,8 @@ ANSIBLE_FORCE_COLOR: False ANSIBLE_COLLECTIONS_PATHS: "{{ collections_destination }}" - when: collections_enabled|bool - delegate_to: localhost + when: + - "ansible_version.full is version_compare('2.8', '>=')" + - collections_enabled|bool + tags: + - install_collections