Add scm_refspec field

Update migration syntax to Django 2

fix status bug where canceled switched to error

awx/api/serializers.py

@@ -1285,7 +1285,7 @@ class OrganizationSerializer(BaseSerializer):
 class ProjectOptionsSerializer(BaseSerializer):
 
     class Meta:
-        fields = ('*', 'local_path', 'scm_type', 'scm_url', 'scm_branch',
+        fields = ('*', 'local_path', 'scm_type', 'scm_url', 'scm_branch', 'scm_refspec',
                   'scm_clean', 'scm_delete_on_update', 'credential', 'timeout', 'scm_revision')
 
     def get_related(self, obj):
@@ -1311,6 +1311,8 @@ class ProjectOptionsSerializer(BaseSerializer):
             attrs.pop('local_path', None)
         if 'local_path' in attrs and attrs['local_path'] not in valid_local_paths:
             errors['local_path'] = _('This path is already being used by another manual project.')
+        if attrs.get('scm_refspec') and scm_type != 'git':
+            errors['scm_refspec'] = _('SCM refspec can only be used with git projects.')
 
         if errors:
             raise serializers.ValidationError(errors)

awx/main/migrations/0082_v360_job_branch_overrirde.py

@@ -13,6 +13,18 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
+        # Add fields for user-provided project refspec
+        migrations.AddField(
+            model_name='project',
+            name='scm_refspec',
+            field=models.CharField(blank=True, default='', help_text='For git projects, an additional refspec to fetch.', max_length=1024, verbose_name='SCM refspec'),
+        ),
+        migrations.AddField(
+            model_name='projectupdate',
+            name='scm_refspec',
+            field=models.CharField(blank=True, default='', help_text='For git projects, an additional refspec to fetch.', max_length=1024, verbose_name='SCM refspec'),
+        ),
+        # Add fields for job specification of project branch
         migrations.AddField(
             model_name='job',
             name='scm_branch',
@@ -21,7 +33,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='jobtemplate',
             name='ask_scm_branch_on_launch',
-            field=awx.main.fields.AskForField(default=False),
+            field=awx.main.fields.AskForField(blank=True, default=False),
         ),
         migrations.AddField(
             model_name='jobtemplate',
@@ -33,11 +45,13 @@ class Migration(migrations.Migration):
             name='allow_override',
             field=models.BooleanField(default=False, help_text='Allow changing the SCM branch or revision in a job template that uses this project.'),
         ),
+        # Fix typo in help_text
         migrations.AlterField(
             model_name='project',
             name='scm_update_cache_timeout',
             field=models.PositiveIntegerField(blank=True, default=0, help_text='The number of seconds after the last project update ran that a new project update will be launched as a job dependency.'),
         ),
+        # Start tracking the fetched revision on project update model
         migrations.AddField(
             model_name='projectupdate',
             name='scm_revision',

awx/main/models/projects.py

@@ -106,6 +106,13 @@ class ProjectOptions(models.Model):
         verbose_name=_('SCM Branch'),
         help_text=_('Specific branch, tag or commit to checkout.'),
     )
+    scm_refspec = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        verbose_name=_('SCM refspec'),
+        help_text=_('For git projects, an additional refspec to fetch.'),
+    )
     scm_clean = models.BooleanField(
         default=False,
         help_text=_('Discard any local changes before syncing the project.'),
@@ -241,7 +248,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
     SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')]
     FIELDS_TO_PRESERVE_AT_COPY = ['labels', 'instance_groups', 'credentials']
     FIELDS_TO_DISCARD_AT_COPY = ['local_path']
-    FIELDS_TRIGGER_UPDATE = frozenset(['scm_url', 'scm_branch', 'scm_type'])
+    FIELDS_TRIGGER_UPDATE = frozenset(['scm_url', 'scm_branch', 'scm_type', 'scm_refspec'])
 
     class Meta:
         app_label = 'main'

awx/main/tasks.py

@@ -1667,7 +1667,7 @@ class RunJob(BaseTask):
             os.chmod(galaxy_install_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
             project_update_task = local_project_sync._get_task_class()
             try:
-                sync_task = project_update_task(roles_destination=galaxy_install_path)
+                sync_task = project_update_task(job_private_data_dir=private_data_dir, roles_destination=galaxy_install_path)
                 sync_task.run(local_project_sync.id)
                 local_project_sync.refresh_from_db()
                 job = self.update_model(job.pk, scm_revision=local_project_sync.scm_revision)
@@ -1678,6 +1678,9 @@ class RunJob(BaseTask):
                                             job_explanation=('Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' %
                                                              ('project_update', local_project_sync.name, local_project_sync.id)))
                     raise
+                job.refresh_from_db()
+                if job.cancel_flag:
+                    return
         else:
             # Case where a local sync is not needed, meaning that local tree is
             # up-to-date with project, job is running project current version
@@ -1742,13 +1745,14 @@ class RunProjectUpdate(BaseTask):
     @property
     def proot_show_paths(self):
         show_paths = [settings.PROJECTS_ROOT]
-        if self.roles_destination:
+        if self.job_private_data_dir:
-            show_paths.append(self.roles_destination)
+            show_paths.append(self.job_private_data_dir)
         return show_paths
 
-    def __init__(self, *args, roles_destination=None, **kwargs):
+    def __init__(self, *args, job_private_data_dir=None, roles_destination=None, **kwargs):
         super(RunProjectUpdate, self).__init__(*args, **kwargs)
         self.playbook_new_revision = None
+        self.job_private_data_dir = job_private_data_dir
         self.roles_destination = roles_destination
 
     def event_handler(self, event_data):
@@ -1891,11 +1895,12 @@ class RunProjectUpdate(BaseTask):
             'scm_full_checkout': True if project_update.job_type == 'run' else False,
             'roles_enabled': getattr(settings, 'AWX_ROLES_ENABLED', True) if project_update.job_type != 'check' else False
         })
-        # TODO: apply custom refspec from user for PR refs and the like
+        # apply custom refspec from user for PR refs and the like
-        if project_update.project.allow_override:
+        if project_update.scm_refspec:
+            extra_vars['scm_refspec'] = project_update.scm_refspec
+        elif project_update.project.allow_override:
             # If branch is override-able, do extra fetch for all branches
-            # coming feature
+            extra_vars['scm_refspec'] = 'refs/heads/*:refs/remotes/origin/*'
-            extra_vars['git_refspec'] = 'refs/heads/*:refs/remotes/origin/*'
         if self.roles_destination:
             extra_vars['roles_destination'] = self.roles_destination
         self._write_extra_vars_file(private_data_dir, extra_vars)

awx/playbooks/project_update.yml

@@ -11,7 +11,7 @@
 # scm_username: username (only for svn/insights)
 # scm_password: password (only for svn/insights)
 # scm_accept_hostkey: true/false (only for git, set automatically)
-# git_refspec: a refspec to fetch in addition to obtaining version
+# scm_refspec: a refspec to fetch in addition to obtaining version
 # roles_enabled: Allow us to pull roles from a requirements.yml file
 # roles_destination: Path to save roles from galaxy to
 # awx_version: Current running version of the awx or tower as a string
@@ -34,7 +34,7 @@
           dest: "{{project_path|quote}}"
           repo: "{{scm_url}}"
           version: "{{scm_branch|quote}}"
-          refspec: "{{git_refspec|default(omit)}}"
+          refspec: "{{scm_refspec|default(omit)}}"
           force: "{{scm_clean}}"
           accept_hostkey: "{{scm_accept_hostkey|default(omit)}}"
         register: git_result