JT-branch docs and code cleanup

bump migration

fine tune validation of project allow_override
  return highly custom error message

Restore branch after syncs to address bugs
  encountered after changing scm_refspec

remove unused code to determine scm_revision

Check Ansible version before project update and
  do not install collections if Ansible version too old

Add docs related to project branch override
  New file specific to branch override and refspec
Complete docs on collections to reflect current
  implementation and give a folder tree example
Update clustering docs related to project syncs

Fix bug where git depth was ignored during the
  local clone from project folder to run folder

Fix bug where submodules were not copied

awx/api/serializers.py

@@ -1391,9 +1391,17 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
             organization = self.instance.organization
 
         if 'allow_override' in attrs and self.instance:
-            if attrs['allow_override'] != self.instance.allow_override:
-                raise serializers.ValidationError({
-                    'allow_override': _('Branch override behavior of a project cannot be changed after creation.')})
+            # case where user is turning off this project setting
+            if self.instance.allow_override and not attrs['allow_override']:
+                used_by = (
+                    set(JobTemplate.objects.filter(project=self.instance, scm_branch__isnull=False).values_list('pk', flat=True)) |
+                    set(JobTemplate.objects.filter(project=self.instance, ask_scm_branch_on_launch=True).values_list('pk', flat=True))
+                )
+                if used_by:
+                    raise serializers.ValidationError({
+                        'allow_override': _('One or more job templates already specify a branch for this project (ids: {}).').format(
+                            ' '.join([str(pk) for pk in used_by])
+                        )})
 
         view = self.context.get('view', None)
         if not organization and not view.request.user.is_superuser:

awx/main/migrations/0083_v360_job_branch_overrirde.py

@@ -9,7 +9,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('main', '0081_v360_notify_on_start'),
+        ('main', '0082_v360_webhook_http_method'),
     ]
 
     operations = [

awx/main/tasks.py

@@ -73,7 +73,7 @@ from awx.main.utils import (get_ssh_version, update_scm_url,
                             ignore_inventory_computed_fields,
                             ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager,
                             get_awx_version)
-from awx.main.utils.common import _get_ansible_version, get_custom_venv_choices
+from awx.main.utils.common import get_ansible_version, _get_ansible_version, get_custom_venv_choices
 from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja
 from awx.main.utils.reload import stop_local_services
 from awx.main.utils.pglock import advisory_lock
@@ -1709,7 +1709,13 @@ class RunJob(BaseTask):
             if not job.scm_revision:
                 raise RuntimeError('Unexpectedly could not determine a revision to run from project.')
             source_branch = git_repo.create_head(tmp_branch_name, job.scm_revision)
-            git_repo.clone(runner_project_folder, branch=source_branch, depth=1, single_branch=True)
+            # git clone must take file:// syntax for source repo or else options like depth will be ignored
+            source_as_uri = Path(project_path).as_uri()
+            git.Repo.clone_from(
+                source_as_uri, runner_project_folder, branch=source_branch,
+                depth=1, single_branch=True,  # shallow, do not copy full history
+                recursive=True  # include submodules
+            )
             # force option is necessary because remote refs are not counted, although no information is lost
             git_repo.delete_head(tmp_branch_name, force=True)
         else:
@@ -1759,6 +1765,7 @@ class RunProjectUpdate(BaseTask):
     def __init__(self, *args, job_private_data_dir=None, **kwargs):
         super(RunProjectUpdate, self).__init__(*args, **kwargs)
         self.playbook_new_revision = None
+        self.original_branch = None
         self.job_private_data_dir = job_private_data_dir
 
     def event_handler(self, event_data):
@@ -1894,6 +1901,15 @@ class RunProjectUpdate(BaseTask):
             scm_branch = project_update.project.scm_revision
         elif not scm_branch:
             scm_branch = {'hg': 'tip'}.get(project_update.scm_type, 'HEAD')
+        if project_update.job_type == 'check':
+            roles_enabled = False
+            collections_enabled = False
+        else:
+            roles_enabled = getattr(settings, 'AWX_ROLES_ENABLED', True)
+            collections_enabled = getattr(settings, 'AWX_COLLECTIONS_ENABLED', True)
+            # collections were introduced in Ansible version 2.8
+            if Version(get_ansible_version()) <= Version('2.8'):
+                collections_enabled = False
         extra_vars.update({
             'project_path': project_update.get_project_path(check_if_exists=False),
             'insights_url': settings.INSIGHTS_URL_BASE,
@@ -1905,8 +1921,8 @@ class RunProjectUpdate(BaseTask):
             'scm_clean': project_update.scm_clean,
             'scm_delete_on_update': project_update.scm_delete_on_update if project_update.job_type == 'check' else False,
             'scm_full_checkout': True if project_update.job_type == 'run' else False,
-            'roles_enabled': getattr(settings, 'AWX_ROLES_ENABLED', True) if project_update.job_type != 'check' else False,
-            'collections_enabled': getattr(settings, 'AWX_COLLECTIONS_ENABLED', True) if project_update.job_type != 'check' else False,
+            'roles_enabled': roles_enabled,
+            'collections_enabled': collections_enabled,
         })
         if project_update.job_type != 'check' and self.job_private_data_dir:
             extra_vars['collections_destination'] = os.path.join(self.job_private_data_dir, 'requirements_collections')
@@ -2041,31 +2057,26 @@ class RunProjectUpdate(BaseTask):
         if not os.path.exists(settings.PROJECTS_ROOT):
             os.mkdir(settings.PROJECTS_ROOT)
         self.acquire_lock(instance)
+        self.original_branch = None
+        if instance.scm_type == 'git' and instance.job_type == 'run' and instance.project:
+            project_path = instance.project.get_project_path(check_if_exists=False)
+            if os.path.exists(project_path):
+                git_repo = git.Repo(project_path)
+                self.original_branch = git_repo.active_branch
 
     def post_run_hook(self, instance, status):
-        # TODO: find the effective revision and save to scm_revision
+        if self.original_branch:
+            # for git project syncs, non-default branches can be problems
+            # restore to branch the repo was on before this run
+            try:
+                self.original_branch.checkout()
+            except Exception:
+                # this could have failed due to dirty tree, but difficult to predict all cases
+                logger.exception('Failed to restore project repo to prior state after {}'.format(instance.log_format))
         self.release_lock(instance)
         p = instance.project
         if self.playbook_new_revision:
             instance.scm_revision = self.playbook_new_revision
-            # If branch of the update differs from project, then its revision will differ
-            if instance.scm_branch != p.scm_branch and p.scm_type == 'git':
-                project_path = p.get_project_path(check_if_exists=False)
-                git_repo = git.Repo(project_path)
-                try:
-                    commit = git_repo.commit(instance.scm_branch)
-                    instance.scm_revision = commit.hexsha  # obtain 40 char long-form of SHA1
-                    logger.debug('Evaluated {} to be a valid commit for {}'.format(instance.scm_branch, instance.log_format))
-                except (ValueError, BadGitName):
-                    # not a commit, see if it is a ref
-                    try:
-                        user_branch = getattr(git_repo.remotes.origin.refs, instance.scm_branch)
-                        instance.scm_revision = user_branch.commit.hexsha  # head of ref
-                        logger.debug('Evaluated {} to be a valid ref for {}'.format(instance.scm_branch, instance.log_format))
-                    except (git.exc.NoSuchPathError, AttributeError) as exc:
-                        raise RuntimeError('Could not find specified version {}, error: {}'.format(
-                            instance.scm_branch, exc
-                        ))
             instance.save(update_fields=['scm_revision'])
         if instance.job_type == 'check' and status not in ('failed', 'canceled',):
             if self.playbook_new_revision:

awx/main/tests/functional/api/test_project.py

@@ -5,6 +5,7 @@ from django.conf import settings
 import pytest
 
 from awx.api.versioning import reverse
+from awx.main.models import Project, JobTemplate
 
 
 @pytest.mark.django_db
@@ -47,7 +48,7 @@ def test_project_unset_custom_virtualenv(get, patch, project, admin, value):
 
 
 @pytest.mark.django_db
-def test_no_changing_overwrite_behavior(post, patch, organization, admin_user):
+def test_no_changing_overwrite_behavior_if_used(post, patch, organization, admin_user):
     r1 = post(
         url=reverse('api:project_list'),
         data={
@@ -58,10 +59,38 @@ def test_no_changing_overwrite_behavior(post, patch, organization, admin_user):
         user=admin_user,
         expect=201
     )
+    JobTemplate.objects.create(
+        name='provides branch', project_id=r1.data['id'],
+        playbook='helloworld.yml',
+        scm_branch='foobar'
+    )
     r2 = patch(
         url=reverse('api:project_detail', kwargs={'pk': r1.data['id']}),
         data={'allow_override': False},
         user=admin_user,
         expect=400
     )
-    assert 'cannot be changed' in str(r2.data['allow_override'])
+    assert 'job templates already specify a branch for this project' in str(r2.data['allow_override'])
+    assert 'ids: 2' in str(r2.data['allow_override'])
+    assert Project.objects.get(pk=r1.data['id']).allow_override is True
+
+
+@pytest.mark.django_db
+def test_changing_overwrite_behavior_okay_if_not_used(post, patch, organization, admin_user):
+    r1 = post(
+        url=reverse('api:project_list'),
+        data={
+            'name': 'fooo',
+            'organization': organization.id,
+            'allow_override': True
+        },
+        user=admin_user,
+        expect=201
+    )
+    patch(
+        url=reverse('api:project_detail', kwargs={'pk': r1.data['id']}),
+        data={'allow_override': False},
+        user=admin_user,
+        expect=200
+    )
+    assert Project.objects.get(pk=r1.data['id']).allow_override is False

docs/clustering.md

@@ -272,7 +272,7 @@ If an Instance Group is configured but all instances in that group are offline o
 
 #### Project synchronization behavior
 
-Project updates behave differently than they did before. Previously they were ordinary jobs that ran on a single instance. It's now important that they run successfully on any instance that could potentially run a job. Projects will sync themselves to the correct version on the instance immediately prior to running the job. If the needed revision is already locally available and galaxy or collections updates are not needed, then a sync may not be performed.
+Project updates behave differently than they did before. Previously they were ordinary jobs that ran on a single instance. It's now important that they run successfully on any instance that could potentially run a job. Projects will sync themselves to the correct version on the instance immediately prior to running the job. If the needed revision is already locally checked out and galaxy or collections updates are not needed, then a sync may not be performed.
 
 When the sync happens, it is recorded in the database as a project update with a `launch_type` of "sync" and a `job_type` of "run". Project syncs will not change the status or version of the project; instead, they will update the source tree _only_ on the instance where they run. The only exception to this behavior is when the project is in the "never updated" state (meaning that no project updates of any type have been run), in which case a sync should fill in the project's initial revision and status, and subsequent syncs should not make such changes.
 

docs/collections.md

@@ -1,6 +1,62 @@
-## Collections Support
+## Collections
 
-AWX supports Ansible collections by appending the directories specified in `AWX_ANSIBLE_COLLECTIONS_PATHS` 
+AWX supports using Ansible collections.
+This section will give ways to use collections in job runs.
+
+### Project Collections Requirements
+
+If you specify a collections requirements file in SCM at `collections/requirements.yml`,
+then AWX will install collections in that file in the implicit project sync
+before a job run. The invocation is:
+
+```
+ansible-galaxy collection install -r requirements.yml -p <job tmp location>
+```
+
+Example of tmp directory where job is running:
+
+```
+├── project
+│   ├── ansible.cfg
+│   └── debug.yml
+├── requirements_collections
+│   └── ansible_collections
+│       └── username
+│           └── collection_name
+│               ├── FILES.json
+│               ├── MANIFEST.json
+│               ├── README.md
+│               ├── roles
+│               │   ├── role_in_collection_name
+│               │   │   ├── defaults
+│               │   │   │   └── main.yml
+│               │   │   ├── tasks
+│               │   │   │   └── main.yml
+│               │   │   └── templates
+│               │   │       └── stuff.j2
+│               └── tests
+│                   └── main.yml
+├── requirements_roles
+│   └── username.role_name
+│       ├── defaults
+│       │   └── main.yml
+│       ├── meta
+│       │   └── main.yml
+│       ├── README.md
+│       ├── tasks
+│       │   ├── main.yml
+│       │   └── some_role.yml
+│       ├── templates
+│       │   └── stuff.j2
+│       └── vars
+│           └── Archlinux.yml
+└── tmp_6wod58k
+
+```
+
+### Global Collections Path
+
+AWX appends the directories specified in `AWX_ANSIBLE_COLLECTIONS_PATHS`
 to the environment variable `ANSIBLE_COLLECTIONS_PATHS`. The default value of `AWX_ANSIBLE_COLLECTIONS_PATHS`
-contains `/var/lib/awx/collections`. It is recommended that place your collections that you wish to call in 
+contains `/var/lib/awx/collections`. It is recommended that place your collections that you wish to call in
 your playbooks into this path.

docs/job_branch_override.md

@@ -0,0 +1,66 @@
+## Job Branch Specification
+
+Background: Projects specify the branch to use from source control
+in the `scm_branch` field.
+
+This feature allows project admins to delegate branch selection to
+admins of job templates that use that project (requiring only project
+`use_role`). Admins of job templates can further
+delegate that ability to users executing the job template
+(requiring only job template `execute_role`) by enabling
+`ask_scm_branch_on_launch` on the job template.
+
+### Source Tree Copy Behavior
+
+Every job run has its own private data directory. This folder is temporary,
+cleaned up at the end of the job run.
+
+This directory contains a copy of the project source tree for the given
+branch the job is running.
+
+A new copy is made for every job run.
+
+This folder will not contain the full git history for git project types.
+
+### Git Refspec
+
+The field `scm_refspec` has been added to projects. This is provided by
+the user or left blank.
+
+A non-blank `scm_refspec` field will cause project updates (of any type)
+to pass the `refspec` field when running the Ansible
+git module inside of the `project_update.yml` playbook. When the git module
+is provided with this field, it performs an extra `git fetch` command
+to pull that refspec from the remote.
+
+The refspec specifies what references the update will download from the remote.
+Examples:
+
+ - `refs/*:refs/remotes/origin/*`
+    This will fetch all references, including even remotes of the remote
+ - `refs/pull/*:refs/remotes/origin/pull/*`
+    Github-specific, this will fetch all refs for all pull requests
+ - `refs/pull/62/head:refs/remotes/origin/pull/62/head`
+    This will fetch only the ref for that one github pull request
+
+For large projects, users should consider performance when
+using the first or second examples here. For example, if a github project
+has over 40,000 pull requests, that refspec will fetch them all
+during a project update.
+
+This parameter affects availability of the project branch, and can allow
+access to references not otherwise available. For example, the third example
+will allow the user to use the branch `refs/pull/62/head`, which would
+not be possible without the refspec field.
+
+The Ansible git module always fetches `refs/heads/*`. It will do this
+whether or not a custom refspec is provided. This means that a project's
+branches and tags (and commit hashes therein) can be used as `scm_branch`
+no matter what is used for `scm_refspec`.
+
+The `scm_refspec` will affect which `scm_branch` fields can be used as overrides.
+For example, you could set up a project that allows branch override with a refspec
+of `refs/pull/*:refs/remotes/origin/pull/*`, then use this in a job template
+that prompts for `scm_branch`, then a client could launch the job template when
+a new pull request is created, providing the branch `refs/pull/N/head`,
+then the job template would run against the provided github pull request reference.