From 9a44dc4ba0427b6ed5ec030945a9d12950623663 Mon Sep 17 00:00:00 2001
From: AlanCoding <arominge@redhat.com>
Date: Thu, 13 Oct 2016 15:18:29 -0400
Subject: [PATCH 01/21] add can_read method to JobAccess

---
 awx/main/access.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/awx/main/access.py b/awx/main/access.py
index c70e6331c9..e11ed4a13a 100644
--- a/awx/main/access.py
+++ b/awx/main/access.py
@@ -1168,6 +1168,22 @@ class JobAccess(BaseAccess):
             Q(inventory__organization__in=org_access_qs) |
             Q(project__organization__in=org_access_qs)).distinct()
 
+    def org_access(self, obj):
+        """
+        Via the organization of a related resource, user has a claim to org_admin access of this job
+        """
+        if obj.inventory and obj.inventory.organization and self.user in obj.inventory.organization.admin_role:
+            return True
+        elif obj.project and obj.project.organization and self.user in obj.project.organization.admin_role:
+            return True
+        return False
+
+    @check_superuser
+    def can_read(self, obj):
+        if obj.job_template and self.user in obj.job_template.read_role:
+            return True
+        return self.org_access(obj)
+
     def can_add(self, data):
         if not data:  # So the browseable API will work
             return True
@@ -1196,12 +1212,7 @@ class JobAccess(BaseAccess):
 
     @check_superuser
     def can_delete(self, obj):
-        if obj.inventory is not None and self.user in obj.inventory.organization.admin_role:
-            return True
-        if (obj.project is not None and obj.project.organization is not None and
-                self.user in obj.project.organization.admin_role):
-            return True
-        return False
+        return self.org_access(obj)
 
     def can_start(self, obj, validate_license=True):
         if validate_license:

From 5e4a4b972230859c458ebd9be482782e4019d822 Mon Sep 17 00:00:00 2001
From: AlanCoding <arominge@redhat.com>
Date: Fri, 14 Oct 2016 15:52:41 -0400
Subject: [PATCH 02/21] refactor Job can_read to allow for org admins and
 auditors to read

---
 awx/main/access.py | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/awx/main/access.py b/awx/main/access.py
index e11ed4a13a..875ab33b7b 100644
--- a/awx/main/access.py
+++ b/awx/main/access.py
@@ -1168,21 +1168,28 @@ class JobAccess(BaseAccess):
             Q(inventory__organization__in=org_access_qs) |
             Q(project__organization__in=org_access_qs)).distinct()
 
-    def org_access(self, obj):
-        """
-        Via the organization of a related resource, user has a claim to org_admin access of this job
-        """
-        if obj.inventory and obj.inventory.organization and self.user in obj.inventory.organization.admin_role:
-            return True
-        elif obj.project and obj.project.organization and self.user in obj.project.organization.admin_role:
-            return True
+    def related_orgs(self, obj):
+        orgs = []
+        if obj.inventory and obj.inventory.organization:
+            orgs.append(obj.inventory.organization)
+        if obj.project and obj.project.organization and obj.project.organization not in orgs:
+            orgs.append(obj.project.organization)
+        return orgs
+
+    def org_access(self, obj, role_types=['admin_role']):
+        orgs = self.related_orgs(obj)
+        for org in orgs:
+            for role_type in role_types:
+                role = getattr(org, role_type)
+                if self.user in role:
+                    return True
         return False
 
     @check_superuser
     def can_read(self, obj):
         if obj.job_template and self.user in obj.job_template.read_role:
             return True
-        return self.org_access(obj)
+        return self.org_access(obj, role_types=['auditor_role', 'admin_role'])
 
     def can_add(self, data):
         if not data:  # So the browseable API will work

From f3cbf714576fb1daa372df2c1152821d52c45e4c Mon Sep 17 00:00:00 2001
From: AlanCoding <arominge@redhat.com>
Date: Mon, 17 Oct 2016 15:08:06 -0400
Subject: [PATCH 03/21] WFJ node-node relationships should work correctly now

---
 awx/api/views.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/awx/api/views.py b/awx/api/views.py
index 1343b34d3b..90a7389239 100644
--- a/awx/api/views.py
+++ b/awx/api/views.py
@@ -2675,13 +2675,9 @@ class WorkflowJobNodeChildrenBaseList(SubListAPIView):
 
     model = WorkflowJobNode
     serializer_class = WorkflowJobNodeListSerializer
-    always_allow_superuser = True # TODO: RBAC
-    parent_model = Job
+    parent_model = WorkflowJobNode
     relationship = ''
-    '''
-    enforce_parent_relationship = 'workflow_job_template'
     new_in_310 = True
-    '''
 
     #
     #Limit the set of WorkflowJobeNodes to the related nodes of specified by

From e8669b0cbf42fb0d7b95f12eeea27c60df3bd458 Mon Sep 17 00:00:00 2001
From: Matthew Jones <matburt@redhat.com>
Date: Mon, 17 Oct 2016 16:20:29 -0400
Subject: [PATCH 04/21] Support celeryd autoreloading for dev environment

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 3967fe5845..d74468a9e7 100644
--- a/Makefile
+++ b/Makefile
@@ -428,7 +428,7 @@ celeryd:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/tower/bin/activate; \
 	fi; \
-	$(PYTHON) manage.py celeryd -l DEBUG -B --autoscale=20,3 --schedule=$(CELERY_SCHEDULE_FILE) -Q projects,jobs,default,scheduler,$(COMPOSE_HOST)
+	$(PYTHON) manage.py celeryd -l DEBUG -B --autoreload --autoscale=20,3 --schedule=$(CELERY_SCHEDULE_FILE) -Q projects,jobs,default,scheduler,$(COMPOSE_HOST)
 	#$(PYTHON) manage.py celery multi show projects jobs default -l DEBUG -Q:projects projects -Q:jobs jobs -Q:default default -c:projects 1 -c:jobs 3 -c:default 3 -Ofair -B --schedule=$(CELERY_SCHEDULE_FILE)
 
 # Run to start the zeromq callback receiver

From b906469b405a299da89c95582c64d71017261da7 Mon Sep 17 00:00:00 2001
From: Matthew Jones <matburt@redhat.com>
Date: Thu, 20 Oct 2016 13:24:12 -0400
Subject: [PATCH 05/21] Add execution node information

---
 awx/api/serializers.py          |  2 +-
 awx/main/managers.py            | 11 ++++-------
 awx/main/models/unified_jobs.py |  7 ++++++-
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/awx/api/serializers.py b/awx/api/serializers.py
index 2a22a6b04f..82e44d4f24 100644
--- a/awx/api/serializers.py
+++ b/awx/api/serializers.py
@@ -562,7 +562,7 @@ class UnifiedJobSerializer(BaseSerializer):
         fields = ('*', 'unified_job_template', 'launch_type', 'status',
                   'failed', 'started', 'finished', 'elapsed', 'job_args',
                   'job_cwd', 'job_env', 'job_explanation', 'result_stdout',
-                  'result_traceback')
+                  'execution_node', 'result_traceback')
         extra_kwargs = {
             'unified_job_template': {
                 'source': 'unified_job_template_id',
diff --git a/awx/main/managers.py b/awx/main/managers.py
index 86c3367140..176deb9483 100644
--- a/awx/main/managers.py
+++ b/awx/main/managers.py
@@ -31,13 +31,10 @@ class InstanceManager(models.Manager):
                               hostname='localhost',
                               uuid='00000000-0000-0000-0000-000000000000')
 
-        # If we can determine the instance we are on then return
-        # that, otherwise None which would be the standalone
-        # case
-        # TODO: Replace, this doesn't work if the hostname
-        #       is different from the Instance.name
-        # node = self.filter(hostname=socket.gethostname())
-        return self.all()[0]
+        node = self.filter(hostname=settings.CLUSTER_HOST_ID)
+        if node.exists():
+            return node[0]
+        raise RuntimeError("No instance found with the current cluster host id")
 
     def my_role(self):
         # NOTE: TODO: Likely to repurpose this once standalone ramparts are a thing
diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py
index 8c953c61e6..677be8136d 100644
--- a/awx/main/models/unified_jobs.py
+++ b/awx/main/models/unified_jobs.py
@@ -438,6 +438,11 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
         editable=False,
         related_name='%(class)s_blocked_jobs+',
     )
+    execution_node = models.TextField(
+        blank=True,
+        default='',
+        editable=False,
+    )
     notifications = models.ManyToManyField(
         'Notification',
         editable=False,
@@ -801,7 +806,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
 
     def pre_start(self, **kwargs):
         if not self.can_start:
-            self.job_explanation = u'%s is not in a startable status: %s, expecting one of %s' % (self._meta.verbose_name, self.status, str(('new', 'waiting')))
+            self.job_explanation = u'%s is not in a startable state: %s, expecting one of %s' % (self._meta.verbose_name, self.status, str(('new', 'waiting')))
             self.save(update_fields=['job_explanation'])
             return (False, None)
 

From e6bcc039f2ce5e8147ec4803724c55a8531db329 Mon Sep 17 00:00:00 2001
From: Matthew Jones <matburt@redhat.com>
Date: Thu, 20 Oct 2016 13:24:42 -0400
Subject: [PATCH 06/21] Rearchitect project update strategy

* Instead of using a fanout for project updates initial project
  updates will sync the latest commit hash
* Prior to a node running a job it will ensure that the latest project
  is synced
---
 awx/api/serializers.py           |  4 +-
 awx/main/models/base.py          |  8 ++-
 awx/main/models/projects.py      | 18 +++++++
 awx/main/tasks.py                | 56 +++++++++++++++-----
 awx/playbooks/project_update.yml | 89 +++++++++++++++++++++++++++++---
 awx/settings/defaults.py         | 31 +++++------
 6 files changed, 169 insertions(+), 37 deletions(-)

diff --git a/awx/api/serializers.py b/awx/api/serializers.py
index 82e44d4f24..861e245a75 100644
--- a/awx/api/serializers.py
+++ b/awx/api/serializers.py
@@ -914,7 +914,7 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
     class Meta:
         model = Project
         fields = ('*', 'organization', 'scm_delete_on_next_update', 'scm_update_on_launch',
-                  'scm_update_cache_timeout', 'timeout') + \
+                  'scm_update_cache_timeout', 'scm_revision', 'timeout',) + \
                  ('last_update_failed', 'last_updated')  # Backwards compatibility
         read_only_fields = ('scm_delete_on_next_update',)
 
@@ -986,7 +986,7 @@ class ProjectUpdateSerializer(UnifiedJobSerializer, ProjectOptionsSerializer):
 
     class Meta:
         model = ProjectUpdate
-        fields = ('*', 'project')
+        fields = ('*', 'project', 'job_type')
 
     def get_related(self, obj):
         res = super(ProjectUpdateSerializer, self).get_related(obj)
diff --git a/awx/main/models/base.py b/awx/main/models/base.py
index c4914cdd20..691b4532fe 100644
--- a/awx/main/models/base.py
+++ b/awx/main/models/base.py
@@ -29,7 +29,8 @@ __all__ = ['VarsDictProperty', 'BaseModel', 'CreatedModifiedModel',
            'PERM_INVENTORY_ADMIN', 'PERM_INVENTORY_READ',
            'PERM_INVENTORY_WRITE', 'PERM_INVENTORY_DEPLOY', 'PERM_INVENTORY_SCAN',
            'PERM_INVENTORY_CHECK', 'PERM_JOBTEMPLATE_CREATE', 'JOB_TYPE_CHOICES',
-           'AD_HOC_JOB_TYPE_CHOICES', 'PERMISSION_TYPE_CHOICES', 'CLOUD_INVENTORY_SOURCES',
+           'AD_HOC_JOB_TYPE_CHOICES', 'PROJECT_UPDATE_JOB_TYPE_CHOICES',
+           'PERMISSION_TYPE_CHOICES', 'CLOUD_INVENTORY_SOURCES',
            'VERBOSITY_CHOICES']
 
 PERM_INVENTORY_ADMIN  = 'admin'
@@ -51,6 +52,11 @@ AD_HOC_JOB_TYPE_CHOICES = [
     (PERM_INVENTORY_CHECK, _('Check')),
 ]
 
+PROJECT_UPDATE_JOB_TYPE_CHOICES = [
+    (PERM_INVENTORY_DEPLOY, _('Run')),
+    (PERM_INVENTORY_CHECK, _('Check')),
+]
+
 PERMISSION_TYPE_CHOICES = [
     (PERM_INVENTORY_READ, _('Read Inventory')),
     (PERM_INVENTORY_WRITE, _('Edit Inventory')),
diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py
index 256809d4ac..d96bcec98b 100644
--- a/awx/main/models/projects.py
+++ b/awx/main/models/projects.py
@@ -7,6 +7,9 @@ import os
 import re
 import urlparse
 
+# Celery
+from celery import group, chord
+
 # Django
 from django.conf import settings
 from django.db import models
@@ -227,6 +230,15 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin):
         blank=True,
     )
 
+    scm_revision = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        editable=False,
+        verbose_name=_('SCM Revision'),
+        help_text=_('The last revision fetched by a project update'),
+    )
+
     admin_role = ImplicitRoleField(parent_role=[
         'organization.admin_role',
         'singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
@@ -393,6 +405,12 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin):
         editable=False,
     )
 
+    job_type = models.CharField(
+        max_length=64,
+        choices=PROJECT_UPDATE_JOB_TYPE_CHOICES,
+        default='check',
+    )
+
     @classmethod
     def _get_parent_field_name(cls):
         return 'project'
diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index 65c687e1d4..814700afd3 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -53,10 +53,9 @@ from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field,
 from awx.main.consumers import emit_channel_notification
 
 __all__ = ['RunJob', 'RunSystemJob', 'RunProjectUpdate', 'RunInventoryUpdate',
-           'RunAdHocCommand', 'RunWorkflowJob', 'handle_work_error',
+           'RunAdHocCommand', 'handle_work_error',
            'handle_work_success', 'update_inventory_computed_fields',
-           'send_notifications', 'run_administrative_checks',
-           'RunJobLaunch']
+           'send_notifications', 'run_administrative_checks']
 
 HIDDEN_PASSWORD = '**********'
 
@@ -234,8 +233,9 @@ def handle_work_error(self, task_id, subtasks=None):
             if instance.celery_task_id != task_id:
                 instance.status = 'failed'
                 instance.failed = True
-                instance.job_explanation = 'Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % \
-                    (first_instance_type, first_instance.name, first_instance.id)
+                if not instance.job_explanation:
+                    instance.job_explanation = 'Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % \
+                                               (first_instance_type, first_instance.name, first_instance.id)
                 instance.save()
                 instance.websocket_emit_status("failed")
 
@@ -538,6 +538,7 @@ class BaseTask(Task):
             expect_passwords[n] = passwords.get(item[1], '') or ''
         expect_list.extend([pexpect.TIMEOUT, pexpect.EOF])
         instance = self.update_model(instance.pk, status='running',
+                                     execution_node=settings.CLUSTER_HOST_ID,
                                      output_replacements=output_replacements)
         job_start = time.time()
         while child.isalive():
@@ -608,7 +609,7 @@ class BaseTask(Task):
         Hook for any steps to run before the job/task starts
         '''
 
-    def post_run_hook(self, instance, **kwargs):
+    def post_run_hook(self, instance, status, **kwargs):
         '''
         Hook for any steps to run after job/task is complete.
         '''
@@ -617,7 +618,7 @@ class BaseTask(Task):
         '''
         Run the job/task and capture its output.
         '''
-        instance = self.update_model(pk, status='running', celery_task_id=self.request.id)
+        instance = self.update_model(pk, status='running', celery_task_id='' if self.request.id is None else self.request.id)
 
         instance.websocket_emit_status("running")
         status, rc, tb = 'error', None, ''
@@ -690,7 +691,7 @@ class BaseTask(Task):
         instance = self.update_model(pk, status=status, result_traceback=tb,
                                      output_replacements=output_replacements,
                                      **extra_update_fields)
-        self.post_run_hook(instance, **kwargs)
+        self.post_run_hook(instance, status, **kwargs)
         instance.websocket_emit_status(status)
         if status != 'successful' and not hasattr(settings, 'CELERY_UNIT_TEST'):
             # Raising an exception will mark the job as 'failed' in celery
@@ -990,11 +991,26 @@ class RunJob(BaseTask):
         '''
         return getattr(settings, 'AWX_PROOT_ENABLED', False)
 
-    def post_run_hook(self, job, **kwargs):
+    def pre_run_hook(self, job, **kwargs):
+        if job.project.scm_type:
+            local_project_sync = job.project.create_project_update()
+            local_project_sync.job_type = 'run'
+            local_project_sync.save()
+            project_update_task = local_project_sync._get_task_class()
+            try:
+                project_update_task().run(local_project_sync.id)
+            except Exception:
+                job.status = 'failed'
+                job.job_explanation = 'Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % \
+                                      ('project_update', local_project_sync.name, local_project_sync.id)
+                job.save()
+                raise
+
+    def post_run_hook(self, job, status, **kwargs):
         '''
         Hook for actions to run after job/task has completed.
         '''
-        super(RunJob, self).post_run_hook(job, **kwargs)
+        super(RunJob, self).post_run_hook(job, status, **kwargs)
         try:
             inventory = job.inventory
         except Inventory.DoesNotExist:
@@ -1095,7 +1111,10 @@ class RunProjectUpdate(BaseTask):
             args.append('-v')
         scm_url, extra_vars = self._build_scm_url_extra_vars(project_update,
                                                              **kwargs)
-        scm_branch = project_update.scm_branch or {'hg': 'tip'}.get(project_update.scm_type, 'HEAD')
+        if project_update.project.scm_revision and project_update.job_type == 'check':
+            scm_branch = project_update.project.scm_revision
+        else:
+            scm_branch = project_update.scm_branch or {'hg': 'tip'}.get(project_update.scm_type, 'HEAD')
         extra_vars.update({
             'project_path': project_update.get_project_path(check_if_exists=False),
             'scm_type': project_update.scm_type,
@@ -1103,6 +1122,8 @@ class RunProjectUpdate(BaseTask):
             'scm_branch': scm_branch,
             'scm_clean': project_update.scm_clean,
             'scm_delete_on_update': project_update.scm_delete_on_update,
+            'scm_full_checkout': True if project_update.job_type == 'run' else False,
+            'scm_revision_output': '/tmp/_{}_syncrev'.format(project_update.id) # TODO: TempFile
         })
         args.extend(['-e', json.dumps(extra_vars)])
         args.append('project_update.yml')
@@ -1176,6 +1197,17 @@ class RunProjectUpdate(BaseTask):
         '''
         return kwargs.get('private_data_files', {}).get('scm_credential', '')
 
+    def post_run_hook(self, instance, status, **kwargs):
+        if instance.job_type == 'check':
+            p = instance.project
+            fd = open('/tmp/_{}_syncrev'.format(instance.id), 'r')
+            lines = fd.readlines()
+            if lines:
+                p.scm_revision = lines[0].strip()
+                p.save()
+            else:
+                logger.error("Could not find scm revision in check")
+
 class RunInventoryUpdate(BaseTask):
 
     name = 'awx.main.tasks.run_inventory_update'
@@ -1670,7 +1702,7 @@ class RunAdHocCommand(BaseTask):
         '''
         return getattr(settings, 'AWX_PROOT_ENABLED', False)
 
-    def post_run_hook(self, ad_hoc_command, **kwargs):
+    def post_run_hook(self, ad_hoc_command, status, **kwargs):
         '''
         Hook for actions to run after ad hoc command has completed.
         '''
diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml
index 1b2f4520f3..9ab7f1a277 100644
--- a/awx/playbooks/project_update.yml
+++ b/awx/playbooks/project_update.yml
@@ -17,28 +17,93 @@
   tasks:
 
     - name: delete project directory before update
-      file: path={{project_path|quote}} state=absent
+      file:
+        path: "{{project_path|quote}}"
+        state: absent
       when: scm_delete_on_update|default('')
 
     - name: update project using git and accept hostkey
-      git: dest={{project_path|quote}} repo={{scm_url|quote}} version={{scm_branch|quote}} force={{scm_clean}} accept_hostkey={{scm_accept_hostkey}}
+      git:
+        dest: "{{project_path|quote}}"
+        repo: "{{scm_url|quote}}"
+        version: "{{scm_branch|quote}}"
+        force: "{{scm_clean}}"
+        accept_hostkey: "{{scm_accept_hostkey}}"
+        clone: "{{ scm_full_checkout }}"
+        update: "{{ scm_full_checkout }}"
       when: scm_type == 'git' and scm_accept_hostkey is defined
+      register: scm_result
+
+    - name: Set the git repository version
+      set_fact:
+        scm_version: "{{ scm_result['after'] }}"
+      when: "'after' in scm_result"
 
     - name: update project using git
-      git: dest={{project_path|quote}} repo={{scm_url|quote}} version={{scm_branch|quote}} force={{scm_clean}}
+      git:
+        dest: "{{project_path|quote}}"
+        repo: "{{scm_url|quote}}"
+        version: "{{scm_branch|quote}}"
+        force: "{{scm_clean}}"
+        clone: "{{ scm_full_checkout }}"
+        update: "{{ scm_full_checkout }}"        
       when: scm_type == 'git' and scm_accept_hostkey is not defined
+      register: scm_result
+
+    - name: Set the git repository version
+      set_fact:
+        scm_version: "{{ scm_result['after'] }}"
+      when: "'after' in scm_result"
 
     - name: update project using hg
-      hg: dest={{project_path|quote}} repo={{scm_url|quote}} revision={{scm_branch|quote}} force={{scm_clean}}
+      hg:
+        dest: "{{project_path|quote}}"
+        repo: "{{scm_url|quote}}"
+        revision: "{{scm_branch|quote}}"
+        force: "{{scm_clean}}"
+        #clone: "{{ scm_full_checkout }}"
+        #update: "{{ scm_full_checkout }}"
       when: scm_type == 'hg'
+      register: scm_result
+
+    - name: Set the hg repository version
+      set_fact:
+        scm_version: "{{ scm_result['after'] }}"
+      when: "'after' in scm_result"
 
     - name: update project using svn
-      subversion: dest={{project_path|quote}} repo={{scm_url|quote}} revision={{scm_branch|quote}} force={{scm_clean}}
+      subversion:
+        dest: "{{project_path|quote}}"
+        repo: "{{scm_url|quote}}"
+        revision: "{{scm_branch|quote}}"
+        force: "{{scm_clean}}"
+        #checkout: "{{ scm_full_checkout }}"
+        #update: "{{ scm_full_checkout }}"
       when: scm_type == 'svn' and not scm_username|default('')
+      register: scm_result
+
+    - name: Set the svn repository version
+      set_fact:
+        scm_version: "{{ scm_result['after'] }}"
+      when: "'after' in scm_result"
 
     - name: update project using svn with auth
-      subversion: dest={{project_path|quote}} repo={{scm_url|quote}} revision={{scm_branch|quote}} force={{scm_clean}} username={{scm_username|quote}} password={{scm_password|quote}}
+      subversion:
+        dest: "{{project_path|quote}}"
+        repo: "{{scm_url|quote}}"
+        revision: "{{scm_branch|quote}}"
+        force: "{{scm_clean}}"
+        username: "{{scm_username|quote}}"
+        password: "{{scm_password|quote}}"
+        #checkout: "{{ scm_full_checkout }}"
+        #update: "{{ scm_full_checkout }}"
       when: scm_type == 'svn' and scm_username|default('')
+      register: scm_result
+
+    - name: Set the svn repository version
+      set_fact:
+        scm_version: "{{ scm_result['after'] }}"
+      when: "'after' in scm_result"
 
     - name: detect requirements.yml
       stat: path={{project_path|quote}}/roles/requirements.yml
@@ -48,4 +113,14 @@
       command: ansible-galaxy install -r requirements.yml -p {{project_path|quote}}/roles/ --force
       args:
         chdir: "{{project_path|quote}}/roles"
-      when: doesRequirementsExist.stat.exists
+      when: doesRequirementsExist.stat.exists and scm_full_checkout|bool
+
+    - name: Repository Version
+      debug: msg="Repository Version {{ scm_version }}"
+      when: scm_version is defined
+
+    - name: Write Repository Version
+      copy:
+        dest: "{{ scm_revision_output }}"
+        content: "{{ scm_version }}"
+      when: scm_version is defined and scm_revision_output is defined
diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py
index 630b5ee6ee..11095a61f5 100644
--- a/awx/settings/defaults.py
+++ b/awx/settings/defaults.py
@@ -358,23 +358,24 @@ CELERY_QUEUES = (
     Queue('jobs', Exchange('jobs'), routing_key='jobs'),
     Queue('scheduler', Exchange('scheduler', type='topic'), routing_key='scheduler.job.#', durable=False),
     # Projects use a fanout queue, this isn't super well supported
-    Broadcast('projects'),
 )
 CELERY_ROUTES = {'awx.main.tasks.run_job': {'queue': 'jobs',
-                                            'routing_key': 'jobs'},
-                 'awx.main.tasks.run_project_update': {'queue': 'projects'},
-                 'awx.main.tasks.run_inventory_update': {'queue': 'jobs',
-                                                         'routing_key': 'jobs'},
-                 'awx.main.tasks.run_ad_hoc_command': {'queue': 'jobs',
-                                                       'routing_key': 'jobs'},
-                 'awx.main.tasks.run_system_job': {'queue': 'jobs',
-                                                   'routing_key': 'jobs'},
-                 'awx.main.scheduler.tasks.run_job_launch': {'queue': 'scheduler',
-                                                             'routing_key': 'scheduler.job.launch'},
-                 'awx.main.scheduler.tasks.run_job_complete': {'queue': 'scheduler',
-                                                               'routing_key': 'scheduler.job.complete'},
-                 'awx.main.tasks.cluster_node_heartbeat': {'queue': 'default',
-                                                           'routing_key': 'cluster.heartbeat'},}
+                                             'routing_key': 'jobs'},
+                  'awx.main.tasks.run_project_update': {'queue': 'jobs',
+                                                        'routing_key': 'jobs'},
+                  'awx.main.tasks.run_inventory_update': {'queue': 'jobs',
+                                                          'routing_key': 'jobs'},
+                  'awx.main.tasks.run_ad_hoc_command': {'queue': 'jobs',
+                                                        'routing_key': 'jobs'},
+                  'awx.main.tasks.run_system_job': {'queue': 'jobs',
+                                                    'routing_key': 'jobs'},
+                  'awx.main.scheduler.tasks.run_job_launch': {'queue': 'scheduler',
+                                                              'routing_key': 'scheduler.job.launch'},
+                  'awx.main.scheduler.tasks.run_job_complete': {'queue': 'scheduler',
+                                                                'routing_key': 'scheduler.job.complete'},
+                  'awx.main.tasks.cluster_node_heartbeat': {'queue': 'default',
+                                                            'routing_key': 'cluster.heartbeat'},
+}
 
 CELERYBEAT_SCHEDULE = {
     'tower_scheduler': {

From 0f93b875ca7307a34e3d440f5fc4c08feafcc38d Mon Sep 17 00:00:00 2001
From: Matthew Jones <matburt@redhat.com>
Date: Thu, 20 Oct 2016 13:25:47 -0400
Subject: [PATCH 07/21] Add migrations for execution node and project upds

---
 .../migrations/0041_v310_executionnode.py     | 19 +++++++++++++++
 awx/main/migrations/0042_v310_scm_revision.py | 24 +++++++++++++++++++
 2 files changed, 43 insertions(+)
 create mode 100644 awx/main/migrations/0041_v310_executionnode.py
 create mode 100644 awx/main/migrations/0042_v310_scm_revision.py

diff --git a/awx/main/migrations/0041_v310_executionnode.py b/awx/main/migrations/0041_v310_executionnode.py
new file mode 100644
index 0000000000..abcf07e59a
--- /dev/null
+++ b/awx/main/migrations/0041_v310_executionnode.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0040_v310_artifacts'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='unifiedjob',
+            name='execution_node',
+            field=models.TextField(default=b'', editable=False, blank=True),
+        ),
+    ]
diff --git a/awx/main/migrations/0042_v310_scm_revision.py b/awx/main/migrations/0042_v310_scm_revision.py
new file mode 100644
index 0000000000..2de25c082c
--- /dev/null
+++ b/awx/main/migrations/0042_v310_scm_revision.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0041_v310_executionnode'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='scm_revision',
+            field=models.CharField(default=b'', editable=False, max_length=1024, blank=True, help_text='The last revision fetched by a project update', verbose_name='SCM Revision'),
+        ),
+        migrations.AddField(
+            model_name='projectupdate',
+            name='job_type',
+            field=models.CharField(default=b'check', max_length=64, choices=[(b'run', 'Run'), (b'check', 'Check')]),
+        ),
+    ]

From ba469a8c13d3fb0e4e2bdc970dcd08cdb41d35d4 Mon Sep 17 00:00:00 2001
From: Matthew Jones <matburt@redhat.com>
Date: Thu, 20 Oct 2016 13:34:10 -0400
Subject: [PATCH 08/21] Remove spurious workflow job class

---
 awx/main/tasks.py | 35 -----------------------------------
 1 file changed, 35 deletions(-)

diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index 814700afd3..bc8013be57 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -1739,38 +1739,3 @@ class RunSystemJob(BaseTask):
 
     def build_cwd(self, instance, **kwargs):
         return settings.BASE_DIR
-
-'''
-class RunWorkflowJob(BaseTask):
-
-    name = 'awx.main.tasks.run_workflow_job'
-    model = WorkflowJob
-
-    def run(self, pk, **kwargs):
-        #Run the job/task and capture its output.
-        instance = self.update_model(pk, status='running', celery_task_id=self.request.id)
-        instance.websocket_emit_status("running")
-
-        # FIXME: Currently, the workflow job busy waits until the graph run is
-        # complete. Instead, the workflow job should return or never even run,
-        # because all of the "launch logic" can be done schedule().
-
-        # However, other aspects of our system depend on a 1-1 relationship
-        # between a Job and a Celery Task.
-        #
-        # * If we let the workflow job task (RunWorkflowJob.run()) complete
-        #   then how do we trigger the handle_work_error and
-        #   handle_work_success subtasks?
-        #
-        # * How do we handle the recovery process? (i.e. there is an entry in
-        #   the database but not in celery).
-        while True:
-            dag = WorkflowDAG(instance)
-            if dag.is_workflow_done():
-                # TODO: update with accurate finish status (i.e. canceled, error, etc.)
-                instance = self.update_model(instance.pk, status='successful')
-                break
-            time.sleep(1)
-        instance.websocket_emit_status(instance.status)
-        # TODO: Handle cancel
-'''

From 03ca8a571a50ed977eb2aa25d437114e818ccc0a Mon Sep 17 00:00:00 2001
From: Matthew Jones <matburt@redhat.com>
Date: Thu, 20 Oct 2016 13:37:22 -0400
Subject: [PATCH 09/21] Update clustering acceptance docs

---
 docs/clustering.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/clustering.md b/docs/clustering.md
index 196d7fd6e6..7f5e6c0e75 100644
--- a/docs/clustering.md
+++ b/docs/clustering.md
@@ -21,6 +21,7 @@ It's important to point out a few existing things:
   by its needs. Thus we are pretty inflexible to customization beyond what our setup playbook allows. Each Tower node has a
   deployment of RabbitMQ that will cluster with the other nodes' RabbitMQ instances.
 * Existing old-style HA deployments will be transitioned automatically to the new HA system during the upgrade process.
+* Manual projects will need to be synced to all nodes by the customer
 
 ## Important Changes
 
@@ -168,6 +169,7 @@ When verifying acceptance we should ensure the following statements are true
     can communicate with the database.
 * Crucially when network partitioning is resolved all nodes should recover into a consistent state
 * Upgrade Testing, verify behavior before and after are the same for the end user.
+* Project Updates should be thoroughly tested for all scm types (git, svn, hg) and for manual projects.
 
 ## Performance Testing
 

From 2a504ff0402920e280c19edc08a5e0e382e2ec11 Mon Sep 17 00:00:00 2001
From: Matthew Jones <matburt@redhat.com>
Date: Thu, 20 Oct 2016 14:42:12 -0400
Subject: [PATCH 10/21] Shift migration files around for conflict

---
 .../{0041_v310_executionnode.py => 0042_v310_executionnode.py}  | 2 +-
 .../{0042_v310_scm_revision.py => 0043_v310_scm_revision.py}    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename awx/main/migrations/{0041_v310_executionnode.py => 0042_v310_executionnode.py} (90%)
 rename awx/main/migrations/{0042_v310_scm_revision.py => 0043_v310_scm_revision.py} (94%)

diff --git a/awx/main/migrations/0041_v310_executionnode.py b/awx/main/migrations/0042_v310_executionnode.py
similarity index 90%
rename from awx/main/migrations/0041_v310_executionnode.py
rename to awx/main/migrations/0042_v310_executionnode.py
index abcf07e59a..f696d4b95d 100644
--- a/awx/main/migrations/0041_v310_executionnode.py
+++ b/awx/main/migrations/0042_v310_executionnode.py
@@ -7,7 +7,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('main', '0040_v310_artifacts'),
+        ('main', '0041_v310_job_timeout'),
     ]
 
     operations = [
diff --git a/awx/main/migrations/0042_v310_scm_revision.py b/awx/main/migrations/0043_v310_scm_revision.py
similarity index 94%
rename from awx/main/migrations/0042_v310_scm_revision.py
rename to awx/main/migrations/0043_v310_scm_revision.py
index 2de25c082c..7936d8b0a1 100644
--- a/awx/main/migrations/0042_v310_scm_revision.py
+++ b/awx/main/migrations/0043_v310_scm_revision.py
@@ -7,7 +7,7 @@ from django.db import migrations, models
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('main', '0041_v310_executionnode'),
+        ('main', '0042_v310_executionnode'),
     ]
 
     operations = [

From 4562faa85800831561a5e067fc84db21fdfcb88a Mon Sep 17 00:00:00 2001
From: Matthew Jones <matburt@redhat.com>
Date: Thu, 20 Oct 2016 14:42:28 -0400
Subject: [PATCH 11/21] Pass scm revision in env and extra var for job run

---
 awx/main/tasks.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index bc8013be57..6215b83f14 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -782,6 +782,7 @@ class RunJob(BaseTask):
         # callbacks to work.
         env['JOB_ID'] = str(job.pk)
         env['INVENTORY_ID'] = str(job.inventory.pk)
+        env['PROJECT_REVISION'] = job.project.scm_revision
         env['ANSIBLE_CALLBACK_PLUGINS'] = plugin_path
         env['REST_API_URL'] = settings.INTERNAL_API_URL
         env['REST_API_TOKEN'] = job.task_auth_token or ''
@@ -915,6 +916,10 @@ class RunJob(BaseTask):
             'tower_job_id': job.pk,
             'tower_job_launch_type': job.launch_type,
         }
+        if job.project:
+            extra_vars.update({
+                'tower_project_revision': job.project.scm_revision,
+                })
         if job.job_template:
             extra_vars.update({
                 'tower_job_template_id': job.job_template.pk,

From ca32c5fd2122dc25b178386e10f14d8efee47d7f Mon Sep 17 00:00:00 2001
From: Matthew Jones <matburt@redhat.com>
Date: Thu, 20 Oct 2016 14:56:15 -0400
Subject: [PATCH 12/21] scm_revision should be used for job_type `run`

---
 awx/main/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index 6215b83f14..456ad6293c 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -1116,7 +1116,7 @@ class RunProjectUpdate(BaseTask):
             args.append('-v')
         scm_url, extra_vars = self._build_scm_url_extra_vars(project_update,
                                                              **kwargs)
-        if project_update.project.scm_revision and project_update.job_type == 'check':
+        if project_update.project.scm_revision and project_update.job_type == 'run':
             scm_branch = project_update.project.scm_revision
         else:
             scm_branch = project_update.scm_branch or {'hg': 'tip'}.get(project_update.scm_type, 'HEAD')

From 6e22460f1ed52f43e80081a8fea21feaa14924e5 Mon Sep 17 00:00:00 2001
From: Matthew Jones <matburt@redhat.com>
Date: Thu, 20 Oct 2016 15:18:05 -0400
Subject: [PATCH 13/21] Add scm revision to the job model

This also cleans up flake8 issues
---
 awx/api/serializers.py                        |  2 +-
 awx/main/migrations/0043_v310_scm_revision.py |  6 ++++
 awx/main/models/jobs.py                       |  9 ++++++
 awx/main/models/projects.py                   |  3 --
 awx/main/tasks.py                             |  4 ++-
 awx/settings/defaults.py                      | 29 +++++++++----------
 6 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/awx/api/serializers.py b/awx/api/serializers.py
index 861e245a75..5053c5aa96 100644
--- a/awx/api/serializers.py
+++ b/awx/api/serializers.py
@@ -1930,7 +1930,7 @@ class JobSerializer(UnifiedJobSerializer, JobOptionsSerializer):
         fields = ('*', 'job_template', 'passwords_needed_to_start', 'ask_variables_on_launch',
                   'ask_limit_on_launch', 'ask_tags_on_launch', 'ask_skip_tags_on_launch',
                   'ask_job_type_on_launch', 'ask_inventory_on_launch', 'ask_credential_on_launch',
-                  'allow_simultaneous', 'artifacts',)
+                  'allow_simultaneous', 'artifacts', 'scm_revision',)
 
     def get_related(self, obj):
         res = super(JobSerializer, self).get_related(obj)
diff --git a/awx/main/migrations/0043_v310_scm_revision.py b/awx/main/migrations/0043_v310_scm_revision.py
index 7936d8b0a1..08db6be47e 100644
--- a/awx/main/migrations/0043_v310_scm_revision.py
+++ b/awx/main/migrations/0043_v310_scm_revision.py
@@ -21,4 +21,10 @@ class Migration(migrations.Migration):
             name='job_type',
             field=models.CharField(default=b'check', max_length=64, choices=[(b'run', 'Run'), (b'check', 'Check')]),
         ),
+        migrations.AddField(
+            model_name='job',
+            name='scm_revision',
+            field=models.CharField(default=b'', editable=False, max_length=1024, blank=True, help_text='The SCM Revision from the Project used for this job, if available', verbose_name='SCM Revision'),
+        ),
+
     ]
diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py
index 9c216fc526..1f11e5dee9 100644
--- a/awx/main/models/jobs.py
+++ b/awx/main/models/jobs.py
@@ -559,6 +559,15 @@ class Job(UnifiedJob, JobOptions, JobNotificationMixin):
         default={},
         editable=False,
     )
+    scm_revision = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        editable=False,
+        verbose_name=_('SCM Revision'),
+        help_text=_('The SCM Revision from the Project used for this job, if available'),
+    )
+
 
     @classmethod
     def _get_parent_field_name(cls):
diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py
index d96bcec98b..5686716d73 100644
--- a/awx/main/models/projects.py
+++ b/awx/main/models/projects.py
@@ -7,9 +7,6 @@ import os
 import re
 import urlparse
 
-# Celery
-from celery import group, chord
-
 # Django
 from django.conf import settings
 from django.db import models
diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index 456ad6293c..f834e32ff3 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -919,7 +919,7 @@ class RunJob(BaseTask):
         if job.project:
             extra_vars.update({
                 'tower_project_revision': job.project.scm_revision,
-                })
+            })
         if job.job_template:
             extra_vars.update({
                 'tower_job_template_id': job.job_template.pk,
@@ -1004,6 +1004,8 @@ class RunJob(BaseTask):
             project_update_task = local_project_sync._get_task_class()
             try:
                 project_update_task().run(local_project_sync.id)
+                job.scm_revision = job.project.scm_revision
+                job.save()
             except Exception:
                 job.status = 'failed'
                 job.job_explanation = 'Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % \
diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py
index 11095a61f5..d0aebcefa1 100644
--- a/awx/settings/defaults.py
+++ b/awx/settings/defaults.py
@@ -9,7 +9,6 @@ import djcelery
 from datetime import timedelta
 
 from kombu import Queue, Exchange
-from kombu.common import Broadcast
 
 # Update this module's local settings from the global settings module.
 from django.conf import global_settings
@@ -361,20 +360,20 @@ CELERY_QUEUES = (
 )
 CELERY_ROUTES = {'awx.main.tasks.run_job': {'queue': 'jobs',
                                              'routing_key': 'jobs'},
-                  'awx.main.tasks.run_project_update': {'queue': 'jobs',
-                                                        'routing_key': 'jobs'},
-                  'awx.main.tasks.run_inventory_update': {'queue': 'jobs',
-                                                          'routing_key': 'jobs'},
-                  'awx.main.tasks.run_ad_hoc_command': {'queue': 'jobs',
-                                                        'routing_key': 'jobs'},
-                  'awx.main.tasks.run_system_job': {'queue': 'jobs',
-                                                    'routing_key': 'jobs'},
-                  'awx.main.scheduler.tasks.run_job_launch': {'queue': 'scheduler',
-                                                              'routing_key': 'scheduler.job.launch'},
-                  'awx.main.scheduler.tasks.run_job_complete': {'queue': 'scheduler',
-                                                                'routing_key': 'scheduler.job.complete'},
-                  'awx.main.tasks.cluster_node_heartbeat': {'queue': 'default',
-                                                            'routing_key': 'cluster.heartbeat'},
+                 'awx.main.tasks.run_project_update': {'queue': 'jobs',
+                                                       'routing_key': 'jobs'},
+                 'awx.main.tasks.run_inventory_update': {'queue': 'jobs',
+                                                         'routing_key': 'jobs'},
+                 'awx.main.tasks.run_ad_hoc_command': {'queue': 'jobs',
+                                                       'routing_key': 'jobs'},
+                 'awx.main.tasks.run_system_job': {'queue': 'jobs',
+                                                   'routing_key': 'jobs'},
+                 'awx.main.scheduler.tasks.run_job_launch': {'queue': 'scheduler',
+                                                             'routing_key': 'scheduler.job.launch'},
+                 'awx.main.scheduler.tasks.run_job_complete': {'queue': 'scheduler',
+                                                               'routing_key': 'scheduler.job.complete'},
+                 'awx.main.tasks.cluster_node_heartbeat': {'queue': 'default',
+                                                           'routing_key': 'cluster.heartbeat'},
 }
 
 CELERYBEAT_SCHEDULE = {

From 73b5b47e9445129f8ba26852e03dabc79268fbe1 Mon Sep 17 00:00:00 2001
From: Matthew Jones <matburt@redhat.com>
Date: Thu, 20 Oct 2016 15:27:59 -0400
Subject: [PATCH 14/21] flake8 cleanup

---
 awx/settings/defaults.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py
index d0aebcefa1..928ac39afa 100644
--- a/awx/settings/defaults.py
+++ b/awx/settings/defaults.py
@@ -359,7 +359,7 @@ CELERY_QUEUES = (
     # Projects use a fanout queue, this isn't super well supported
 )
 CELERY_ROUTES = {'awx.main.tasks.run_job': {'queue': 'jobs',
-                                             'routing_key': 'jobs'},
+                                            'routing_key': 'jobs'},
                  'awx.main.tasks.run_project_update': {'queue': 'jobs',
                                                        'routing_key': 'jobs'},
                  'awx.main.tasks.run_inventory_update': {'queue': 'jobs',
@@ -373,8 +373,7 @@ CELERY_ROUTES = {'awx.main.tasks.run_job': {'queue': 'jobs',
                  'awx.main.scheduler.tasks.run_job_complete': {'queue': 'scheduler',
                                                                'routing_key': 'scheduler.job.complete'},
                  'awx.main.tasks.cluster_node_heartbeat': {'queue': 'default',
-                                                           'routing_key': 'cluster.heartbeat'},
-}
+                                                           'routing_key': 'cluster.heartbeat'}}
 
 CELERYBEAT_SCHEDULE = {
     'tower_scheduler': {

From 3d8eb489863cf8b425187e8b78e0b56a642313a3 Mon Sep 17 00:00:00 2001
From: Matthew Jones <matburt@redhat.com>
Date: Thu, 20 Oct 2016 16:50:25 -0400
Subject: [PATCH 15/21] Test for HA license before allowing cluster job start

---
 awx/main/access.py                 |  2 ++
 awx/main/managers.py               |  4 ++++
 awx/main/tests/unit/test_access.py | 15 ++++++++++++++-
 3 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/awx/main/access.py b/awx/main/access.py
index 7be1838017..70346d8b4b 100644
--- a/awx/main/access.py
+++ b/awx/main/access.py
@@ -1044,6 +1044,8 @@ class JobTemplateAccess(BaseAccess):
                 self.check_license(feature='system_tracking')
             if obj.survey_enabled:
                 self.check_license(feature='surveys')
+            if Instance.objects.active_count() > 1:
+                self.check_license(feature='ha')
 
         # Super users can start any job
         if self.user.is_superuser:
diff --git a/awx/main/managers.py b/awx/main/managers.py
index 176deb9483..c054584b0c 100644
--- a/awx/main/managers.py
+++ b/awx/main/managers.py
@@ -36,6 +36,10 @@ class InstanceManager(models.Manager):
             return node[0]
         raise RuntimeError("No instance found with the current cluster host id")
 
+    def active_count(self):
+        """Return count of active Tower nodes for licensing."""
+        return self.all().count()
+
     def my_role(self):
         # NOTE: TODO: Likely to repurpose this once standalone ramparts are a thing
         return "tower"
diff --git a/awx/main/tests/unit/test_access.py b/awx/main/tests/unit/test_access.py
index 650ed19864..fa6c34b95e 100644
--- a/awx/main/tests/unit/test_access.py
+++ b/awx/main/tests/unit/test_access.py
@@ -10,7 +10,8 @@ from awx.main.access import (
     JobTemplateAccess,
     WorkflowJobTemplateAccess,
 )
-from awx.main.models import Credential, Inventory, Project, Role, Organization
+from awx.conf.license import LicenseForbids
+from awx.main.models import Credential, Inventory, Project, Role, Organization, Instance
 
 
 @pytest.fixture
@@ -106,6 +107,18 @@ def test_jt_add_scan_job_check(job_template_with_ids, user_unit):
                     'job_type': 'scan'
                 })
 
+def mock_raise_license_forbids(self, add_host=False, feature=None, check_expiration=True):
+    raise LicenseForbids("Feature not enabled")
+
+def mock_raise_none(self, add_host=False, feature=None, check_expiration=True):
+    return None
+    
+def test_jt_can_start_ha(job_template_with_ids):
+    with mock.patch.object(Instance.objects, 'active_count', return_value=2):
+        with mock.patch('awx.main.access.BaseAccess.check_license', new=mock_raise_license_forbids):
+            with pytest.raises(LicenseForbids):
+                JobTemplateAccess(user_unit).can_start(job_template_with_ids)
+
 def test_jt_can_add_bad_data(user_unit):
     "Assure that no server errors are returned if we call JT can_add with bad data"
     access = JobTemplateAccess(user_unit)

From 7050c4e09effb3155c1e8e65f9c056e9fdedbc0e Mon Sep 17 00:00:00 2001
From: Chris Meyers <chris.meyers.fsu@gmail.com>
Date: Fri, 21 Oct 2016 10:04:50 -0400
Subject: [PATCH 16/21] Revert "no one knows wtf the files in this dir a for,
 so don't serve them"

This reverts commit b3cccea70368ba1550366380e583751df1df061b.
---
 tools/docker-compose/nginx.vh.default.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/docker-compose/nginx.vh.default.conf b/tools/docker-compose/nginx.vh.default.conf
index 7de0ac073b..2325057378 100644
--- a/tools/docker-compose/nginx.vh.default.conf
+++ b/tools/docker-compose/nginx.vh.default.conf
@@ -21,7 +21,7 @@ server {
 
     location /static/ {
         root /tower_devel;
-        try_files /awx/ui/$uri /awx/$uri =404;
+        try_files /awx/ui/$uri /awx/$uri /awx/public/$uri =404;
         access_log off;
         sendfile off;
     }

From 0f0d9953b34ae509328cf8045832ccc9b4884aa6 Mon Sep 17 00:00:00 2001
From: Chris Meyers <chrismeyersfsu@users.noreply.github.com>
Date: Sat, 22 Oct 2016 09:51:12 -0400
Subject: [PATCH 17/21] bump os-client-config for shade

---
 requirements/requirements_ansible.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt
index 6fe35d5d89..8d5cfcb4e2 100644
--- a/requirements/requirements_ansible.txt
+++ b/requirements/requirements_ansible.txt
@@ -33,7 +33,7 @@ msgpack-python==0.4.7
 munch==2.0.4
 netaddr==0.7.18
 netifaces==0.10.4
-os-client-config==1.14.0
+os-client-config==1.22.0
 os-diskconfig-python-novaclient-ext==0.1.3
 os-networksv2-python-novaclient-ext==0.25
 os-virtual-interfacesv2-python-novaclient-ext==0.19

From 312183970434358cf761c375fe68b84783d76452 Mon Sep 17 00:00:00 2001
From: Alan Rominger <arominge@redhat.com>
Date: Mon, 24 Oct 2016 09:43:17 -0400
Subject: [PATCH 18/21] only fill scm_revision var if project for job exists

---
 awx/main/tasks.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index f834e32ff3..c88ce05f1f 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -782,7 +782,8 @@ class RunJob(BaseTask):
         # callbacks to work.
         env['JOB_ID'] = str(job.pk)
         env['INVENTORY_ID'] = str(job.inventory.pk)
-        env['PROJECT_REVISION'] = job.project.scm_revision
+        if job.project:
+            env['PROJECT_REVISION'] = job.project.scm_revision
         env['ANSIBLE_CALLBACK_PLUGINS'] = plugin_path
         env['REST_API_URL'] = settings.INTERNAL_API_URL
         env['REST_API_TOKEN'] = job.task_auth_token or ''

From 7c7d2e37edbf9d39ecfbf1a8a403ec176ca2e5a0 Mon Sep 17 00:00:00 2001
From: Matthew Jones <matburt@redhat.com>
Date: Mon, 24 Oct 2016 11:31:39 -0400
Subject: [PATCH 19/21] Fix some issues syncing playbooks

* Build a list of playbooks and store it in the database at sync time
* Fix an issue running playbook sync on jobs for scan jobs
* Remove a TODO that was unneeded
---
 awx/api/serializers.py                        |  5 ++++-
 .../0044_v310_project_playbook_files.py       | 20 +++++++++++++++++++
 awx/main/models/projects.py                   | 11 ++++++++++
 awx/main/tasks.py                             | 11 +++-------
 awx/playbooks/project_update.yml              |  8 ++++----
 5 files changed, 42 insertions(+), 13 deletions(-)
 create mode 100644 awx/main/migrations/0044_v310_project_playbook_files.py

diff --git a/awx/api/serializers.py b/awx/api/serializers.py
index 5053c5aa96..d6aa39f706 100644
--- a/awx/api/serializers.py
+++ b/awx/api/serializers.py
@@ -961,12 +961,15 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
 
 class ProjectPlaybooksSerializer(ProjectSerializer):
 
-    playbooks = serializers.ReadOnlyField(help_text='Array of playbooks available within this project.')
+    playbooks = serializers.SerializerMethodField(help_text='Array of playbooks available within this project.')
 
     class Meta:
         model = Project
         fields = ('playbooks',)
 
+    def get_playbooks(self, obj):
+        return obj.playbook_files
+
     @property
     def data(self):
         ret = super(ProjectPlaybooksSerializer, self).data
diff --git a/awx/main/migrations/0044_v310_project_playbook_files.py b/awx/main/migrations/0044_v310_project_playbook_files.py
new file mode 100644
index 0000000000..cdf059faec
--- /dev/null
+++ b/awx/main/migrations/0044_v310_project_playbook_files.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import jsonfield.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0043_v310_scm_revision'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='playbook_files',
+            field=jsonfield.fields.JSONField(default=[], help_text='List of playbooks found in the project', verbose_name='Playbook Files', editable=False, blank=True),
+        ),
+    ]
diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py
index 5686716d73..1e007a7125 100644
--- a/awx/main/models/projects.py
+++ b/awx/main/models/projects.py
@@ -7,6 +7,9 @@ import os
 import re
 import urlparse
 
+# JSONField
+from jsonfield import JSONField
+
 # Django
 from django.conf import settings
 from django.db import models
@@ -236,6 +239,14 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin):
         help_text=_('The last revision fetched by a project update'),
     )
 
+    playbook_files = JSONField(
+        blank=True,
+        default=[],
+        editable=False,
+        verbose_name=_('Playbook Files'),
+        help_text=_('List of playbooks found in the project'),
+    )
+
     admin_role = ImplicitRoleField(parent_role=[
         'organization.admin_role',
         'singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index f834e32ff3..c952cdc83e 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -160,12 +160,6 @@ def tower_periodic_scheduler(self):
     logger.debug("Last run was: %s", last_run)
     write_last_run(run_now)
 
-    # Sanity check: If this is a secondary machine, there is nothing
-    # on the schedule.
-    # TODO: Fix for clustering/ha
-    if Instance.objects.my_role() == 'secondary':
-        return
-
     old_schedules = Schedule.objects.enabled().before(last_run)
     for schedule in old_schedules:
         schedule.save()
@@ -997,7 +991,7 @@ class RunJob(BaseTask):
         return getattr(settings, 'AWX_PROOT_ENABLED', False)
 
     def pre_run_hook(self, job, **kwargs):
-        if job.project.scm_type:
+        if job.project and job.project.scm_type:
             local_project_sync = job.project.create_project_update()
             local_project_sync.job_type = 'run'
             local_project_sync.save()
@@ -1205,12 +1199,13 @@ class RunProjectUpdate(BaseTask):
         return kwargs.get('private_data_files', {}).get('scm_credential', '')
 
     def post_run_hook(self, instance, status, **kwargs):
-        if instance.job_type == 'check':
+        if instance.job_type == 'check' and status not in ('failed', 'canceled',):
             p = instance.project
             fd = open('/tmp/_{}_syncrev'.format(instance.id), 'r')
             lines = fd.readlines()
             if lines:
                 p.scm_revision = lines[0].strip()
+                p.playbook_files = p.playbooks
                 p.save()
             else:
                 logger.error("Could not find scm revision in check")
diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml
index 9ab7f1a277..30eff5f6bc 100644
--- a/awx/playbooks/project_update.yml
+++ b/awx/playbooks/project_update.yml
@@ -29,8 +29,8 @@
         version: "{{scm_branch|quote}}"
         force: "{{scm_clean}}"
         accept_hostkey: "{{scm_accept_hostkey}}"
-        clone: "{{ scm_full_checkout }}"
-        update: "{{ scm_full_checkout }}"
+        #clone: "{{ scm_full_checkout }}"
+        #update: "{{ scm_full_checkout }}"
       when: scm_type == 'git' and scm_accept_hostkey is defined
       register: scm_result
 
@@ -45,8 +45,8 @@
         repo: "{{scm_url|quote}}"
         version: "{{scm_branch|quote}}"
         force: "{{scm_clean}}"
-        clone: "{{ scm_full_checkout }}"
-        update: "{{ scm_full_checkout }}"        
+        #clone: "{{ scm_full_checkout }}"
+        #update: "{{ scm_full_checkout }}"
       when: scm_type == 'git' and scm_accept_hostkey is not defined
       register: scm_result
 

From 18cb20ebb62f586f3a6bcdecadb4944e18f89ab9 Mon Sep 17 00:00:00 2001
From: AlanCoding <arominge@redhat.com>
Date: Mon, 24 Oct 2016 12:42:04 -0400
Subject: [PATCH 20/21] remove code that checks for local project directory to
 give status

---
 awx/api/views.py            | 9 ---------
 awx/main/models/projects.py | 4 ----
 2 files changed, 13 deletions(-)

diff --git a/awx/api/views.py b/awx/api/views.py
index 7a82408401..66980e141d 100644
--- a/awx/api/views.py
+++ b/awx/api/views.py
@@ -954,15 +954,6 @@ class ProjectList(ListCreateAPIView):
         )
         return projects_qs
 
-    def get(self, request, *args, **kwargs):
-        # Not optimal, but make sure the project status and last_updated fields
-        # are up to date here...
-        projects_qs = Project.objects
-        projects_qs = projects_qs.select_related('current_job', 'last_job')
-        for project in projects_qs:
-            project._set_status_and_last_job_run()
-        return super(ProjectList, self).get(request, *args, **kwargs)
-
 class ProjectDetail(RetrieveUpdateDestroyAPIView):
 
     model = Project
diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py
index 1e007a7125..9888afaaa7 100644
--- a/awx/main/models/projects.py
+++ b/awx/main/models/projects.py
@@ -318,10 +318,6 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin):
             # inherit the child job status on failure
             elif self.last_job_failed:
                 return self.last_job.status
-            # Even on a successful child run, a missing project path overides
-            # the successful status
-            elif not self.get_project_path():
-                return 'missing'
             # Return the successful status
             else:
                 return self.last_job.status

From b143f538888584a40ab6d7cfe91c6eb175178958 Mon Sep 17 00:00:00 2001
From: Graham Mainwaring <gmainwaring@ansible.com>
Date: Mon, 24 Oct 2016 18:50:08 +0000
Subject: [PATCH 21/21] Fix RPM builds and setup for 3.1

---
 config/awx-nginx.conf      | 2 +-
 tools/scripts/tower-python | 9 ---------
 2 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/config/awx-nginx.conf b/config/awx-nginx.conf
index 6089cb7a3c..3df3155ec8 100644
--- a/config/awx-nginx.conf
+++ b/config/awx-nginx.conf
@@ -48,7 +48,7 @@ http {
         server_name _;
         keepalive_timeout 70;
 
-        ssl_certificate /etc/tower/tower.crt;
+        ssl_certificate /etc/tower/tower.cert;
         ssl_certificate_key /etc/tower/tower.key;
         ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
         ssl_ciphers HIGH:!aNULL:!MD5;
diff --git a/tools/scripts/tower-python b/tools/scripts/tower-python
index 3e94c3a59b..d96abe967f 100755
--- a/tools/scripts/tower-python
+++ b/tools/scripts/tower-python
@@ -1,14 +1,5 @@
 #!/bin/bash
 
-# Enable needed Software Collections, if installed
-for scl in python27 httpd24; do
-  if [ -f /etc/scl/prefixes/$scl ]; then
-    if [ -f `cat /etc/scl/prefixes/$scl`/$scl/enable ]; then
-      . `cat /etc/scl/prefixes/$scl`/$scl/enable
-    fi
-  fi
-done
-
 # Enable Tower virtualenv
 if [ -f /var/lib/awx/venv/tower/bin/activate ]; then
   . /var/lib/awx/venv/tower/bin/activate