Merge pull request #5985 from AlanCoding/scm_inv5

SCM file-based inventory sources

awx/api/serializers.py

@@ -81,6 +81,7 @@ SUMMARIZABLE_FK_FIELDS = {
                                        'groups_with_active_failures',
                                        'has_inventory_sources'),
     'project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'),
+    'scm_project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'),
     'project_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed',),
     'credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud'),
     'cloud_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud'),
@@ -960,8 +961,10 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
         res.update(dict(
             teams = self.reverse('api:project_teams_list', kwargs={'pk': obj.pk}),
             playbooks = self.reverse('api:project_playbooks', kwargs={'pk': obj.pk}),
+            inventory_files = self.reverse('api:project_inventories', kwargs={'pk': obj.pk}),
             update = self.reverse('api:project_update_view', kwargs={'pk': obj.pk}),
             project_updates = self.reverse('api:project_updates_list', kwargs={'pk': obj.pk}),
+            scm_inventories = self.reverse('api:project_scm_inventory_sources', kwargs={'pk': obj.pk}),
             schedules = self.reverse('api:project_schedules_list', kwargs={'pk': obj.pk}),
             activity_stream = self.reverse('api:project_activity_stream_list', kwargs={'pk': obj.pk}),
             notification_templates_any = self.reverse('api:project_notification_templates_any_list', kwargs={'pk': obj.pk}),
@@ -1027,6 +1030,23 @@ class ProjectPlaybooksSerializer(ProjectSerializer):
         return ReturnList(ret, serializer=self)
 
 
+class ProjectInventoriesSerializer(ProjectSerializer):
+
+    inventory_files = serializers.ReadOnlyField(help_text=_(
+        'Array of inventory files and directories available within this project, '
+        'not comprehensive.'))
+
+    class Meta:
+        model = Project
+        fields = ('inventory_files',)
+
+    @property
+    def data(self):
+        ret = super(ProjectInventoriesSerializer, self).data
+        ret = ret.get('inventory_files', [])
+        return ReturnList(ret, serializer=self)
+
+
 class ProjectUpdateViewSerializer(ProjectSerializer):
 
     can_update = serializers.BooleanField(read_only=True)
@@ -1046,6 +1066,7 @@ class ProjectUpdateSerializer(UnifiedJobSerializer, ProjectOptionsSerializer):
         res.update(dict(
             project = self.reverse('api:project_detail', kwargs={'pk': obj.project.pk}),
             cancel = self.reverse('api:project_update_cancel', kwargs={'pk': obj.pk}),
+            scm_inventory_updates = self.reverse('api:project_update_scm_inventory_updates', kwargs={'pk': obj.pk}),
             notifications = self.reverse('api:project_update_notifications_list', kwargs={'pk': obj.pk}),
         ))
         return res
@@ -1481,7 +1502,7 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
 
     class Meta:
         model = InventorySource
-        fields = ('*', 'name', 'inventory', 'update_on_launch', 'update_cache_timeout') + \
+        fields = ('*', 'name', 'inventory', 'update_on_launch', 'update_cache_timeout', 'scm_project') + \
                  ('last_update_failed', 'last_updated', 'group') # Backwards compatibility.
 
     def get_related(self, obj):
@@ -1499,6 +1520,8 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
         ))
         if obj.inventory:
             res['inventory'] = self.reverse('api:inventory_detail', kwargs={'pk': obj.inventory.pk})
+        if obj.scm_project_id is not None:
+            res['scm_project'] = self.reverse('api:project_detail', kwargs={'pk': obj.scm_project.pk})
         # Backwards compatibility.
         if obj.current_update:
             res['current_update'] = self.reverse('api:inventory_update_detail',
@@ -1530,6 +1553,14 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
             return True
         return False
 
+    def build_relational_field(self, field_name, relation_info):
+        field_class, field_kwargs = super(InventorySourceSerializer, self).build_relational_field(field_name, relation_info)
+        # SCM Project and inventory are read-only unless creating a new inventory.
+        if self.instance and field_name in ['scm_project', 'inventory']:
+            field_kwargs['read_only'] = True
+            field_kwargs.pop('queryset', None)
+        return field_class, field_kwargs
+
     def to_representation(self, obj):
         ret = super(InventorySourceSerializer, self).to_representation(obj)
         if obj is None:
@@ -1538,6 +1569,16 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
             ret['inventory'] = None
         return ret
 
+    def validate(self, attrs):
+        # source_path = attrs.get('source_path', self.instance and self.instance.source_path)
+        update_on_launch = attrs.get('update_on_launch', self.instance and self.instance.update_on_launch)
+        scm_project = attrs.get('scm_project', self.instance and self.instance.scm_project)
+        if attrs.get('source_path', None) and not scm_project:
+            raise serializers.ValidationError({"detail": _("Cannot set source_path if not SCM type.")})
+        elif update_on_launch and scm_project:
+            raise serializers.ValidationError({"detail": _("Cannot update SCM-based inventory source on launch.")})
+        return super(InventorySourceSerializer, self).validate(attrs)
+
 
 class InventorySourceUpdateSerializer(InventorySourceSerializer):
 

awx/api/urls.py

@@ -48,6 +48,8 @@ project_urls = patterns('awx.api.views',
     url(r'^$',                                          'project_list'),
     url(r'^(?P<pk>[0-9]+)/$',                           'project_detail'),
     url(r'^(?P<pk>[0-9]+)/playbooks/$',                 'project_playbooks'),
+    url(r'^(?P<pk>[0-9]+)/inventories/$',               'project_inventories'),
+    url(r'^(?P<pk>[0-9]+)/scm_inventory_sources/$',     'project_scm_inventory_sources'),
     url(r'^(?P<pk>[0-9]+)/teams/$',                     'project_teams_list'),
     url(r'^(?P<pk>[0-9]+)/update/$',                    'project_update_view'),
     url(r'^(?P<pk>[0-9]+)/project_updates/$',           'project_updates_list'),
@@ -65,6 +67,7 @@ project_update_urls = patterns('awx.api.views',
     url(r'^(?P<pk>[0-9]+)/$',                           'project_update_detail'),
     url(r'^(?P<pk>[0-9]+)/cancel/$',                    'project_update_cancel'),
     url(r'^(?P<pk>[0-9]+)/stdout/$',                    'project_update_stdout'),
+    url(r'^(?P<pk>[0-9]+)/scm_inventory_updates/$',     'project_update_scm_inventory_updates'),
     url(r'^(?P<pk>[0-9]+)/notifications/$',             'project_update_notifications_list'),
 )
 

awx/api/views.py

@@ -1074,6 +1074,12 @@ class ProjectPlaybooks(RetrieveAPIView):
     serializer_class = ProjectPlaybooksSerializer
 
 
+class ProjectInventories(RetrieveAPIView):
+
+    model = Project
+    serializer_class = ProjectInventoriesSerializer
+
+
 class ProjectTeamsList(ListAPIView):
 
     model = Team
@@ -1101,6 +1107,17 @@ class ProjectSchedulesList(SubListCreateAPIView):
     new_in_148 = True
 
 
+class ProjectScmInventorySources(SubListCreateAPIView):
+
+    view_name = _("Project SCM Inventory Sources")
+    model = Inventory
+    serializer_class = InventorySourceSerializer
+    parent_model = Project
+    relationship = 'scm_inventory_sources'
+    parent_key = 'scm_project'
+    new_in_320 = True
+
+
 class ProjectActivityStreamList(ActivityStreamEnforcementMixin, SubListAPIView):
 
     model = ActivityStream
@@ -1226,6 +1243,17 @@ class ProjectUpdateNotificationsList(SubListAPIView):
     new_in_300 = True
 
 
+class ProjectUpdateScmInventoryUpdates(SubListCreateAPIView):
+
+    view_name = _("Project Update SCM Inventory Updates")
+    model = InventoryUpdate
+    serializer_class = InventoryUpdateSerializer
+    parent_model = ProjectUpdate
+    relationship = 'scm_inventory_updates'
+    parent_key = 'scm_project_update'
+    new_in_320 = True
+
+
 class ProjectAccessList(ResourceAccessList):
 
     model = User # needs to be User for AccessLists's
@@ -2185,7 +2213,7 @@ class InventoryInventorySourcesList(SubListCreateAPIView):
     new_in_14 = True
 
 
-class InventorySourceList(ListAPIView):
+class InventorySourceList(ListCreateAPIView):
 
     model = InventorySource
     serializer_class = InventorySourceSerializer
@@ -2292,6 +2320,10 @@ class InventorySourceUpdateView(RetrieveAPIView):
 
     def post(self, request, *args, **kwargs):
         obj = self.get_object()
+        if obj.source == 'file' and obj.scm_project_id is not None:
+            raise PermissionDenied(detail=_(
+                'Update the project `{}` in order to update this inventory source.'.format(
+                    obj.scm_project.name)))
         if obj.can_update:
             inventory_update = obj.update()
             if not inventory_update:

awx/main/access.py

@@ -762,9 +762,12 @@ class InventorySourceAccess(BaseAccess):
         else:
             return False
 
+    @check_superuser
     def can_add(self, data):
         if not data or 'inventory' not in data:
             return False
+        if not self.check_related('scm_project', Project, data, role_field='admin_role'):
+            return False
         # Checks for admin or change permission on inventory.
         return self.check_related('inventory', Inventory, data)
 

awx/main/migrations/0037_v320_release.py

@@ -51,4 +51,51 @@ class Migration(migrations.Migration):
         migrations.RunSQL([("CREATE INDEX host_ansible_facts_default_gin ON %s USING gin"
                             "(ansible_facts jsonb_path_ops);", [AsIs(Host._meta.db_table)])],
                           [('DROP INDEX host_ansible_facts_default_gin;', None)]),
+
+        # SCM file-based inventories
+        migrations.AddField(
+            model_name='inventorysource',
+            name='scm_last_revision',
+            field=models.CharField(default=b'', max_length=1024, editable=False, blank=True),
+        ),
+        migrations.AddField(
+            model_name='inventorysource',
+            name='scm_project',
+            field=models.ForeignKey(related_name='scm_inventory_sources', default=None, blank=True, to='main.Project', help_text='Project containing inventory file used as source.', null=True),
+        ),
+        migrations.AddField(
+            model_name='inventoryupdate',
+            name='scm_project_update',
+            field=models.ForeignKey(related_name='scm_inventory_updates', default=None, blank=True, to='main.ProjectUpdate', help_text='Inventory files from this Project Update were used for the inventory update.', null=True),
+        ),
+        migrations.AddField(
+            model_name='project',
+            name='inventory_files',
+            field=awx.main.fields.JSONField(default=[], help_text='Suggested list of content that could be Ansible inventory in the project', verbose_name='Inventory Files', editable=False, blank=True),
+        ),
+        migrations.AlterField(
+            model_name='inventorysource',
+            name='source',
+            field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'File, Directory or Script Locally or in Project'), (b'rax', 'Rackspace Cloud Servers'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'satellite6', 'Red Hat Satellite 6'), (b'cloudforms', 'Red Hat CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]),
+        ),
+        migrations.AlterField(
+            model_name='inventoryupdate',
+            name='source',
+            field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'File, Directory or Script Locally or in Project'), (b'rax', 'Rackspace Cloud Servers'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'satellite6', 'Red Hat Satellite 6'), (b'cloudforms', 'Red Hat CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]),
+        ),
+        migrations.AlterField(
+            model_name='inventorysource',
+            name='source_path',
+            field=models.CharField(default=b'', max_length=1024, blank=True),
+        ),
+        migrations.AlterField(
+            model_name='inventoryupdate',
+            name='source_path',
+            field=models.CharField(default=b'', max_length=1024, blank=True),
+        ),
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='launch_type',
+            field=models.CharField(default=b'manual', max_length=20, editable=False, choices=[(b'manual', 'Manual'), (b'relaunch', 'Relaunch'), (b'callback', 'Callback'), (b'scheduled', 'Scheduled'), (b'dependency', 'Dependency'), (b'workflow', 'Workflow'), (b'sync', 'Sync'), (b'scm', 'SCM Update')]),
+        ),
     ]

awx/main/models/inventory.py

@@ -7,6 +7,7 @@ import logging
 import re
 import copy
 from urlparse import urljoin
+import os.path
 
 # Django
 from django.conf import settings
@@ -719,7 +720,7 @@ class InventorySourceOptions(BaseModel):
 
     SOURCE_CHOICES = [
         ('', _('Manual')),
-        ('file', _('Local File, Directory or Script')),
+        ('file', _('File, Directory or Script Locally or in Project')),
         ('rax', _('Rackspace Cloud Servers')),
         ('ec2', _('Amazon EC2')),
         ('gce', _('Google Compute Engine')),
@@ -828,7 +829,6 @@ class InventorySourceOptions(BaseModel):
         max_length=1024,
         blank=True,
         default='',
-        editable=False,
     )
     source_script = models.ForeignKey(
         'CustomInventoryScript',
@@ -1086,6 +1086,21 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions):
         on_delete=models.CASCADE,
     )
 
+    scm_project = models.ForeignKey(
+        'Project',
+        related_name='scm_inventory_sources',
+        help_text=_('Project containing inventory file used as source.'),
+        on_delete=models.CASCADE,
+        blank=True,
+        default=None,
+        null=True
+    )
+    scm_last_revision = models.CharField(
+        max_length=1024,
+        blank=True,
+        default='',
+        editable=False,
+    )
     update_on_launch = models.BooleanField(
         default=False,
     )
@@ -1101,12 +1116,14 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions):
     def _get_unified_job_field_names(cls):
         return ['name', 'description', 'source', 'source_path', 'source_script', 'source_vars', 'schedule',
                 'credential', 'source_regions', 'instance_filters', 'group_by', 'overwrite', 'overwrite_vars',
-                'timeout', 'launch_type',]
+                'timeout', 'launch_type', 'scm_project_update',]
 
     def save(self, *args, **kwargs):
         # If update_fields has been specified, add our field names to it,
         # if it hasn't been specified, then we're just doing a normal save.
         update_fields = kwargs.get('update_fields', [])
+        is_new_instance = not bool(self.pk)
+        is_scm_type = self.scm_project_id is not None
 
         # Set name automatically. Include PK (or placeholder) to make sure the names are always unique.
         replace_text = '__replace_%s__' % now()
@@ -1116,18 +1133,33 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions):
                 self.name = '%s (%s)' % (self.inventory.name, self.pk)
             elif self.inventory:
                 self.name = '%s (%s)' % (self.inventory.name, replace_text)
-            elif self.pk:
+            elif not is_new_instance:
                 self.name = 'inventory source (%s)' % self.pk
             else:
                 self.name = 'inventory source (%s)' % replace_text
             if 'name' not in update_fields:
                 update_fields.append('name')
+        # Reset revision if SCM source has changed parameters
+        if is_scm_type and not is_new_instance:
+            before_is = self.__class__.objects.get(pk=self.pk)
+            if before_is.source_path != self.source_path or before_is.scm_project_id != self.scm_project_id:
+                # Reset the scm_revision if file changed to force update
+                self.scm_revision = None
+                if 'scm_revision' not in update_fields:
+                    update_fields.append('scm_revision')
+
         # Do the actual save.
         super(InventorySource, self).save(*args, **kwargs)
+
         # Add the PK to the name.
         if replace_text in self.name:
             self.name = self.name.replace(replace_text, str(self.pk))
             super(InventorySource, self).save(update_fields=['name'])
+        if is_scm_type and is_new_instance:
+            # Schedule a new Project update if one is not already queued
+            if not self.scm_project.project_updates.filter(
+                    status__in=['new', 'pending', 'waiting']).exists():
+                self.scm_project.update()
         if not getattr(_inventory_updates, 'is_updating', False):
             if self.inventory is not None:
                 self.inventory.update_computed_fields(update_groups=False, update_hosts=False)
@@ -1222,6 +1254,15 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin):
         default=False,
         editable=False,
     )
+    scm_project_update = models.ForeignKey(
+        'ProjectUpdate',
+        related_name='scm_inventory_updates',
+        help_text=_('Inventory files from this Project Update were used for the inventory update.'),
+        on_delete=models.CASCADE,
+        blank=True,
+        default=None,
+        null=True
+    )
 
     @classmethod
     def _get_parent_field_name(cls):
@@ -1256,6 +1297,14 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin):
     def get_ui_url(self):
         return urljoin(settings.TOWER_URL_BASE, "/#/inventory_sync/{}".format(self.pk))
 
+    def get_actual_source_path(self):
+        '''Alias to source_path that combines with project path for for SCM file based sources'''
+        if self.inventory_source_id is None or self.inventory_source.scm_project_id is None:
+            return self.source_path
+        return os.path.join(
+            self.inventory_source.scm_project.get_project_path(check_if_exists=False),
+            self.source_path)
+
     @property
     def task_impact(self):
         return 50

awx/main/models/projects.py

@@ -4,7 +4,6 @@
 # Python
 import datetime
 import os
-import re
 import urlparse
 
 # Django
@@ -26,6 +25,7 @@ from awx.main.models.notifications import (
 from awx.main.models.unified_jobs import * # noqa
 from awx.main.models.mixins import ResourceMixin
 from awx.main.utils import update_scm_url
+from awx.main.utils.ansible import could_be_inventory, could_be_playbook
 from awx.main.fields import ImplicitRoleField
 from awx.main.models.rbac import (
     ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
@@ -173,39 +173,35 @@ class ProjectOptions(models.Model):
 
     @property
     def playbooks(self):
-        valid_re = re.compile(r'^\s*?-?\s*?(?:hosts|include):\s*?.*?$')
         results = []
         project_path = self.get_project_path()
         if project_path:
             for dirpath, dirnames, filenames in os.walk(smart_str(project_path)):
                 for filename in filenames:
-                    if os.path.splitext(filename)[-1] not in ['.yml', '.yaml']:
-                        continue
-                    playbook = os.path.join(dirpath, filename)
-                    # Filter files that do not have either hosts or top-level
-                    # includes. Use regex to allow files with invalid YAML to
-                    # show up.
-                    matched = False
-                    try:
-                        for n, line in enumerate(file(playbook)):
-                            if valid_re.match(line):
-                                matched = True
-                            # Any YAML file can also be encrypted with vault;
-                            # allow these to be used as the main playbook.
-                            elif n == 0 and line.startswith('$ANSIBLE_VAULT;'):
-                                matched = True
-                    except IOError:
-                        continue
-                    if not matched:
-                        continue
-                    playbook = os.path.relpath(playbook, smart_str(project_path))
-                    # Filter files in a roles subdirectory.
-                    if 'roles' in playbook.split(os.sep):
-                        continue
-                    # Filter files in a tasks subdirectory.
-                    if 'tasks' in playbook.split(os.sep):
-                        continue
-                    results.append(smart_text(playbook))
+                    playbook = could_be_playbook(project_path, dirpath, filename)
+                    if playbook is not None:
+                        results.append(smart_text(playbook))
+        return sorted(results, key=lambda x: smart_str(x).lower())
+
+
+    @property
+    def inventories(self):
+        results = []
+        project_path = self.get_project_path()
+        if project_path:
+            # Cap the number of results, because it could include lots
+            max_inventory_listing = 50
+            for dirpath, dirnames, filenames in os.walk(smart_str(project_path)):
+                if dirpath.startswith('.'):
+                    continue
+                for filename in filenames:
+                    inv_path = could_be_inventory(project_path, dirpath, filename)
+                    if inv_path is not None:
+                        results.append(smart_text(inv_path))
+                        if len(results) > max_inventory_listing:
+                            break
+                if len(results) > max_inventory_listing:
+                    break
         return sorted(results, key=lambda x: smart_str(x).lower())
 
 
@@ -257,6 +253,14 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin):
         help_text=_('List of playbooks found in the project'),
     )
 
+    inventory_files = JSONField(
+        blank=True,
+        default=[],
+        editable=False,
+        verbose_name=_('Inventory Files'),
+        help_text=_('Suggested list of content that could be Ansible inventory in the project'),
+    )
+
     admin_role = ImplicitRoleField(parent_role=[
         'organization.admin_role',
         'singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,

awx/main/models/unified_jobs.py

@@ -333,6 +333,10 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, Notificatio
         unified_job_class = self._get_unified_job_class()
         fields = self._get_unified_job_field_names()
         unified_job = copy_model_by_class(self, unified_job_class, fields, kwargs)
+        eager_fields = kwargs.get('_eager_fields', None)
+        if eager_fields:
+            for fd, val in eager_fields.items():
+                setattr(unified_job, fd, val)
 
         # Set the unified job template back-link on the job
         parent_field_name = unified_job_class._get_parent_field_name()
@@ -418,6 +422,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
         ('dependency', _('Dependency')),    # Job was started as a dependency of another job.
         ('workflow', _('Workflow')),        # Job was started from a workflow job.
         ('sync', _('Sync')),                # Job was started from a project sync.
+        ('scm', _('SCM Update'))            # Job was created as an Inventory SCM sync.
     ]
 
     PASSWORD_FIELDS = ('start_args',)

awx/main/signals.py

@@ -268,7 +268,7 @@ def migrate_children_from_deleted_group_to_parent_groups(sender, **kwargs):
                     try:
                         inventory = Inventory.objects.get(pk=inventory_pk)
                         inventory.update_computed_fields()
-                    except Inventory.DoesNotExist:
+                    except (Inventory.DoesNotExist, Project.DoesNotExist):
                         pass
 
 

awx/main/tasks.py

@@ -48,7 +48,7 @@ from django.core.exceptions import ObjectDoesNotExist
 # AWX
 from awx.main.constants import CLOUD_PROVIDERS
 from awx.main.models import * # noqa
-from awx.main.models import UnifiedJob
+from awx.main.models.unified_jobs import ACTIVE_STATES
 from awx.main.queue import CallbackQueueDispatcher
 from awx.main.task_engine import TaskEnhancer
 from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, update_scm_url,
@@ -1099,16 +1099,22 @@ class RunJob(BaseTask):
 
     def pre_run_hook(self, job, **kwargs):
         if job.project and job.project.scm_type:
-            local_project_sync = job.project.create_project_update(launch_type="sync")
-            local_project_sync.job_type = 'run'
-            local_project_sync.save()
-            # save the associated project update before calling run() so that a
+            job_request_id = '' if self.request.id is None else self.request.id
+            local_project_sync = job.project.create_project_update(
+                launch_type="sync",
+                _eager_params=dict(
+                    job_type='run',
+                    status='running',
+                    celery_task_id=job_request_id))
+            # save the associated job before calling run() so that a
             # cancel() call on the job can cancel the project update
             job = self.update_model(job.pk, project_update=local_project_sync)
 
             project_update_task = local_project_sync._get_task_class()
             try:
-                project_update_task().run(local_project_sync.id)
+                task_instance = project_update_task()
+                task_instance.request.id = job_request_id
+                task_instance.run(local_project_sync.id)
                 job = self.update_model(job.pk, scm_revision=job.project.scm_revision)
             except Exception:
                 job = self.update_model(job.pk, status='failed',
@@ -1317,9 +1323,43 @@ class RunProjectUpdate(BaseTask):
             instance_actual.save()
         return OutputEventFilter(stdout_handle, raw_callback=raw_callback)
 
+    def _update_dependent_inventories(self, project_update, dependent_inventory_sources):
+        for inv_src in dependent_inventory_sources:
+            if inv_src.scm_last_revision == project_update.project.scm_revision:
+                logger.debug('Skipping SCM inventory update for `{}` because '
+                             'project has not changed.'.format(inv_src.name))
+                continue
+            logger.debug('Local dependent inventory update for `{}`.'.format(inv_src.name))
+            with transaction.atomic():
+                if InventoryUpdate.objects.filter(inventory_source=inv_src,
+                                                  status__in=ACTIVE_STATES).exists():
+                    logger.info('Skipping SCM inventory update for `{}` because '
+                                'another update is already active.'.format(inv.name))
+                    continue
+                project_request_id = '' if self.request.id is None else self.request.id
+                local_inv_update = inv_src.create_inventory_update(
+                    scm_project_update_id=project_update.id,
+                    launch_type='scm',
+                    _eager_fields=dict(
+                        status='running',
+                        celery_task_id=str(project_request_id)))
+            inv_update_task = local_inv_update._get_task_class()
+            try:
+                task_instance = inv_update_task()
+                # Runs in the same Celery task as project update
+                task_instance.request.id = project_request_id
+                task_instance.run(local_inv_update.id)
+                inv_src.scm_last_revision = project_update.project.scm_revision
+                inv_src.save(update_fields=['scm_last_revision'])
+            except Exception as e:
+                # A failed file update does not block other actions
+                logger.error('Encountered error updating project dependent inventory: {}'.format(e))
+                continue
+
     def post_run_hook(self, instance, status, **kwargs):
+        p = instance.project
+        dependent_inventory_sources = p.scm_inventory_sources.all()
         if instance.job_type == 'check' and status not in ('failed', 'canceled',):
-            p = instance.project
             fd = open(self.revision_path, 'r')
             lines = fd.readlines()
             if lines:
@@ -1327,11 +1367,17 @@ class RunProjectUpdate(BaseTask):
             else:
                 logger.info("Could not find scm revision in check")
             p.playbook_files = p.playbooks
+            if len(dependent_inventory_sources) > 0:
+                p.inventory_files = p.inventories
             p.save()
         try:
             os.remove(self.revision_path)
         except Exception, e:
             logger.error("Failed removing revision tmp file: {}".format(e))
+        # Update any inventories that depend on this project
+        if len(dependent_inventory_sources) > 0:
+            if status == 'successful' and instance.launch_type != 'sync':
+                self._update_dependent_inventories(p, dependent_inventory_sources)
 
 
 class RunInventoryUpdate(BaseTask):
@@ -1580,8 +1626,8 @@ class RunInventoryUpdate(BaseTask):
         elif inventory_update.source == 'cloudforms':
             env['CLOUDFORMS_INI_PATH'] = cloud_credential
         elif inventory_update.source == 'file':
-            # FIXME: Parse source_env to dict, update env.
-            pass
+            # Parse source_vars to dict, update env.
+            env.update(parse_yaml_or_json(inventory_update.source_vars))
         elif inventory_update.source == 'custom':
             for env_k in inventory_update.source_vars_dict:
                 if str(env_k) not in env and str(env_k) not in settings.INV_ENV_VARIABLE_BLACKLIST:
@@ -1650,7 +1696,7 @@ class RunInventoryUpdate(BaseTask):
                 ])
 
         elif inventory_update.source == 'file':
-            args.append(inventory_update.source_path)
+            args.append(inventory_update.get_actual_source_path())
         elif inventory_update.source == 'custom':
             runpath = tempfile.mkdtemp(prefix='ansible_tower_launch_')
             handle, path = tempfile.mkstemp(dir=runpath)

awx/main/tests/functional/conftest.py

@@ -133,7 +133,9 @@ def project(instance, organization):
     prj = Project.objects.create(name="test-proj",
                                  description="test-proj-desc",
                                  organization=organization,
-                                 playbook_files=['helloworld.yml', 'alt-helloworld.yml']
+                                 playbook_files=['helloworld.yml', 'alt-helloworld.yml'],
+                                 local_path='_92__test_proj',
+                                 scm_revision='1234567890123456789012345678901234567890'
                                  )
     return prj
 
@@ -208,6 +210,15 @@ def inventory(organization):
     return organization.inventories.create(name="test-inv")
 
 
+@pytest.fixture
+def scm_inventory_source(inventory, project):
+    return InventorySource.objects.create(
+        name="test-scm-inv",
+        scm_project=project,
+        source_path='inventory_file',
+        inventory=inventory)
+
+
 @pytest.fixture
 def inventory_factory(organization):
     def factory(name, org=organization):

awx/main/tests/functional/models/test_inventory.py

@@ -0,0 +1,33 @@
+import pytest
+import mock
+
+# AWX
+from awx.main.models import InventorySource, InventoryUpdate
+
+
+@pytest.mark.django_db
+class TestSCMUpdateFeatures:
+
+    def test_automatic_project_update_on_create(self, inventory, project):
+        inv_src = InventorySource(
+            scm_project=project,
+            source_path='inventory_file',
+            inventory=inventory)
+        with mock.patch.object(inv_src.scm_project, 'update') as mck_update:
+            inv_src.save()
+            mck_update.assert_called_once_with()
+
+    def test_source_location(self, scm_inventory_source):
+        # Combines project directory with the inventory file specified
+        inventory_update = InventoryUpdate(
+            inventory_source=scm_inventory_source,
+            source_path=scm_inventory_source.source_path)
+        assert inventory_update.get_actual_source_path().endswith('_92__test_proj/inventory_file')
+
+    def test_no_unwanted_updates(self, scm_inventory_source):
+        # Changing the non-sensitive fields should not trigger update
+        with mock.patch.object(scm_inventory_source.scm_project, 'update') as mck_update:
+            scm_inventory_source.name = 'edited_inventory'
+            scm_inventory_source.description = "I'm testing this!"
+            scm_inventory_source.save()
+            assert not mck_update.called

awx/main/tests/functional/test_projects.py

@@ -24,6 +24,13 @@ def team_project_list(organization_factory):
     return objects
 
 
+@pytest.mark.django_db
+def test_get_project_path(project):
+    # Test combining projects root with project local path
+    with mock.patch('awx.main.models.projects.settings.PROJECTS_ROOT', '/var/lib/awx'):
+        assert project.get_project_path(check_if_exists=False) == '/var/lib/awx/_92__test_proj'
+
+
 @pytest.mark.django_db
 def test_user_project_paged_list(get, organization_factory):
     'Test project listing that spans multiple pages'

awx/main/tests/functional/test_tasks.py

@@ -0,0 +1,43 @@
+import pytest
+import mock
+import os
+
+from awx.main.tasks import RunProjectUpdate, RunInventoryUpdate
+from awx.main.models import ProjectUpdate, InventoryUpdate
+
+
+@pytest.fixture
+def scm_revision_file(tmpdir_factory):
+    # Returns path to temporary testing revision file
+    revision_file = tmpdir_factory.mktemp('revisions').join('revision.txt')
+    with open(str(revision_file), 'w') as f:
+        f.write('1234567890123456789012345678901234567890')
+    return os.path.join(revision_file.dirname, 'revision.txt')
+
+
+@pytest.mark.django_db
+class TestDependentInventoryUpdate:
+
+    def test_dependent_inventory_updates_is_called(self, scm_inventory_source, scm_revision_file):
+        task = RunProjectUpdate()
+        task.revision_path = scm_revision_file
+        proj_update = ProjectUpdate.objects.create(project=scm_inventory_source.scm_project)
+        with mock.patch.object(RunProjectUpdate, '_update_dependent_inventories') as inv_update_mck:
+            task.post_run_hook(proj_update, 'successful')
+            inv_update_mck.assert_called_once_with(scm_inventory_source.scm_project, mock.ANY)
+
+    def test_no_unwanted_dependent_inventory_updates(self, project, scm_revision_file):
+        task = RunProjectUpdate()
+        task.revision_path = scm_revision_file
+        proj_update = ProjectUpdate.objects.create(project=project)
+        with mock.patch.object(RunProjectUpdate, '_update_dependent_inventories') as inv_update_mck:
+            task.post_run_hook(proj_update, 'successful')
+            assert not inv_update_mck.called
+
+    def test_dependent_inventory_updates(self, scm_inventory_source):
+        task = RunProjectUpdate()
+        with mock.patch.object(RunInventoryUpdate, 'run') as iu_run_mock:
+            task._update_dependent_inventories(scm_inventory_source.scm_project, [scm_inventory_source])
+            assert InventoryUpdate.objects.count() == 1
+            inv_update = InventoryUpdate.objects.first()
+            iu_run_mock.assert_called_once_with(inv_update.id)

awx/main/utils/ansible.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2017 Ansible by Red Hat
+# All Rights Reserved.
+
+# Python
+import re
+import os
+from itertools import islice
+
+# Django
+from django.utils.encoding import smart_str
+
+
+__all__ = ['could_be_playbook', 'could_be_inventory']
+
+
+valid_playbook_re = re.compile(r'^\s*?-?\s*?(?:hosts|include):\s*?.*?$')
+valid_inventory_re = re.compile(r'^[a-zA-Z0-9_.=\[\]]')
+
+
+def _skip_directory(rel_path):
+    path_elements = rel_path.split(os.sep)
+    # Exclude files in a roles subdirectory.
+    if 'roles' in path_elements:
+        return True
+    # Filter files in a tasks subdirectory.
+    if 'tasks' in path_elements:
+        return True
+    for element in path_elements:
+        # Do not include dot files or dirs
+        if element.startswith('.'):
+            return True
+    return False
+
+
+def could_be_playbook(project_path, dir_path, filename):
+    if os.path.splitext(filename)[-1] not in ['.yml', '.yaml']:
+        return None
+    playbook_path = os.path.join(dir_path, filename)
+    # Filter files that do not have either hosts or top-level
+    # includes. Use regex to allow files with invalid YAML to
+    # show up.
+    matched = False
+    try:
+        for n, line in enumerate(file(playbook_path)):
+            if valid_playbook_re.match(line):
+                matched = True
+            # Any YAML file can also be encrypted with vault;
+            # allow these to be used as the main playbook.
+            elif n == 0 and line.startswith('$ANSIBLE_VAULT;'):
+                matched = True
+    except IOError:
+        return None
+    if not matched:
+        return None
+    playbook = os.path.relpath(playbook_path, smart_str(project_path))
+    if _skip_directory(playbook):
+        return None
+    return playbook
+
+
+def could_be_inventory(project_path, dir_path, filename):
+    suspected_ext = os.path.splitext(filename)[-1]
+    # Allow for files with no extension, or with extensions in a certain set
+    if '.' in suspected_ext and suspected_ext not in ['.yml', '.yaml', '.ini']:
+        return None
+    inventory_path = os.path.join(dir_path, filename)
+    # Filter files that do not use a character set consistent with
+    # Ansible inventory mainly
+    try:
+        # only read through first 10 lines for performance
+        with open(inventory_path) as inv_file:
+            for line in islice(inv_file, 10):
+                if not valid_inventory_re.match(line):
+                    return None
+    except IOError:
+        return None
+    inventory = os.path.relpath(inventory_path, smart_str(project_path))
+    if _skip_directory(inventory):
+        return None
+    return inventory