diff --git a/awx/api/serializers.py b/awx/api/serializers.py index c9db59a166..4a59e97871 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -81,6 +81,7 @@ SUMMARIZABLE_FK_FIELDS = { 'groups_with_active_failures', 'has_inventory_sources'), 'project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'), + 'scm_project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'), 'project_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed',), 'credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud'), 'cloud_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud'), @@ -960,8 +961,10 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer): res.update(dict( teams = self.reverse('api:project_teams_list', kwargs={'pk': obj.pk}), playbooks = self.reverse('api:project_playbooks', kwargs={'pk': obj.pk}), + inventory_files = self.reverse('api:project_inventories', kwargs={'pk': obj.pk}), update = self.reverse('api:project_update_view', kwargs={'pk': obj.pk}), project_updates = self.reverse('api:project_updates_list', kwargs={'pk': obj.pk}), + scm_inventories = self.reverse('api:project_scm_inventory_sources', kwargs={'pk': obj.pk}), schedules = self.reverse('api:project_schedules_list', kwargs={'pk': obj.pk}), activity_stream = self.reverse('api:project_activity_stream_list', kwargs={'pk': obj.pk}), notification_templates_any = self.reverse('api:project_notification_templates_any_list', kwargs={'pk': obj.pk}), @@ -1027,6 +1030,23 @@ class ProjectPlaybooksSerializer(ProjectSerializer): return ReturnList(ret, serializer=self) +class ProjectInventoriesSerializer(ProjectSerializer): + + inventory_files = serializers.ReadOnlyField(help_text=_( + 'Array of inventory files and directories available within this project, ' + 'not comprehensive.')) + + class Meta: + model = Project + fields = ('inventory_files',) + + @property + def data(self): + ret = super(ProjectInventoriesSerializer, self).data + ret = ret.get('inventory_files', []) + return ReturnList(ret, serializer=self) + + class ProjectUpdateViewSerializer(ProjectSerializer): can_update = serializers.BooleanField(read_only=True) @@ -1046,6 +1066,7 @@ class ProjectUpdateSerializer(UnifiedJobSerializer, ProjectOptionsSerializer): res.update(dict( project = self.reverse('api:project_detail', kwargs={'pk': obj.project.pk}), cancel = self.reverse('api:project_update_cancel', kwargs={'pk': obj.pk}), + scm_inventory_updates = self.reverse('api:project_update_scm_inventory_updates', kwargs={'pk': obj.pk}), notifications = self.reverse('api:project_update_notifications_list', kwargs={'pk': obj.pk}), )) return res @@ -1481,7 +1502,7 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt class Meta: model = InventorySource - fields = ('*', 'name', 'inventory', 'update_on_launch', 'update_cache_timeout') + \ + fields = ('*', 'name', 'inventory', 'update_on_launch', 'update_cache_timeout', 'scm_project') + \ ('last_update_failed', 'last_updated', 'group') # Backwards compatibility. def get_related(self, obj): @@ -1499,6 +1520,8 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt )) if obj.inventory: res['inventory'] = self.reverse('api:inventory_detail', kwargs={'pk': obj.inventory.pk}) + if obj.scm_project_id is not None: + res['scm_project'] = self.reverse('api:project_detail', kwargs={'pk': obj.scm_project.pk}) # Backwards compatibility. if obj.current_update: res['current_update'] = self.reverse('api:inventory_update_detail', @@ -1530,6 +1553,14 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt return True return False + def build_relational_field(self, field_name, relation_info): + field_class, field_kwargs = super(InventorySourceSerializer, self).build_relational_field(field_name, relation_info) + # SCM Project and inventory are read-only unless creating a new inventory. + if self.instance and field_name in ['scm_project', 'inventory']: + field_kwargs['read_only'] = True + field_kwargs.pop('queryset', None) + return field_class, field_kwargs + def to_representation(self, obj): ret = super(InventorySourceSerializer, self).to_representation(obj) if obj is None: @@ -1538,6 +1569,16 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt ret['inventory'] = None return ret + def validate(self, attrs): + # source_path = attrs.get('source_path', self.instance and self.instance.source_path) + update_on_launch = attrs.get('update_on_launch', self.instance and self.instance.update_on_launch) + scm_project = attrs.get('scm_project', self.instance and self.instance.scm_project) + if attrs.get('source_path', None) and not scm_project: + raise serializers.ValidationError({"detail": _("Cannot set source_path if not SCM type.")}) + elif update_on_launch and scm_project: + raise serializers.ValidationError({"detail": _("Cannot update SCM-based inventory source on launch.")}) + return super(InventorySourceSerializer, self).validate(attrs) + class InventorySourceUpdateSerializer(InventorySourceSerializer): diff --git a/awx/api/urls.py b/awx/api/urls.py index daa02d4329..fda53adf1d 100644 --- a/awx/api/urls.py +++ b/awx/api/urls.py @@ -48,6 +48,8 @@ project_urls = patterns('awx.api.views', url(r'^$', 'project_list'), url(r'^(?P[0-9]+)/$', 'project_detail'), url(r'^(?P[0-9]+)/playbooks/$', 'project_playbooks'), + url(r'^(?P[0-9]+)/inventories/$', 'project_inventories'), + url(r'^(?P[0-9]+)/scm_inventory_sources/$', 'project_scm_inventory_sources'), url(r'^(?P[0-9]+)/teams/$', 'project_teams_list'), url(r'^(?P[0-9]+)/update/$', 'project_update_view'), url(r'^(?P[0-9]+)/project_updates/$', 'project_updates_list'), @@ -65,6 +67,7 @@ project_update_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/$', 'project_update_detail'), url(r'^(?P[0-9]+)/cancel/$', 'project_update_cancel'), url(r'^(?P[0-9]+)/stdout/$', 'project_update_stdout'), + url(r'^(?P[0-9]+)/scm_inventory_updates/$', 'project_update_scm_inventory_updates'), url(r'^(?P[0-9]+)/notifications/$', 'project_update_notifications_list'), ) diff --git a/awx/api/views.py b/awx/api/views.py index e4e56c1c5c..b14ddf0e5f 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -1074,6 +1074,12 @@ class ProjectPlaybooks(RetrieveAPIView): serializer_class = ProjectPlaybooksSerializer +class ProjectInventories(RetrieveAPIView): + + model = Project + serializer_class = ProjectInventoriesSerializer + + class ProjectTeamsList(ListAPIView): model = Team @@ -1101,6 +1107,17 @@ class ProjectSchedulesList(SubListCreateAPIView): new_in_148 = True +class ProjectScmInventorySources(SubListCreateAPIView): + + view_name = _("Project SCM Inventory Sources") + model = Inventory + serializer_class = InventorySourceSerializer + parent_model = Project + relationship = 'scm_inventory_sources' + parent_key = 'scm_project' + new_in_320 = True + + class ProjectActivityStreamList(ActivityStreamEnforcementMixin, SubListAPIView): model = ActivityStream @@ -1226,6 +1243,17 @@ class ProjectUpdateNotificationsList(SubListAPIView): new_in_300 = True +class ProjectUpdateScmInventoryUpdates(SubListCreateAPIView): + + view_name = _("Project Update SCM Inventory Updates") + model = InventoryUpdate + serializer_class = InventoryUpdateSerializer + parent_model = ProjectUpdate + relationship = 'scm_inventory_updates' + parent_key = 'scm_project_update' + new_in_320 = True + + class ProjectAccessList(ResourceAccessList): model = User # needs to be User for AccessLists's @@ -2185,7 +2213,7 @@ class InventoryInventorySourcesList(SubListCreateAPIView): new_in_14 = True -class InventorySourceList(ListAPIView): +class InventorySourceList(ListCreateAPIView): model = InventorySource serializer_class = InventorySourceSerializer @@ -2292,6 +2320,10 @@ class InventorySourceUpdateView(RetrieveAPIView): def post(self, request, *args, **kwargs): obj = self.get_object() + if obj.source == 'file' and obj.scm_project_id is not None: + raise PermissionDenied(detail=_( + 'Update the project `{}` in order to update this inventory source.'.format( + obj.scm_project.name))) if obj.can_update: inventory_update = obj.update() if not inventory_update: diff --git a/awx/main/access.py b/awx/main/access.py index 0d363ea3c7..79ad7dfaef 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -765,9 +765,12 @@ class InventorySourceAccess(BaseAccess): else: return False + @check_superuser def can_add(self, data): if not data or 'inventory' not in data: return False + if not self.check_related('scm_project', Project, data, role_field='admin_role'): + return False # Checks for admin or change permission on inventory. return self.check_related('inventory', Inventory, data) diff --git a/awx/main/migrations/0037_v320_release.py b/awx/main/migrations/0037_v320_release.py index 6049a9d385..550abcfefe 100644 --- a/awx/main/migrations/0037_v320_release.py +++ b/awx/main/migrations/0037_v320_release.py @@ -51,4 +51,51 @@ class Migration(migrations.Migration): migrations.RunSQL([("CREATE INDEX host_ansible_facts_default_gin ON %s USING gin" "(ansible_facts jsonb_path_ops);", [AsIs(Host._meta.db_table)])], [('DROP INDEX host_ansible_facts_default_gin;', None)]), + + # SCM file-based inventories + migrations.AddField( + model_name='inventorysource', + name='scm_last_revision', + field=models.CharField(default=b'', max_length=1024, editable=False, blank=True), + ), + migrations.AddField( + model_name='inventorysource', + name='scm_project', + field=models.ForeignKey(related_name='scm_inventory_sources', default=None, blank=True, to='main.Project', help_text='Project containing inventory file used as source.', null=True), + ), + migrations.AddField( + model_name='inventoryupdate', + name='scm_project_update', + field=models.ForeignKey(related_name='scm_inventory_updates', default=None, blank=True, to='main.ProjectUpdate', help_text='Inventory files from this Project Update were used for the inventory update.', null=True), + ), + migrations.AddField( + model_name='project', + name='inventory_files', + field=awx.main.fields.JSONField(default=[], help_text='Suggested list of content that could be Ansible inventory in the project', verbose_name='Inventory Files', editable=False, blank=True), + ), + migrations.AlterField( + model_name='inventorysource', + name='source', + field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'File, Directory or Script Locally or in Project'), (b'rax', 'Rackspace Cloud Servers'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'satellite6', 'Red Hat Satellite 6'), (b'cloudforms', 'Red Hat CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]), + ), + migrations.AlterField( + model_name='inventoryupdate', + name='source', + field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'File, Directory or Script Locally or in Project'), (b'rax', 'Rackspace Cloud Servers'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'satellite6', 'Red Hat Satellite 6'), (b'cloudforms', 'Red Hat CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]), + ), + migrations.AlterField( + model_name='inventorysource', + name='source_path', + field=models.CharField(default=b'', max_length=1024, blank=True), + ), + migrations.AlterField( + model_name='inventoryupdate', + name='source_path', + field=models.CharField(default=b'', max_length=1024, blank=True), + ), + migrations.AlterField( + model_name='unifiedjob', + name='launch_type', + field=models.CharField(default=b'manual', max_length=20, editable=False, choices=[(b'manual', 'Manual'), (b'relaunch', 'Relaunch'), (b'callback', 'Callback'), (b'scheduled', 'Scheduled'), (b'dependency', 'Dependency'), (b'workflow', 'Workflow'), (b'sync', 'Sync'), (b'scm', 'SCM Update')]), + ), ] diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 789804e8b5..ae1701c686 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -7,6 +7,7 @@ import logging import re import copy from urlparse import urljoin +import os.path # Django from django.conf import settings @@ -719,7 +720,7 @@ class InventorySourceOptions(BaseModel): SOURCE_CHOICES = [ ('', _('Manual')), - ('file', _('Local File, Directory or Script')), + ('file', _('File, Directory or Script Locally or in Project')), ('rax', _('Rackspace Cloud Servers')), ('ec2', _('Amazon EC2')), ('gce', _('Google Compute Engine')), @@ -828,7 +829,6 @@ class InventorySourceOptions(BaseModel): max_length=1024, blank=True, default='', - editable=False, ) source_script = models.ForeignKey( 'CustomInventoryScript', @@ -1086,6 +1086,21 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions): on_delete=models.CASCADE, ) + scm_project = models.ForeignKey( + 'Project', + related_name='scm_inventory_sources', + help_text=_('Project containing inventory file used as source.'), + on_delete=models.CASCADE, + blank=True, + default=None, + null=True + ) + scm_last_revision = models.CharField( + max_length=1024, + blank=True, + default='', + editable=False, + ) update_on_launch = models.BooleanField( default=False, ) @@ -1101,12 +1116,14 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions): def _get_unified_job_field_names(cls): return ['name', 'description', 'source', 'source_path', 'source_script', 'source_vars', 'schedule', 'credential', 'source_regions', 'instance_filters', 'group_by', 'overwrite', 'overwrite_vars', - 'timeout', 'launch_type',] + 'timeout', 'launch_type', 'scm_project_update',] def save(self, *args, **kwargs): # If update_fields has been specified, add our field names to it, # if it hasn't been specified, then we're just doing a normal save. update_fields = kwargs.get('update_fields', []) + is_new_instance = not bool(self.pk) + is_scm_type = self.scm_project_id is not None # Set name automatically. Include PK (or placeholder) to make sure the names are always unique. replace_text = '__replace_%s__' % now() @@ -1116,18 +1133,33 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions): self.name = '%s (%s)' % (self.inventory.name, self.pk) elif self.inventory: self.name = '%s (%s)' % (self.inventory.name, replace_text) - elif self.pk: + elif not is_new_instance: self.name = 'inventory source (%s)' % self.pk else: self.name = 'inventory source (%s)' % replace_text if 'name' not in update_fields: update_fields.append('name') + # Reset revision if SCM source has changed parameters + if is_scm_type and not is_new_instance: + before_is = self.__class__.objects.get(pk=self.pk) + if before_is.source_path != self.source_path or before_is.scm_project_id != self.scm_project_id: + # Reset the scm_revision if file changed to force update + self.scm_revision = None + if 'scm_revision' not in update_fields: + update_fields.append('scm_revision') + # Do the actual save. super(InventorySource, self).save(*args, **kwargs) + # Add the PK to the name. if replace_text in self.name: self.name = self.name.replace(replace_text, str(self.pk)) super(InventorySource, self).save(update_fields=['name']) + if is_scm_type and is_new_instance: + # Schedule a new Project update if one is not already queued + if not self.scm_project.project_updates.filter( + status__in=['new', 'pending', 'waiting']).exists(): + self.scm_project.update() if not getattr(_inventory_updates, 'is_updating', False): if self.inventory is not None: self.inventory.update_computed_fields(update_groups=False, update_hosts=False) @@ -1222,6 +1254,15 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin): default=False, editable=False, ) + scm_project_update = models.ForeignKey( + 'ProjectUpdate', + related_name='scm_inventory_updates', + help_text=_('Inventory files from this Project Update were used for the inventory update.'), + on_delete=models.CASCADE, + blank=True, + default=None, + null=True + ) @classmethod def _get_parent_field_name(cls): @@ -1256,6 +1297,14 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin): def get_ui_url(self): return urljoin(settings.TOWER_URL_BASE, "/#/inventory_sync/{}".format(self.pk)) + def get_actual_source_path(self): + '''Alias to source_path that combines with project path for for SCM file based sources''' + if self.inventory_source_id is None or self.inventory_source.scm_project_id is None: + return self.source_path + return os.path.join( + self.inventory_source.scm_project.get_project_path(check_if_exists=False), + self.source_path) + @property def task_impact(self): return 50 diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index 36fca1d370..1de195d08e 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -4,7 +4,6 @@ # Python import datetime import os -import re import urlparse # Django @@ -26,6 +25,7 @@ from awx.main.models.notifications import ( from awx.main.models.unified_jobs import * # noqa from awx.main.models.mixins import ResourceMixin from awx.main.utils import update_scm_url +from awx.main.utils.ansible import could_be_inventory, could_be_playbook from awx.main.fields import ImplicitRoleField from awx.main.models.rbac import ( ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, @@ -173,39 +173,35 @@ class ProjectOptions(models.Model): @property def playbooks(self): - valid_re = re.compile(r'^\s*?-?\s*?(?:hosts|include):\s*?.*?$') results = [] project_path = self.get_project_path() if project_path: for dirpath, dirnames, filenames in os.walk(smart_str(project_path)): for filename in filenames: - if os.path.splitext(filename)[-1] not in ['.yml', '.yaml']: - continue - playbook = os.path.join(dirpath, filename) - # Filter files that do not have either hosts or top-level - # includes. Use regex to allow files with invalid YAML to - # show up. - matched = False - try: - for n, line in enumerate(file(playbook)): - if valid_re.match(line): - matched = True - # Any YAML file can also be encrypted with vault; - # allow these to be used as the main playbook. - elif n == 0 and line.startswith('$ANSIBLE_VAULT;'): - matched = True - except IOError: - continue - if not matched: - continue - playbook = os.path.relpath(playbook, smart_str(project_path)) - # Filter files in a roles subdirectory. - if 'roles' in playbook.split(os.sep): - continue - # Filter files in a tasks subdirectory. - if 'tasks' in playbook.split(os.sep): - continue - results.append(smart_text(playbook)) + playbook = could_be_playbook(project_path, dirpath, filename) + if playbook is not None: + results.append(smart_text(playbook)) + return sorted(results, key=lambda x: smart_str(x).lower()) + + + @property + def inventories(self): + results = [] + project_path = self.get_project_path() + if project_path: + # Cap the number of results, because it could include lots + max_inventory_listing = 50 + for dirpath, dirnames, filenames in os.walk(smart_str(project_path)): + if dirpath.startswith('.'): + continue + for filename in filenames: + inv_path = could_be_inventory(project_path, dirpath, filename) + if inv_path is not None: + results.append(smart_text(inv_path)) + if len(results) > max_inventory_listing: + break + if len(results) > max_inventory_listing: + break return sorted(results, key=lambda x: smart_str(x).lower()) @@ -257,6 +253,14 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin): help_text=_('List of playbooks found in the project'), ) + inventory_files = JSONField( + blank=True, + default=[], + editable=False, + verbose_name=_('Inventory Files'), + help_text=_('Suggested list of content that could be Ansible inventory in the project'), + ) + admin_role = ImplicitRoleField(parent_role=[ 'organization.admin_role', 'singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 51d8f50aae..23f98c6b7d 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -333,6 +333,10 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, Notificatio unified_job_class = self._get_unified_job_class() fields = self._get_unified_job_field_names() unified_job = copy_model_by_class(self, unified_job_class, fields, kwargs) + eager_fields = kwargs.get('_eager_fields', None) + if eager_fields: + for fd, val in eager_fields.items(): + setattr(unified_job, fd, val) # Set the unified job template back-link on the job parent_field_name = unified_job_class._get_parent_field_name() @@ -418,6 +422,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique ('dependency', _('Dependency')), # Job was started as a dependency of another job. ('workflow', _('Workflow')), # Job was started from a workflow job. ('sync', _('Sync')), # Job was started from a project sync. + ('scm', _('SCM Update')) # Job was created as an Inventory SCM sync. ] PASSWORD_FIELDS = ('start_args',) diff --git a/awx/main/signals.py b/awx/main/signals.py index 39c2822fed..3dda0873b1 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -268,7 +268,7 @@ def migrate_children_from_deleted_group_to_parent_groups(sender, **kwargs): try: inventory = Inventory.objects.get(pk=inventory_pk) inventory.update_computed_fields() - except Inventory.DoesNotExist: + except (Inventory.DoesNotExist, Project.DoesNotExist): pass diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 55d8f5f755..02d875c327 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -48,7 +48,7 @@ from django.core.exceptions import ObjectDoesNotExist # AWX from awx.main.constants import CLOUD_PROVIDERS from awx.main.models import * # noqa -from awx.main.models import UnifiedJob +from awx.main.models.unified_jobs import ACTIVE_STATES from awx.main.queue import CallbackQueueDispatcher from awx.main.task_engine import TaskEnhancer from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, update_scm_url, @@ -1099,16 +1099,22 @@ class RunJob(BaseTask): def pre_run_hook(self, job, **kwargs): if job.project and job.project.scm_type: - local_project_sync = job.project.create_project_update(launch_type="sync") - local_project_sync.job_type = 'run' - local_project_sync.save() - # save the associated project update before calling run() so that a + job_request_id = '' if self.request.id is None else self.request.id + local_project_sync = job.project.create_project_update( + launch_type="sync", + _eager_params=dict( + job_type='run', + status='running', + celery_task_id=job_request_id)) + # save the associated job before calling run() so that a # cancel() call on the job can cancel the project update job = self.update_model(job.pk, project_update=local_project_sync) project_update_task = local_project_sync._get_task_class() try: - project_update_task().run(local_project_sync.id) + task_instance = project_update_task() + task_instance.request.id = job_request_id + task_instance.run(local_project_sync.id) job = self.update_model(job.pk, scm_revision=job.project.scm_revision) except Exception: job = self.update_model(job.pk, status='failed', @@ -1317,9 +1323,43 @@ class RunProjectUpdate(BaseTask): instance_actual.save() return OutputEventFilter(stdout_handle, raw_callback=raw_callback) + def _update_dependent_inventories(self, project_update, dependent_inventory_sources): + for inv_src in dependent_inventory_sources: + if inv_src.scm_last_revision == project_update.project.scm_revision: + logger.debug('Skipping SCM inventory update for `{}` because ' + 'project has not changed.'.format(inv_src.name)) + continue + logger.debug('Local dependent inventory update for `{}`.'.format(inv_src.name)) + with transaction.atomic(): + if InventoryUpdate.objects.filter(inventory_source=inv_src, + status__in=ACTIVE_STATES).exists(): + logger.info('Skipping SCM inventory update for `{}` because ' + 'another update is already active.'.format(inv.name)) + continue + project_request_id = '' if self.request.id is None else self.request.id + local_inv_update = inv_src.create_inventory_update( + scm_project_update_id=project_update.id, + launch_type='scm', + _eager_fields=dict( + status='running', + celery_task_id=str(project_request_id))) + inv_update_task = local_inv_update._get_task_class() + try: + task_instance = inv_update_task() + # Runs in the same Celery task as project update + task_instance.request.id = project_request_id + task_instance.run(local_inv_update.id) + inv_src.scm_last_revision = project_update.project.scm_revision + inv_src.save(update_fields=['scm_last_revision']) + except Exception as e: + # A failed file update does not block other actions + logger.error('Encountered error updating project dependent inventory: {}'.format(e)) + continue + def post_run_hook(self, instance, status, **kwargs): + p = instance.project + dependent_inventory_sources = p.scm_inventory_sources.all() if instance.job_type == 'check' and status not in ('failed', 'canceled',): - p = instance.project fd = open(self.revision_path, 'r') lines = fd.readlines() if lines: @@ -1327,11 +1367,17 @@ class RunProjectUpdate(BaseTask): else: logger.info("Could not find scm revision in check") p.playbook_files = p.playbooks + if len(dependent_inventory_sources) > 0: + p.inventory_files = p.inventories p.save() try: os.remove(self.revision_path) except Exception, e: logger.error("Failed removing revision tmp file: {}".format(e)) + # Update any inventories that depend on this project + if len(dependent_inventory_sources) > 0: + if status == 'successful' and instance.launch_type != 'sync': + self._update_dependent_inventories(p, dependent_inventory_sources) class RunInventoryUpdate(BaseTask): @@ -1580,8 +1626,8 @@ class RunInventoryUpdate(BaseTask): elif inventory_update.source == 'cloudforms': env['CLOUDFORMS_INI_PATH'] = cloud_credential elif inventory_update.source == 'file': - # FIXME: Parse source_env to dict, update env. - pass + # Parse source_vars to dict, update env. + env.update(parse_yaml_or_json(inventory_update.source_vars)) elif inventory_update.source == 'custom': for env_k in inventory_update.source_vars_dict: if str(env_k) not in env and str(env_k) not in settings.INV_ENV_VARIABLE_BLACKLIST: @@ -1650,7 +1696,7 @@ class RunInventoryUpdate(BaseTask): ]) elif inventory_update.source == 'file': - args.append(inventory_update.source_path) + args.append(inventory_update.get_actual_source_path()) elif inventory_update.source == 'custom': runpath = tempfile.mkdtemp(prefix='ansible_tower_launch_') handle, path = tempfile.mkstemp(dir=runpath) diff --git a/awx/main/tests/functional/api/__init__.py b/awx/main/tests/functional/api/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/awx/main/tests/functional/conftest.py b/awx/main/tests/functional/conftest.py index fa87ed4df7..fd89e3df15 100644 --- a/awx/main/tests/functional/conftest.py +++ b/awx/main/tests/functional/conftest.py @@ -133,7 +133,9 @@ def project(instance, organization): prj = Project.objects.create(name="test-proj", description="test-proj-desc", organization=organization, - playbook_files=['helloworld.yml', 'alt-helloworld.yml'] + playbook_files=['helloworld.yml', 'alt-helloworld.yml'], + local_path='_92__test_proj', + scm_revision='1234567890123456789012345678901234567890' ) return prj @@ -208,6 +210,15 @@ def inventory(organization): return organization.inventories.create(name="test-inv") +@pytest.fixture +def scm_inventory_source(inventory, project): + return InventorySource.objects.create( + name="test-scm-inv", + scm_project=project, + source_path='inventory_file', + inventory=inventory) + + @pytest.fixture def inventory_factory(organization): def factory(name, org=organization): diff --git a/awx/main/tests/functional/models/test_inventory.py b/awx/main/tests/functional/models/test_inventory.py new file mode 100644 index 0000000000..8187c9aea6 --- /dev/null +++ b/awx/main/tests/functional/models/test_inventory.py @@ -0,0 +1,33 @@ +import pytest +import mock + +# AWX +from awx.main.models import InventorySource, InventoryUpdate + + +@pytest.mark.django_db +class TestSCMUpdateFeatures: + + def test_automatic_project_update_on_create(self, inventory, project): + inv_src = InventorySource( + scm_project=project, + source_path='inventory_file', + inventory=inventory) + with mock.patch.object(inv_src.scm_project, 'update') as mck_update: + inv_src.save() + mck_update.assert_called_once_with() + + def test_source_location(self, scm_inventory_source): + # Combines project directory with the inventory file specified + inventory_update = InventoryUpdate( + inventory_source=scm_inventory_source, + source_path=scm_inventory_source.source_path) + assert inventory_update.get_actual_source_path().endswith('_92__test_proj/inventory_file') + + def test_no_unwanted_updates(self, scm_inventory_source): + # Changing the non-sensitive fields should not trigger update + with mock.patch.object(scm_inventory_source.scm_project, 'update') as mck_update: + scm_inventory_source.name = 'edited_inventory' + scm_inventory_source.description = "I'm testing this!" + scm_inventory_source.save() + assert not mck_update.called diff --git a/awx/main/tests/functional/test_projects.py b/awx/main/tests/functional/test_projects.py index 5be0e6c3a3..e8b86502f5 100644 --- a/awx/main/tests/functional/test_projects.py +++ b/awx/main/tests/functional/test_projects.py @@ -24,6 +24,13 @@ def team_project_list(organization_factory): return objects +@pytest.mark.django_db +def test_get_project_path(project): + # Test combining projects root with project local path + with mock.patch('awx.main.models.projects.settings.PROJECTS_ROOT', '/var/lib/awx'): + assert project.get_project_path(check_if_exists=False) == '/var/lib/awx/_92__test_proj' + + @pytest.mark.django_db def test_user_project_paged_list(get, organization_factory): 'Test project listing that spans multiple pages' diff --git a/awx/main/tests/functional/test_tasks.py b/awx/main/tests/functional/test_tasks.py new file mode 100644 index 0000000000..6d6d21819d --- /dev/null +++ b/awx/main/tests/functional/test_tasks.py @@ -0,0 +1,43 @@ +import pytest +import mock +import os + +from awx.main.tasks import RunProjectUpdate, RunInventoryUpdate +from awx.main.models import ProjectUpdate, InventoryUpdate + + +@pytest.fixture +def scm_revision_file(tmpdir_factory): + # Returns path to temporary testing revision file + revision_file = tmpdir_factory.mktemp('revisions').join('revision.txt') + with open(str(revision_file), 'w') as f: + f.write('1234567890123456789012345678901234567890') + return os.path.join(revision_file.dirname, 'revision.txt') + + +@pytest.mark.django_db +class TestDependentInventoryUpdate: + + def test_dependent_inventory_updates_is_called(self, scm_inventory_source, scm_revision_file): + task = RunProjectUpdate() + task.revision_path = scm_revision_file + proj_update = ProjectUpdate.objects.create(project=scm_inventory_source.scm_project) + with mock.patch.object(RunProjectUpdate, '_update_dependent_inventories') as inv_update_mck: + task.post_run_hook(proj_update, 'successful') + inv_update_mck.assert_called_once_with(scm_inventory_source.scm_project, mock.ANY) + + def test_no_unwanted_dependent_inventory_updates(self, project, scm_revision_file): + task = RunProjectUpdate() + task.revision_path = scm_revision_file + proj_update = ProjectUpdate.objects.create(project=project) + with mock.patch.object(RunProjectUpdate, '_update_dependent_inventories') as inv_update_mck: + task.post_run_hook(proj_update, 'successful') + assert not inv_update_mck.called + + def test_dependent_inventory_updates(self, scm_inventory_source): + task = RunProjectUpdate() + with mock.patch.object(RunInventoryUpdate, 'run') as iu_run_mock: + task._update_dependent_inventories(scm_inventory_source.scm_project, [scm_inventory_source]) + assert InventoryUpdate.objects.count() == 1 + inv_update = InventoryUpdate.objects.first() + iu_run_mock.assert_called_once_with(inv_update.id) diff --git a/awx/main/utils/ansible.py b/awx/main/utils/ansible.py new file mode 100644 index 0000000000..561ac7b5c9 --- /dev/null +++ b/awx/main/utils/ansible.py @@ -0,0 +1,80 @@ +# Copyright (c) 2017 Ansible by Red Hat +# All Rights Reserved. + +# Python +import re +import os +from itertools import islice + +# Django +from django.utils.encoding import smart_str + + +__all__ = ['could_be_playbook', 'could_be_inventory'] + + +valid_playbook_re = re.compile(r'^\s*?-?\s*?(?:hosts|include):\s*?.*?$') +valid_inventory_re = re.compile(r'^[a-zA-Z0-9_.=\[\]]') + + +def _skip_directory(rel_path): + path_elements = rel_path.split(os.sep) + # Exclude files in a roles subdirectory. + if 'roles' in path_elements: + return True + # Filter files in a tasks subdirectory. + if 'tasks' in path_elements: + return True + for element in path_elements: + # Do not include dot files or dirs + if element.startswith('.'): + return True + return False + + +def could_be_playbook(project_path, dir_path, filename): + if os.path.splitext(filename)[-1] not in ['.yml', '.yaml']: + return None + playbook_path = os.path.join(dir_path, filename) + # Filter files that do not have either hosts or top-level + # includes. Use regex to allow files with invalid YAML to + # show up. + matched = False + try: + for n, line in enumerate(file(playbook_path)): + if valid_playbook_re.match(line): + matched = True + # Any YAML file can also be encrypted with vault; + # allow these to be used as the main playbook. + elif n == 0 and line.startswith('$ANSIBLE_VAULT;'): + matched = True + except IOError: + return None + if not matched: + return None + playbook = os.path.relpath(playbook_path, smart_str(project_path)) + if _skip_directory(playbook): + return None + return playbook + + +def could_be_inventory(project_path, dir_path, filename): + suspected_ext = os.path.splitext(filename)[-1] + # Allow for files with no extension, or with extensions in a certain set + if '.' in suspected_ext and suspected_ext not in ['.yml', '.yaml', '.ini']: + return None + inventory_path = os.path.join(dir_path, filename) + # Filter files that do not use a character set consistent with + # Ansible inventory mainly + try: + # only read through first 10 lines for performance + with open(inventory_path) as inv_file: + for line in islice(inv_file, 10): + if not valid_inventory_re.match(line): + return None + except IOError: + return None + inventory = os.path.relpath(inventory_path, smart_str(project_path)) + if _skip_directory(inventory): + return None + return inventory diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 331d7f6c65..53fe4067d7 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -12,3 +12,5 @@ group serializer and added `user_capabilities` to inventory source serializer, allow user creation and naming of inventory sources [[#5741](https://github.com/ansible/ansible-tower/issues/5741)] +* support sourcing inventory from a file inside of a project's source + tree [[#2477](https://github.com/ansible/ansible-tower/issues/2477)] diff --git a/docs/scm_file_inventory.md b/docs/scm_file_inventory.md new file mode 100644 index 0000000000..8aa26cfc1b --- /dev/null +++ b/docs/scm_file_inventory.md @@ -0,0 +1,63 @@ +# SCM Flat File Inventory + +Users can create inventory sources that use content in the source tree of +a project as an Ansible inventory file. + +## Usage Details + +Fields that should be specified on creation of SCM inventory source: + + - `scm_project` - project to use + - `source_path` - relative path inside of the project indicating a + directory or a file, if left blank, "" is still a relative path + indicating the root directory of the project + +A user should not be able to update this inventory source via through +the endpoint `/inventory_sources/N/update/`. Instead, they should update +the linked project. + +An update of the project automatically triggers an inventory update within +the proper context. An update _of the project_ is scheduled immediately +after creation of the inventory source. + +The project should show a listing of suggested inventory locations, at the +endpoint `/projects/N/inventories/`, but this is not a comprehensive list of +all paths that could be used as an Ansible inventory because of the wide +range of inclusion criteria. The list will also max out at 50 entries. +The user should be allowed to specify a location manually in the UI. +This listing should be refreshed to latest SCM info on a project update. +If no inventory sources use a project as an SCM inventory source, then +the inventory listing may not be refreshed on update. + +### Still-to-come 3.2 Changes + +As a part of a different feature, it is planned to have all inventory sources +inside of an inventory all update with a single button click. When this +happens for an inventory containing an SCM inventory source, it should +update the project. + +## Supported File Syntax + +> Any Inventory Ansible supports should be supported by this feature + +This statement is the overall goal and should hold true absolutely for +Ansible version 2.4 and beyond due to the use of `ansible-inventory`. +Versions of Ansible before that may not support all valid inventory syntax +because the internal mechanism is different. + +Documentation should reflect the limitations of inventory file syntax +support in old Ansible versions. + +# Acceptance Criteria Notes + +Some test scenarios to look at: + - Obviously use a git repo with examples of host patterns, etc. + - Test multiple inventories that use the same project, pointing to different + files / directories inside of the project + - Feature works correctly even if project doesn't have any playbook files + - File related errors should surface as inventory import failures + + missing file + + invalid syntax in file + - If the project SCM update encounters errors, it should not run the + inventory updates +