mirror of
https://github.com/ansible/awx.git
synced 2026-01-11 10:00:01 -03:30
SCM Inventory model, view, and task system changes
Inventory source file-type combined with a linked project will allow the inventory source to be updated when the project is updated. The inventory update runs in the post-run hook of the project update.
This commit is contained in:
AlanCoding
parent
2c34aef661
commit
6d92c56da5
@ -81,6 +81,7 @@ SUMMARIZABLE_FK_FIELDS = {
|
||||
'groups_with_active_failures',
|
||||
'has_inventory_sources'),
|
||||
'project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'),
|
||||
'scm_project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'),
|
||||
'project_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed',),
|
||||
'credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud'),
|
||||
'cloud_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud'),
|
||||
@ -960,8 +961,10 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
|
||||
res.update(dict(
|
||||
teams = self.reverse('api:project_teams_list', kwargs={'pk': obj.pk}),
|
||||
playbooks = self.reverse('api:project_playbooks', kwargs={'pk': obj.pk}),
|
||||
inventory_files = self.reverse('api:project_inventories', kwargs={'pk': obj.pk}),
|
||||
update = self.reverse('api:project_update_view', kwargs={'pk': obj.pk}),
|
||||
project_updates = self.reverse('api:project_updates_list', kwargs={'pk': obj.pk}),
|
||||
scm_inventories = self.reverse('api:project_scm_inventory_sources', kwargs={'pk': obj.pk}),
|
||||
schedules = self.reverse('api:project_schedules_list', kwargs={'pk': obj.pk}),
|
||||
activity_stream = self.reverse('api:project_activity_stream_list', kwargs={'pk': obj.pk}),
|
||||
notification_templates_any = self.reverse('api:project_notification_templates_any_list', kwargs={'pk': obj.pk}),
|
||||
@ -1027,6 +1030,23 @@ class ProjectPlaybooksSerializer(ProjectSerializer):
|
||||
return ReturnList(ret, serializer=self)
|
||||
|
||||
|
||||
class ProjectInventoriesSerializer(ProjectSerializer):
|
||||
|
||||
inventory_files = serializers.ReadOnlyField(help_text=_(
|
||||
'Array of inventory files and directories available within this project, '
|
||||
'not comprehensive.'))
|
||||
|
||||
class Meta:
|
||||
model = Project
|
||||
fields = ('inventory_files',)
|
||||
|
||||
@property
|
||||
def data(self):
|
||||
ret = super(ProjectInventoriesSerializer, self).data
|
||||
ret = ret.get('inventory_files', [])
|
||||
return ReturnList(ret, serializer=self)
|
||||
|
||||
|
||||
class ProjectUpdateViewSerializer(ProjectSerializer):
|
||||
|
||||
can_update = serializers.BooleanField(read_only=True)
|
||||
@ -1046,6 +1066,7 @@ class ProjectUpdateSerializer(UnifiedJobSerializer, ProjectOptionsSerializer):
|
||||
res.update(dict(
|
||||
project = self.reverse('api:project_detail', kwargs={'pk': obj.project.pk}),
|
||||
cancel = self.reverse('api:project_update_cancel', kwargs={'pk': obj.pk}),
|
||||
scm_inventory_updates = self.reverse('api:project_update_scm_inventory_updates', kwargs={'pk': obj.pk}),
|
||||
notifications = self.reverse('api:project_update_notifications_list', kwargs={'pk': obj.pk}),
|
||||
))
|
||||
return res
|
||||
@ -1481,7 +1502,7 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
|
||||
|
||||
class Meta:
|
||||
model = InventorySource
|
||||
fields = ('*', 'name', 'inventory', 'update_on_launch', 'update_cache_timeout') + \
|
||||
fields = ('*', 'name', 'inventory', 'update_on_launch', 'update_cache_timeout', 'scm_project') + \
|
||||
('last_update_failed', 'last_updated', 'group') # Backwards compatibility.
|
||||
|
||||
def get_related(self, obj):
|
||||
@ -1499,6 +1520,8 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
|
||||
))
|
||||
if obj.inventory:
|
||||
res['inventory'] = self.reverse('api:inventory_detail', kwargs={'pk': obj.inventory.pk})
|
||||
if obj.scm_project_id is not None:
|
||||
res['scm_project'] = self.reverse('api:project_detail', kwargs={'pk': obj.scm_project.pk})
|
||||
# Backwards compatibility.
|
||||
if obj.current_update:
|
||||
res['current_update'] = self.reverse('api:inventory_update_detail',
|
||||
@ -1530,6 +1553,14 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
|
||||
return True
|
||||
return False
|
||||
|
||||
def build_relational_field(self, field_name, relation_info):
|
||||
field_class, field_kwargs = super(InventorySourceSerializer, self).build_relational_field(field_name, relation_info)
|
||||
# SCM Project and inventory are read-only unless creating a new inventory.
|
||||
if self.instance and field_name in ['scm_project', 'inventory']:
|
||||
field_kwargs['read_only'] = True
|
||||
field_kwargs.pop('queryset', None)
|
||||
return field_class, field_kwargs
|
||||
|
||||
def to_representation(self, obj):
|
||||
ret = super(InventorySourceSerializer, self).to_representation(obj)
|
||||
if obj is None:
|
||||
@ -1538,6 +1569,16 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
|
||||
ret['inventory'] = None
|
||||
return ret
|
||||
|
||||
def validate(self, attrs):
|
||||
# source_path = attrs.get('source_path', self.instance and self.instance.source_path)
|
||||
update_on_launch = attrs.get('update_on_launch', self.instance and self.instance.update_on_launch)
|
||||
scm_project = attrs.get('scm_project', self.instance and self.instance.scm_project)
|
||||
if attrs.get('source_path', None) and not scm_project:
|
||||
raise serializers.ValidationError({"detail": _("Cannot set source_path if not SCM type.")})
|
||||
elif update_on_launch and scm_project:
|
||||
raise serializers.ValidationError({"detail": _("Cannot update SCM-based inventory source on launch.")})
|
||||
return super(InventorySourceSerializer, self).validate(attrs)
|
||||
|
||||
|
||||
class InventorySourceUpdateSerializer(InventorySourceSerializer):
|
||||
|
||||
|
||||
@ -48,6 +48,8 @@ project_urls = patterns('awx.api.views',
|
||||
url(r'^$', 'project_list'),
|
||||
url(r'^(?P<pk>[0-9]+)/$', 'project_detail'),
|
||||
url(r'^(?P<pk>[0-9]+)/playbooks/$', 'project_playbooks'),
|
||||
url(r'^(?P<pk>[0-9]+)/inventories/$', 'project_inventories'),
|
||||
url(r'^(?P<pk>[0-9]+)/scm_inventory_sources/$', 'project_scm_inventory_sources'),
|
||||
url(r'^(?P<pk>[0-9]+)/teams/$', 'project_teams_list'),
|
||||
url(r'^(?P<pk>[0-9]+)/update/$', 'project_update_view'),
|
||||
url(r'^(?P<pk>[0-9]+)/project_updates/$', 'project_updates_list'),
|
||||
@ -65,6 +67,7 @@ project_update_urls = patterns('awx.api.views',
|
||||
url(r'^(?P<pk>[0-9]+)/$', 'project_update_detail'),
|
||||
url(r'^(?P<pk>[0-9]+)/cancel/$', 'project_update_cancel'),
|
||||
url(r'^(?P<pk>[0-9]+)/stdout/$', 'project_update_stdout'),
|
||||
url(r'^(?P<pk>[0-9]+)/scm_inventory_updates/$', 'project_update_scm_inventory_updates'),
|
||||
url(r'^(?P<pk>[0-9]+)/notifications/$', 'project_update_notifications_list'),
|
||||
)
|
||||
|
||||
|
||||
@ -1074,6 +1074,12 @@ class ProjectPlaybooks(RetrieveAPIView):
|
||||
serializer_class = ProjectPlaybooksSerializer
|
||||
|
||||
|
||||
class ProjectInventories(RetrieveAPIView):
|
||||
|
||||
model = Project
|
||||
serializer_class = ProjectInventoriesSerializer
|
||||
|
||||
|
||||
class ProjectTeamsList(ListAPIView):
|
||||
|
||||
model = Team
|
||||
@ -1101,6 +1107,17 @@ class ProjectSchedulesList(SubListCreateAPIView):
|
||||
new_in_148 = True
|
||||
|
||||
|
||||
class ProjectScmInventorySources(SubListCreateAPIView):
|
||||
|
||||
view_name = _("Project SCM Inventory Sources")
|
||||
model = Inventory
|
||||
serializer_class = InventorySourceSerializer
|
||||
parent_model = Project
|
||||
relationship = 'scm_inventory_sources'
|
||||
parent_key = 'scm_project'
|
||||
new_in_320 = True
|
||||
|
||||
|
||||
class ProjectActivityStreamList(ActivityStreamEnforcementMixin, SubListAPIView):
|
||||
|
||||
model = ActivityStream
|
||||
@ -1226,6 +1243,17 @@ class ProjectUpdateNotificationsList(SubListAPIView):
|
||||
new_in_300 = True
|
||||
|
||||
|
||||
class ProjectUpdateScmInventoryUpdates(SubListCreateAPIView):
|
||||
|
||||
view_name = _("Project Update SCM Inventory Updates")
|
||||
model = InventoryUpdate
|
||||
serializer_class = InventoryUpdateSerializer
|
||||
parent_model = ProjectUpdate
|
||||
relationship = 'scm_inventory_updates'
|
||||
parent_key = 'scm_project_update'
|
||||
new_in_320 = True
|
||||
|
||||
|
||||
class ProjectAccessList(ResourceAccessList):
|
||||
|
||||
model = User # needs to be User for AccessLists's
|
||||
@ -2185,7 +2213,7 @@ class InventoryInventorySourcesList(SubListCreateAPIView):
|
||||
new_in_14 = True
|
||||
|
||||
|
||||
class InventorySourceList(ListAPIView):
|
||||
class InventorySourceList(ListCreateAPIView):
|
||||
|
||||
model = InventorySource
|
||||
serializer_class = InventorySourceSerializer
|
||||
@ -2292,6 +2320,10 @@ class InventorySourceUpdateView(RetrieveAPIView):
|
||||
|
||||
def post(self, request, *args, **kwargs):
|
||||
obj = self.get_object()
|
||||
if obj.source == 'file' and obj.scm_project_id is not None:
|
||||
raise PermissionDenied(detail=_(
|
||||
'Update the project `{}` in order to update this inventory source.'.format(
|
||||
obj.scm_project.name)))
|
||||
if obj.can_update:
|
||||
inventory_update = obj.update()
|
||||
if not inventory_update:
|
||||
|
||||
@ -765,9 +765,12 @@ class InventorySourceAccess(BaseAccess):
|
||||
else:
|
||||
return False
|
||||
|
||||
@check_superuser
|
||||
def can_add(self, data):
|
||||
if not data or 'inventory' not in data:
|
||||
return False
|
||||
if not self.check_related('scm_project', Project, data, role_field='admin_role'):
|
||||
return False
|
||||
# Checks for admin or change permission on inventory.
|
||||
return self.check_related('inventory', Inventory, data)
|
||||
|
||||
|
||||
@ -51,4 +51,51 @@ class Migration(migrations.Migration):
|
||||
migrations.RunSQL([("CREATE INDEX host_ansible_facts_default_gin ON %s USING gin"
|
||||
"(ansible_facts jsonb_path_ops);", [AsIs(Host._meta.db_table)])],
|
||||
[('DROP INDEX host_ansible_facts_default_gin;', None)]),
|
||||
|
||||
# SCM file-based inventories
|
||||
migrations.AddField(
|
||||
model_name='inventorysource',
|
||||
name='scm_last_revision',
|
||||
field=models.CharField(default=b'', max_length=1024, editable=False, blank=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='inventorysource',
|
||||
name='scm_project',
|
||||
field=models.ForeignKey(related_name='scm_inventory_sources', default=None, blank=True, to='main.Project', help_text='Project containing inventory file used as source.', null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='inventoryupdate',
|
||||
name='scm_project_update',
|
||||
field=models.ForeignKey(related_name='scm_inventory_updates', default=None, blank=True, to='main.ProjectUpdate', help_text='Inventory files from this Project Update were used for the inventory update.', null=True),
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name='project',
|
||||
name='inventory_files',
|
||||
field=awx.main.fields.JSONField(default=[], help_text='Suggested list of content that could be Ansible inventory in the project', verbose_name='Inventory Files', editable=False, blank=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='inventorysource',
|
||||
name='source',
|
||||
field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'File, Directory or Script Locally or in Project'), (b'rax', 'Rackspace Cloud Servers'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'satellite6', 'Red Hat Satellite 6'), (b'cloudforms', 'Red Hat CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='inventoryupdate',
|
||||
name='source',
|
||||
field=models.CharField(default=b'', max_length=32, blank=True, choices=[(b'', 'Manual'), (b'file', 'File, Directory or Script Locally or in Project'), (b'rax', 'Rackspace Cloud Servers'), (b'ec2', 'Amazon EC2'), (b'gce', 'Google Compute Engine'), (b'azure', 'Microsoft Azure Classic (deprecated)'), (b'azure_rm', 'Microsoft Azure Resource Manager'), (b'vmware', 'VMware vCenter'), (b'satellite6', 'Red Hat Satellite 6'), (b'cloudforms', 'Red Hat CloudForms'), (b'openstack', 'OpenStack'), (b'custom', 'Custom Script')]),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='inventorysource',
|
||||
name='source_path',
|
||||
field=models.CharField(default=b'', max_length=1024, blank=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='inventoryupdate',
|
||||
name='source_path',
|
||||
field=models.CharField(default=b'', max_length=1024, blank=True),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name='unifiedjob',
|
||||
name='launch_type',
|
||||
field=models.CharField(default=b'manual', max_length=20, editable=False, choices=[(b'manual', 'Manual'), (b'relaunch', 'Relaunch'), (b'callback', 'Callback'), (b'scheduled', 'Scheduled'), (b'dependency', 'Dependency'), (b'workflow', 'Workflow'), (b'sync', 'Sync'), (b'scm', 'SCM Update')]),
|
||||
),
|
||||
]
|
||||
|
||||
@ -7,6 +7,7 @@ import logging
|
||||
import re
|
||||
import copy
|
||||
from urlparse import urljoin
|
||||
import os.path
|
||||
|
||||
# Django
|
||||
from django.conf import settings
|
||||
@ -719,7 +720,7 @@ class InventorySourceOptions(BaseModel):
|
||||
|
||||
SOURCE_CHOICES = [
|
||||
('', _('Manual')),
|
||||
('file', _('Local File, Directory or Script')),
|
||||
('file', _('File, Directory or Script Locally or in Project')),
|
||||
('rax', _('Rackspace Cloud Servers')),
|
||||
('ec2', _('Amazon EC2')),
|
||||
('gce', _('Google Compute Engine')),
|
||||
@ -828,7 +829,6 @@ class InventorySourceOptions(BaseModel):
|
||||
max_length=1024,
|
||||
blank=True,
|
||||
default='',
|
||||
editable=False,
|
||||
)
|
||||
source_script = models.ForeignKey(
|
||||
'CustomInventoryScript',
|
||||
@ -1086,6 +1086,21 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions):
|
||||
on_delete=models.CASCADE,
|
||||
)
|
||||
|
||||
scm_project = models.ForeignKey(
|
||||
'Project',
|
||||
related_name='scm_inventory_sources',
|
||||
help_text=_('Project containing inventory file used as source.'),
|
||||
on_delete=models.CASCADE,
|
||||
blank=True,
|
||||
default=None,
|
||||
null=True
|
||||
)
|
||||
scm_last_revision = models.CharField(
|
||||
max_length=1024,
|
||||
blank=True,
|
||||
default='',
|
||||
editable=False,
|
||||
)
|
||||
update_on_launch = models.BooleanField(
|
||||
default=False,
|
||||
)
|
||||
@ -1101,12 +1116,14 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions):
|
||||
def _get_unified_job_field_names(cls):
|
||||
return ['name', 'description', 'source', 'source_path', 'source_script', 'source_vars', 'schedule',
|
||||
'credential', 'source_regions', 'instance_filters', 'group_by', 'overwrite', 'overwrite_vars',
|
||||
'timeout', 'launch_type',]
|
||||
'timeout', 'launch_type', 'scm_project_update',]
|
||||
|
||||
def save(self, *args, **kwargs):
|
||||
# If update_fields has been specified, add our field names to it,
|
||||
# if it hasn't been specified, then we're just doing a normal save.
|
||||
update_fields = kwargs.get('update_fields', [])
|
||||
is_new_instance = not bool(self.pk)
|
||||
is_scm_type = self.scm_project_id is not None
|
||||
|
||||
# Set name automatically. Include PK (or placeholder) to make sure the names are always unique.
|
||||
replace_text = '__replace_%s__' % now()
|
||||
@ -1116,18 +1133,33 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions):
|
||||
self.name = '%s (%s)' % (self.inventory.name, self.pk)
|
||||
elif self.inventory:
|
||||
self.name = '%s (%s)' % (self.inventory.name, replace_text)
|
||||
elif self.pk:
|
||||
elif not is_new_instance:
|
||||
self.name = 'inventory source (%s)' % self.pk
|
||||
else:
|
||||
self.name = 'inventory source (%s)' % replace_text
|
||||
if 'name' not in update_fields:
|
||||
update_fields.append('name')
|
||||
# Reset revision if SCM source has changed parameters
|
||||
if is_scm_type and not is_new_instance:
|
||||
before_is = self.__class__.objects.get(pk=self.pk)
|
||||
if before_is.source_path != self.source_path or before_is.scm_project_id != self.scm_project_id:
|
||||
# Reset the scm_revision if file changed to force update
|
||||
self.scm_revision = None
|
||||
if 'scm_revision' not in update_fields:
|
||||
update_fields.append('scm_revision')
|
||||
|
||||
# Do the actual save.
|
||||
super(InventorySource, self).save(*args, **kwargs)
|
||||
|
||||
# Add the PK to the name.
|
||||
if replace_text in self.name:
|
||||
self.name = self.name.replace(replace_text, str(self.pk))
|
||||
super(InventorySource, self).save(update_fields=['name'])
|
||||
if is_scm_type and is_new_instance:
|
||||
# Schedule a new Project update if one is not already queued
|
||||
if not self.scm_project.project_updates.filter(
|
||||
status__in=['new', 'pending', 'waiting']).exists():
|
||||
self.scm_project.update()
|
||||
if not getattr(_inventory_updates, 'is_updating', False):
|
||||
if self.inventory is not None:
|
||||
self.inventory.update_computed_fields(update_groups=False, update_hosts=False)
|
||||
@ -1222,6 +1254,15 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin):
|
||||
default=False,
|
||||
editable=False,
|
||||
)
|
||||
scm_project_update = models.ForeignKey(
|
||||
'ProjectUpdate',
|
||||
related_name='scm_inventory_updates',
|
||||
help_text=_('Inventory files from this Project Update were used for the inventory update.'),
|
||||
on_delete=models.CASCADE,
|
||||
blank=True,
|
||||
default=None,
|
||||
null=True
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _get_parent_field_name(cls):
|
||||
@ -1256,6 +1297,14 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin):
|
||||
def get_ui_url(self):
|
||||
return urljoin(settings.TOWER_URL_BASE, "/#/inventory_sync/{}".format(self.pk))
|
||||
|
||||
def get_actual_source_path(self):
|
||||
'''Alias to source_path that combines with project path for for SCM file based sources'''
|
||||
if self.inventory_source_id is None or self.inventory_source.scm_project_id is None:
|
||||
return self.source_path
|
||||
return os.path.join(
|
||||
self.inventory_source.scm_project.get_project_path(check_if_exists=False),
|
||||
self.source_path)
|
||||
|
||||
@property
|
||||
def task_impact(self):
|
||||
return 50
|
||||
|
||||
@ -4,7 +4,6 @@
|
||||
# Python
|
||||
import datetime
|
||||
import os
|
||||
import re
|
||||
import urlparse
|
||||
|
||||
# Django
|
||||
@ -26,6 +25,7 @@ from awx.main.models.notifications import (
|
||||
from awx.main.models.unified_jobs import * # noqa
|
||||
from awx.main.models.mixins import ResourceMixin
|
||||
from awx.main.utils import update_scm_url
|
||||
from awx.main.utils.ansible import could_be_inventory, could_be_playbook
|
||||
from awx.main.fields import ImplicitRoleField
|
||||
from awx.main.models.rbac import (
|
||||
ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
|
||||
@ -173,39 +173,35 @@ class ProjectOptions(models.Model):
|
||||
|
||||
@property
|
||||
def playbooks(self):
|
||||
valid_re = re.compile(r'^\s*?-?\s*?(?:hosts|include):\s*?.*?$')
|
||||
results = []
|
||||
project_path = self.get_project_path()
|
||||
if project_path:
|
||||
for dirpath, dirnames, filenames in os.walk(smart_str(project_path)):
|
||||
for filename in filenames:
|
||||
if os.path.splitext(filename)[-1] not in ['.yml', '.yaml']:
|
||||
continue
|
||||
playbook = os.path.join(dirpath, filename)
|
||||
# Filter files that do not have either hosts or top-level
|
||||
# includes. Use regex to allow files with invalid YAML to
|
||||
# show up.
|
||||
matched = False
|
||||
try:
|
||||
for n, line in enumerate(file(playbook)):
|
||||
if valid_re.match(line):
|
||||
matched = True
|
||||
# Any YAML file can also be encrypted with vault;
|
||||
# allow these to be used as the main playbook.
|
||||
elif n == 0 and line.startswith('$ANSIBLE_VAULT;'):
|
||||
matched = True
|
||||
except IOError:
|
||||
continue
|
||||
if not matched:
|
||||
continue
|
||||
playbook = os.path.relpath(playbook, smart_str(project_path))
|
||||
# Filter files in a roles subdirectory.
|
||||
if 'roles' in playbook.split(os.sep):
|
||||
continue
|
||||
# Filter files in a tasks subdirectory.
|
||||
if 'tasks' in playbook.split(os.sep):
|
||||
continue
|
||||
results.append(smart_text(playbook))
|
||||
playbook = could_be_playbook(project_path, dirpath, filename)
|
||||
if playbook is not None:
|
||||
results.append(smart_text(playbook))
|
||||
return sorted(results, key=lambda x: smart_str(x).lower())
|
||||
|
||||
|
||||
@property
|
||||
def inventories(self):
|
||||
results = []
|
||||
project_path = self.get_project_path()
|
||||
if project_path:
|
||||
# Cap the number of results, because it could include lots
|
||||
max_inventory_listing = 50
|
||||
for dirpath, dirnames, filenames in os.walk(smart_str(project_path)):
|
||||
if dirpath.startswith('.'):
|
||||
continue
|
||||
for filename in filenames:
|
||||
inv_path = could_be_inventory(project_path, dirpath, filename)
|
||||
if inv_path is not None:
|
||||
results.append(smart_text(inv_path))
|
||||
if len(results) > max_inventory_listing:
|
||||
break
|
||||
if len(results) > max_inventory_listing:
|
||||
break
|
||||
return sorted(results, key=lambda x: smart_str(x).lower())
|
||||
|
||||
|
||||
@ -257,6 +253,14 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin):
|
||||
help_text=_('List of playbooks found in the project'),
|
||||
)
|
||||
|
||||
inventory_files = JSONField(
|
||||
blank=True,
|
||||
default=[],
|
||||
editable=False,
|
||||
verbose_name=_('Inventory Files'),
|
||||
help_text=_('Suggested list of content that could be Ansible inventory in the project'),
|
||||
)
|
||||
|
||||
admin_role = ImplicitRoleField(parent_role=[
|
||||
'organization.admin_role',
|
||||
'singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
|
||||
|
||||
@ -333,6 +333,10 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, Notificatio
|
||||
unified_job_class = self._get_unified_job_class()
|
||||
fields = self._get_unified_job_field_names()
|
||||
unified_job = copy_model_by_class(self, unified_job_class, fields, kwargs)
|
||||
eager_fields = kwargs.get('_eager_fields', None)
|
||||
if eager_fields:
|
||||
for fd, val in eager_fields.items():
|
||||
setattr(unified_job, fd, val)
|
||||
|
||||
# Set the unified job template back-link on the job
|
||||
parent_field_name = unified_job_class._get_parent_field_name()
|
||||
@ -418,6 +422,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
|
||||
('dependency', _('Dependency')), # Job was started as a dependency of another job.
|
||||
('workflow', _('Workflow')), # Job was started from a workflow job.
|
||||
('sync', _('Sync')), # Job was started from a project sync.
|
||||
('scm', _('SCM Update')) # Job was created as an Inventory SCM sync.
|
||||
]
|
||||
|
||||
PASSWORD_FIELDS = ('start_args',)
|
||||
|
||||
@ -268,7 +268,7 @@ def migrate_children_from_deleted_group_to_parent_groups(sender, **kwargs):
|
||||
try:
|
||||
inventory = Inventory.objects.get(pk=inventory_pk)
|
||||
inventory.update_computed_fields()
|
||||
except Inventory.DoesNotExist:
|
||||
except (Inventory.DoesNotExist, Project.DoesNotExist):
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@ -48,7 +48,7 @@ from django.core.exceptions import ObjectDoesNotExist
|
||||
# AWX
|
||||
from awx.main.constants import CLOUD_PROVIDERS
|
||||
from awx.main.models import * # noqa
|
||||
from awx.main.models import UnifiedJob
|
||||
from awx.main.models.unified_jobs import ACTIVE_STATES
|
||||
from awx.main.queue import CallbackQueueDispatcher
|
||||
from awx.main.task_engine import TaskEnhancer
|
||||
from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, update_scm_url,
|
||||
@ -1099,16 +1099,22 @@ class RunJob(BaseTask):
|
||||
|
||||
def pre_run_hook(self, job, **kwargs):
|
||||
if job.project and job.project.scm_type:
|
||||
local_project_sync = job.project.create_project_update(launch_type="sync")
|
||||
local_project_sync.job_type = 'run'
|
||||
local_project_sync.save()
|
||||
# save the associated project update before calling run() so that a
|
||||
job_request_id = '' if self.request.id is None else self.request.id
|
||||
local_project_sync = job.project.create_project_update(
|
||||
launch_type="sync",
|
||||
_eager_params=dict(
|
||||
job_type='run',
|
||||
status='running',
|
||||
celery_task_id=job_request_id))
|
||||
# save the associated job before calling run() so that a
|
||||
# cancel() call on the job can cancel the project update
|
||||
job = self.update_model(job.pk, project_update=local_project_sync)
|
||||
|
||||
project_update_task = local_project_sync._get_task_class()
|
||||
try:
|
||||
project_update_task().run(local_project_sync.id)
|
||||
task_instance = project_update_task()
|
||||
task_instance.request.id = job_request_id
|
||||
task_instance.run(local_project_sync.id)
|
||||
job = self.update_model(job.pk, scm_revision=job.project.scm_revision)
|
||||
except Exception:
|
||||
job = self.update_model(job.pk, status='failed',
|
||||
@ -1317,9 +1323,43 @@ class RunProjectUpdate(BaseTask):
|
||||
instance_actual.save()
|
||||
return OutputEventFilter(stdout_handle, raw_callback=raw_callback)
|
||||
|
||||
def _update_dependent_inventories(self, project_update, dependent_inventory_sources):
|
||||
for inv_src in dependent_inventory_sources:
|
||||
if inv_src.scm_last_revision == project_update.project.scm_revision:
|
||||
logger.debug('Skipping SCM inventory update for `{}` because '
|
||||
'project has not changed.'.format(inv_src.name))
|
||||
continue
|
||||
logger.debug('Local dependent inventory update for `{}`.'.format(inv_src.name))
|
||||
with transaction.atomic():
|
||||
if InventoryUpdate.objects.filter(inventory_source=inv_src,
|
||||
status__in=ACTIVE_STATES).exists():
|
||||
logger.info('Skipping SCM inventory update for `{}` because '
|
||||
'another update is already active.'.format(inv.name))
|
||||
continue
|
||||
project_request_id = '' if self.request.id is None else self.request.id
|
||||
local_inv_update = inv_src.create_inventory_update(
|
||||
scm_project_update_id=project_update.id,
|
||||
launch_type='scm',
|
||||
_eager_fields=dict(
|
||||
status='running',
|
||||
celery_task_id=str(project_request_id)))
|
||||
inv_update_task = local_inv_update._get_task_class()
|
||||
try:
|
||||
task_instance = inv_update_task()
|
||||
# Runs in the same Celery task as project update
|
||||
task_instance.request.id = project_request_id
|
||||
task_instance.run(local_inv_update.id)
|
||||
inv_src.scm_last_revision = project_update.project.scm_revision
|
||||
inv_src.save(update_fields=['scm_last_revision'])
|
||||
except Exception as e:
|
||||
# A failed file update does not block other actions
|
||||
logger.error('Encountered error updating project dependent inventory: {}'.format(e))
|
||||
continue
|
||||
|
||||
def post_run_hook(self, instance, status, **kwargs):
|
||||
p = instance.project
|
||||
dependent_inventory_sources = p.scm_inventory_sources.all()
|
||||
if instance.job_type == 'check' and status not in ('failed', 'canceled',):
|
||||
p = instance.project
|
||||
fd = open(self.revision_path, 'r')
|
||||
lines = fd.readlines()
|
||||
if lines:
|
||||
@ -1327,11 +1367,17 @@ class RunProjectUpdate(BaseTask):
|
||||
else:
|
||||
logger.info("Could not find scm revision in check")
|
||||
p.playbook_files = p.playbooks
|
||||
if len(dependent_inventory_sources) > 0:
|
||||
p.inventory_files = p.inventories
|
||||
p.save()
|
||||
try:
|
||||
os.remove(self.revision_path)
|
||||
except Exception, e:
|
||||
logger.error("Failed removing revision tmp file: {}".format(e))
|
||||
# Update any inventories that depend on this project
|
||||
if len(dependent_inventory_sources) > 0:
|
||||
if status == 'successful' and instance.launch_type != 'sync':
|
||||
self._update_dependent_inventories(p, dependent_inventory_sources)
|
||||
|
||||
|
||||
class RunInventoryUpdate(BaseTask):
|
||||
@ -1580,8 +1626,8 @@ class RunInventoryUpdate(BaseTask):
|
||||
elif inventory_update.source == 'cloudforms':
|
||||
env['CLOUDFORMS_INI_PATH'] = cloud_credential
|
||||
elif inventory_update.source == 'file':
|
||||
# FIXME: Parse source_env to dict, update env.
|
||||
pass
|
||||
# Parse source_vars to dict, update env.
|
||||
env.update(parse_yaml_or_json(inventory_update.source_vars))
|
||||
elif inventory_update.source == 'custom':
|
||||
for env_k in inventory_update.source_vars_dict:
|
||||
if str(env_k) not in env and str(env_k) not in settings.INV_ENV_VARIABLE_BLACKLIST:
|
||||
@ -1650,7 +1696,7 @@ class RunInventoryUpdate(BaseTask):
|
||||
])
|
||||
|
||||
elif inventory_update.source == 'file':
|
||||
args.append(inventory_update.source_path)
|
||||
args.append(inventory_update.get_actual_source_path())
|
||||
elif inventory_update.source == 'custom':
|
||||
runpath = tempfile.mkdtemp(prefix='ansible_tower_launch_')
|
||||
handle, path = tempfile.mkstemp(dir=runpath)
|
||||
|
||||
0
awx/main/tests/functional/api/__init__.py
Normal file
0
awx/main/tests/functional/api/__init__.py
Normal file
@ -133,7 +133,9 @@ def project(instance, organization):
|
||||
prj = Project.objects.create(name="test-proj",
|
||||
description="test-proj-desc",
|
||||
organization=organization,
|
||||
playbook_files=['helloworld.yml', 'alt-helloworld.yml']
|
||||
playbook_files=['helloworld.yml', 'alt-helloworld.yml'],
|
||||
local_path='_92__test_proj',
|
||||
scm_revision='1234567890123456789012345678901234567890'
|
||||
)
|
||||
return prj
|
||||
|
||||
@ -208,6 +210,15 @@ def inventory(organization):
|
||||
return organization.inventories.create(name="test-inv")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scm_inventory_source(inventory, project):
|
||||
return InventorySource.objects.create(
|
||||
name="test-scm-inv",
|
||||
scm_project=project,
|
||||
source_path='inventory_file',
|
||||
inventory=inventory)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def inventory_factory(organization):
|
||||
def factory(name, org=organization):
|
||||
|
||||
33
awx/main/tests/functional/models/test_inventory.py
Normal file
33
awx/main/tests/functional/models/test_inventory.py
Normal file
@ -0,0 +1,33 @@
|
||||
import pytest
|
||||
import mock
|
||||
|
||||
# AWX
|
||||
from awx.main.models import InventorySource, InventoryUpdate
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestSCMUpdateFeatures:
|
||||
|
||||
def test_automatic_project_update_on_create(self, inventory, project):
|
||||
inv_src = InventorySource(
|
||||
scm_project=project,
|
||||
source_path='inventory_file',
|
||||
inventory=inventory)
|
||||
with mock.patch.object(inv_src.scm_project, 'update') as mck_update:
|
||||
inv_src.save()
|
||||
mck_update.assert_called_once_with()
|
||||
|
||||
def test_source_location(self, scm_inventory_source):
|
||||
# Combines project directory with the inventory file specified
|
||||
inventory_update = InventoryUpdate(
|
||||
inventory_source=scm_inventory_source,
|
||||
source_path=scm_inventory_source.source_path)
|
||||
assert inventory_update.get_actual_source_path().endswith('_92__test_proj/inventory_file')
|
||||
|
||||
def test_no_unwanted_updates(self, scm_inventory_source):
|
||||
# Changing the non-sensitive fields should not trigger update
|
||||
with mock.patch.object(scm_inventory_source.scm_project, 'update') as mck_update:
|
||||
scm_inventory_source.name = 'edited_inventory'
|
||||
scm_inventory_source.description = "I'm testing this!"
|
||||
scm_inventory_source.save()
|
||||
assert not mck_update.called
|
||||
@ -24,6 +24,13 @@ def team_project_list(organization_factory):
|
||||
return objects
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_get_project_path(project):
|
||||
# Test combining projects root with project local path
|
||||
with mock.patch('awx.main.models.projects.settings.PROJECTS_ROOT', '/var/lib/awx'):
|
||||
assert project.get_project_path(check_if_exists=False) == '/var/lib/awx/_92__test_proj'
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_user_project_paged_list(get, organization_factory):
|
||||
'Test project listing that spans multiple pages'
|
||||
|
||||
43
awx/main/tests/functional/test_tasks.py
Normal file
43
awx/main/tests/functional/test_tasks.py
Normal file
@ -0,0 +1,43 @@
|
||||
import pytest
|
||||
import mock
|
||||
import os
|
||||
|
||||
from awx.main.tasks import RunProjectUpdate, RunInventoryUpdate
|
||||
from awx.main.models import ProjectUpdate, InventoryUpdate
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scm_revision_file(tmpdir_factory):
|
||||
# Returns path to temporary testing revision file
|
||||
revision_file = tmpdir_factory.mktemp('revisions').join('revision.txt')
|
||||
with open(str(revision_file), 'w') as f:
|
||||
f.write('1234567890123456789012345678901234567890')
|
||||
return os.path.join(revision_file.dirname, 'revision.txt')
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestDependentInventoryUpdate:
|
||||
|
||||
def test_dependent_inventory_updates_is_called(self, scm_inventory_source, scm_revision_file):
|
||||
task = RunProjectUpdate()
|
||||
task.revision_path = scm_revision_file
|
||||
proj_update = ProjectUpdate.objects.create(project=scm_inventory_source.scm_project)
|
||||
with mock.patch.object(RunProjectUpdate, '_update_dependent_inventories') as inv_update_mck:
|
||||
task.post_run_hook(proj_update, 'successful')
|
||||
inv_update_mck.assert_called_once_with(scm_inventory_source.scm_project, mock.ANY)
|
||||
|
||||
def test_no_unwanted_dependent_inventory_updates(self, project, scm_revision_file):
|
||||
task = RunProjectUpdate()
|
||||
task.revision_path = scm_revision_file
|
||||
proj_update = ProjectUpdate.objects.create(project=project)
|
||||
with mock.patch.object(RunProjectUpdate, '_update_dependent_inventories') as inv_update_mck:
|
||||
task.post_run_hook(proj_update, 'successful')
|
||||
assert not inv_update_mck.called
|
||||
|
||||
def test_dependent_inventory_updates(self, scm_inventory_source):
|
||||
task = RunProjectUpdate()
|
||||
with mock.patch.object(RunInventoryUpdate, 'run') as iu_run_mock:
|
||||
task._update_dependent_inventories(scm_inventory_source.scm_project, [scm_inventory_source])
|
||||
assert InventoryUpdate.objects.count() == 1
|
||||
inv_update = InventoryUpdate.objects.first()
|
||||
iu_run_mock.assert_called_once_with(inv_update.id)
|
||||
80
awx/main/utils/ansible.py
Normal file
80
awx/main/utils/ansible.py
Normal file
@ -0,0 +1,80 @@
|
||||
# Copyright (c) 2017 Ansible by Red Hat
|
||||
# All Rights Reserved.
|
||||
|
||||
# Python
|
||||
import re
|
||||
import os
|
||||
from itertools import islice
|
||||
|
||||
# Django
|
||||
from django.utils.encoding import smart_str
|
||||
|
||||
|
||||
__all__ = ['could_be_playbook', 'could_be_inventory']
|
||||
|
||||
|
||||
valid_playbook_re = re.compile(r'^\s*?-?\s*?(?:hosts|include):\s*?.*?$')
|
||||
valid_inventory_re = re.compile(r'^[a-zA-Z0-9_.=\[\]]')
|
||||
|
||||
|
||||
def _skip_directory(rel_path):
|
||||
path_elements = rel_path.split(os.sep)
|
||||
# Exclude files in a roles subdirectory.
|
||||
if 'roles' in path_elements:
|
||||
return True
|
||||
# Filter files in a tasks subdirectory.
|
||||
if 'tasks' in path_elements:
|
||||
return True
|
||||
for element in path_elements:
|
||||
# Do not include dot files or dirs
|
||||
if element.startswith('.'):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def could_be_playbook(project_path, dir_path, filename):
|
||||
if os.path.splitext(filename)[-1] not in ['.yml', '.yaml']:
|
||||
return None
|
||||
playbook_path = os.path.join(dir_path, filename)
|
||||
# Filter files that do not have either hosts or top-level
|
||||
# includes. Use regex to allow files with invalid YAML to
|
||||
# show up.
|
||||
matched = False
|
||||
try:
|
||||
for n, line in enumerate(file(playbook_path)):
|
||||
if valid_playbook_re.match(line):
|
||||
matched = True
|
||||
# Any YAML file can also be encrypted with vault;
|
||||
# allow these to be used as the main playbook.
|
||||
elif n == 0 and line.startswith('$ANSIBLE_VAULT;'):
|
||||
matched = True
|
||||
except IOError:
|
||||
return None
|
||||
if not matched:
|
||||
return None
|
||||
playbook = os.path.relpath(playbook_path, smart_str(project_path))
|
||||
if _skip_directory(playbook):
|
||||
return None
|
||||
return playbook
|
||||
|
||||
|
||||
def could_be_inventory(project_path, dir_path, filename):
|
||||
suspected_ext = os.path.splitext(filename)[-1]
|
||||
# Allow for files with no extension, or with extensions in a certain set
|
||||
if '.' in suspected_ext and suspected_ext not in ['.yml', '.yaml', '.ini']:
|
||||
return None
|
||||
inventory_path = os.path.join(dir_path, filename)
|
||||
# Filter files that do not use a character set consistent with
|
||||
# Ansible inventory mainly
|
||||
try:
|
||||
# only read through first 10 lines for performance
|
||||
with open(inventory_path) as inv_file:
|
||||
for line in islice(inv_file, 10):
|
||||
if not valid_inventory_re.match(line):
|
||||
return None
|
||||
except IOError:
|
||||
return None
|
||||
inventory = os.path.relpath(inventory_path, smart_str(project_path))
|
||||
if _skip_directory(inventory):
|
||||
return None
|
||||
return inventory
|
||||
@ -12,3 +12,5 @@
|
||||
group serializer and added `user_capabilities` to inventory source
|
||||
serializer, allow user creation and naming of inventory sources
|
||||
[[#5741](https://github.com/ansible/ansible-tower/issues/5741)]
|
||||
* support sourcing inventory from a file inside of a project's source
|
||||
tree [[#2477](https://github.com/ansible/ansible-tower/issues/2477)]
|
||||
|
||||
63
docs/scm_file_inventory.md
Normal file
63
docs/scm_file_inventory.md
Normal file
@ -0,0 +1,63 @@
|
||||
# SCM Flat File Inventory
|
||||
|
||||
Users can create inventory sources that use content in the source tree of
|
||||
a project as an Ansible inventory file.
|
||||
|
||||
## Usage Details
|
||||
|
||||
Fields that should be specified on creation of SCM inventory source:
|
||||
|
||||
- `scm_project` - project to use
|
||||
- `source_path` - relative path inside of the project indicating a
|
||||
directory or a file, if left blank, "" is still a relative path
|
||||
indicating the root directory of the project
|
||||
|
||||
A user should not be able to update this inventory source via through
|
||||
the endpoint `/inventory_sources/N/update/`. Instead, they should update
|
||||
the linked project.
|
||||
|
||||
An update of the project automatically triggers an inventory update within
|
||||
the proper context. An update _of the project_ is scheduled immediately
|
||||
after creation of the inventory source.
|
||||
|
||||
The project should show a listing of suggested inventory locations, at the
|
||||
endpoint `/projects/N/inventories/`, but this is not a comprehensive list of
|
||||
all paths that could be used as an Ansible inventory because of the wide
|
||||
range of inclusion criteria. The list will also max out at 50 entries.
|
||||
The user should be allowed to specify a location manually in the UI.
|
||||
This listing should be refreshed to latest SCM info on a project update.
|
||||
If no inventory sources use a project as an SCM inventory source, then
|
||||
the inventory listing may not be refreshed on update.
|
||||
|
||||
### Still-to-come 3.2 Changes
|
||||
|
||||
As a part of a different feature, it is planned to have all inventory sources
|
||||
inside of an inventory all update with a single button click. When this
|
||||
happens for an inventory containing an SCM inventory source, it should
|
||||
update the project.
|
||||
|
||||
## Supported File Syntax
|
||||
|
||||
> Any Inventory Ansible supports should be supported by this feature
|
||||
|
||||
This statement is the overall goal and should hold true absolutely for
|
||||
Ansible version 2.4 and beyond due to the use of `ansible-inventory`.
|
||||
Versions of Ansible before that may not support all valid inventory syntax
|
||||
because the internal mechanism is different.
|
||||
|
||||
Documentation should reflect the limitations of inventory file syntax
|
||||
support in old Ansible versions.
|
||||
|
||||
# Acceptance Criteria Notes
|
||||
|
||||
Some test scenarios to look at:
|
||||
- Obviously use a git repo with examples of host patterns, etc.
|
||||
- Test multiple inventories that use the same project, pointing to different
|
||||
files / directories inside of the project
|
||||
- Feature works correctly even if project doesn't have any playbook files
|
||||
- File related errors should surface as inventory import failures
|
||||
+ missing file
|
||||
+ invalid syntax in file
|
||||
- If the project SCM update encounters errors, it should not run the
|
||||
inventory updates
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user