External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-10 15:32:07 -03:30
Code Issues Packages Projects Releases Wiki Activity

Implement item copy feature

Browse Source 
See acceptance doc for implement details.

Signed-off-by: Aaron Tan <jangsutsr@gmail.com>
This commit is contained in:
Aaron Tan 2017-10-18 11:50:29 -04:00
parent 28c612ae9c
commit a2fd78add4
19 changed files with 763 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
awx/api/generics.py
View File

@ -5,6 +5,7 @@
import inspect
import logging
import time
import six
# Django
from django.conf import settings
@ -26,6 +27,10 @@ from rest_framework import generics
from rest_framework.response import Response
from rest_framework import status
from rest_framework import views
from rest_framework.permissions import AllowAny
# cryptography
from cryptography.fernet import InvalidToken
# AWX
from awx.api.filters import FieldLookupBackend
@ -33,9 +38,9 @@ from awx.main.models import * # noqa
from awx.main.access import access_registry
from awx.main.utils import * # noqa
from awx.main.utils.db import get_all_field_names
from awx.api.serializers import ResourceAccessListElementSerializer
from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer
from awx.api.versioning import URLPathVersioning, get_request_version
from awx.api.metadata import SublistAttachDetatchMetadata
from awx.api.metadata import SublistAttachDetatchMetadata, Metadata
__all__ = ['APIView', 'GenericAPIView', 'ListAPIView', 'SimpleListAPIView',
'ListCreateAPIView', 'SubListAPIView', 'SubListCreateAPIView',
@ -47,7 +52,8 @@ __all__ = ['APIView', 'GenericAPIView', 'ListAPIView', 'SimpleListAPIView',
'ResourceAccessList',
'ParentMixin',
'DeleteLastUnattachLabelMixin',
'SubListAttachDetachAPIView',]
'SubListAttachDetachAPIView',
'CopyAPIView']
logger = logging.getLogger('awx.api.generics')
analytics_logger = logging.getLogger('awx.analytics.performance')
@ -747,3 +753,152 @@ class ResourceAccessList(ParentMixin, ListAPIView):
for r in roles:
ancestors.update(set(r.ancestors.all()))
return User.objects.filter(roles__in=list(ancestors)).distinct()
def trigger_delayed_deep_copy(*args, **kwargs):
from awx.main.tasks import deep_copy_model_obj
connection.on_commit(lambda: deep_copy_model_obj.delay(*args, **kwargs))
class CopyAPIView(GenericAPIView):
serializer_class = CopySerializer
permission_classes = (AllowAny,)
copy_return_serializer_class = None
new_in_330 = True
new_in_api_v2 = True
def _get_copy_return_serializer(self, *args, **kwargs):
if not self.copy_return_serializer_class:
return self.get_serializer(*args, **kwargs)
serializer_class_store = self.serializer_class
self.serializer_class = self.copy_return_serializer_class
ret = self.get_serializer(*args, **kwargs)
self.serializer_class = serializer_class_store
return ret
@staticmethod
def _decrypt_model_field_if_needed(obj, field_name, field_val):
if field_name in getattr(type(obj), 'REENCRYPTION_BLACKLIST_AT_COPY', []):
return field_val
if isinstance(field_val, dict):
for sub_field in field_val:
if isinstance(sub_field, six.string_types) \
and isinstance(field_val[sub_field], six.string_types):
try:
field_val[sub_field] = decrypt_field(obj, field_name, sub_field)
except InvalidToken:
# Catching the corner case with v1 credential fields
field_val[sub_field] = decrypt_field(obj, sub_field)
elif isinstance(field_val, six.string_types):
field_val = decrypt_field(obj, field_name)
return field_val
def _build_create_dict(self, obj):
ret = {}
if self.copy_return_serializer_class:
all_fields = Metadata().get_serializer_info(
self._get_copy_return_serializer(), method='POST'
)
for field_name, field_info in all_fields.items():
if not hasattr(obj, field_name) or field_info.get('read_only', True):
continue
ret[field_name] = CopyAPIView._decrypt_model_field_if_needed(
obj, field_name, getattr(obj, field_name)
)
return ret
@staticmethod
def copy_model_obj(old_parent, new_parent, model, obj, creater, copy_name='', create_kwargs=None):
fields_to_preserve = set(getattr(model, 'FIELDS_TO_PRESERVE_AT_COPY', []))
fields_to_discard = set(getattr(model, 'FIELDS_TO_DISCARD_AT_COPY', []))
m2m_to_preserve = {}
o2m_to_preserve = {}
create_kwargs = create_kwargs or {}
for field_name in fields_to_discard:
create_kwargs.pop(field_name, None)
for field in model._meta.get_fields():
try:
field_val = getattr(obj, field.name)
except AttributeError:
continue
# Adjust copy blacklist fields here.
if field.name in fields_to_discard or field.name in [
'id', 'pk', 'polymorphic_ctype', 'unifiedjobtemplate_ptr', 'created_by', 'modified_by'
] or field.name.endswith('_role'):
create_kwargs.pop(field.name, None)
continue
if field.one_to_many:
if field.name in fields_to_preserve:
o2m_to_preserve[field.name] = field_val
elif field.many_to_many:
if field.name in fields_to_preserve and not old_parent:
m2m_to_preserve[field.name] = field_val
elif field.many_to_one and not field_val:
create_kwargs.pop(field.name, None)
elif field.many_to_one and field_val == old_parent:
create_kwargs[field.name] = new_parent
elif field.name == 'name' and not old_parent:
create_kwargs[field.name] = copy_name or field_val + ' copy'
elif field.name in fields_to_preserve:
create_kwargs[field.name] = CopyAPIView._decrypt_model_field_if_needed(
obj, field.name, field_val
)
new_obj = model.objects.create(**create_kwargs)
# Need to save separatedly because Djang-crum get_current_user would
# not work properly in non-request-response-cycle context.
new_obj.created_by = creater
new_obj.save()
for m2m in m2m_to_preserve:
for related_obj in m2m_to_preserve[m2m].all():
getattr(new_obj, m2m).add(related_obj)
if not old_parent:
sub_objects = []
for o2m in o2m_to_preserve:
for sub_obj in o2m_to_preserve[o2m].all():
sub_model = type(sub_obj)
sub_objects.append((sub_model.__module__, sub_model.__name__, sub_obj.pk))
return new_obj, sub_objects
ret = {obj: new_obj}
for o2m in o2m_to_preserve:
for sub_obj in o2m_to_preserve[o2m].all():
ret.update(CopyAPIView.copy_model_obj(obj, new_obj, type(sub_obj), sub_obj, creater))
return ret
def get(self, request, *args, **kwargs):
obj = self.get_object()
create_kwargs = self._build_create_dict(obj)
for key in create_kwargs:
create_kwargs[key] = getattr(create_kwargs[key], 'pk', None) or create_kwargs[key]
return Response({'can_copy': request.user.can_access(self.model, 'add', create_kwargs)})
def post(self, request, *args, **kwargs):
obj = self.get_object()
create_kwargs = self._build_create_dict(obj)
create_kwargs_check = {}
for key in create_kwargs:
create_kwargs_check[key] = getattr(create_kwargs[key], 'pk', None) or create_kwargs[key]
if not request.user.can_access(self.model, 'add', create_kwargs_check):
raise PermissionDenied()
serializer = self.get_serializer(data=request.data)
if not serializer.is_valid():
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
new_obj, sub_objs = CopyAPIView.copy_model_obj(
None, None, self.model, obj, request.user, create_kwargs=create_kwargs,
copy_name=serializer.validated_data.get('name', '')
)
if hasattr(new_obj, 'admin_role') and request.user not in new_obj.admin_role:
new_obj.admin_role.members.add(request.user)
if sub_objs:
permission_check_func = None
if hasattr(type(self), 'deep_copy_permission_check_func'):
permission_check_func = (
type(self).__module__, type(self).__name__, 'deep_copy_permission_check_func'
)
trigger_delayed_deep_copy(
self.model.__module__, self.model.__name__,
obj.pk, new_obj.pk, request.user.pk, sub_objs,
permission_check_func=permission_check_func
)
serializer = self._get_copy_return_serializer(new_obj)
return Response(serializer.data, status=status.HTTP_201_CREATED)

22
awx/api/serializers.py
View File

@ -131,6 +131,22 @@ def reverse_gfk(content_object, request):
}
class CopySerializer(serializers.Serializer):
name = serializers.CharField()
def validate(self, attrs):
name = attrs.get('name')
view = self.context.get('view', None)
obj = view.get_object()
if name == obj.name:
raise serializers.ValidationError(_(
'The original object is already named {}, a copy from'
' it cannot have the same name.'.format(name)
))
return attrs
class BaseSerializerMetaclass(serializers.SerializerMetaclass):
'''
Custom metaclass to enable attribute inheritance from Meta objects on
@ -1022,6 +1038,7 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer):
notification_templates_error = self.reverse('api:project_notification_templates_error_list', kwargs={'pk': obj.pk}),
access_list = self.reverse('api:project_access_list', kwargs={'pk': obj.pk}),
object_roles = self.reverse('api:project_object_roles_list', kwargs={'pk': obj.pk}),
copy = self.reverse('api:project_copy', kwargs={'pk': obj.pk}),
))
if obj.organization:
res['organization'] = self.reverse('api:organization_detail',
@ -1174,6 +1191,7 @@ class InventorySerializer(BaseSerializerWithVariables):
access_list = self.reverse('api:inventory_access_list', kwargs={'pk': obj.pk}),
object_roles = self.reverse('api:inventory_object_roles_list', kwargs={'pk': obj.pk}),
instance_groups = self.reverse('api:inventory_instance_groups_list', kwargs={'pk': obj.pk}),
copy = self.reverse('api:inventory_copy', kwargs={'pk': obj.pk}),
))
if obj.insights_credential:
res['insights_credential'] = self.reverse('api:credential_detail', kwargs={'pk': obj.insights_credential.pk})
@ -1530,6 +1548,7 @@ class CustomInventoryScriptSerializer(BaseSerializer):
res = super(CustomInventoryScriptSerializer, self).get_related(obj)
res.update(dict(
object_roles = self.reverse('api:inventory_script_object_roles_list', kwargs={'pk': obj.pk}),
copy = self.reverse('api:inventory_script_copy', kwargs={'pk': obj.pk}),
))
if obj.organization:
@ -2086,6 +2105,7 @@ class CredentialSerializer(BaseSerializer):
object_roles = self.reverse('api:credential_object_roles_list', kwargs={'pk': obj.pk}),
owner_users = self.reverse('api:credential_owner_users_list', kwargs={'pk': obj.pk}),
owner_teams = self.reverse('api:credential_owner_teams_list', kwargs={'pk': obj.pk}),
copy = self.reverse('api:credential_copy', kwargs={'pk': obj.pk}),
))
# TODO: remove when API v1 is removed
@ -2563,6 +2583,7 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO
labels = self.reverse('api:job_template_label_list', kwargs={'pk': obj.pk}),
object_roles = self.reverse('api:job_template_object_roles_list', kwargs={'pk': obj.pk}),
instance_groups = self.reverse('api:job_template_instance_groups_list', kwargs={'pk': obj.pk}),
copy = self.reverse('api:job_template_copy', kwargs={'pk': obj.pk}),
))
if obj.host_config_key:
res['callback'] = self.reverse('api:job_template_callback', kwargs={'pk': obj.pk})
@ -3686,6 +3707,7 @@ class NotificationTemplateSerializer(BaseSerializer):
res.update(dict(
test = self.reverse('api:notification_template_test', kwargs={'pk': obj.pk}),
notifications = self.reverse('api:notification_template_notification_list', kwargs={'pk': obj.pk}),
copy = self.reverse('api:notification_template_copy', kwargs={'pk': obj.pk}),
))
if obj.organization:
res['organization'] = self.reverse('api:organization_detail', kwargs={'pk': obj.organization.pk})

2
awx/api/urls/credential.py
View File

@ -11,6 +11,7 @@ from awx.api.views import (
CredentialObjectRolesList,
CredentialOwnerUsersList,
CredentialOwnerTeamsList,
CredentialCopy,
)
@ -22,6 +23,7 @@ urls = [
url(r'^(?P<pk>[0-9]+)/object_roles/$', CredentialObjectRolesList.as_view(), name='credential_object_roles_list'),
url(r'^(?P<pk>[0-9]+)/owner_users/$', CredentialOwnerUsersList.as_view(), name='credential_owner_users_list'),
url(r'^(?P<pk>[0-9]+)/owner_teams/$', CredentialOwnerTeamsList.as_view(), name='credential_owner_teams_list'),
url(r'^(?P<pk>[0-9]+)/copy/$', CredentialCopy.as_view(), name='credential_copy'),
]
__all__ = ['urls']

2
awx/api/urls/inventory.py
View File

@ -20,6 +20,7 @@ from awx.api.views import (
InventoryAccessList,
InventoryObjectRolesList,
InventoryInstanceGroupsList,
InventoryCopy,
)
@ -40,6 +41,7 @@ urls = [
url(r'^(?P<pk>[0-9]+)/access_list/$', InventoryAccessList.as_view(), name='inventory_access_list'),
url(r'^(?P<pk>[0-9]+)/object_roles/$', InventoryObjectRolesList.as_view(), name='inventory_object_roles_list'),
url(r'^(?P<pk>[0-9]+)/instance_groups/$', InventoryInstanceGroupsList.as_view(), name='inventory_instance_groups_list'),
url(r'^(?P<pk>[0-9]+)/copy/$', InventoryCopy.as_view(), name='inventory_copy'),
]
__all__ = ['urls']

2
awx/api/urls/inventory_script.py
View File

@ -7,6 +7,7 @@ from awx.api.views import (
InventoryScriptList,
InventoryScriptDetail,
InventoryScriptObjectRolesList,
InventoryScriptCopy,
)
@ -14,6 +15,7 @@ urls = [
url(r'^$', InventoryScriptList.as_view(), name='inventory_script_list'),
url(r'^(?P<pk>[0-9]+)/$', InventoryScriptDetail.as_view(), name='inventory_script_detail'),
url(r'^(?P<pk>[0-9]+)/object_roles/$', InventoryScriptObjectRolesList.as_view(), name='inventory_script_object_roles_list'),
url(r'^(?P<pk>[0-9]+)/copy/$', InventoryScriptCopy.as_view(), name='inventory_script_copy'),
]
__all__ = ['urls']

2
awx/api/urls/job_template.py
View File

@ -19,6 +19,7 @@ from awx.api.views import (
JobTemplateAccessList,
JobTemplateObjectRolesList,
JobTemplateLabelList,
JobTemplateCopy,
)
@ -41,6 +42,7 @@ urls = [
url(r'^(?P<pk>[0-9]+)/access_list/$', JobTemplateAccessList.as_view(), name='job_template_access_list'),
url(r'^(?P<pk>[0-9]+)/object_roles/$', JobTemplateObjectRolesList.as_view(), name='job_template_object_roles_list'),
url(r'^(?P<pk>[0-9]+)/labels/$', JobTemplateLabelList.as_view(), name='job_template_label_list'),
url(r'^(?P<pk>[0-9]+)/copy/$', JobTemplateCopy.as_view(), name='job_template_copy'),
]
__all__ = ['urls']

2
awx/api/urls/notification_template.py
View File

@ -8,6 +8,7 @@ from awx.api.views import (
NotificationTemplateDetail,
NotificationTemplateTest,
NotificationTemplateNotificationList,
NotificationTemplateCopy,
)
@ -16,6 +17,7 @@ urls = [
url(r'^(?P<pk>[0-9]+)/$', NotificationTemplateDetail.as_view(), name='notification_template_detail'),
url(r'^(?P<pk>[0-9]+)/test/$', NotificationTemplateTest.as_view(), name='notification_template_test'),
url(r'^(?P<pk>[0-9]+)/notifications/$', NotificationTemplateNotificationList.as_view(), name='notification_template_notification_list'),
url(r'^(?P<pk>[0-9]+)/copy/$', NotificationTemplateCopy.as_view(), name='notification_template_copy'),
]
__all__ = ['urls']

4
awx/api/urls/project.py
View File

@ -19,10 +19,11 @@ from awx.api.views import (
ProjectNotificationTemplatesSuccessList,
ProjectObjectRolesList,
ProjectAccessList,
ProjectCopy,
)
urls = [
urls = [
url(r'^$', ProjectList.as_view(), name='project_list'),
url(r'^(?P<pk>[0-9]+)/$', ProjectDetail.as_view(), name='project_detail'),
url(r'^(?P<pk>[0-9]+)/playbooks/$', ProjectPlaybooks.as_view(), name='project_playbooks'),
@ -39,6 +40,7 @@ urls = [
name='project_notification_templates_success_list'),
url(r'^(?P<pk>[0-9]+)/object_roles/$', ProjectObjectRolesList.as_view(), name='project_object_roles_list'),
url(r'^(?P<pk>[0-9]+)/access_list/$', ProjectAccessList.as_view(), name='project_access_list'),
url(r'^(?P<pk>[0-9]+)/copy/$', ProjectCopy.as_view(), name='project_copy'),
]
__all__ = ['urls']

65
awx/api/views.py
View File

@ -1424,6 +1424,12 @@ class ProjectObjectRolesList(SubListAPIView):
return Role.objects.filter(content_type=content_type, object_id=po.pk)
class ProjectCopy(CopyAPIView):
model = Project
copy_return_serializer_class = ProjectSerializer
class UserList(ListCreateAPIView):
model = User
@ -1805,6 +1811,12 @@ class CredentialObjectRolesList(SubListAPIView):
return Role.objects.filter(content_type=content_type, object_id=po.pk)
class CredentialCopy(CopyAPIView):
model = Credential
copy_return_serializer_class = CredentialSerializer
class InventoryScriptList(ListCreateAPIView):
model = CustomInventoryScript
@ -1842,6 +1854,12 @@ class InventoryScriptObjectRolesList(SubListAPIView):
return Role.objects.filter(content_type=content_type, object_id=po.pk)
class InventoryScriptCopy(CopyAPIView):
model = CustomInventoryScript
copy_return_serializer_class = CustomInventoryScriptSerializer
class InventoryList(ListCreateAPIView):
model = Inventory
@ -1969,6 +1987,12 @@ class InventoryJobTemplateList(SubListAPIView):
return qs.filter(inventory=parent)
class InventoryCopy(CopyAPIView):
model = Inventory
copy_return_serializer_class = InventorySerializer
class HostRelatedSearchMixin(object):
@property
@ -3337,6 +3361,12 @@ class JobTemplateObjectRolesList(SubListAPIView):
return Role.objects.filter(content_type=content_type, object_id=po.pk)
class JobTemplateCopy(CopyAPIView):
model = JobTemplate
copy_return_serializer_class = JobTemplateSerializer
class WorkflowJobNodeList(WorkflowsEnforcementMixin, ListAPIView):
model = WorkflowJobNode
@ -3500,10 +3530,10 @@ class WorkflowJobTemplateDetail(WorkflowsEnforcementMixin, RetrieveUpdateDestroy
new_in_310 = True
class WorkflowJobTemplateCopy(WorkflowsEnforcementMixin, GenericAPIView):
class WorkflowJobTemplateCopy(WorkflowsEnforcementMixin, CopyAPIView):
model = WorkflowJobTemplate
serializer_class = EmptySerializer
copy_return_serializer_class = WorkflowJobTemplateSerializer
new_in_310 = True
def get(self, request, *args, **kwargs):
@ -3520,17 +3550,20 @@ class WorkflowJobTemplateCopy(WorkflowsEnforcementMixin, GenericAPIView):
data.update(messages)
return Response(data)
def post(self, request, *args, **kwargs):
obj = self.get_object()
if not request.user.can_access(self.model, 'copy', obj):
raise PermissionDenied()
new_obj = obj.user_copy(request.user)
if request.user not in new_obj.admin_role:
new_obj.admin_role.members.add(request.user)
data = OrderedDict()
data.update(WorkflowJobTemplateSerializer(
new_obj, context=self.get_serializer_context()).to_representation(new_obj))
return Response(data, status=status.HTTP_201_CREATED)
@staticmethod
def deep_copy_permission_check_func(user, new_objs):
for obj in new_objs:
for field_name in obj._get_workflow_job_field_names():
item = getattr(obj, field_name, None)
if item is None:
continue
if field_name in ['inventory']:
if not user.can_access(item.__class__, 'use', item):
setattr(obj, field_name, None)
if field_name in ['unified_job_template']:
if not user.can_access(item.__class__, 'start', item, validate_license=False):
setattr(obj, field_name, None)
obj.save()
class WorkflowJobTemplateLabelList(WorkflowsEnforcementMixin, JobTemplateLabelList):
@ -4704,6 +4737,12 @@ class NotificationTemplateNotificationList(SubListAPIView):
new_in_300 = True
class NotificationTemplateCopy(CopyAPIView):
model = NotificationTemplate
copy_return_serializer_class = NotificationTemplateSerializer
class NotificationList(ListAPIView):
model = Notification

9
awx/main/models/inventory.py
View File

@ -48,6 +48,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin):
an inventory source contains lists and hosts.
'''
FIELDS_TO_PRESERVE_AT_COPY = ['hosts', 'groups', 'instance_groups']
KIND_CHOICES = [
('', _('Hosts have a direct link to this inventory.')),
('smart', _('Hosts for inventory generated using the host_filter property.')),
@ -503,6 +504,10 @@ class Host(CommonModelNameNotUnique):
A managed node
'''
FIELDS_TO_PRESERVE_AT_COPY = [
'name', 'description', 'groups', 'inventory', 'enabled', 'instance_id', 'variables'
]
class Meta:
app_label = 'main'
unique_together = (("name", "inventory"),) # FIXME: Add ('instance_id', 'inventory') after migration.
@ -690,6 +695,10 @@ class Group(CommonModelNameNotUnique):
groups.
'''
FIELDS_TO_PRESERVE_AT_COPY = [
'name', 'description', 'inventory', 'children', 'parents', 'hosts', 'variables'
]
class Meta:
app_label = 'main'
unique_together = (("name", "inventory"),)

4
awx/main/models/jobs.py
View File

@ -227,6 +227,10 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
A job template is a reusable job definition for applying a project (with
playbook) to an inventory source with a given credential.
'''
FIELDS_TO_PRESERVE_AT_COPY = [
'labels', 'instance_groups', 'credentials', 'survey_spec'
]
FIELDS_TO_DISCARD_AT_COPY = ['vault_credential', 'credential']
SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name')]
class Meta:

2
awx/main/models/projects.py
View File

@ -228,6 +228,8 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin):
'''
SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')]
FIELDS_TO_PRESERVE_AT_COPY = ['labels', 'instance_groups', 'credentials']
FIELDS_TO_DISCARD_AT_COPY = ['local_path']
class Meta:
app_label = 'main'

15
awx/main/models/workflow.py
View File

@ -110,6 +110,13 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig):
class WorkflowJobTemplateNode(WorkflowNodeBase):
FIELDS_TO_PRESERVE_AT_COPY = [
'unified_job_template', 'workflow_job_template', 'success_nodes', 'failure_nodes',
'always_nodes', 'credentials', 'inventory', 'extra_data', 'survey_passwords',
'char_prompts'
]
REENCRYPTION_BLACKLIST_AT_COPY = ['extra_data', 'survey_passwords']
workflow_job_template = models.ForeignKey(
'WorkflowJobTemplate',
related_name='workflow_job_template_nodes',
@ -283,6 +290,9 @@ class WorkflowJobOptions(BaseModel):
class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTemplateMixin, ResourceMixin):
SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')]
FIELDS_TO_PRESERVE_AT_COPY = [
'labels', 'instance_groups', 'workflow_job_template_nodes', 'credentials', 'survey_spec'
]
class Meta:
app_label = 'main'
@ -393,11 +403,6 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
node_list.append(node.pk)
return node_list
def user_copy(self, user):
new_wfjt = self.copy_unified_jt()
new_wfjt.copy_nodes_from_original(original=self, user=user)
return new_wfjt
class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificationMixin):
class Meta:

58
awx/main/tasks.py
View File

@ -7,6 +7,7 @@ from collections import OrderedDict
import ConfigParser
import cStringIO
import functools
import importlib
import json
import logging
import os
@ -33,6 +34,7 @@ from celery.signals import celeryd_init, worker_process_init, worker_shutdown
# Django
from django.conf import settings
from django.db import transaction, DatabaseError, IntegrityError
from django.db.models.fields.related import ForeignKey
from django.utils.timezone import now, timedelta
from django.utils.encoding import smart_str
from django.core.mail import send_mail
@ -2279,6 +2281,62 @@ class RunSystemJob(BaseTask):
return settings.BASE_DIR
def _reconstruct_relationships(copy_mapping):
for old_obj, new_obj in copy_mapping.items():
model = type(old_obj)
for field_name in getattr(model, 'FIELDS_TO_PRESERVE_AT_COPY', []):
field = model._meta.get_field(field_name)
if isinstance(field, ForeignKey):
if getattr(new_obj, field_name, None):
continue
related_obj = getattr(old_obj, field_name)
related_obj = copy_mapping.get(related_obj, related_obj)
setattr(new_obj, field_name, related_obj)
elif field.many_to_many:
for related_obj in getattr(old_obj, field_name).all():
getattr(new_obj, field_name).add(copy_mapping.get(related_obj, related_obj))
new_obj.save()
@shared_task(bind=True, queue='tower', base=LogErrorsTask)
def deep_copy_model_obj(
self, model_module, model_name, obj_pk, new_obj_pk,
user_pk, sub_obj_list, permission_check_func=None
):
logger.info('Deep copy {} from {} to {}.'.format(model_name, obj_pk, new_obj_pk))
from awx.api.generics import CopyAPIView
model = getattr(importlib.import_module(model_module), model_name, None)
if model is None:
return
try:
obj = model.objects.get(pk=obj_pk)
new_obj = model.objects.get(pk=new_obj_pk)
creater = User.objects.get(pk=user_pk)
except ObjectDoesNotExist:
logger.warning("Object or user no longer exists.")
return
with transaction.atomic():
copy_mapping = {}
for sub_obj_setup in sub_obj_list:
sub_model = getattr(importlib.import_module(sub_obj_setup[0]),
sub_obj_setup[1], None)
if sub_model is None:
continue
try:
sub_obj = sub_model.objects.get(pk=sub_obj_setup[2])
except ObjectDoesNotExist:
continue
copy_mapping.update(CopyAPIView.copy_model_obj(
obj, new_obj, sub_model, sub_obj, creater
))
_reconstruct_relationships(copy_mapping)
if permission_check_func:
permission_check_func = getattr(getattr(
importlib.import_module(permission_check_func[0]), permission_check_func[1]
), permission_check_func[2])
permission_check_func(creater, copy_mapping.values())
celery_app.register_task(RunJob())
celery_app.register_task(RunProjectUpdate())
celery_app.register_task(RunInventoryUpdate())

10
awx/main/tests/functional/conftest.py
View File

@ -33,7 +33,8 @@ from awx.main.models.inventory import (
Group,
Inventory,
InventoryUpdate,
InventorySource
InventorySource,
CustomInventoryScript
)
from awx.main.models.organization import (
Organization,
@ -489,6 +490,13 @@ def inventory_update(inventory_source):
return InventoryUpdate.objects.create(inventory_source=inventory_source)
@pytest.fixture
def inventory_script(organization):
return CustomInventoryScript.objects.create(name='test inv script',
organization=organization,
script='#!/usr/bin/python')
@pytest.fixture
def host(group, inventory):
return group.hosts.create(name='single-host', inventory=inventory)

19
awx/main/tests/functional/models/test_workflow.py
View File

@ -191,25 +191,6 @@ class TestWorkflowJobTemplate:
assert (test_view.is_valid_relation(nodes[2], node_assoc_1) ==
{'Error': 'Cannot associate failure_nodes when always_nodes have been associated.'})
def test_wfjt_copy(self, wfjt, job_template, inventory, admin_user):
old_nodes = wfjt.workflow_job_template_nodes.all()
node1 = old_nodes[1]
node1.unified_job_template = job_template
node1.save()
node2 = old_nodes[2]
node2.inventory = inventory
node2.save()
new_wfjt = wfjt.user_copy(admin_user)
for fd in ['description', 'survey_spec', 'survey_enabled', 'extra_vars']:
assert getattr(wfjt, fd) == getattr(new_wfjt, fd)
assert new_wfjt.organization == wfjt.organization
assert len(new_wfjt.workflow_job_template_nodes.all()) == 3
nodes = new_wfjt.workflow_job_template_nodes.all()
assert nodes[0].success_nodes.all()[0] == nodes[1]
assert nodes[1].failure_nodes.all()[0] == nodes[2]
assert nodes[1].unified_job_template == job_template
assert nodes[2].inventory == inventory
def test_wfjt_unique_together_with_org(self, organization):
wfjt1 = WorkflowJobTemplate(name='foo', organization=organization)
wfjt1.save()

214
awx/main/tests/functional/test_copy.py Normal file
View File

@ -0,0 +1,214 @@
import pytest
import mock
from awx.api.versioning import reverse
from awx.main.utils import decrypt_field
from awx.main.models.workflow import WorkflowJobTemplateNode
from awx.main.models.jobs import JobTemplate
from awx.main.tasks import deep_copy_model_obj
@pytest.mark.django_db
def test_job_template_copy(post, get, project, inventory, machine_credential, vault_credential,
credential, alice, job_template_with_survey_passwords, admin):
job_template_with_survey_passwords.project = project
job_template_with_survey_passwords.inventory = inventory
job_template_with_survey_passwords.save()
job_template_with_survey_passwords.credentials.add(credential)
job_template_with_survey_passwords.credentials.add(machine_credential)
job_template_with_survey_passwords.credentials.add(vault_credential)
job_template_with_survey_passwords.admin_role.members.add(alice)
assert get(
reverse('api:job_template_copy', kwargs={'pk': job_template_with_survey_passwords.pk}),
alice, expect=200
).data['can_copy'] is False
assert get(
reverse('api:job_template_copy', kwargs={'pk': job_template_with_survey_passwords.pk}),
admin, expect=200
).data['can_copy'] is True
jt_copy_pk = post(
reverse('api:job_template_copy', kwargs={'pk': job_template_with_survey_passwords.pk}),
{'name': 'new jt name'}, admin, expect=201
).data['id']
jt_copy = type(job_template_with_survey_passwords).objects.get(pk=jt_copy_pk)
assert jt_copy.created_by == admin
assert jt_copy.name == 'new jt name'
assert jt_copy.project == project
assert jt_copy.inventory == inventory
assert jt_copy.playbook == job_template_with_survey_passwords.playbook
assert jt_copy.credentials.count() == 3
assert credential in jt_copy.credentials.all()
assert vault_credential in jt_copy.credentials.all()
assert machine_credential in jt_copy.credentials.all()
assert job_template_with_survey_passwords.survey_spec == jt_copy.survey_spec
@pytest.mark.django_db
def test_project_copy(post, get, project, organization, scm_credential, alice):
project.credential = scm_credential
project.save()
project.admin_role.members.add(alice)
assert get(
reverse('api:project_copy', kwargs={'pk': project.pk}), alice, expect=200
).data['can_copy'] is False
project.organization.admin_role.members.add(alice)
assert get(
reverse('api:project_copy', kwargs={'pk': project.pk}), alice, expect=200
).data['can_copy'] is True
project_copy_pk = post(
reverse('api:project_copy', kwargs={'pk': project.pk}),
{'name': 'copied project'}, alice, expect=201
).data['id']
project_copy = type(project).objects.get(pk=project_copy_pk)
assert project_copy.created_by == alice
assert project_copy.name == 'copied project'
assert project_copy.organization == organization
assert project_copy.credential == scm_credential
@pytest.mark.django_db
def test_inventory_copy(inventory, group_factory, post, get, alice, organization):
group_1_1 = group_factory('g_1_1')
group_2_1 = group_factory('g_2_1')
group_2_2 = group_factory('g_2_2')
group_2_1.parents.add(group_1_1)
group_2_2.parents.add(group_1_1)
group_2_2.parents.add(group_2_1)
host = group_1_1.hosts.create(name='host', inventory=inventory)
group_2_1.hosts.add(host)
inventory.admin_role.members.add(alice)
assert get(
reverse('api:inventory_copy', kwargs={'pk': inventory.pk}), alice, expect=200
).data['can_copy'] is False
inventory.organization.admin_role.members.add(alice)
assert get(
reverse('api:inventory_copy', kwargs={'pk': inventory.pk}), alice, expect=200
).data['can_copy'] is True
with mock.patch('awx.api.generics.trigger_delayed_deep_copy') as deep_copy_mock:
inv_copy_pk = post(
reverse('api:inventory_copy', kwargs={'pk': inventory.pk}),
{'name': 'new inv name'}, alice, expect=201
).data['id']
inventory_copy = type(inventory).objects.get(pk=inv_copy_pk)
args, kwargs = deep_copy_mock.call_args
deep_copy_model_obj(*args, **kwargs)
group_1_1_copy = inventory_copy.groups.get(name='g_1_1')
group_2_1_copy = inventory_copy.groups.get(name='g_2_1')
group_2_2_copy = inventory_copy.groups.get(name='g_2_2')
host_copy = inventory_copy.hosts.get(name='host')
assert inventory_copy.organization == organization
assert inventory_copy.created_by == alice
assert inventory_copy.name == 'new inv name'
assert set(group_1_1_copy.parents.all()) == set()
assert set(group_2_1_copy.parents.all()) == set([group_1_1_copy])
assert set(group_2_2_copy.parents.all()) == set([group_1_1_copy, group_2_1_copy])
assert set(group_1_1_copy.hosts.all()) == set([host_copy])
assert set(group_2_1_copy.hosts.all()) == set([host_copy])
assert set(group_2_2_copy.hosts.all()) == set()
@pytest.mark.django_db
def test_workflow_job_template_copy(workflow_job_template, post, get, admin, organization):
workflow_job_template.organization = organization
workflow_job_template.save()
jts = [JobTemplate.objects.create(name='test-jt-{}'.format(i)) for i in range(0, 5)]
nodes = [
WorkflowJobTemplateNode.objects.create(
workflow_job_template=workflow_job_template, unified_job_template=jts[i]
) for i in range(0, 5)
]
nodes[0].success_nodes.add(nodes[1])
nodes[1].success_nodes.add(nodes[2])
nodes[0].failure_nodes.add(nodes[3])
nodes[3].failure_nodes.add(nodes[4])
with mock.patch('awx.api.generics.trigger_delayed_deep_copy') as deep_copy_mock:
wfjt_copy_id = post(
reverse('api:workflow_job_template_copy', kwargs={'pk': workflow_job_template.pk}),
{'name': 'new wfjt name'}, admin, expect=201
).data['id']
wfjt_copy = type(workflow_job_template).objects.get(pk=wfjt_copy_id)
args, kwargs = deep_copy_mock.call_args
deep_copy_model_obj(*args, **kwargs)
assert wfjt_copy.organization == organization
assert wfjt_copy.created_by == admin
assert wfjt_copy.name == 'new wfjt name'
copied_node_list = [x for x in wfjt_copy.workflow_job_template_nodes.all()]
copied_node_list.sort(key=lambda x: int(x.unified_job_template.name[-1]))
for node, success_count, failure_count, always_count in zip(
copied_node_list,
[1, 1, 0, 0, 0],
[1, 0, 0, 1, 0],
[0, 0, 0, 0, 0]
):
assert node.success_nodes.count() == success_count
assert node.failure_nodes.count() == failure_count
assert node.always_nodes.count() == always_count
assert copied_node_list[1] in copied_node_list[0].success_nodes.all()
assert copied_node_list[2] in copied_node_list[1].success_nodes.all()
assert copied_node_list[3] in copied_node_list[0].failure_nodes.all()
assert copied_node_list[4] in copied_node_list[3].failure_nodes.all()
@pytest.mark.django_db
def test_credential_copy(post, get, machine_credential, credentialtype_ssh, admin):
assert get(
reverse('api:credential_copy', kwargs={'pk': machine_credential.pk}), admin, expect=200
).data['can_copy'] is True
credential_copy_pk = post(
reverse('api:credential_copy', kwargs={'pk': machine_credential.pk}),
{'name': 'copied credential'}, admin, expect=201
).data['id']
credential_copy = type(machine_credential).objects.get(pk=credential_copy_pk)
assert credential_copy.created_by == admin
assert credential_copy.name == 'copied credential'
assert credential_copy.credential_type == credentialtype_ssh
assert credential_copy.inputs['username'] == machine_credential.inputs['username']
assert (decrypt_field(credential_copy, 'password') ==
decrypt_field(machine_credential, 'password'))
@pytest.mark.django_db
def test_notification_template_copy(post, get, notification_template_with_encrypt,
organization, alice):
#notification_template_with_encrypt.admin_role.members.add(alice)
assert get(
reverse(
'api:notification_template_copy', kwargs={'pk': notification_template_with_encrypt.pk}
), alice, expect=200
).data['can_copy'] is False
notification_template_with_encrypt.organization.admin_role.members.add(alice)
assert get(
reverse(
'api:notification_template_copy', kwargs={'pk': notification_template_with_encrypt.pk}
), alice, expect=200
).data['can_copy'] is True
nt_copy_pk = post(
reverse(
'api:notification_template_copy', kwargs={'pk': notification_template_with_encrypt.pk}
), {'name': 'copied nt'}, alice, expect=201
).data['id']
notification_template_copy = type(notification_template_with_encrypt).objects.get(pk=nt_copy_pk)
assert notification_template_copy.created_by == alice
assert notification_template_copy.name == 'copied nt'
assert notification_template_copy.organization == organization
assert (decrypt_field(notification_template_with_encrypt, 'notification_configuration', 'token') ==
decrypt_field(notification_template_copy, 'notification_configuration', 'token'))
@pytest.mark.django_db
def test_inventory_script_copy(post, get, inventory_script, organization, alice):
assert get(
reverse('api:inventory_script_copy', kwargs={'pk': inventory_script.pk}), alice, expect=200
).data['can_copy'] is False
inventory_script.organization.admin_role.members.add(alice)
assert get(
reverse('api:inventory_script_copy', kwargs={'pk': inventory_script.pk}), alice, expect=200
).data['can_copy'] is True
is_copy_pk = post(
reverse('api:inventory_script_copy', kwargs={'pk': inventory_script.pk}),
{'name': 'copied inv script'}, alice, expect=201
).data['id']
inventory_script_copy = type(inventory_script).objects.get(pk=is_copy_pk)
assert inventory_script_copy.created_by == alice
assert inventory_script_copy.name == 'copied inv script'
assert inventory_script_copy.organization == organization

166
docs/resource_copy.md Normal file
View File

@ -0,0 +1,166 @@
Starting from Tower 3.3 and API v2, user are able to copy some existing resource objects to quickly
create new resource objects via POSTing to corresponding `/copy/` endpoint. A new `CopyAPIView` class
is introduced as the base view class for `/copy/` endpoints. It mimics the process of manually fetching
fields from the existing object to create a new object, plus the ability to automatically detect sub
structures of existing objects and make a background task-based deep copy when necessary.
## Usage
If an AWX resource is copiable, all of its object detail API views will have a related URL field
`"copy"`, which has form `/api/<version>/<resource name>/<object pk>/copy/`. GET to this endpoint
will return `can_copy`, which is a boolean indicating whether the current user can execute a copy
operation; POST to this endpoint actually copies the resource object. One field `name` is required
which will later be used as the name of the created copy. Upon success, 201 will be returned, along
with the created copy.
For some resources like credential, the copy process is not time-consuming, thus the entire copy
process will take place in the request-response cycle, and the created object copy is returned as
POST response.
For some other resources like inventory, the copy process can take longer, depending on the number
of sub-objects to copy (will explain later). Thus, although the created copy will be returned, the
copy process is not finished yet. All sub-objects (like all hosts and groups of an inventory) will
not be created until after the background copy task is finished in success.
Currently the available list of copiable resources are:
- job templates
- projects
- inventories
- workflows
- credentials
- notifications
- inventory scripts
For most of the resources above, only the object to be copied itself will be copied; For some resources
like inventories, however, sub resources belonging to the resource will also be copied to maintain the
full functionality of the copied new resource. In specific:
- When an inventory is copied, all its hosts, groups and inventory sources are copied.
- When a workflow job template is copied, all its workflow job template nodes are copied.
## How to add a copy end-point for a resource
The copy behavior of different resources largely follow the same pattern, therefore a unified way of
enabling copy capability for resources is available for developers:
Firstly, create a `/copy/` url endpoint for the target resource.
Secondly, create a view class as handler to `/copy/` endpoint. This view class should be subclassed
from `awx.api.generics.CopyAPIView`. Here is an example:
```python
class JobTemplateCopy(CopyAPIView):
model = JobTemplate
copy_return_serializer_class = JobTemplateSerializer
```
Note the above example declares a custom class attribute `copy_return_serializer_class`. This attribute
is used by `CopyAPIView` to render the created copy in POST response, so in most cases the value should
be the same as `serializer_class` of corresponding resource detail view, like here the value is the
`serializer_class` of `JobTemplateDetail`.
Thirdly, for the underlying model of the resource, Add 2 macros, `FIELDS_TO_PRESERVE_AT_COPY` and
`FIELDS_TO_DISCARD_AT_COPY`, as needed. Here is an example:
```python
class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, ResourceMixin):
'''
A job template is a reusable job definition for applying a project (with
playbook) to an inventory source with a given credential.
'''
FIELDS_TO_PRESERVE_AT_COPY = [
'labels', 'instance_groups', 'credentials', 'survey_spec'
]
FIELDS_TO_DISCARD_AT_COPY = ['vault_credential', 'credential']
```
When copying a resource object, basically all fields necessary for creating a new resource (fields
composing a valid POST body for creating new resources) are extracted from the original object and
used to create the copy.
However, sometimes we need more fields to be copied, like `credentials` of a job template, which
cannot be provided during creation. In this case we list such fields in `FIELDS_TO_PRESERVE_AT_COPY`
so that these fields won't be missed.
On the other hand, sometimes we do not want to include some fields provided in create POST body,
like `vault_credential` and `credential` fields used for creating a job template, which do not have
tangible field correspondence in `JobTemplate` model. In this case we list such fields in
`FIELDS_TO_DISCARD_AT_COPY` so that those fields won't be included.
For models that will be part of a deep copy, like hosts and workflow job template nodes, the related
POST body for creating a new object is not available. Therefore all necessary fields for creating
a new resource should also be included in `FIELDS_TO_PRESERVE_AT_COPY`.
Lastly, unit test copy behavior of the new endpoint in `/awx/main/tests/functional/test_copy.py` and
update docs (like this doc).
Fields in `FIELDS_TO_PRESERVE_AT_COPY` must be solid model fields, while fields in
`FIELDS_TO_DISCARD_AT_COPY` do not need to be. Note there are hidden fields not visible from model
definition, namely reverse relationships and fields inherited from super classes or mix-ins. A help
script `tools/scripts/list_fields.py` is available to inspect a model and list details of all its
available fields.
```
# In shell_plus
>>> from list_fields import pretty_print_model_fields
>>> pretty_print_model_fields(JobTemplate)
```
`CopyAPIView` will automatically detect sub objects of an object, and do a deep copy of all sub objects
as a background celery task. There are sometimes permission issues with sub object copy. For example,
when copying nodes of a workflow job template, there are cases where the user performing copy has no use
permission of related credential and inventory of some nodes, and it is desired those fields will be
`None`. In order to do that, developer should provide a static method `deep_copy_permission_check_func`
under corresponding specific copy view. Like
```python
class WorkflowJobTemplateCopy(WorkflowsEnforcementMixin, CopyAPIView):
model = WorkflowJobTemplate
copy_return_serializer_class = WorkflowJobTemplateSerializer
# Other code
@staticmethod
def deep_copy_permission_check_func(user, new_objs):
# method body
# Other code
```
Static method `deep_copy_permission_check_func` must have and only have two arguments: `user`, the
user performing the copy; `new_objs`, a list of all sub objects of the created copy. Sub objects in
`new_objs` are initially populated disregarding any permission constraints, developer shall check
`user`'s permission against these new sub objects and react like unlink related objects or sending
warning logs. `deep_copy_permission_check_func` should not return anything.
Lastly, macro `REENCRYPTION_BLACKLIST_AT_COPY` is available as part of a model definition. It is a
list of field names which will escape re-encryption during copy. For example, `extra_data` field
of workflow job template nodes.
## Acceptance Criteria
* Credentials should be able to copy themselves. The behavior of copying credential A shall be exactly
the same as creating a credential B with all needed fields for creation coming from credential A.
* Inventories should be able to copy themselves. The behavior of copying inventory A shall be exactly
the same as creating an inventory B with all needed fields for creation coming from inventory A. Other
than that, inventory B should inherit A's `instance_groups`, and have exactly the same host and group
structures as A.
* Inventory scripts should be able to copy themselves. The behavior of copying inventory script A
shall be exactly the same as creating an inventory script B with all needed fields for creation
coming from inventory script A.
* Job templates should be able to copy themselves. The behavior of copying job template A
shall be exactly the same as creating a job template B with all needed fields for creation
coming from job template A. Other than that, job template B should inherit A's `labels`,
`instance_groups`, `credentials` and `survey_spec`.
* Notification templates should be able to copy themselves. The behavior of copying notification
template A shall be exactly the same as creating a notification template B with all needed fields
for creation coming from notification template A.
* Projects should be able to copy themselves. The behavior of copying project A shall be the
same as creating a project B with all needed fields for creation coming from project A, except for
`local_path`, which will be populated by triggered project update. Other than that, project B
should inherit A's `labels`, `instance_groups` and `credentials`.
* Workflow Job templates should be able to copy themselves. The behavior of copying workflow job
template A shall be exactly the same as creating a workflow job template B with all needed fields
for creation coming from workflow job template A. Other than that, workflow job template B should
inherit A's `labels`, `instance_groups`, `credentials` and `survey_spec`, and have exactly the
same workflow job template node structure as A.
* In all copy processes, `name` field of the created copy of the original object should be able to
customize in the POST body.
* The permission for a user to make a copy for an existing resource object should be the same as the
permission for a user to create a brand new resource object using fields from the existing object.
* The RBAC behavior of original workflow job template `/copy/` should be pertained. That is, if the
user has no necessary permission to the related project and credential of a workflow job template
node, the copied workflow job template node should have those fields empty.

46
tools/scripts/list_fields.py Executable file
View File

@ -0,0 +1,46 @@
__all__ = ['pretty_print_model_fields']
def _get_class_full_name(cls_):
return cls_.__module__ + '.' + cls_.__name__
class _ModelFieldRow(object):
def __init__(self, field):
self.field = field
self.name = field.name
self.type_ = _get_class_full_name(type(field))
if self.field.many_to_many\
or self.field.many_to_one\
or self.field.one_to_many\
or self.field.one_to_one:
self.related_model = _get_class_full_name(self.field.remote_field.model)
else:
self.related_model = 'N/A'
def pretty_print(self, max_name_len, max_type_len, max_rel_model_len):
row = []
row.append(self.name)
row.append(' ' * (max_name_len - len(self.name)))
row.append('|')
row.append(self.type_)
row.append(' ' * (max_type_len - len(self.type_)))
row.append('|')
row.append(self.related_model)
row.append(' ' * (max_rel_model_len - len(self.related_model)))
print(''.join(row))
def pretty_print_model_fields(model):
field_info_rows = []
max_lens = [0, 0, 0]
for field in model._meta.get_fields():
field_info_rows.append(_ModelFieldRow(field))
max_lens[0] = max(max_lens[0], len(field_info_rows[-1].name))
max_lens[1] = max(max_lens[1], len(field_info_rows[-1].type_))
max_lens[2] = max(max_lens[2], len(field_info_rows[-1].related_model))
print('=' * (sum(max_lens) + len(max_lens) - 1))
for row in field_info_rows:
row.pretty_print(*max_lens)
print('=' * (sum(max_lens) + len(max_lens) - 1))