From a2fd78add4abc32149023c808d9510f662fe1d51 Mon Sep 17 00:00:00 2001 From: Aaron Tan Date: Wed, 18 Oct 2017 11:50:29 -0400 Subject: [PATCH] Implement item copy feature See acceptance doc for implement details. Signed-off-by: Aaron Tan --- awx/api/generics.py | 161 ++++++++++++- awx/api/serializers.py | 22 ++ awx/api/urls/credential.py | 2 + awx/api/urls/inventory.py | 2 + awx/api/urls/inventory_script.py | 2 + awx/api/urls/job_template.py | 2 + awx/api/urls/notification_template.py | 2 + awx/api/urls/project.py | 4 +- awx/api/views.py | 65 ++++-- awx/main/models/inventory.py | 9 + awx/main/models/jobs.py | 4 + awx/main/models/projects.py | 2 + awx/main/models/workflow.py | 15 +- awx/main/tasks.py | 58 +++++ awx/main/tests/functional/conftest.py | 10 +- .../tests/functional/models/test_workflow.py | 19 -- awx/main/tests/functional/test_copy.py | 214 ++++++++++++++++++ docs/resource_copy.md | 166 ++++++++++++++ tools/scripts/list_fields.py | 46 ++++ 19 files changed, 763 insertions(+), 42 deletions(-) create mode 100644 awx/main/tests/functional/test_copy.py create mode 100644 docs/resource_copy.md create mode 100755 tools/scripts/list_fields.py diff --git a/awx/api/generics.py b/awx/api/generics.py index 73c8ddd1db..bee184c838 100644 --- a/awx/api/generics.py +++ b/awx/api/generics.py @@ -5,6 +5,7 @@ import inspect import logging import time +import six # Django from django.conf import settings @@ -26,6 +27,10 @@ from rest_framework import generics from rest_framework.response import Response from rest_framework import status from rest_framework import views +from rest_framework.permissions import AllowAny + +# cryptography +from cryptography.fernet import InvalidToken # AWX from awx.api.filters import FieldLookupBackend @@ -33,9 +38,9 @@ from awx.main.models import * # noqa from awx.main.access import access_registry from awx.main.utils import * # noqa from awx.main.utils.db import get_all_field_names -from awx.api.serializers import ResourceAccessListElementSerializer +from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer from awx.api.versioning import URLPathVersioning, get_request_version -from awx.api.metadata import SublistAttachDetatchMetadata +from awx.api.metadata import SublistAttachDetatchMetadata, Metadata __all__ = ['APIView', 'GenericAPIView', 'ListAPIView', 'SimpleListAPIView', 'ListCreateAPIView', 'SubListAPIView', 'SubListCreateAPIView', @@ -47,7 +52,8 @@ __all__ = ['APIView', 'GenericAPIView', 'ListAPIView', 'SimpleListAPIView', 'ResourceAccessList', 'ParentMixin', 'DeleteLastUnattachLabelMixin', - 'SubListAttachDetachAPIView',] + 'SubListAttachDetachAPIView', + 'CopyAPIView'] logger = logging.getLogger('awx.api.generics') analytics_logger = logging.getLogger('awx.analytics.performance') @@ -747,3 +753,152 @@ class ResourceAccessList(ParentMixin, ListAPIView): for r in roles: ancestors.update(set(r.ancestors.all())) return User.objects.filter(roles__in=list(ancestors)).distinct() + + +def trigger_delayed_deep_copy(*args, **kwargs): + from awx.main.tasks import deep_copy_model_obj + connection.on_commit(lambda: deep_copy_model_obj.delay(*args, **kwargs)) + + +class CopyAPIView(GenericAPIView): + + serializer_class = CopySerializer + permission_classes = (AllowAny,) + copy_return_serializer_class = None + new_in_330 = True + new_in_api_v2 = True + + def _get_copy_return_serializer(self, *args, **kwargs): + if not self.copy_return_serializer_class: + return self.get_serializer(*args, **kwargs) + serializer_class_store = self.serializer_class + self.serializer_class = self.copy_return_serializer_class + ret = self.get_serializer(*args, **kwargs) + self.serializer_class = serializer_class_store + return ret + + @staticmethod + def _decrypt_model_field_if_needed(obj, field_name, field_val): + if field_name in getattr(type(obj), 'REENCRYPTION_BLACKLIST_AT_COPY', []): + return field_val + if isinstance(field_val, dict): + for sub_field in field_val: + if isinstance(sub_field, six.string_types) \ + and isinstance(field_val[sub_field], six.string_types): + try: + field_val[sub_field] = decrypt_field(obj, field_name, sub_field) + except InvalidToken: + # Catching the corner case with v1 credential fields + field_val[sub_field] = decrypt_field(obj, sub_field) + elif isinstance(field_val, six.string_types): + field_val = decrypt_field(obj, field_name) + return field_val + + def _build_create_dict(self, obj): + ret = {} + if self.copy_return_serializer_class: + all_fields = Metadata().get_serializer_info( + self._get_copy_return_serializer(), method='POST' + ) + for field_name, field_info in all_fields.items(): + if not hasattr(obj, field_name) or field_info.get('read_only', True): + continue + ret[field_name] = CopyAPIView._decrypt_model_field_if_needed( + obj, field_name, getattr(obj, field_name) + ) + return ret + + @staticmethod + def copy_model_obj(old_parent, new_parent, model, obj, creater, copy_name='', create_kwargs=None): + fields_to_preserve = set(getattr(model, 'FIELDS_TO_PRESERVE_AT_COPY', [])) + fields_to_discard = set(getattr(model, 'FIELDS_TO_DISCARD_AT_COPY', [])) + m2m_to_preserve = {} + o2m_to_preserve = {} + create_kwargs = create_kwargs or {} + for field_name in fields_to_discard: + create_kwargs.pop(field_name, None) + for field in model._meta.get_fields(): + try: + field_val = getattr(obj, field.name) + except AttributeError: + continue + # Adjust copy blacklist fields here. + if field.name in fields_to_discard or field.name in [ + 'id', 'pk', 'polymorphic_ctype', 'unifiedjobtemplate_ptr', 'created_by', 'modified_by' + ] or field.name.endswith('_role'): + create_kwargs.pop(field.name, None) + continue + if field.one_to_many: + if field.name in fields_to_preserve: + o2m_to_preserve[field.name] = field_val + elif field.many_to_many: + if field.name in fields_to_preserve and not old_parent: + m2m_to_preserve[field.name] = field_val + elif field.many_to_one and not field_val: + create_kwargs.pop(field.name, None) + elif field.many_to_one and field_val == old_parent: + create_kwargs[field.name] = new_parent + elif field.name == 'name' and not old_parent: + create_kwargs[field.name] = copy_name or field_val + ' copy' + elif field.name in fields_to_preserve: + create_kwargs[field.name] = CopyAPIView._decrypt_model_field_if_needed( + obj, field.name, field_val + ) + new_obj = model.objects.create(**create_kwargs) + # Need to save separatedly because Djang-crum get_current_user would + # not work properly in non-request-response-cycle context. + new_obj.created_by = creater + new_obj.save() + for m2m in m2m_to_preserve: + for related_obj in m2m_to_preserve[m2m].all(): + getattr(new_obj, m2m).add(related_obj) + if not old_parent: + sub_objects = [] + for o2m in o2m_to_preserve: + for sub_obj in o2m_to_preserve[o2m].all(): + sub_model = type(sub_obj) + sub_objects.append((sub_model.__module__, sub_model.__name__, sub_obj.pk)) + return new_obj, sub_objects + ret = {obj: new_obj} + for o2m in o2m_to_preserve: + for sub_obj in o2m_to_preserve[o2m].all(): + ret.update(CopyAPIView.copy_model_obj(obj, new_obj, type(sub_obj), sub_obj, creater)) + return ret + + def get(self, request, *args, **kwargs): + obj = self.get_object() + create_kwargs = self._build_create_dict(obj) + for key in create_kwargs: + create_kwargs[key] = getattr(create_kwargs[key], 'pk', None) or create_kwargs[key] + return Response({'can_copy': request.user.can_access(self.model, 'add', create_kwargs)}) + + def post(self, request, *args, **kwargs): + obj = self.get_object() + create_kwargs = self._build_create_dict(obj) + create_kwargs_check = {} + for key in create_kwargs: + create_kwargs_check[key] = getattr(create_kwargs[key], 'pk', None) or create_kwargs[key] + if not request.user.can_access(self.model, 'add', create_kwargs_check): + raise PermissionDenied() + serializer = self.get_serializer(data=request.data) + if not serializer.is_valid(): + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + new_obj, sub_objs = CopyAPIView.copy_model_obj( + None, None, self.model, obj, request.user, create_kwargs=create_kwargs, + copy_name=serializer.validated_data.get('name', '') + ) + if hasattr(new_obj, 'admin_role') and request.user not in new_obj.admin_role: + new_obj.admin_role.members.add(request.user) + if sub_objs: + permission_check_func = None + if hasattr(type(self), 'deep_copy_permission_check_func'): + permission_check_func = ( + type(self).__module__, type(self).__name__, 'deep_copy_permission_check_func' + ) + trigger_delayed_deep_copy( + self.model.__module__, self.model.__name__, + obj.pk, new_obj.pk, request.user.pk, sub_objs, + permission_check_func=permission_check_func + ) + serializer = self._get_copy_return_serializer(new_obj) + return Response(serializer.data, status=status.HTTP_201_CREATED) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 3923e5c9ef..5844cbdfed 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -131,6 +131,22 @@ def reverse_gfk(content_object, request): } +class CopySerializer(serializers.Serializer): + + name = serializers.CharField() + + def validate(self, attrs): + name = attrs.get('name') + view = self.context.get('view', None) + obj = view.get_object() + if name == obj.name: + raise serializers.ValidationError(_( + 'The original object is already named {}, a copy from' + ' it cannot have the same name.'.format(name) + )) + return attrs + + class BaseSerializerMetaclass(serializers.SerializerMetaclass): ''' Custom metaclass to enable attribute inheritance from Meta objects on @@ -1022,6 +1038,7 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer): notification_templates_error = self.reverse('api:project_notification_templates_error_list', kwargs={'pk': obj.pk}), access_list = self.reverse('api:project_access_list', kwargs={'pk': obj.pk}), object_roles = self.reverse('api:project_object_roles_list', kwargs={'pk': obj.pk}), + copy = self.reverse('api:project_copy', kwargs={'pk': obj.pk}), )) if obj.organization: res['organization'] = self.reverse('api:organization_detail', @@ -1174,6 +1191,7 @@ class InventorySerializer(BaseSerializerWithVariables): access_list = self.reverse('api:inventory_access_list', kwargs={'pk': obj.pk}), object_roles = self.reverse('api:inventory_object_roles_list', kwargs={'pk': obj.pk}), instance_groups = self.reverse('api:inventory_instance_groups_list', kwargs={'pk': obj.pk}), + copy = self.reverse('api:inventory_copy', kwargs={'pk': obj.pk}), )) if obj.insights_credential: res['insights_credential'] = self.reverse('api:credential_detail', kwargs={'pk': obj.insights_credential.pk}) @@ -1530,6 +1548,7 @@ class CustomInventoryScriptSerializer(BaseSerializer): res = super(CustomInventoryScriptSerializer, self).get_related(obj) res.update(dict( object_roles = self.reverse('api:inventory_script_object_roles_list', kwargs={'pk': obj.pk}), + copy = self.reverse('api:inventory_script_copy', kwargs={'pk': obj.pk}), )) if obj.organization: @@ -2086,6 +2105,7 @@ class CredentialSerializer(BaseSerializer): object_roles = self.reverse('api:credential_object_roles_list', kwargs={'pk': obj.pk}), owner_users = self.reverse('api:credential_owner_users_list', kwargs={'pk': obj.pk}), owner_teams = self.reverse('api:credential_owner_teams_list', kwargs={'pk': obj.pk}), + copy = self.reverse('api:credential_copy', kwargs={'pk': obj.pk}), )) # TODO: remove when API v1 is removed @@ -2563,6 +2583,7 @@ class JobTemplateSerializer(JobTemplateMixin, UnifiedJobTemplateSerializer, JobO labels = self.reverse('api:job_template_label_list', kwargs={'pk': obj.pk}), object_roles = self.reverse('api:job_template_object_roles_list', kwargs={'pk': obj.pk}), instance_groups = self.reverse('api:job_template_instance_groups_list', kwargs={'pk': obj.pk}), + copy = self.reverse('api:job_template_copy', kwargs={'pk': obj.pk}), )) if obj.host_config_key: res['callback'] = self.reverse('api:job_template_callback', kwargs={'pk': obj.pk}) @@ -3686,6 +3707,7 @@ class NotificationTemplateSerializer(BaseSerializer): res.update(dict( test = self.reverse('api:notification_template_test', kwargs={'pk': obj.pk}), notifications = self.reverse('api:notification_template_notification_list', kwargs={'pk': obj.pk}), + copy = self.reverse('api:notification_template_copy', kwargs={'pk': obj.pk}), )) if obj.organization: res['organization'] = self.reverse('api:organization_detail', kwargs={'pk': obj.organization.pk}) diff --git a/awx/api/urls/credential.py b/awx/api/urls/credential.py index b8480ab4e8..c444da9090 100644 --- a/awx/api/urls/credential.py +++ b/awx/api/urls/credential.py @@ -11,6 +11,7 @@ from awx.api.views import ( CredentialObjectRolesList, CredentialOwnerUsersList, CredentialOwnerTeamsList, + CredentialCopy, ) @@ -22,6 +23,7 @@ urls = [ url(r'^(?P[0-9]+)/object_roles/$', CredentialObjectRolesList.as_view(), name='credential_object_roles_list'), url(r'^(?P[0-9]+)/owner_users/$', CredentialOwnerUsersList.as_view(), name='credential_owner_users_list'), url(r'^(?P[0-9]+)/owner_teams/$', CredentialOwnerTeamsList.as_view(), name='credential_owner_teams_list'), + url(r'^(?P[0-9]+)/copy/$', CredentialCopy.as_view(), name='credential_copy'), ] __all__ = ['urls'] diff --git a/awx/api/urls/inventory.py b/awx/api/urls/inventory.py index 0d8e2ca8d5..c2f67ab457 100644 --- a/awx/api/urls/inventory.py +++ b/awx/api/urls/inventory.py @@ -20,6 +20,7 @@ from awx.api.views import ( InventoryAccessList, InventoryObjectRolesList, InventoryInstanceGroupsList, + InventoryCopy, ) @@ -40,6 +41,7 @@ urls = [ url(r'^(?P[0-9]+)/access_list/$', InventoryAccessList.as_view(), name='inventory_access_list'), url(r'^(?P[0-9]+)/object_roles/$', InventoryObjectRolesList.as_view(), name='inventory_object_roles_list'), url(r'^(?P[0-9]+)/instance_groups/$', InventoryInstanceGroupsList.as_view(), name='inventory_instance_groups_list'), + url(r'^(?P[0-9]+)/copy/$', InventoryCopy.as_view(), name='inventory_copy'), ] __all__ = ['urls'] diff --git a/awx/api/urls/inventory_script.py b/awx/api/urls/inventory_script.py index 088ccf21ca..03852e78cb 100644 --- a/awx/api/urls/inventory_script.py +++ b/awx/api/urls/inventory_script.py @@ -7,6 +7,7 @@ from awx.api.views import ( InventoryScriptList, InventoryScriptDetail, InventoryScriptObjectRolesList, + InventoryScriptCopy, ) @@ -14,6 +15,7 @@ urls = [ url(r'^$', InventoryScriptList.as_view(), name='inventory_script_list'), url(r'^(?P[0-9]+)/$', InventoryScriptDetail.as_view(), name='inventory_script_detail'), url(r'^(?P[0-9]+)/object_roles/$', InventoryScriptObjectRolesList.as_view(), name='inventory_script_object_roles_list'), + url(r'^(?P[0-9]+)/copy/$', InventoryScriptCopy.as_view(), name='inventory_script_copy'), ] __all__ = ['urls'] diff --git a/awx/api/urls/job_template.py b/awx/api/urls/job_template.py index 32b11444be..b11dbf4fea 100644 --- a/awx/api/urls/job_template.py +++ b/awx/api/urls/job_template.py @@ -19,6 +19,7 @@ from awx.api.views import ( JobTemplateAccessList, JobTemplateObjectRolesList, JobTemplateLabelList, + JobTemplateCopy, ) @@ -41,6 +42,7 @@ urls = [ url(r'^(?P[0-9]+)/access_list/$', JobTemplateAccessList.as_view(), name='job_template_access_list'), url(r'^(?P[0-9]+)/object_roles/$', JobTemplateObjectRolesList.as_view(), name='job_template_object_roles_list'), url(r'^(?P[0-9]+)/labels/$', JobTemplateLabelList.as_view(), name='job_template_label_list'), + url(r'^(?P[0-9]+)/copy/$', JobTemplateCopy.as_view(), name='job_template_copy'), ] __all__ = ['urls'] diff --git a/awx/api/urls/notification_template.py b/awx/api/urls/notification_template.py index eba6be5ef3..8473878922 100644 --- a/awx/api/urls/notification_template.py +++ b/awx/api/urls/notification_template.py @@ -8,6 +8,7 @@ from awx.api.views import ( NotificationTemplateDetail, NotificationTemplateTest, NotificationTemplateNotificationList, + NotificationTemplateCopy, ) @@ -16,6 +17,7 @@ urls = [ url(r'^(?P[0-9]+)/$', NotificationTemplateDetail.as_view(), name='notification_template_detail'), url(r'^(?P[0-9]+)/test/$', NotificationTemplateTest.as_view(), name='notification_template_test'), url(r'^(?P[0-9]+)/notifications/$', NotificationTemplateNotificationList.as_view(), name='notification_template_notification_list'), + url(r'^(?P[0-9]+)/copy/$', NotificationTemplateCopy.as_view(), name='notification_template_copy'), ] __all__ = ['urls'] diff --git a/awx/api/urls/project.py b/awx/api/urls/project.py index 629ec1ce05..263014e6e2 100644 --- a/awx/api/urls/project.py +++ b/awx/api/urls/project.py @@ -19,10 +19,11 @@ from awx.api.views import ( ProjectNotificationTemplatesSuccessList, ProjectObjectRolesList, ProjectAccessList, + ProjectCopy, ) -urls = [ +urls = [ url(r'^$', ProjectList.as_view(), name='project_list'), url(r'^(?P[0-9]+)/$', ProjectDetail.as_view(), name='project_detail'), url(r'^(?P[0-9]+)/playbooks/$', ProjectPlaybooks.as_view(), name='project_playbooks'), @@ -39,6 +40,7 @@ urls = [ name='project_notification_templates_success_list'), url(r'^(?P[0-9]+)/object_roles/$', ProjectObjectRolesList.as_view(), name='project_object_roles_list'), url(r'^(?P[0-9]+)/access_list/$', ProjectAccessList.as_view(), name='project_access_list'), + url(r'^(?P[0-9]+)/copy/$', ProjectCopy.as_view(), name='project_copy'), ] __all__ = ['urls'] diff --git a/awx/api/views.py b/awx/api/views.py index a6067d3268..a8dec129ca 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -1424,6 +1424,12 @@ class ProjectObjectRolesList(SubListAPIView): return Role.objects.filter(content_type=content_type, object_id=po.pk) +class ProjectCopy(CopyAPIView): + + model = Project + copy_return_serializer_class = ProjectSerializer + + class UserList(ListCreateAPIView): model = User @@ -1805,6 +1811,12 @@ class CredentialObjectRolesList(SubListAPIView): return Role.objects.filter(content_type=content_type, object_id=po.pk) +class CredentialCopy(CopyAPIView): + + model = Credential + copy_return_serializer_class = CredentialSerializer + + class InventoryScriptList(ListCreateAPIView): model = CustomInventoryScript @@ -1842,6 +1854,12 @@ class InventoryScriptObjectRolesList(SubListAPIView): return Role.objects.filter(content_type=content_type, object_id=po.pk) +class InventoryScriptCopy(CopyAPIView): + + model = CustomInventoryScript + copy_return_serializer_class = CustomInventoryScriptSerializer + + class InventoryList(ListCreateAPIView): model = Inventory @@ -1969,6 +1987,12 @@ class InventoryJobTemplateList(SubListAPIView): return qs.filter(inventory=parent) +class InventoryCopy(CopyAPIView): + + model = Inventory + copy_return_serializer_class = InventorySerializer + + class HostRelatedSearchMixin(object): @property @@ -3337,6 +3361,12 @@ class JobTemplateObjectRolesList(SubListAPIView): return Role.objects.filter(content_type=content_type, object_id=po.pk) +class JobTemplateCopy(CopyAPIView): + + model = JobTemplate + copy_return_serializer_class = JobTemplateSerializer + + class WorkflowJobNodeList(WorkflowsEnforcementMixin, ListAPIView): model = WorkflowJobNode @@ -3500,10 +3530,10 @@ class WorkflowJobTemplateDetail(WorkflowsEnforcementMixin, RetrieveUpdateDestroy new_in_310 = True -class WorkflowJobTemplateCopy(WorkflowsEnforcementMixin, GenericAPIView): +class WorkflowJobTemplateCopy(WorkflowsEnforcementMixin, CopyAPIView): model = WorkflowJobTemplate - serializer_class = EmptySerializer + copy_return_serializer_class = WorkflowJobTemplateSerializer new_in_310 = True def get(self, request, *args, **kwargs): @@ -3520,17 +3550,20 @@ class WorkflowJobTemplateCopy(WorkflowsEnforcementMixin, GenericAPIView): data.update(messages) return Response(data) - def post(self, request, *args, **kwargs): - obj = self.get_object() - if not request.user.can_access(self.model, 'copy', obj): - raise PermissionDenied() - new_obj = obj.user_copy(request.user) - if request.user not in new_obj.admin_role: - new_obj.admin_role.members.add(request.user) - data = OrderedDict() - data.update(WorkflowJobTemplateSerializer( - new_obj, context=self.get_serializer_context()).to_representation(new_obj)) - return Response(data, status=status.HTTP_201_CREATED) + @staticmethod + def deep_copy_permission_check_func(user, new_objs): + for obj in new_objs: + for field_name in obj._get_workflow_job_field_names(): + item = getattr(obj, field_name, None) + if item is None: + continue + if field_name in ['inventory']: + if not user.can_access(item.__class__, 'use', item): + setattr(obj, field_name, None) + if field_name in ['unified_job_template']: + if not user.can_access(item.__class__, 'start', item, validate_license=False): + setattr(obj, field_name, None) + obj.save() class WorkflowJobTemplateLabelList(WorkflowsEnforcementMixin, JobTemplateLabelList): @@ -4704,6 +4737,12 @@ class NotificationTemplateNotificationList(SubListAPIView): new_in_300 = True +class NotificationTemplateCopy(CopyAPIView): + + model = NotificationTemplate + copy_return_serializer_class = NotificationTemplateSerializer + + class NotificationList(ListAPIView): model = Notification diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 5f136ee5b2..e1c10de3a7 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -48,6 +48,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin): an inventory source contains lists and hosts. ''' + FIELDS_TO_PRESERVE_AT_COPY = ['hosts', 'groups', 'instance_groups'] KIND_CHOICES = [ ('', _('Hosts have a direct link to this inventory.')), ('smart', _('Hosts for inventory generated using the host_filter property.')), @@ -503,6 +504,10 @@ class Host(CommonModelNameNotUnique): A managed node ''' + FIELDS_TO_PRESERVE_AT_COPY = [ + 'name', 'description', 'groups', 'inventory', 'enabled', 'instance_id', 'variables' + ] + class Meta: app_label = 'main' unique_together = (("name", "inventory"),) # FIXME: Add ('instance_id', 'inventory') after migration. @@ -690,6 +695,10 @@ class Group(CommonModelNameNotUnique): groups. ''' + FIELDS_TO_PRESERVE_AT_COPY = [ + 'name', 'description', 'inventory', 'children', 'parents', 'hosts', 'variables' + ] + class Meta: app_label = 'main' unique_together = (("name", "inventory"),) diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 2bb2f53ca2..baded4fba0 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -227,6 +227,10 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour A job template is a reusable job definition for applying a project (with playbook) to an inventory source with a given credential. ''' + FIELDS_TO_PRESERVE_AT_COPY = [ + 'labels', 'instance_groups', 'credentials', 'survey_spec' + ] + FIELDS_TO_DISCARD_AT_COPY = ['vault_credential', 'credential'] SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name')] class Meta: diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index a8578b97bd..9390f6d042 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -228,6 +228,8 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin): ''' SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')] + FIELDS_TO_PRESERVE_AT_COPY = ['labels', 'instance_groups', 'credentials'] + FIELDS_TO_DISCARD_AT_COPY = ['local_path'] class Meta: app_label = 'main' diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index 78978f998c..ade3544dc2 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -110,6 +110,13 @@ class WorkflowNodeBase(CreatedModifiedModel, LaunchTimeConfig): class WorkflowJobTemplateNode(WorkflowNodeBase): + FIELDS_TO_PRESERVE_AT_COPY = [ + 'unified_job_template', 'workflow_job_template', 'success_nodes', 'failure_nodes', + 'always_nodes', 'credentials', 'inventory', 'extra_data', 'survey_passwords', + 'char_prompts' + ] + REENCRYPTION_BLACKLIST_AT_COPY = ['extra_data', 'survey_passwords'] + workflow_job_template = models.ForeignKey( 'WorkflowJobTemplate', related_name='workflow_job_template_nodes', @@ -283,6 +290,9 @@ class WorkflowJobOptions(BaseModel): class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTemplateMixin, ResourceMixin): SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name', 'organization')] + FIELDS_TO_PRESERVE_AT_COPY = [ + 'labels', 'instance_groups', 'workflow_job_template_nodes', 'credentials', 'survey_spec' + ] class Meta: app_label = 'main' @@ -393,11 +403,6 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl node_list.append(node.pk) return node_list - def user_copy(self, user): - new_wfjt = self.copy_unified_jt() - new_wfjt.copy_nodes_from_original(original=self, user=user) - return new_wfjt - class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificationMixin): class Meta: diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 6375a532dd..afaae03cb9 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -7,6 +7,7 @@ from collections import OrderedDict import ConfigParser import cStringIO import functools +import importlib import json import logging import os @@ -33,6 +34,7 @@ from celery.signals import celeryd_init, worker_process_init, worker_shutdown # Django from django.conf import settings from django.db import transaction, DatabaseError, IntegrityError +from django.db.models.fields.related import ForeignKey from django.utils.timezone import now, timedelta from django.utils.encoding import smart_str from django.core.mail import send_mail @@ -2279,6 +2281,62 @@ class RunSystemJob(BaseTask): return settings.BASE_DIR +def _reconstruct_relationships(copy_mapping): + for old_obj, new_obj in copy_mapping.items(): + model = type(old_obj) + for field_name in getattr(model, 'FIELDS_TO_PRESERVE_AT_COPY', []): + field = model._meta.get_field(field_name) + if isinstance(field, ForeignKey): + if getattr(new_obj, field_name, None): + continue + related_obj = getattr(old_obj, field_name) + related_obj = copy_mapping.get(related_obj, related_obj) + setattr(new_obj, field_name, related_obj) + elif field.many_to_many: + for related_obj in getattr(old_obj, field_name).all(): + getattr(new_obj, field_name).add(copy_mapping.get(related_obj, related_obj)) + new_obj.save() + + +@shared_task(bind=True, queue='tower', base=LogErrorsTask) +def deep_copy_model_obj( + self, model_module, model_name, obj_pk, new_obj_pk, + user_pk, sub_obj_list, permission_check_func=None +): + logger.info('Deep copy {} from {} to {}.'.format(model_name, obj_pk, new_obj_pk)) + from awx.api.generics import CopyAPIView + model = getattr(importlib.import_module(model_module), model_name, None) + if model is None: + return + try: + obj = model.objects.get(pk=obj_pk) + new_obj = model.objects.get(pk=new_obj_pk) + creater = User.objects.get(pk=user_pk) + except ObjectDoesNotExist: + logger.warning("Object or user no longer exists.") + return + with transaction.atomic(): + copy_mapping = {} + for sub_obj_setup in sub_obj_list: + sub_model = getattr(importlib.import_module(sub_obj_setup[0]), + sub_obj_setup[1], None) + if sub_model is None: + continue + try: + sub_obj = sub_model.objects.get(pk=sub_obj_setup[2]) + except ObjectDoesNotExist: + continue + copy_mapping.update(CopyAPIView.copy_model_obj( + obj, new_obj, sub_model, sub_obj, creater + )) + _reconstruct_relationships(copy_mapping) + if permission_check_func: + permission_check_func = getattr(getattr( + importlib.import_module(permission_check_func[0]), permission_check_func[1] + ), permission_check_func[2]) + permission_check_func(creater, copy_mapping.values()) + + celery_app.register_task(RunJob()) celery_app.register_task(RunProjectUpdate()) celery_app.register_task(RunInventoryUpdate()) diff --git a/awx/main/tests/functional/conftest.py b/awx/main/tests/functional/conftest.py index e0d793d190..4eb4bcca2f 100644 --- a/awx/main/tests/functional/conftest.py +++ b/awx/main/tests/functional/conftest.py @@ -33,7 +33,8 @@ from awx.main.models.inventory import ( Group, Inventory, InventoryUpdate, - InventorySource + InventorySource, + CustomInventoryScript ) from awx.main.models.organization import ( Organization, @@ -489,6 +490,13 @@ def inventory_update(inventory_source): return InventoryUpdate.objects.create(inventory_source=inventory_source) +@pytest.fixture +def inventory_script(organization): + return CustomInventoryScript.objects.create(name='test inv script', + organization=organization, + script='#!/usr/bin/python') + + @pytest.fixture def host(group, inventory): return group.hosts.create(name='single-host', inventory=inventory) diff --git a/awx/main/tests/functional/models/test_workflow.py b/awx/main/tests/functional/models/test_workflow.py index af6724d8b7..61882f2097 100644 --- a/awx/main/tests/functional/models/test_workflow.py +++ b/awx/main/tests/functional/models/test_workflow.py @@ -191,25 +191,6 @@ class TestWorkflowJobTemplate: assert (test_view.is_valid_relation(nodes[2], node_assoc_1) == {'Error': 'Cannot associate failure_nodes when always_nodes have been associated.'}) - def test_wfjt_copy(self, wfjt, job_template, inventory, admin_user): - old_nodes = wfjt.workflow_job_template_nodes.all() - node1 = old_nodes[1] - node1.unified_job_template = job_template - node1.save() - node2 = old_nodes[2] - node2.inventory = inventory - node2.save() - new_wfjt = wfjt.user_copy(admin_user) - for fd in ['description', 'survey_spec', 'survey_enabled', 'extra_vars']: - assert getattr(wfjt, fd) == getattr(new_wfjt, fd) - assert new_wfjt.organization == wfjt.organization - assert len(new_wfjt.workflow_job_template_nodes.all()) == 3 - nodes = new_wfjt.workflow_job_template_nodes.all() - assert nodes[0].success_nodes.all()[0] == nodes[1] - assert nodes[1].failure_nodes.all()[0] == nodes[2] - assert nodes[1].unified_job_template == job_template - assert nodes[2].inventory == inventory - def test_wfjt_unique_together_with_org(self, organization): wfjt1 = WorkflowJobTemplate(name='foo', organization=organization) wfjt1.save() diff --git a/awx/main/tests/functional/test_copy.py b/awx/main/tests/functional/test_copy.py new file mode 100644 index 0000000000..99e123a8fa --- /dev/null +++ b/awx/main/tests/functional/test_copy.py @@ -0,0 +1,214 @@ +import pytest +import mock + +from awx.api.versioning import reverse +from awx.main.utils import decrypt_field +from awx.main.models.workflow import WorkflowJobTemplateNode +from awx.main.models.jobs import JobTemplate +from awx.main.tasks import deep_copy_model_obj + + +@pytest.mark.django_db +def test_job_template_copy(post, get, project, inventory, machine_credential, vault_credential, + credential, alice, job_template_with_survey_passwords, admin): + job_template_with_survey_passwords.project = project + job_template_with_survey_passwords.inventory = inventory + job_template_with_survey_passwords.save() + job_template_with_survey_passwords.credentials.add(credential) + job_template_with_survey_passwords.credentials.add(machine_credential) + job_template_with_survey_passwords.credentials.add(vault_credential) + job_template_with_survey_passwords.admin_role.members.add(alice) + assert get( + reverse('api:job_template_copy', kwargs={'pk': job_template_with_survey_passwords.pk}), + alice, expect=200 + ).data['can_copy'] is False + assert get( + reverse('api:job_template_copy', kwargs={'pk': job_template_with_survey_passwords.pk}), + admin, expect=200 + ).data['can_copy'] is True + jt_copy_pk = post( + reverse('api:job_template_copy', kwargs={'pk': job_template_with_survey_passwords.pk}), + {'name': 'new jt name'}, admin, expect=201 + ).data['id'] + jt_copy = type(job_template_with_survey_passwords).objects.get(pk=jt_copy_pk) + assert jt_copy.created_by == admin + assert jt_copy.name == 'new jt name' + assert jt_copy.project == project + assert jt_copy.inventory == inventory + assert jt_copy.playbook == job_template_with_survey_passwords.playbook + assert jt_copy.credentials.count() == 3 + assert credential in jt_copy.credentials.all() + assert vault_credential in jt_copy.credentials.all() + assert machine_credential in jt_copy.credentials.all() + assert job_template_with_survey_passwords.survey_spec == jt_copy.survey_spec + + +@pytest.mark.django_db +def test_project_copy(post, get, project, organization, scm_credential, alice): + project.credential = scm_credential + project.save() + project.admin_role.members.add(alice) + assert get( + reverse('api:project_copy', kwargs={'pk': project.pk}), alice, expect=200 + ).data['can_copy'] is False + project.organization.admin_role.members.add(alice) + assert get( + reverse('api:project_copy', kwargs={'pk': project.pk}), alice, expect=200 + ).data['can_copy'] is True + project_copy_pk = post( + reverse('api:project_copy', kwargs={'pk': project.pk}), + {'name': 'copied project'}, alice, expect=201 + ).data['id'] + project_copy = type(project).objects.get(pk=project_copy_pk) + assert project_copy.created_by == alice + assert project_copy.name == 'copied project' + assert project_copy.organization == organization + assert project_copy.credential == scm_credential + + +@pytest.mark.django_db +def test_inventory_copy(inventory, group_factory, post, get, alice, organization): + group_1_1 = group_factory('g_1_1') + group_2_1 = group_factory('g_2_1') + group_2_2 = group_factory('g_2_2') + group_2_1.parents.add(group_1_1) + group_2_2.parents.add(group_1_1) + group_2_2.parents.add(group_2_1) + host = group_1_1.hosts.create(name='host', inventory=inventory) + group_2_1.hosts.add(host) + inventory.admin_role.members.add(alice) + assert get( + reverse('api:inventory_copy', kwargs={'pk': inventory.pk}), alice, expect=200 + ).data['can_copy'] is False + inventory.organization.admin_role.members.add(alice) + assert get( + reverse('api:inventory_copy', kwargs={'pk': inventory.pk}), alice, expect=200 + ).data['can_copy'] is True + with mock.patch('awx.api.generics.trigger_delayed_deep_copy') as deep_copy_mock: + inv_copy_pk = post( + reverse('api:inventory_copy', kwargs={'pk': inventory.pk}), + {'name': 'new inv name'}, alice, expect=201 + ).data['id'] + inventory_copy = type(inventory).objects.get(pk=inv_copy_pk) + args, kwargs = deep_copy_mock.call_args + deep_copy_model_obj(*args, **kwargs) + group_1_1_copy = inventory_copy.groups.get(name='g_1_1') + group_2_1_copy = inventory_copy.groups.get(name='g_2_1') + group_2_2_copy = inventory_copy.groups.get(name='g_2_2') + host_copy = inventory_copy.hosts.get(name='host') + assert inventory_copy.organization == organization + assert inventory_copy.created_by == alice + assert inventory_copy.name == 'new inv name' + assert set(group_1_1_copy.parents.all()) == set() + assert set(group_2_1_copy.parents.all()) == set([group_1_1_copy]) + assert set(group_2_2_copy.parents.all()) == set([group_1_1_copy, group_2_1_copy]) + assert set(group_1_1_copy.hosts.all()) == set([host_copy]) + assert set(group_2_1_copy.hosts.all()) == set([host_copy]) + assert set(group_2_2_copy.hosts.all()) == set() + + +@pytest.mark.django_db +def test_workflow_job_template_copy(workflow_job_template, post, get, admin, organization): + workflow_job_template.organization = organization + workflow_job_template.save() + jts = [JobTemplate.objects.create(name='test-jt-{}'.format(i)) for i in range(0, 5)] + nodes = [ + WorkflowJobTemplateNode.objects.create( + workflow_job_template=workflow_job_template, unified_job_template=jts[i] + ) for i in range(0, 5) + ] + nodes[0].success_nodes.add(nodes[1]) + nodes[1].success_nodes.add(nodes[2]) + nodes[0].failure_nodes.add(nodes[3]) + nodes[3].failure_nodes.add(nodes[4]) + with mock.patch('awx.api.generics.trigger_delayed_deep_copy') as deep_copy_mock: + wfjt_copy_id = post( + reverse('api:workflow_job_template_copy', kwargs={'pk': workflow_job_template.pk}), + {'name': 'new wfjt name'}, admin, expect=201 + ).data['id'] + wfjt_copy = type(workflow_job_template).objects.get(pk=wfjt_copy_id) + args, kwargs = deep_copy_mock.call_args + deep_copy_model_obj(*args, **kwargs) + assert wfjt_copy.organization == organization + assert wfjt_copy.created_by == admin + assert wfjt_copy.name == 'new wfjt name' + copied_node_list = [x for x in wfjt_copy.workflow_job_template_nodes.all()] + copied_node_list.sort(key=lambda x: int(x.unified_job_template.name[-1])) + for node, success_count, failure_count, always_count in zip( + copied_node_list, + [1, 1, 0, 0, 0], + [1, 0, 0, 1, 0], + [0, 0, 0, 0, 0] + ): + assert node.success_nodes.count() == success_count + assert node.failure_nodes.count() == failure_count + assert node.always_nodes.count() == always_count + assert copied_node_list[1] in copied_node_list[0].success_nodes.all() + assert copied_node_list[2] in copied_node_list[1].success_nodes.all() + assert copied_node_list[3] in copied_node_list[0].failure_nodes.all() + assert copied_node_list[4] in copied_node_list[3].failure_nodes.all() + + +@pytest.mark.django_db +def test_credential_copy(post, get, machine_credential, credentialtype_ssh, admin): + assert get( + reverse('api:credential_copy', kwargs={'pk': machine_credential.pk}), admin, expect=200 + ).data['can_copy'] is True + credential_copy_pk = post( + reverse('api:credential_copy', kwargs={'pk': machine_credential.pk}), + {'name': 'copied credential'}, admin, expect=201 + ).data['id'] + credential_copy = type(machine_credential).objects.get(pk=credential_copy_pk) + assert credential_copy.created_by == admin + assert credential_copy.name == 'copied credential' + assert credential_copy.credential_type == credentialtype_ssh + assert credential_copy.inputs['username'] == machine_credential.inputs['username'] + assert (decrypt_field(credential_copy, 'password') == + decrypt_field(machine_credential, 'password')) + + +@pytest.mark.django_db +def test_notification_template_copy(post, get, notification_template_with_encrypt, + organization, alice): + #notification_template_with_encrypt.admin_role.members.add(alice) + assert get( + reverse( + 'api:notification_template_copy', kwargs={'pk': notification_template_with_encrypt.pk} + ), alice, expect=200 + ).data['can_copy'] is False + notification_template_with_encrypt.organization.admin_role.members.add(alice) + assert get( + reverse( + 'api:notification_template_copy', kwargs={'pk': notification_template_with_encrypt.pk} + ), alice, expect=200 + ).data['can_copy'] is True + nt_copy_pk = post( + reverse( + 'api:notification_template_copy', kwargs={'pk': notification_template_with_encrypt.pk} + ), {'name': 'copied nt'}, alice, expect=201 + ).data['id'] + notification_template_copy = type(notification_template_with_encrypt).objects.get(pk=nt_copy_pk) + assert notification_template_copy.created_by == alice + assert notification_template_copy.name == 'copied nt' + assert notification_template_copy.organization == organization + assert (decrypt_field(notification_template_with_encrypt, 'notification_configuration', 'token') == + decrypt_field(notification_template_copy, 'notification_configuration', 'token')) + + +@pytest.mark.django_db +def test_inventory_script_copy(post, get, inventory_script, organization, alice): + assert get( + reverse('api:inventory_script_copy', kwargs={'pk': inventory_script.pk}), alice, expect=200 + ).data['can_copy'] is False + inventory_script.organization.admin_role.members.add(alice) + assert get( + reverse('api:inventory_script_copy', kwargs={'pk': inventory_script.pk}), alice, expect=200 + ).data['can_copy'] is True + is_copy_pk = post( + reverse('api:inventory_script_copy', kwargs={'pk': inventory_script.pk}), + {'name': 'copied inv script'}, alice, expect=201 + ).data['id'] + inventory_script_copy = type(inventory_script).objects.get(pk=is_copy_pk) + assert inventory_script_copy.created_by == alice + assert inventory_script_copy.name == 'copied inv script' + assert inventory_script_copy.organization == organization diff --git a/docs/resource_copy.md b/docs/resource_copy.md new file mode 100644 index 0000000000..da85d55225 --- /dev/null +++ b/docs/resource_copy.md @@ -0,0 +1,166 @@ +Starting from Tower 3.3 and API v2, user are able to copy some existing resource objects to quickly +create new resource objects via POSTing to corresponding `/copy/` endpoint. A new `CopyAPIView` class +is introduced as the base view class for `/copy/` endpoints. It mimics the process of manually fetching +fields from the existing object to create a new object, plus the ability to automatically detect sub +structures of existing objects and make a background task-based deep copy when necessary. + +## Usage +If an AWX resource is copiable, all of its object detail API views will have a related URL field +`"copy"`, which has form `/api////copy/`. GET to this endpoint +will return `can_copy`, which is a boolean indicating whether the current user can execute a copy +operation; POST to this endpoint actually copies the resource object. One field `name` is required +which will later be used as the name of the created copy. Upon success, 201 will be returned, along +with the created copy. + +For some resources like credential, the copy process is not time-consuming, thus the entire copy +process will take place in the request-response cycle, and the created object copy is returned as +POST response. + +For some other resources like inventory, the copy process can take longer, depending on the number +of sub-objects to copy (will explain later). Thus, although the created copy will be returned, the +copy process is not finished yet. All sub-objects (like all hosts and groups of an inventory) will +not be created until after the background copy task is finished in success. + +Currently the available list of copiable resources are: + +- job templates +- projects +- inventories +- workflows +- credentials +- notifications +- inventory scripts + +For most of the resources above, only the object to be copied itself will be copied; For some resources +like inventories, however, sub resources belonging to the resource will also be copied to maintain the +full functionality of the copied new resource. In specific: + +- When an inventory is copied, all its hosts, groups and inventory sources are copied. +- When a workflow job template is copied, all its workflow job template nodes are copied. + +## How to add a copy end-point for a resource +The copy behavior of different resources largely follow the same pattern, therefore a unified way of +enabling copy capability for resources is available for developers: + +Firstly, create a `/copy/` url endpoint for the target resource. + +Secondly, create a view class as handler to `/copy/` endpoint. This view class should be subclassed +from `awx.api.generics.CopyAPIView`. Here is an example: +```python +class JobTemplateCopy(CopyAPIView): + + model = JobTemplate + copy_return_serializer_class = JobTemplateSerializer +``` +Note the above example declares a custom class attribute `copy_return_serializer_class`. This attribute +is used by `CopyAPIView` to render the created copy in POST response, so in most cases the value should +be the same as `serializer_class` of corresponding resource detail view, like here the value is the +`serializer_class` of `JobTemplateDetail`. + +Thirdly, for the underlying model of the resource, Add 2 macros, `FIELDS_TO_PRESERVE_AT_COPY` and +`FIELDS_TO_DISCARD_AT_COPY`, as needed. Here is an example: +```python +class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, ResourceMixin): + ''' + A job template is a reusable job definition for applying a project (with + playbook) to an inventory source with a given credential. + ''' + FIELDS_TO_PRESERVE_AT_COPY = [ + 'labels', 'instance_groups', 'credentials', 'survey_spec' + ] + FIELDS_TO_DISCARD_AT_COPY = ['vault_credential', 'credential'] +``` +When copying a resource object, basically all fields necessary for creating a new resource (fields +composing a valid POST body for creating new resources) are extracted from the original object and +used to create the copy. + +However, sometimes we need more fields to be copied, like `credentials` of a job template, which +cannot be provided during creation. In this case we list such fields in `FIELDS_TO_PRESERVE_AT_COPY` +so that these fields won't be missed. + +On the other hand, sometimes we do not want to include some fields provided in create POST body, +like `vault_credential` and `credential` fields used for creating a job template, which do not have +tangible field correspondence in `JobTemplate` model. In this case we list such fields in +`FIELDS_TO_DISCARD_AT_COPY` so that those fields won't be included. + +For models that will be part of a deep copy, like hosts and workflow job template nodes, the related +POST body for creating a new object is not available. Therefore all necessary fields for creating +a new resource should also be included in `FIELDS_TO_PRESERVE_AT_COPY`. + +Lastly, unit test copy behavior of the new endpoint in `/awx/main/tests/functional/test_copy.py` and +update docs (like this doc). + +Fields in `FIELDS_TO_PRESERVE_AT_COPY` must be solid model fields, while fields in +`FIELDS_TO_DISCARD_AT_COPY` do not need to be. Note there are hidden fields not visible from model +definition, namely reverse relationships and fields inherited from super classes or mix-ins. A help +script `tools/scripts/list_fields.py` is available to inspect a model and list details of all its +available fields. +``` +# In shell_plus +>>> from list_fields import pretty_print_model_fields +>>> pretty_print_model_fields(JobTemplate) +``` + +`CopyAPIView` will automatically detect sub objects of an object, and do a deep copy of all sub objects +as a background celery task. There are sometimes permission issues with sub object copy. For example, +when copying nodes of a workflow job template, there are cases where the user performing copy has no use +permission of related credential and inventory of some nodes, and it is desired those fields will be +`None`. In order to do that, developer should provide a static method `deep_copy_permission_check_func` +under corresponding specific copy view. Like +```python +class WorkflowJobTemplateCopy(WorkflowsEnforcementMixin, CopyAPIView): + + model = WorkflowJobTemplate + copy_return_serializer_class = WorkflowJobTemplateSerializer + + # Other code + + @staticmethod + def deep_copy_permission_check_func(user, new_objs): + # method body + + # Other code +``` +Static method `deep_copy_permission_check_func` must have and only have two arguments: `user`, the +user performing the copy; `new_objs`, a list of all sub objects of the created copy. Sub objects in +`new_objs` are initially populated disregarding any permission constraints, developer shall check +`user`'s permission against these new sub objects and react like unlink related objects or sending +warning logs. `deep_copy_permission_check_func` should not return anything. + +Lastly, macro `REENCRYPTION_BLACKLIST_AT_COPY` is available as part of a model definition. It is a +list of field names which will escape re-encryption during copy. For example, `extra_data` field +of workflow job template nodes. + +## Acceptance Criteria +* Credentials should be able to copy themselves. The behavior of copying credential A shall be exactly + the same as creating a credential B with all needed fields for creation coming from credential A. +* Inventories should be able to copy themselves. The behavior of copying inventory A shall be exactly + the same as creating an inventory B with all needed fields for creation coming from inventory A. Other + than that, inventory B should inherit A's `instance_groups`, and have exactly the same host and group + structures as A. +* Inventory scripts should be able to copy themselves. The behavior of copying inventory script A + shall be exactly the same as creating an inventory script B with all needed fields for creation + coming from inventory script A. +* Job templates should be able to copy themselves. The behavior of copying job template A + shall be exactly the same as creating a job template B with all needed fields for creation + coming from job template A. Other than that, job template B should inherit A's `labels`, + `instance_groups`, `credentials` and `survey_spec`. +* Notification templates should be able to copy themselves. The behavior of copying notification + template A shall be exactly the same as creating a notification template B with all needed fields + for creation coming from notification template A. +* Projects should be able to copy themselves. The behavior of copying project A shall be the + same as creating a project B with all needed fields for creation coming from project A, except for + `local_path`, which will be populated by triggered project update. Other than that, project B + should inherit A's `labels`, `instance_groups` and `credentials`. +* Workflow Job templates should be able to copy themselves. The behavior of copying workflow job + template A shall be exactly the same as creating a workflow job template B with all needed fields + for creation coming from workflow job template A. Other than that, workflow job template B should + inherit A's `labels`, `instance_groups`, `credentials` and `survey_spec`, and have exactly the + same workflow job template node structure as A. +* In all copy processes, `name` field of the created copy of the original object should be able to + customize in the POST body. +* The permission for a user to make a copy for an existing resource object should be the same as the + permission for a user to create a brand new resource object using fields from the existing object. +* The RBAC behavior of original workflow job template `/copy/` should be pertained. That is, if the + user has no necessary permission to the related project and credential of a workflow job template + node, the copied workflow job template node should have those fields empty. diff --git a/tools/scripts/list_fields.py b/tools/scripts/list_fields.py new file mode 100755 index 0000000000..1e637e9c1b --- /dev/null +++ b/tools/scripts/list_fields.py @@ -0,0 +1,46 @@ +__all__ = ['pretty_print_model_fields'] + + +def _get_class_full_name(cls_): + return cls_.__module__ + '.' + cls_.__name__ + + +class _ModelFieldRow(object): + + def __init__(self, field): + self.field = field + self.name = field.name + self.type_ = _get_class_full_name(type(field)) + if self.field.many_to_many\ + or self.field.many_to_one\ + or self.field.one_to_many\ + or self.field.one_to_one: + self.related_model = _get_class_full_name(self.field.remote_field.model) + else: + self.related_model = 'N/A' + + def pretty_print(self, max_name_len, max_type_len, max_rel_model_len): + row = [] + row.append(self.name) + row.append(' ' * (max_name_len - len(self.name))) + row.append('|') + row.append(self.type_) + row.append(' ' * (max_type_len - len(self.type_))) + row.append('|') + row.append(self.related_model) + row.append(' ' * (max_rel_model_len - len(self.related_model))) + print(''.join(row)) + + +def pretty_print_model_fields(model): + field_info_rows = [] + max_lens = [0, 0, 0] + for field in model._meta.get_fields(): + field_info_rows.append(_ModelFieldRow(field)) + max_lens[0] = max(max_lens[0], len(field_info_rows[-1].name)) + max_lens[1] = max(max_lens[1], len(field_info_rows[-1].type_)) + max_lens[2] = max(max_lens[2], len(field_info_rows[-1].related_model)) + print('=' * (sum(max_lens) + len(max_lens) - 1)) + for row in field_info_rows: + row.pretty_print(*max_lens) + print('=' * (sum(max_lens) + len(max_lens) - 1))