From 855cb162b7b01e30bf38e1f7bd2be3d4950a3a5f Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 5 Apr 2021 15:12:13 -0400 Subject: [PATCH 01/10] Change inventory source factory defaults Also set source_path to a newly merged test-playbooks script this gives similar behavior to the prior default script --- awxkit/awxkit/api/pages/inventory.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/awxkit/awxkit/api/pages/inventory.py b/awxkit/awxkit/api/pages/inventory.py index 1d0be5d805..4378cc47ce 100644 --- a/awxkit/awxkit/api/pages/inventory.py +++ b/awxkit/awxkit/api/pages/inventory.py @@ -359,7 +359,7 @@ class InventorySource(HasCreate, HasNotifications, UnifiedJobTemplate): optional_dependencies = [Credential, InventoryScript, Project] NATURAL_KEY = ('organization', 'name', 'inventory') - def payload(self, inventory, source='custom', credential=None, source_script=None, project=None, **kwargs): + def payload(self, inventory, source='scm', credential=None, source_script=None, project=None, **kwargs): payload = PseudoNamespace( name=kwargs.get('name') or 'InventorySource - {}'.format(random_title()), description=kwargs.get('description') or random_title(10), @@ -391,12 +391,13 @@ class InventorySource(HasCreate, HasNotifications, UnifiedJobTemplate): return payload def create_payload( - self, name='', description='', source='custom', inventory=Inventory, credential=None, source_script=InventoryScript, project=None, **kwargs + self, name='', description='', source='scm', inventory=Inventory, credential=None, source_script=InventoryScript, project=None, **kwargs ): if source != 'custom' and source_script == InventoryScript: source_script = None if source == 'scm': kwargs.setdefault('overwrite_vars', True) + kwargs.setdefault('source_path', 'inventories/script_migrations/script_source.py') if project is None: project = Project @@ -425,7 +426,7 @@ class InventorySource(HasCreate, HasNotifications, UnifiedJobTemplate): payload.ds = DSAdapter(self.__class__.__name__, self._dependency_store) return payload - def create(self, name='', description='', source='custom', inventory=Inventory, credential=None, source_script=InventoryScript, project=None, **kwargs): + def create(self, name='', description='', source='scm', inventory=Inventory, credential=None, source_script=InventoryScript, project=None, **kwargs): payload = self.create_payload( name=name, description=description, From 38352063e8caadcc0b641d87bd95f350529d51bf Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 12 Apr 2021 13:57:03 -0400 Subject: [PATCH 02/10] Remove custom inventory script API --- awx/api/serializers.py | 47 --------- awx/api/urls/inventory_script.py | 16 --- awx/api/urls/urls.py | 2 - awx/api/views/__init__.py | 4 - awx/api/views/inventory.py | 52 ---------- awx/api/views/root.py | 1 - awx/main/access.py | 30 +----- awx/main/analytics/collectors.py | 97 +++++++++---------- awx/main/migrations/_rbac.py | 1 - awx/main/models/__init__.py | 3 +- awx/main/models/activity_stream.py | 1 + awx/main/models/inventory.py | 3 +- awx/main/signals.py | 1 - .../tests/functional/analytics/test_counts.py | 3 - .../functional/analytics/test_metrics.py | 2 - .../tests/functional/api/test_inventory.py | 8 +- awx/main/tests/functional/conftest.py | 7 +- awx/main/tests/functional/test_copy.py | 13 --- awx/main/tests/functional/test_named_url.py | 14 --- .../tests/functional/test_rbac_inventory.py | 47 --------- .../serializers/test_inventory_serializers.py | 43 +------- awx/main/tests/unit/api/test_filters.py | 2 - awx/main/tests/unit/api/test_views.py | 1 - awx/main/tests/unit/test_tasks.py | 50 ---------- .../plugins/modules/tower_inventory_source.py | 2 - .../plugins/modules/tower_receive.py | 7 -- awx_collection/test/awx/test_completeness.py | 4 +- awxkit/awxkit/api/pages/inventory.py | 60 ------------ awxkit/awxkit/api/resources.py | 3 - awxkit/awxkit/awx/utils.py | 1 - awxkit/awxkit/cli/custom.py | 1 - awxkit/awxkit/cli/resource.py | 1 - 32 files changed, 60 insertions(+), 467 deletions(-) delete mode 100644 awx/api/urls/inventory_script.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 5f45d14937..1fb0c493f9 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -51,7 +51,6 @@ from awx.main.models import ( Credential, CredentialInputSource, CredentialType, - CustomInventoryScript, ExecutionEnvironment, Group, Host, @@ -167,7 +166,6 @@ SUMMARIZABLE_FK_FIELDS = { 'current_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'license_error'), 'current_job': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'license_error'), 'inventory_source': ('id', 'name', 'source', 'last_updated', 'status'), - 'custom_inventory_script': DEFAULT_SUMMARY_FIELDS, 'source_script': DEFAULT_SUMMARY_FIELDS, 'role': ('id', 'role_field'), 'notification_template': DEFAULT_SUMMARY_FIELDS, @@ -1984,49 +1982,6 @@ class GroupVariableDataSerializer(BaseVariableDataSerializer): model = Group -class CustomInventoryScriptSerializer(BaseSerializer): - - script = serializers.CharField(trim_whitespace=False) - show_capabilities = ['edit', 'delete', 'copy'] - capabilities_prefetch = [{'edit': 'admin'}] - - class Meta: - model = CustomInventoryScript - fields = ('*', "script", "organization") - - def validate_script(self, value): - if not value.startswith("#!"): - raise serializers.ValidationError(_('Script must begin with a hashbang sequence: i.e.... #!/usr/bin/env python')) - return value - - def to_representation(self, obj): - ret = super(CustomInventoryScriptSerializer, self).to_representation(obj) - if obj is None: - return ret - request = self.context.get('request', None) - if ( - request.user not in obj.admin_role - and not request.user.is_superuser - and not request.user.is_system_auditor - and not (obj.organization is not None and request.user in obj.organization.auditor_role) - ): - ret['script'] = None - return ret - - def get_related(self, obj): - res = super(CustomInventoryScriptSerializer, self).get_related(obj) - res.update( - dict( - object_roles=self.reverse('api:inventory_script_object_roles_list', kwargs={'pk': obj.pk}), - copy=self.reverse('api:inventory_script_copy', kwargs={'pk': obj.pk}), - ) - ) - - if obj.organization: - res['organization'] = self.reverse('api:organization_detail', kwargs={'pk': obj.organization.pk}) - return res - - class InventorySourceOptionsSerializer(BaseSerializer): credential = DeprecatedCredentialField(help_text=_('Cloud credential to use for inventory updates.')) @@ -2053,8 +2008,6 @@ class InventorySourceOptionsSerializer(BaseSerializer): res = super(InventorySourceOptionsSerializer, self).get_related(obj) if obj.credential: # TODO: remove when 'credential' field is removed res['credential'] = self.reverse('api:credential_detail', kwargs={'pk': obj.credential}) - if obj.source_script: - res['source_script'] = self.reverse('api:inventory_script_detail', kwargs={'pk': obj.source_script.pk}) return res def validate_source_vars(self, value): diff --git a/awx/api/urls/inventory_script.py b/awx/api/urls/inventory_script.py deleted file mode 100644 index a3e72e27f6..0000000000 --- a/awx/api/urls/inventory_script.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2017 Ansible, Inc. -# All Rights Reserved. - -from django.conf.urls import url - -from awx.api.views import InventoryScriptList, InventoryScriptDetail, InventoryScriptObjectRolesList, InventoryScriptCopy - - -urls = [ - url(r'^$', InventoryScriptList.as_view(), name='inventory_script_list'), - url(r'^(?P[0-9]+)/$', InventoryScriptDetail.as_view(), name='inventory_script_detail'), - url(r'^(?P[0-9]+)/object_roles/$', InventoryScriptObjectRolesList.as_view(), name='inventory_script_object_roles_list'), - url(r'^(?P[0-9]+)/copy/$', InventoryScriptCopy.as_view(), name='inventory_script_copy'), -] - -__all__ = ['urls'] diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index 1df8641e5d..9985de6ee1 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -43,7 +43,6 @@ from .host import urls as host_urls from .group import urls as group_urls from .inventory_source import urls as inventory_source_urls from .inventory_update import urls as inventory_update_urls -from .inventory_script import urls as inventory_script_urls from .credential_type import urls as credential_type_urls from .credential import urls as credential_urls from .credential_input_source import urls as credential_input_source_urls @@ -111,7 +110,6 @@ v2_urls = [ url(r'^groups/', include(group_urls)), url(r'^inventory_sources/', include(inventory_source_urls)), url(r'^inventory_updates/', include(inventory_update_urls)), - url(r'^inventory_scripts/', include(inventory_script_urls)), url(r'^credentials/', include(credential_urls)), url(r'^roles/', include(role_urls)), url(r'^job_templates/', include(job_template_urls)), diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index cf64e4114c..789abe11e6 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -152,10 +152,6 @@ from awx.api.views.inventory import ( # noqa InventoryList, InventoryDetail, InventoryUpdateEventsList, - InventoryScriptList, - InventoryScriptDetail, - InventoryScriptObjectRolesList, - InventoryScriptCopy, InventoryList, InventoryDetail, InventoryActivityStreamList, diff --git a/awx/api/views/inventory.py b/awx/api/views/inventory.py index 9a484285cc..f494833927 100644 --- a/awx/api/views/inventory.py +++ b/awx/api/views/inventory.py @@ -25,8 +25,6 @@ from awx.main.models import ( InstanceGroup, InventoryUpdateEvent, InventoryUpdate, - InventorySource, - CustomInventoryScript, ) from awx.api.generics import ListCreateAPIView, RetrieveUpdateDestroyAPIView, SubListAPIView, SubListAttachDetachAPIView, ResourceAccessList, CopyAPIView @@ -36,7 +34,6 @@ from awx.api.serializers import ( RoleSerializer, InstanceGroupSerializer, InventoryUpdateEventSerializer, - CustomInventoryScriptSerializer, JobTemplateSerializer, ) from awx.api.views.mixin import RelatedJobsPreventDeleteMixin, ControlledByScmMixin @@ -58,55 +55,6 @@ class InventoryUpdateEventsList(SubListAPIView): return super(InventoryUpdateEventsList, self).finalize_response(request, response, *args, **kwargs) -class InventoryScriptList(ListCreateAPIView): - - deprecated = True - - model = CustomInventoryScript - serializer_class = CustomInventoryScriptSerializer - - -class InventoryScriptDetail(RetrieveUpdateDestroyAPIView): - - deprecated = True - - model = CustomInventoryScript - serializer_class = CustomInventoryScriptSerializer - - def destroy(self, request, *args, **kwargs): - instance = self.get_object() - can_delete = request.user.can_access(self.model, 'delete', instance) - if not can_delete: - raise PermissionDenied(_("Cannot delete inventory script.")) - for inv_src in InventorySource.objects.filter(source_script=instance): - inv_src.source_script = None - inv_src.save() - return super(InventoryScriptDetail, self).destroy(request, *args, **kwargs) - - -class InventoryScriptObjectRolesList(SubListAPIView): - - deprecated = True - - model = Role - serializer_class = RoleSerializer - parent_model = CustomInventoryScript - search_fields = ('role_field', 'content_type__model') - - def get_queryset(self): - po = self.get_parent_object() - content_type = ContentType.objects.get_for_model(self.parent_model) - return Role.objects.filter(content_type=content_type, object_id=po.pk) - - -class InventoryScriptCopy(CopyAPIView): - - deprecated = True - - model = CustomInventoryScript - copy_return_serializer_class = CustomInventoryScriptSerializer - - class InventoryList(ListCreateAPIView): model = Inventory diff --git a/awx/api/views/root.py b/awx/api/views/root.py index 7a7ea649b1..fdbf56580f 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -100,7 +100,6 @@ class ApiVersionRootView(APIView): data['tokens'] = reverse('api:o_auth2_token_list', request=request) data['metrics'] = reverse('api:metrics_view', request=request) data['inventory'] = reverse('api:inventory_list', request=request) - data['inventory_scripts'] = reverse('api:inventory_script_list', request=request) data['inventory_sources'] = reverse('api:inventory_source_list', request=request) data['inventory_updates'] = reverse('api:inventory_update_list', request=request) data['groups'] = reverse('api:group_list', request=request) diff --git a/awx/main/access.py b/awx/main/access.py index 4d09ee98a1..e6fc29fdc8 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -34,7 +34,6 @@ from awx.main.models import ( Credential, CredentialType, CredentialInputSource, - CustomInventoryScript, ExecutionEnvironment, Group, Host, @@ -465,7 +464,7 @@ class BaseAccess(object): if display_method == 'schedule': user_capabilities['schedule'] = user_capabilities['start'] continue - elif display_method == 'delete' and not isinstance(obj, (User, UnifiedJob, CustomInventoryScript, CredentialInputSource)): + elif display_method == 'delete' and not isinstance(obj, (User, UnifiedJob, CredentialInputSource)): user_capabilities['delete'] = user_capabilities['edit'] continue elif display_method == 'copy' and isinstance(obj, (Group, Host)): @@ -2755,33 +2754,6 @@ class ActivityStreamAccess(BaseAccess): return False -class CustomInventoryScriptAccess(BaseAccess): - - model = CustomInventoryScript - prefetch_related = ('created_by', 'modified_by', 'organization') - - def filtered_queryset(self): - return self.model.accessible_objects(self.user, 'read_role').all() - - @check_superuser - def can_add(self, data): - if not data: # So the browseable API will work - return Organization.accessible_objects(self.user, 'admin_role').exists() - return self.check_related('organization', Organization, data, mandatory=True) - - @check_superuser - def can_admin(self, obj, data=None): - return self.check_related('organization', Organization, data, obj=obj) and self.user in obj.admin_role - - @check_superuser - def can_change(self, obj, data): - return self.can_admin(obj, data=data) - - @check_superuser - def can_delete(self, obj): - return self.can_admin(obj) - - class RoleAccess(BaseAccess): """ - I can see roles when diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index d178208cab..e7fc7a8c90 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -135,7 +135,6 @@ def counts(since, **kwargs): models.WorkflowJobTemplate, models.Host, models.Schedule, - models.CustomInventoryScript, models.NotificationTemplate, ): counts[camelcase_to_underscore(cls.__name__)] = cls.objects.count() @@ -347,29 +346,29 @@ def _copy_table(table, query, path): @register('events_table', '1.2', format='csv', description=_('Automation task records'), expensive=events_slicing) def events_table(since, full_path, until, **kwargs): def query(event_data): - return f'''COPY (SELECT main_jobevent.id, + return f'''COPY (SELECT main_jobevent.id, main_jobevent.created, main_jobevent.modified, main_jobevent.uuid, main_jobevent.parent_uuid, - main_jobevent.event, + main_jobevent.event, {event_data}->'task_action' AS task_action, (CASE WHEN event = 'playbook_on_stats' THEN event_data END) as playbook_on_stats, - main_jobevent.failed, - main_jobevent.changed, - main_jobevent.playbook, + main_jobevent.failed, + main_jobevent.changed, + main_jobevent.playbook, main_jobevent.play, main_jobevent.task, - main_jobevent.role, - main_jobevent.job_id, - main_jobevent.host_id, + main_jobevent.role, + main_jobevent.job_id, + main_jobevent.host_id, main_jobevent.host_name, CAST({event_data}->>'start' AS TIMESTAMP WITH TIME ZONE) AS start, CAST({event_data}->>'end' AS TIMESTAMP WITH TIME ZONE) AS end, {event_data}->'duration' AS duration, {event_data}->'res'->'warnings' AS warnings, {event_data}->'res'->'deprecations' AS deprecations - FROM main_jobevent + FROM main_jobevent WHERE (main_jobevent.id > {since} AND main_jobevent.id <= {until}) ORDER BY main_jobevent.id ASC) TO STDOUT WITH CSV HEADER''' @@ -386,22 +385,22 @@ def unified_jobs_table(since, full_path, until, **kwargs): django_content_type.model, main_unifiedjob.organization_id, main_organization.name as organization_name, - main_job.inventory_id, + main_job.inventory_id, main_inventory.name as inventory_name, - main_unifiedjob.created, - main_unifiedjob.name, - main_unifiedjob.unified_job_template_id, - main_unifiedjob.launch_type, - main_unifiedjob.schedule_id, - main_unifiedjob.execution_node, - main_unifiedjob.controller_node, - main_unifiedjob.cancel_flag, - main_unifiedjob.status, - main_unifiedjob.failed, - main_unifiedjob.started, - main_unifiedjob.finished, - main_unifiedjob.elapsed, - main_unifiedjob.job_explanation, + main_unifiedjob.created, + main_unifiedjob.name, + main_unifiedjob.unified_job_template_id, + main_unifiedjob.launch_type, + main_unifiedjob.schedule_id, + main_unifiedjob.execution_node, + main_unifiedjob.controller_node, + main_unifiedjob.cancel_flag, + main_unifiedjob.status, + main_unifiedjob.failed, + main_unifiedjob.started, + main_unifiedjob.finished, + main_unifiedjob.elapsed, + main_unifiedjob.job_explanation, main_unifiedjob.instance_group_id, main_unifiedjob.installed_collections, main_unifiedjob.ansible_version @@ -422,21 +421,21 @@ def unified_jobs_table(since, full_path, until, **kwargs): @register('unified_job_template_table', '1.0', format='csv', description=_('Data on job templates')) def unified_job_template_table(since, full_path, **kwargs): - unified_job_template_query = '''COPY (SELECT main_unifiedjobtemplate.id, + unified_job_template_query = '''COPY (SELECT main_unifiedjobtemplate.id, main_unifiedjobtemplate.polymorphic_ctype_id, django_content_type.model, - main_unifiedjobtemplate.created, - main_unifiedjobtemplate.modified, - main_unifiedjobtemplate.created_by_id, - main_unifiedjobtemplate.modified_by_id, - main_unifiedjobtemplate.name, - main_unifiedjobtemplate.current_job_id, - main_unifiedjobtemplate.last_job_id, - main_unifiedjobtemplate.last_job_failed, - main_unifiedjobtemplate.last_job_run, - main_unifiedjobtemplate.next_job_run, - main_unifiedjobtemplate.next_schedule_id, - main_unifiedjobtemplate.status + main_unifiedjobtemplate.created, + main_unifiedjobtemplate.modified, + main_unifiedjobtemplate.created_by_id, + main_unifiedjobtemplate.modified_by_id, + main_unifiedjobtemplate.name, + main_unifiedjobtemplate.current_job_id, + main_unifiedjobtemplate.last_job_id, + main_unifiedjobtemplate.last_job_failed, + main_unifiedjobtemplate.last_job_run, + main_unifiedjobtemplate.next_job_run, + main_unifiedjobtemplate.next_schedule_id, + main_unifiedjobtemplate.status FROM main_unifiedjobtemplate, django_content_type WHERE main_unifiedjobtemplate.polymorphic_ctype_id = django_content_type.id ORDER BY main_unifiedjobtemplate.id ASC) TO STDOUT WITH CSV HEADER''' @@ -447,15 +446,15 @@ def unified_job_template_table(since, full_path, **kwargs): def workflow_job_node_table(since, full_path, until, **kwargs): workflow_job_node_query = '''COPY (SELECT main_workflowjobnode.id, main_workflowjobnode.created, - main_workflowjobnode.modified, - main_workflowjobnode.job_id, - main_workflowjobnode.unified_job_template_id, - main_workflowjobnode.workflow_job_id, - main_workflowjobnode.inventory_id, + main_workflowjobnode.modified, + main_workflowjobnode.job_id, + main_workflowjobnode.unified_job_template_id, + main_workflowjobnode.workflow_job_id, + main_workflowjobnode.inventory_id, success_nodes.nodes AS success_nodes, failure_nodes.nodes AS failure_nodes, always_nodes.nodes AS always_nodes, - main_workflowjobnode.do_not_run, + main_workflowjobnode.do_not_run, main_workflowjobnode.all_parents_must_converge FROM main_workflowjobnode LEFT JOIN ( @@ -483,12 +482,12 @@ def workflow_job_node_table(since, full_path, until, **kwargs): @register('workflow_job_template_node_table', '1.0', format='csv', description=_('Data on workflows')) def workflow_job_template_node_table(since, full_path, **kwargs): - workflow_job_template_node_query = '''COPY (SELECT main_workflowjobtemplatenode.id, + workflow_job_template_node_query = '''COPY (SELECT main_workflowjobtemplatenode.id, main_workflowjobtemplatenode.created, - main_workflowjobtemplatenode.modified, - main_workflowjobtemplatenode.unified_job_template_id, - main_workflowjobtemplatenode.workflow_job_template_id, - main_workflowjobtemplatenode.inventory_id, + main_workflowjobtemplatenode.modified, + main_workflowjobtemplatenode.unified_job_template_id, + main_workflowjobtemplatenode.workflow_job_template_id, + main_workflowjobtemplatenode.inventory_id, success_nodes.nodes AS success_nodes, failure_nodes.nodes AS failure_nodes, always_nodes.nodes AS always_nodes, diff --git a/awx/main/migrations/_rbac.py b/awx/main/migrations/_rbac.py index d6aabe49fd..07284b64cd 100644 --- a/awx/main/migrations/_rbac.py +++ b/awx/main/migrations/_rbac.py @@ -28,7 +28,6 @@ def create_roles(apps, schema_editor): 'Inventory', 'Project', 'Credential', - 'CustomInventoryScript', 'JobTemplate', ] ] diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 824f0807c4..d181222e73 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -12,7 +12,7 @@ from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutM from awx.main.models.organization import Organization, Profile, Team, UserSessionMembership # noqa from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env # noqa from awx.main.models.projects import Project, ProjectUpdate # noqa -from awx.main.models.inventory import CustomInventoryScript, Group, Host, Inventory, InventorySource, InventoryUpdate, SmartInventoryMembership # noqa +from awx.main.models.inventory import Group, Host, Inventory, InventorySource, InventoryUpdate, SmartInventoryMembership # noqa from awx.main.models.jobs import ( # noqa Job, JobHostSummary, @@ -224,7 +224,6 @@ activity_stream_registrar.connect(AdHocCommand) # activity_stream_registrar.connect(JobEvent) # activity_stream_registrar.connect(Profile) activity_stream_registrar.connect(Schedule) -activity_stream_registrar.connect(CustomInventoryScript) activity_stream_registrar.connect(NotificationTemplate) activity_stream_registrar.connect(Notification) activity_stream_registrar.connect(Label) diff --git a/awx/main/models/activity_stream.py b/awx/main/models/activity_stream.py index 16f3dcb5c2..16e73bef56 100644 --- a/awx/main/models/activity_stream.py +++ b/awx/main/models/activity_stream.py @@ -74,6 +74,7 @@ class ActivityStream(models.Model): unified_job = models.ManyToManyField("UnifiedJob", blank=True, related_name='activity_stream_as_unified_job+') ad_hoc_command = models.ManyToManyField("AdHocCommand", blank=True) schedule = models.ManyToManyField("Schedule", blank=True) + # TODO: migrate away custom_inventory_script = models.ManyToManyField("CustomInventoryScript", blank=True) execution_environment = models.ManyToManyField("ExecutionEnvironment", blank=True) notification_template = models.ManyToManyField("NotificationTemplate", blank=True) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 6fabdf7567..58465ba78b 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -52,7 +52,7 @@ from awx.main.utils import _inventory_updates from awx.main.utils.safe_yaml import sanitize_jinja -__all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'CustomInventoryScript', 'SmartInventoryMembership'] +__all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'SmartInventoryMembership'] logger = logging.getLogger('awx.main.models.inventory') @@ -845,6 +845,7 @@ class InventorySourceOptions(BaseModel): blank=True, default='', ) + # TODO: migrate away source_script = models.ForeignKey( 'CustomInventoryScript', null=True, diff --git a/awx/main/signals.py b/awx/main/signals.py index 82cd8f43a7..492da999bc 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -378,7 +378,6 @@ def model_serializer_mapping(): models.Group: serializers.GroupSerializer, models.InstanceGroup: serializers.InstanceGroupSerializer, models.InventorySource: serializers.InventorySourceSerializer, - models.CustomInventoryScript: serializers.CustomInventoryScriptSerializer, models.Credential: serializers.CredentialSerializer, models.Team: serializers.TeamSerializer, models.Project: serializers.ProjectSerializer, diff --git a/awx/main/tests/functional/analytics/test_counts.py b/awx/main/tests/functional/analytics/test_counts.py index a4ce6a3a38..146afb604c 100644 --- a/awx/main/tests/functional/analytics/test_counts.py +++ b/awx/main/tests/functional/analytics/test_counts.py @@ -12,7 +12,6 @@ def test_empty(): "active_sessions": 0, "active_host_count": 0, "credential": 0, - "custom_inventory_script": 0, "custom_virtualenvs": 0, # dev env ansible3 "host": 0, "inventory": 0, @@ -48,7 +47,6 @@ def test_database_counts(organization_factory, job_template_factory, workflow_jo rrule="DTSTART;TZID=America/New_York:20300504T150000", unified_job_template=jt.job_template, ).save() - models.CustomInventoryScript(organization=objs.organization).save() counts = collectors.counts(None) for key in ( @@ -62,7 +60,6 @@ def test_database_counts(organization_factory, job_template_factory, workflow_jo "workflow_job_template", "host", "schedule", - "custom_inventory_script", ): assert counts[key] == 1 diff --git a/awx/main/tests/functional/analytics/test_metrics.py b/awx/main/tests/functional/analytics/test_metrics.py index 442c83699c..2a04adcfce 100644 --- a/awx/main/tests/functional/analytics/test_metrics.py +++ b/awx/main/tests/functional/analytics/test_metrics.py @@ -18,7 +18,6 @@ EXPECTED_VALUES = { 'awx_hosts_total': 1.0, 'awx_hosts_total': 1.0, 'awx_schedules_total': 1.0, - 'awx_inventory_scripts_total': 1.0, 'awx_sessions_total': 0.0, 'awx_sessions_total': 0.0, 'awx_sessions_total': 0.0, @@ -44,7 +43,6 @@ def test_metrics_counts(organization_factory, job_template_factory, workflow_job models.Team(organization=objs.organization).save() models.Host(inventory=jt.inventory).save() models.Schedule(rrule='DTSTART;TZID=America/New_York:20300504T150000', unified_job_template=jt.job_template).save() - models.CustomInventoryScript(organization=objs.organization).save() output = metrics() gauges = text_string_to_metric_families(output.decode('UTF-8')) diff --git a/awx/main/tests/functional/api/test_inventory.py b/awx/main/tests/functional/api/test_inventory.py index 1ed1c633d1..a6db567ed1 100644 --- a/awx/main/tests/functional/api/test_inventory.py +++ b/awx/main/tests/functional/api/test_inventory.py @@ -144,14 +144,10 @@ def test_async_inventory_deletion_deletes_related_jt(delete, get, job_template, assert jdata['inventory'] is None -@pytest.mark.parametrize('order_by', ('script', '-script', 'script,pk', '-script,pk')) +@pytest.mark.parametrize('order_by', ('extra_vars', '-extra_vars', 'extra_vars,pk', '-extra_vars,pk')) @pytest.mark.django_db def test_list_cannot_order_by_unsearchable_field(get, organization, alice, order_by): - for i, script in enumerate(('#!/bin/a', '#!/bin/b', '#!/bin/c')): - custom_script = organization.custom_inventory_scripts.create(name="I%d" % i, script=script) - custom_script.admin_role.members.add(alice) - - get(reverse('api:inventory_script_list'), alice, QUERY_STRING='order_by=%s' % order_by, expect=403) + get(reverse('api:jobs_list'), alice, QUERY_STRING='order_by=%s' % order_by, expect=403) @pytest.mark.parametrize("role_field,expected_status_code", [(None, 403), ('admin_role', 201), ('update_role', 403), ('adhoc_role', 403), ('use_role', 403)]) diff --git a/awx/main/tests/functional/conftest.py b/awx/main/tests/functional/conftest.py index c54b06b86f..1150c025aa 100644 --- a/awx/main/tests/functional/conftest.py +++ b/awx/main/tests/functional/conftest.py @@ -26,7 +26,7 @@ from rest_framework.test import ( from awx.main.models.credential import CredentialType, Credential from awx.main.models.jobs import JobTemplate, SystemJobTemplate -from awx.main.models.inventory import Group, Inventory, InventoryUpdate, InventorySource, CustomInventoryScript +from awx.main.models.inventory import Group, Inventory, InventoryUpdate, InventorySource from awx.main.models.organization import ( Organization, Team, @@ -540,11 +540,6 @@ def inventory_update(inventory_source): return InventoryUpdate.objects.create(inventory_source=inventory_source, source=inventory_source.source) -@pytest.fixture -def inventory_script(organization): - return CustomInventoryScript.objects.create(name='test inv script', organization=organization, script='#!/usr/bin/python') - - @pytest.fixture def host(group, inventory): return group.hosts.create(name='single-host', inventory=inventory) diff --git a/awx/main/tests/functional/test_copy.py b/awx/main/tests/functional/test_copy.py index 31bd250662..86334efd34 100644 --- a/awx/main/tests/functional/test_copy.py +++ b/awx/main/tests/functional/test_copy.py @@ -216,16 +216,3 @@ def test_notification_template_copy(post, get, notification_template_with_encryp assert decrypt_field(notification_template_with_encrypt, 'notification_configuration', 'token') == decrypt_field( notification_template_copy, 'notification_configuration', 'token' ) - - -@pytest.mark.django_db -def test_inventory_script_copy(post, get, inventory_script, organization, alice): - inventory_script.organization.auditor_role.members.add(alice) - assert get(reverse('api:inventory_script_copy', kwargs={'pk': inventory_script.pk}), alice, expect=200).data['can_copy'] is False - inventory_script.organization.admin_role.members.add(alice) - assert get(reverse('api:inventory_script_copy', kwargs={'pk': inventory_script.pk}), alice, expect=200).data['can_copy'] is True - is_copy_pk = post(reverse('api:inventory_script_copy', kwargs={'pk': inventory_script.pk}), {'name': 'copied inv script'}, alice, expect=201).data['id'] - inventory_script_copy = type(inventory_script).objects.get(pk=is_copy_pk) - assert inventory_script_copy.created_by == alice - assert inventory_script_copy.name == 'copied inv script' - assert inventory_script_copy.organization == organization diff --git a/awx/main/tests/functional/test_named_url.py b/awx/main/tests/functional/test_named_url.py index e7bd9b4fae..7df38aa4e1 100644 --- a/awx/main/tests/functional/test_named_url.py +++ b/awx/main/tests/functional/test_named_url.py @@ -8,7 +8,6 @@ from awx.api.versioning import reverse from awx.main.middleware import URLModificationMiddleware from awx.main.models import ( # noqa Credential, - CustomInventoryScript, Group, Host, Instance, @@ -204,19 +203,6 @@ def test_inventory_source(get, admin_user): assert response.data['related']['named_url'].endswith('/test_source++/') -@pytest.mark.django_db -def test_inventory_script(get, admin_user): - test_script = CustomInventoryScript.objects.create(name='test_script') - url = reverse('api:inventory_script_detail', kwargs={'pk': test_script.pk}) - response = get(url, user=admin_user, expect=200) - assert response.data['related']['named_url'].endswith('/test_script++/') - test_org = Organization.objects.create(name='test_org') - test_script.organization = test_org - test_script.save() - response = get(url, user=admin_user, expect=200) - assert response.data['related']['named_url'].endswith('/test_script++test_org/') - - @pytest.mark.django_db def test_credential(get, admin_user, credentialtype_ssh): test_cred = Credential.objects.create(name='test_cred', credential_type=credentialtype_ssh) diff --git a/awx/main/tests/functional/test_rbac_inventory.py b/awx/main/tests/functional/test_rbac_inventory.py index 1e4b47e45e..87804d1c74 100644 --- a/awx/main/tests/functional/test_rbac_inventory.py +++ b/awx/main/tests/functional/test_rbac_inventory.py @@ -2,7 +2,6 @@ import pytest from awx.main.models import ( Host, - CustomInventoryScript, Schedule, ) from awx.main.access import ( @@ -10,56 +9,10 @@ from awx.main.access import ( InventorySourceAccess, HostAccess, InventoryUpdateAccess, - CustomInventoryScriptAccess, ScheduleAccess, ) -@pytest.mark.django_db -def test_custom_inv_script_access(organization, user): - u = user('user', False) - ou = user('oadm', False) - - custom_inv = CustomInventoryScript.objects.create(name='test', script='test', description='test') - custom_inv.organization = organization - custom_inv.save() - assert u not in custom_inv.read_role - - organization.member_role.members.add(u) - assert u in custom_inv.read_role - - organization.admin_role.members.add(ou) - assert ou in custom_inv.admin_role - - -@pytest.fixture -def custom_inv(organization): - return CustomInventoryScript.objects.create(name='test', script='test', description='test', organization=organization) - - -@pytest.mark.django_db -def test_modify_inv_script_foreign_org_admin(org_admin, organization, organization_factory, project, custom_inv): - other_org = organization_factory('not-my-org').organization - access = CustomInventoryScriptAccess(org_admin) - assert not access.can_change(custom_inv, {'organization': other_org.pk, 'name': 'new-project'}) - - -@pytest.mark.django_db -def test_org_member_inventory_script_permissions(org_member, organization, custom_inv): - access = CustomInventoryScriptAccess(org_member) - assert access.can_read(custom_inv) - assert not access.can_delete(custom_inv) - assert not access.can_change(custom_inv, {'name': 'ed-test'}) - - -@pytest.mark.django_db -def test_copy_only_admin(org_member, organization, custom_inv): - custom_inv.admin_role.members.add(org_member) - access = CustomInventoryScriptAccess(org_member) - assert not access.can_copy(custom_inv) - assert access.get_user_capabilities(custom_inv, method_list=['edit', 'delete', 'copy']) == {'edit': True, 'delete': True, 'copy': False} - - @pytest.mark.django_db @pytest.mark.parametrize("role", ["admin_role", "inventory_admin_role"]) def test_access_admin(role, organization, inventory, user): diff --git a/awx/main/tests/unit/api/serializers/test_inventory_serializers.py b/awx/main/tests/unit/api/serializers/test_inventory_serializers.py index dd14d9e0e9..98fa53901f 100644 --- a/awx/main/tests/unit/api/serializers/test_inventory_serializers.py +++ b/awx/main/tests/unit/api/serializers/test_inventory_serializers.py @@ -1,25 +1,12 @@ # Python import pytest from unittest import mock -from unittest.mock import PropertyMock # AWX from awx.api.serializers import ( - CustomInventoryScriptSerializer, InventorySourceSerializer, ) -from awx.main.models import ( - CustomInventoryScript, - InventorySource, - User, -) - -# DRF -from rest_framework.request import Request -from rest_framework.test import ( - APIRequestFactory, - force_authenticate, -) +from awx.main.models import InventorySource @pytest.fixture @@ -30,34 +17,6 @@ def inventory_source(mocker): return obj -class TestCustomInventoryScriptSerializer(object): - @pytest.mark.parametrize( - "superuser,sysaudit,admin_role,value", - ((True, False, False, '#!/python'), (False, True, False, '#!/python'), (False, False, True, '#!/python'), (False, False, False, None)), - ) - def test_to_representation_orphan(self, superuser, sysaudit, admin_role, value): - with mock.patch.object(CustomInventoryScriptSerializer, 'get_summary_fields', return_value={}): - with mock.patch.object(User, 'is_system_auditor', return_value=sysaudit): - user = User(username="root", is_superuser=superuser) - roles = [user] if admin_role else [] - - with mock.patch('awx.main.models.CustomInventoryScript.admin_role', new_callable=PropertyMock, return_value=roles), mock.patch( - 'awx.api.serializers.settings' - ): - cis = CustomInventoryScript(pk=1, script=value) - serializer = CustomInventoryScriptSerializer() - - factory = APIRequestFactory() - wsgi_request = factory.post("/inventory_script/1", {'id': 1}, format="json") - force_authenticate(wsgi_request, user) - - request = Request(wsgi_request) - serializer.context['request'] = request - - representation = serializer.to_representation(cis) - assert representation['script'] == value - - @mock.patch('awx.api.serializers.UnifiedJobTemplateSerializer.get_related', lambda x, y: {}) @mock.patch('awx.api.serializers.InventorySourceOptionsSerializer.get_related', lambda x, y: {}) class TestInventorySourceSerializerGetRelated(object): diff --git a/awx/main/tests/unit/api/test_filters.py b/awx/main/tests/unit/api/test_filters.py index 5b247790e2..c523cd2650 100644 --- a/awx/main/tests/unit/api/test_filters.py +++ b/awx/main/tests/unit/api/test_filters.py @@ -7,7 +7,6 @@ from awx.api.filters import FieldLookupBackend, OrderByBackend, get_field_from_p from awx.main.models import ( AdHocCommand, ActivityStream, - CustomInventoryScript, Credential, Job, JobTemplate, @@ -106,7 +105,6 @@ def test_filter_on_password_field(password_field, lookup_suffix): (WorkflowJob, 'survey_passwords__icontains'), (JobTemplate, 'survey_spec__icontains'), (WorkflowJobTemplate, 'survey_spec__icontains'), - (CustomInventoryScript, 'script__icontains'), (ActivityStream, 'o_auth2_application__client_secret__gt'), (OAuth2Application, 'grant__code__gt'), ], diff --git a/awx/main/tests/unit/api/test_views.py b/awx/main/tests/unit/api/test_views.py index aff0dd2a3b..2be393c460 100644 --- a/awx/main/tests/unit/api/test_views.py +++ b/awx/main/tests/unit/api/test_views.py @@ -32,7 +32,6 @@ class TestApiRootView: 'teams', 'credentials', 'inventory', - 'inventory_scripts', 'inventory_sources', 'groups', 'hosts', diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index e03ca0a7af..e6460dbc07 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -30,7 +30,6 @@ from awx.main.models import ( ProjectUpdate, UnifiedJob, User, - CustomInventoryScript, build_safe_env, ) from awx.main.models.credential import ManagedCredentialType @@ -1631,55 +1630,6 @@ class TestInventoryUpdateCredentials(TestJobExecution): assert 'AWS_ACCESS_KEY_ID' not in env assert 'AWS_SECRET_ACCESS_KEY' not in env - @pytest.mark.parametrize('with_credential', [True, False]) - def test_custom_source(self, with_credential, mocker, inventory_update, private_data_dir): - task = tasks.RunInventoryUpdate() - task.instance = inventory_update - inventory_update.source = 'custom' - inventory_update.source_vars = '{"FOO": "BAR"}' - inventory_update.source_script = CustomInventoryScript(script='#!/bin/sh\necho "Hello, World!"') - - if with_credential: - azure_rm = CredentialType.defaults['azure_rm']() - - def get_creds(): - cred = Credential( - pk=1, - credential_type=azure_rm, - inputs={ - 'client': 'some-client', - 'secret': 'some-secret', - 'tenant': 'some-tenant', - 'subscription': 'some-subscription', - }, - ) - return [cred] - - inventory_update.get_extra_credentials = get_creds - else: - inventory_update.get_extra_credentials = mocker.Mock(return_value=[]) - inventory_update.get_cloud_credential = mocker.Mock(return_value=None) - - env = task.build_env(inventory_update, private_data_dir, False) - args = task.build_args(inventory_update, private_data_dir, {}) - - credentials = task.build_credentials_list(inventory_update) - for credential in credentials: - if credential: - credential.credential_type.inject_credential(credential, env, {}, [], private_data_dir) - - assert '-i' in ' '.join(args) - script = args[args.index('-i') + 1] - host_script = script.replace('/runner', private_data_dir) - with open(host_script, 'r') as f: - assert f.read() == inventory_update.source_script.script - assert env['FOO'] == 'BAR' - if with_credential: - assert env['AZURE_CLIENT_ID'] == 'some-client' - assert env['AZURE_SECRET'] == 'some-secret' - assert env['AZURE_TENANT'] == 'some-tenant' - assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription' - def test_ec2_source(self, private_data_dir, inventory_update, mocker): task = tasks.RunInventoryUpdate() task.instance = inventory_update diff --git a/awx_collection/plugins/modules/tower_inventory_source.py b/awx_collection/plugins/modules/tower_inventory_source.py index 376317c9a4..0353576b02 100644 --- a/awx_collection/plugins/modules/tower_inventory_source.py +++ b/awx_collection/plugins/modules/tower_inventory_source.py @@ -256,8 +256,6 @@ def main(): inventory_source_fields['execution_environment'] = module.resolve_name_to_id('execution_environments', ee) if source_project is not None: inventory_source_fields['source_project'] = module.resolve_name_to_id('projects', source_project) - if source_script is not None: - inventory_source_fields['source_script'] = module.resolve_name_to_id('inventory_scripts', source_script) OPTIONAL_VARS = ( 'description', diff --git a/awx_collection/plugins/modules/tower_receive.py b/awx_collection/plugins/modules/tower_receive.py index 7952c88d5e..d2031d6667 100644 --- a/awx_collection/plugins/modules/tower_receive.py +++ b/awx_collection/plugins/modules/tower_receive.py @@ -66,12 +66,6 @@ options: default: [] type: list elements: str - inventory_script: - description: - - List of inventory script names to export - default: [] - type: list - elements: str inventory: description: - List of inventory names to export @@ -153,7 +147,6 @@ def main(): credential=dict(type='list', default=[], elements='str'), credential_type=dict(type='list', default=[], elements='str'), inventory=dict(type='list', default=[], elements='str'), - inventory_script=dict(type='list', default=[], elements='str'), job_template=dict(type='list', default=[], elements='str'), notification_template=dict(type='list', default=[], elements='str'), organization=dict(type='list', default=[], elements='str'), diff --git a/awx_collection/test/awx/test_completeness.py b/awx_collection/test/awx/test_completeness.py index d639f828cd..66c5ee8604 100644 --- a/awx_collection/test/awx/test_completeness.py +++ b/awx_collection/test/awx/test_completeness.py @@ -69,7 +69,9 @@ no_api_parameter_ok = { # When this tool was created we were not feature complete. Adding something in here indicates a module # that needs to be developed. If the module is found on the file system it will auto-detect that the # work is being done and will bypass this check. At some point this module should be removed from this list. -needs_development = ['tower_inventory_script', 'tower_workflow_approval'] +needs_development = [ + 'tower_workflow_approval', +] needs_param_development = { 'tower_host': ['instance_id'], } diff --git a/awxkit/awxkit/api/pages/inventory.py b/awxkit/awxkit/api/pages/inventory.py index 4378cc47ce..994bc32880 100644 --- a/awxkit/awxkit/api/pages/inventory.py +++ b/awxkit/awxkit/api/pages/inventory.py @@ -125,64 +125,6 @@ class Inventories(page.PageList, Inventory): page.register_page([resources.inventories, resources.related_inventories], Inventories) -class InventoryScript(HasCopy, HasCreate, base.Base): - - dependencies = [Organization] - - def payload(self, organization, **kwargs): - payload = PseudoNamespace( - name=kwargs.get('name') or 'Inventory Script - {}'.format(random_title()), - description=kwargs.get('description') or random_title(10), - organization=organization.id, - script=kwargs.get('script') or self._generate_script(), - ) - return payload - - def create_payload(self, name='', description='', organization=Organization, script='', **kwargs): - self.create_and_update_dependencies(organization) - payload = self.payload(name=name, description=description, organization=self.ds.organization, script=script, **kwargs) - payload.ds = DSAdapter(self.__class__.__name__, self._dependency_store) - return payload - - def create(self, name='', description='', organization=Organization, script='', **kwargs): - payload = self.create_payload(name=name, description=description, organization=organization, script=script, **kwargs) - return self.update_identity(InventoryScripts(self.connection).post(payload)) - - def _generate_script(self): - script = '\n'.join( - [ - '#!/usr/bin/env python', - '# -*- coding: utf-8 -*-', - 'import json', - 'inventory = dict()', - 'inventory["{0}"] = dict()', - 'inventory["{0}"]["hosts"] = list()', - 'inventory["{0}"]["hosts"].append("{1}")', - 'inventory["{0}"]["hosts"].append("{2}")', - 'inventory["{0}"]["hosts"].append("{3}")', - 'inventory["{0}"]["hosts"].append("{4}")', - 'inventory["{0}"]["hosts"].append("{5}")', - 'inventory["{0}"]["vars"] = dict(ansible_host="127.0.0.1", ansible_connection="local")', - 'print(json.dumps(inventory))', - ] - ) - group_name = re.sub(r"[\']", "", "group_{}".format(random_title(non_ascii=False))) - host_names = [re.sub(r"[\':]", "", "host_{}".format(random_utf8())) for _ in range(5)] - - return script.format(group_name, *host_names) - - -page.register_page([resources.inventory_script, (resources.inventory_scripts, 'post'), (resources.inventory_script_copy, 'post')], InventoryScript) - - -class InventoryScripts(page.PageList, InventoryScript): - - pass - - -page.register_page([resources.inventory_scripts], InventoryScripts) - - class Group(HasCreate, HasVariables, base.Base): dependencies = [Inventory] @@ -408,8 +350,6 @@ class InventorySource(HasCreate, HasNotifications, UnifiedJobTemplate): if credential: credential = self.ds.credential - if source_script: - source_script = self.ds.inventory_script if project: project = self.ds.project diff --git a/awxkit/awxkit/api/resources.py b/awxkit/awxkit/api/resources.py index 573c96598f..3d27dbb1df 100644 --- a/awxkit/awxkit/api/resources.py +++ b/awxkit/awxkit/api/resources.py @@ -65,9 +65,6 @@ class Resources(object): _inventory_related_root_groups = r'inventories/\d+/root_groups/' _inventory_related_script = r'inventories/\d+/script/' _inventory_related_update_inventory_sources = r'inventories/\d+/update_inventory_sources/' - _inventory_script = r'inventory_scripts/\d+/' - _inventory_script_copy = r'inventory_scripts/\d+/copy/' - _inventory_scripts = 'inventory_scripts/' _inventory_source = r'inventory_sources/\d+/' _inventory_source_schedule = r'inventory_sources/\d+/schedules/\d+/' _inventory_source_schedules = r'inventory_sources/\d+/schedules/' diff --git a/awxkit/awxkit/awx/utils.py b/awxkit/awxkit/awx/utils.py index 0a96a4cb38..66de4517dc 100644 --- a/awxkit/awxkit/awx/utils.py +++ b/awxkit/awxkit/awx/utils.py @@ -39,7 +39,6 @@ def delete_all(v): v.projects, v.inventory, v.hosts, - v.inventory_scripts, v.labels, v.credentials, v.teams, diff --git a/awxkit/awxkit/cli/custom.py b/awxkit/awxkit/cli/custom.py index d2b9bc6590..e65b20e852 100644 --- a/awxkit/awxkit/cli/custom.py +++ b/awxkit/awxkit/cli/custom.py @@ -349,7 +349,6 @@ class RoleMixin(object): ['organizations', 'organization'], ['projects', 'project'], ['inventories', 'inventory'], - ['inventory_scripts', 'inventory_script'], ['teams', 'team'], ['credentials', 'credential'], ['job_templates', 'job_template'], diff --git a/awxkit/awxkit/cli/resource.py b/awxkit/awxkit/cli/resource.py index 7aa7c32e0a..b4747b6e43 100644 --- a/awxkit/awxkit/cli/resource.py +++ b/awxkit/awxkit/cli/resource.py @@ -23,7 +23,6 @@ DEPRECATED_RESOURCES = { 'instances': 'instance', 'instance_groups': 'instance_group', 'inventory': 'inventories', - 'inventory_scripts': 'inventory_script', 'inventory_sources': 'inventory_source', 'inventory_updates': 'inventory_update', 'jobs': 'job', From a54aab971755e6fc3b5ba91b824939c758980f88 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 12 Apr 2021 14:54:44 -0400 Subject: [PATCH 03/10] Remove old uses of CustomInventoryScript some uses are minor references Fix some test fails --- awx/main/access.py | 1 - awx/main/analytics/metrics.py | 2 - awx/main/conf.py | 6 +-- .../management/commands/inventory_import.py | 3 -- awx/main/tasks.py | 11 +--- .../tests/functional/api/test_inventory.py | 2 +- .../tests/functional/models/test_inventory.py | 2 +- awxkit/awxkit/api/pages/inventory.py | 52 +++++-------------- 8 files changed, 18 insertions(+), 61 deletions(-) diff --git a/awx/main/access.py b/awx/main/access.py index e6fc29fdc8..f8a5bb0cba 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -2670,7 +2670,6 @@ class ActivityStreamAccess(BaseAccess): 'role', 'actor', 'schedule', - 'custom_inventory_script', 'unified_job_template', 'workflow_job_template_node', ) diff --git a/awx/main/analytics/metrics.py b/awx/main/analytics/metrics.py index 9ddb97e13b..0af34a60ea 100644 --- a/awx/main/analytics/metrics.py +++ b/awx/main/analytics/metrics.py @@ -31,7 +31,6 @@ def metrics(): registry=REGISTRY, ) SCHEDULE_COUNT = Gauge('awx_schedules_total', 'Number of schedules', registry=REGISTRY) - INV_SCRIPT_COUNT = Gauge('awx_inventory_scripts_total', 'Number of invetory scripts', registry=REGISTRY) USER_SESSIONS = Gauge( 'awx_sessions_total', 'Number of sessions', @@ -160,7 +159,6 @@ def metrics(): HOST_COUNT.labels(type='active').set(current_counts['active_host_count']) SCHEDULE_COUNT.set(current_counts['schedule']) - INV_SCRIPT_COUNT.set(current_counts['custom_inventory_script']) CUSTOM_VENVS.set(current_counts['custom_virtualenvs']) USER_SESSIONS.labels(type='all').set(current_counts['active_sessions']) diff --git a/awx/main/conf.py b/awx/main/conf.py index 69da05a944..f50f813533 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -235,11 +235,7 @@ register( 'AWX_ISOLATION_BASE_PATH', field_class=fields.CharField, label=_('Job execution path'), - help_text=_( - 'The directory in which Tower will create new temporary ' - 'directories for job execution and isolation ' - '(such as credential files and custom inventory scripts).' - ), + help_text=_('The directory in which Tower will create new temporary directories for job execution and isolation (such as credential files).'), category=_('Jobs'), category_slug='jobs', ) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index bba14c53d1..92509a6980 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -147,9 +147,6 @@ class Command(BaseCommand): parser.add_argument('--overwrite', dest='overwrite', action='store_true', default=False, help='overwrite the destination hosts and groups') parser.add_argument('--overwrite-vars', dest='overwrite_vars', action='store_true', default=False, help='overwrite (rather than merge) variables') parser.add_argument('--keep-vars', dest='keep_vars', action='store_true', default=False, help='DEPRECATED legacy option, has no effect') - parser.add_argument( - '--custom', dest='custom', action='store_true', default=False, help='DEPRECATED indicates a custom inventory script, no longer used' - ) parser.add_argument('--source', dest='source', type=str, default=None, metavar='s', help='inventory directory, file, or script to load') parser.add_argument( '--enabled-var', diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 975614c0d0..063b5f47d7 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2412,7 +2412,7 @@ class RunInventoryUpdate(BaseTask): # All CLOUD_PROVIDERS sources implement as inventory plugin from collection env['ANSIBLE_INVENTORY_ENABLED'] = 'auto' - if inventory_update.source in ['scm', 'custom']: + if inventory_update.source == 'scm': for env_k in inventory_update.source_vars_dict: if str(env_k) not in env and str(env_k) not in settings.INV_ENV_VARIABLE_BLOCKED: env[str(env_k)] = str(inventory_update.source_vars_dict[env_k]) @@ -2513,16 +2513,7 @@ class RunInventoryUpdate(BaseTask): rel_path = injector.filename elif src == 'scm': rel_path = os.path.join('project', inventory_update.source_path) - elif src == 'custom': - handle, inventory_path = tempfile.mkstemp(dir=private_data_dir) - f = os.fdopen(handle, 'w') - if inventory_update.source_script is None: - raise RuntimeError('Inventory Script does not exist') - f.write(inventory_update.source_script.script) - f.close() - os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - rel_path = os.path.split(inventory_path)[-1] return rel_path def build_cwd(self, inventory_update, private_data_dir): diff --git a/awx/main/tests/functional/api/test_inventory.py b/awx/main/tests/functional/api/test_inventory.py index a6db567ed1..5aa6fb8992 100644 --- a/awx/main/tests/functional/api/test_inventory.py +++ b/awx/main/tests/functional/api/test_inventory.py @@ -147,7 +147,7 @@ def test_async_inventory_deletion_deletes_related_jt(delete, get, job_template, @pytest.mark.parametrize('order_by', ('extra_vars', '-extra_vars', 'extra_vars,pk', '-extra_vars,pk')) @pytest.mark.django_db def test_list_cannot_order_by_unsearchable_field(get, organization, alice, order_by): - get(reverse('api:jobs_list'), alice, QUERY_STRING='order_by=%s' % order_by, expect=403) + get(reverse('api:job_list'), alice, QUERY_STRING='order_by=%s' % order_by, expect=403) @pytest.mark.parametrize("role_field,expected_status_code", [(None, 403), ('admin_role', 201), ('update_role', 403), ('adhoc_role', 403), ('use_role', 403)]) diff --git a/awx/main/tests/functional/models/test_inventory.py b/awx/main/tests/functional/models/test_inventory.py index 8493b798fe..eab7e02895 100644 --- a/awx/main/tests/functional/models/test_inventory.py +++ b/awx/main/tests/functional/models/test_inventory.py @@ -174,7 +174,7 @@ class TestSCMClean: @pytest.mark.django_db class TestInventorySourceInjectors: def test_extra_credentials(self, project, credential): - inventory_source = InventorySource.objects.create(name='foo', source='custom', source_project=project) + inventory_source = InventorySource.objects.create(name='foo', source='scm', source_project=project) inventory_source.credentials.add(credential) assert inventory_source.get_cloud_credential() == credential # for serializer assert inventory_source.get_extra_credentials() == [credential] diff --git a/awxkit/awxkit/api/pages/inventory.py b/awxkit/awxkit/api/pages/inventory.py index 994bc32880..7fcc4b4255 100644 --- a/awxkit/awxkit/api/pages/inventory.py +++ b/awxkit/awxkit/api/pages/inventory.py @@ -128,7 +128,7 @@ page.register_page([resources.inventories, resources.related_inventories], Inven class Group(HasCreate, HasVariables, base.Base): dependencies = [Inventory] - optional_dependencies = [Credential, InventoryScript] + optional_dependencies = [Credential] NATURAL_KEY = ('name', 'inventory') @property @@ -157,9 +157,11 @@ class Group(HasCreate, HasVariables, base.Base): return payload - def create_payload(self, name='', description='', inventory=Inventory, credential=None, source_script=None, **kwargs): - credential, source_script = filter_by_class((credential, Credential), (source_script, InventoryScript)) - self.create_and_update_dependencies(inventory, credential, source_script) + def create_payload(self, name='', description='', inventory=Inventory, credential=None, **kwargs): + credential = filter_by_class( + (credential, Credential), + ) + self.create_and_update_dependencies(inventory, credential) credential = self.ds.credential if credential else None payload = self.payload(inventory=self.ds.inventory, credential=credential, name=name, description=description, **kwargs) payload.ds = DSAdapter(self.__class__.__name__, self._dependency_store) @@ -298,10 +300,10 @@ class InventorySource(HasCreate, HasNotifications, UnifiedJobTemplate): optional_schedule_fields = tuple() dependencies = [Inventory] - optional_dependencies = [Credential, InventoryScript, Project] + optional_dependencies = [Credential, Project] NATURAL_KEY = ('organization', 'name', 'inventory') - def payload(self, inventory, source='scm', credential=None, source_script=None, project=None, **kwargs): + def payload(self, inventory, source='scm', credential=None, project=None, **kwargs): payload = PseudoNamespace( name=kwargs.get('name') or 'InventorySource - {}'.format(random_title()), description=kwargs.get('description') or random_title(10), @@ -311,8 +313,6 @@ class InventorySource(HasCreate, HasNotifications, UnifiedJobTemplate): if credential: payload.credential = credential.id - if source_script: - payload.source_script = source_script.id if project: payload.source_project = project.id @@ -332,51 +332,27 @@ class InventorySource(HasCreate, HasNotifications, UnifiedJobTemplate): return payload - def create_payload( - self, name='', description='', source='scm', inventory=Inventory, credential=None, source_script=InventoryScript, project=None, **kwargs - ): - if source != 'custom' and source_script == InventoryScript: - source_script = None + def create_payload(self, name='', description='', source='scm', inventory=Inventory, credential=None, project=None, **kwargs): if source == 'scm': kwargs.setdefault('overwrite_vars', True) kwargs.setdefault('source_path', 'inventories/script_migrations/script_source.py') if project is None: project = Project - inventory, credential, source_script, project = filter_by_class( - (inventory, Inventory), (credential, Credential), (source_script, InventoryScript), (project, Project) - ) - self.create_and_update_dependencies(inventory, credential, source_script, project) + inventory, credential, project = filter_by_class((inventory, Inventory), (credential, Credential), (project, Project)) + self.create_and_update_dependencies(inventory, credential, project) if credential: credential = self.ds.credential if project: project = self.ds.project - payload = self.payload( - inventory=self.ds.inventory, - source=source, - credential=credential, - source_script=source_script, - project=project, - name=name, - description=description, - **kwargs - ) + payload = self.payload(inventory=self.ds.inventory, source=source, credential=credential, project=project, name=name, description=description, **kwargs) payload.ds = DSAdapter(self.__class__.__name__, self._dependency_store) return payload - def create(self, name='', description='', source='scm', inventory=Inventory, credential=None, source_script=InventoryScript, project=None, **kwargs): - payload = self.create_payload( - name=name, - description=description, - source=source, - inventory=inventory, - credential=credential, - source_script=source_script, - project=project, - **kwargs - ) + def create(self, name='', description='', source='scm', inventory=Inventory, credential=None, project=None, **kwargs): + payload = self.create_payload(name=name, description=description, source=source, inventory=inventory, credential=credential, project=project, **kwargs) return self.update_identity(InventorySources(self.connection).post(payload)) def update(self): From 33b6da445605bd644f7c50a966f991a9b55a1ffc Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Mon, 12 Apr 2021 16:06:40 -0400 Subject: [PATCH 04/10] Remove filter_by_class where it was not working --- awxkit/awxkit/api/pages/inventory.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/awxkit/awxkit/api/pages/inventory.py b/awxkit/awxkit/api/pages/inventory.py index 7fcc4b4255..b47be84372 100644 --- a/awxkit/awxkit/api/pages/inventory.py +++ b/awxkit/awxkit/api/pages/inventory.py @@ -158,9 +158,6 @@ class Group(HasCreate, HasVariables, base.Base): return payload def create_payload(self, name='', description='', inventory=Inventory, credential=None, **kwargs): - credential = filter_by_class( - (credential, Credential), - ) self.create_and_update_dependencies(inventory, credential) credential = self.ds.credential if credential else None payload = self.payload(inventory=self.ds.inventory, credential=credential, name=name, description=description, **kwargs) From 8440e3f41d9720b58e536da1ffc61914c739a62c Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 15 Apr 2021 09:45:14 -0400 Subject: [PATCH 05/10] Remove ForeignKey relations involving CustomInventoryScript Add migration to carry out corresponding schema change Add data migration to delete inventory sources which are the custom type Split migration into two files data migration touches same rows as schema migration --- awx/api/serializers.py | 5 +- awx/main/constants.py | 5 -- ...6_custom_inventory_scripts_removal_data.py | 19 +++++ .../0137_custom_inventory_scripts_removal.py | 81 +++++++++++++++++++ awx/main/migrations/_inventory_source.py | 19 +++++ awx/main/migrations/_rbac.py | 12 +++ awx/main/models/activity_stream.py | 2 - awx/main/models/base.py | 2 +- awx/main/models/inventory.py | 25 ------ 9 files changed, 135 insertions(+), 35 deletions(-) create mode 100644 awx/main/migrations/0136_custom_inventory_scripts_removal_data.py create mode 100644 awx/main/migrations/0137_custom_inventory_scripts_removal.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 1fb0c493f9..266cb01191 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -43,7 +43,7 @@ from polymorphic.models import PolymorphicModel # AWX from awx.main.access import get_user_capabilities -from awx.main.constants import SCHEDULEABLE_PROVIDERS, ACTIVE_STATES, CENSOR_VALUE +from awx.main.constants import ACTIVE_STATES, CENSOR_VALUE from awx.main.models import ( ActivityStream, AdHocCommand, @@ -91,6 +91,7 @@ from awx.main.models import ( WorkflowJobTemplate, WorkflowJobTemplateNode, StdoutMaxBytesExceeded, + CLOUD_INVENTORY_SOURCES, ) from awx.main.models.base import VERBOSITY_CHOICES, NEW_JOB_TYPE_CHOICES from awx.main.models.rbac import get_roles_on_resource, role_summary_fields_generator @@ -4761,7 +4762,7 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria return summary_fields def validate_unified_job_template(self, value): - if type(value) == InventorySource and value.source not in SCHEDULEABLE_PROVIDERS: + if type(value) == InventorySource and value.source not in CLOUD_INVENTORY_SOURCES: raise serializers.ValidationError(_('Inventory Source must be a cloud resource.')) elif type(value) == Project and value.scm_type == '': raise serializers.ValidationError(_('Manual Project cannot have a schedule set.')) diff --git a/awx/main/constants.py b/awx/main/constants.py index 6a44087c28..b2d62d5ec9 100644 --- a/awx/main/constants.py +++ b/awx/main/constants.py @@ -7,7 +7,6 @@ from django.utils.translation import ugettext_lazy as _ __all__ = [ 'CLOUD_PROVIDERS', - 'SCHEDULEABLE_PROVIDERS', 'PRIVILEGE_ESCALATION_METHODS', 'ANSI_SGR_PATTERN', 'CAN_CANCEL', @@ -16,10 +15,6 @@ __all__ = [ ] CLOUD_PROVIDERS = ('azure_rm', 'ec2', 'gce', 'vmware', 'openstack', 'rhv', 'satellite6', 'tower') -SCHEDULEABLE_PROVIDERS = CLOUD_PROVIDERS + ( - 'custom', - 'scm', -) PRIVILEGE_ESCALATION_METHODS = [ ('sudo', _('Sudo')), ('su', _('Su')), diff --git a/awx/main/migrations/0136_custom_inventory_scripts_removal_data.py b/awx/main/migrations/0136_custom_inventory_scripts_removal_data.py new file mode 100644 index 0000000000..7ade393641 --- /dev/null +++ b/awx/main/migrations/0136_custom_inventory_scripts_removal_data.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.16 on 2021-04-13 19:51 + +from django.db import migrations + +# AWX migration utils +from awx.main.migrations._rbac import delete_all_custom_script_roles +from awx.main.migrations._inventory_source import delete_custom_inv_source + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0135_schedule_sort_fallback_to_id'), + ] + + operations = [ + migrations.RunPython(delete_custom_inv_source), + migrations.RunPython(delete_all_custom_script_roles), + ] diff --git a/awx/main/migrations/0137_custom_inventory_scripts_removal.py b/awx/main/migrations/0137_custom_inventory_scripts_removal.py new file mode 100644 index 0000000000..10bfdc9484 --- /dev/null +++ b/awx/main/migrations/0137_custom_inventory_scripts_removal.py @@ -0,0 +1,81 @@ +# Generated by Django 2.2.16 on 2021-04-13 19:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0136_custom_inventory_scripts_removal_data'), + ] + + operations = [ + migrations.RemoveField( + model_name='activitystream', + name='custom_inventory_script', + ), + migrations.RemoveField( + model_name='inventorysource', + name='source_script', + ), + migrations.RemoveField( + model_name='inventoryupdate', + name='source_script', + ), + migrations.AlterField( + model_name='inventorysource', + name='source', + field=models.CharField( + choices=[ + ('file', 'File, Directory or Script'), + ('scm', 'Sourced from a Project'), + ('ec2', 'Amazon EC2'), + ('gce', 'Google Compute Engine'), + ('azure_rm', 'Microsoft Azure Resource Manager'), + ('vmware', 'VMware vCenter'), + ('satellite6', 'Red Hat Satellite 6'), + ('openstack', 'OpenStack'), + ('rhv', 'Red Hat Virtualization'), + ('tower', 'Ansible Tower'), + ], + default=None, + max_length=32, + ), + ), + migrations.AlterField( + model_name='inventoryupdate', + name='source', + field=models.CharField( + choices=[ + ('file', 'File, Directory or Script'), + ('scm', 'Sourced from a Project'), + ('ec2', 'Amazon EC2'), + ('gce', 'Google Compute Engine'), + ('azure_rm', 'Microsoft Azure Resource Manager'), + ('vmware', 'VMware vCenter'), + ('satellite6', 'Red Hat Satellite 6'), + ('openstack', 'OpenStack'), + ('rhv', 'Red Hat Virtualization'), + ('tower', 'Ansible Tower'), + ], + default=None, + max_length=32, + ), + ), + migrations.AlterUniqueTogether( + name='custominventoryscript', + unique_together=set(), + ), + migrations.RemoveField( + model_name='custominventoryscript', + name='admin_role', + ), + migrations.RemoveField( + model_name='custominventoryscript', + name='organization', + ), + migrations.RemoveField( + model_name='custominventoryscript', + name='read_role', + ), + ] diff --git a/awx/main/migrations/_inventory_source.py b/awx/main/migrations/_inventory_source.py index 40769a0cf9..b12cfa0918 100644 --- a/awx/main/migrations/_inventory_source.py +++ b/awx/main/migrations/_inventory_source.py @@ -90,3 +90,22 @@ def delete_cloudforms_inv_source(apps, schema_editor): if ct: ct.credentials.all().delete() ct.delete() + + +def delete_custom_inv_source(apps, schema_editor): + set_current_apps(apps) + InventorySource = apps.get_model('main', 'InventorySource') + InventoryUpdate = apps.get_model('main', 'InventoryUpdate') + ct, deletions = InventoryUpdate.objects.filter(source='custom').delete() + if ct: + logger.info('deleted {}'.format((ct, deletions))) + update_ct = deletions['main.InventoryUpdate'] + if update_ct: + logger.info('Deleted {} custom inventory script sources.'.format(update_ct)) + ct, deletions = InventorySource.objects.filter(source='custom').delete() + if ct: + logger.info('deleted {}'.format((ct, deletions))) + src_ct = deletions['main.InventorySource'] + if src_ct: + logger.info('Deleted {} custom inventory script updates.'.format(src_ct)) + logger.warning('Custom inventory scripts have been removed, see awx-manage XXXXX') diff --git a/awx/main/migrations/_rbac.py b/awx/main/migrations/_rbac.py index 07284b64cd..480fdd8929 100644 --- a/awx/main/migrations/_rbac.py +++ b/awx/main/migrations/_rbac.py @@ -47,6 +47,18 @@ def delete_all_user_roles(apps, schema_editor): role.delete() +def delete_all_custom_script_roles(apps, schema_editor): + ContentType = apps.get_model('contenttypes', "ContentType") + Role = apps.get_model('main', "Role") + cis_type = ContentType.objects.get(model='custominventoryscript') + role_ct = 0 + for role in Role.objects.filter(content_type=cis_type).iterator(): + role.delete() + role_ct += 1 + if role_ct: + logger.debug('Deleted roles corresponding to custom inventory sources.') + + UNIFIED_ORG_LOOKUPS = { # Job Templates had an implicit organization via their project 'jobtemplate': 'project', diff --git a/awx/main/models/activity_stream.py b/awx/main/models/activity_stream.py index 16e73bef56..e0eafbcfbb 100644 --- a/awx/main/models/activity_stream.py +++ b/awx/main/models/activity_stream.py @@ -74,8 +74,6 @@ class ActivityStream(models.Model): unified_job = models.ManyToManyField("UnifiedJob", blank=True, related_name='activity_stream_as_unified_job+') ad_hoc_command = models.ManyToManyField("AdHocCommand", blank=True) schedule = models.ManyToManyField("Schedule", blank=True) - # TODO: migrate away - custom_inventory_script = models.ManyToManyField("CustomInventoryScript", blank=True) execution_environment = models.ManyToManyField("ExecutionEnvironment", blank=True) notification_template = models.ManyToManyField("NotificationTemplate", blank=True) notification = models.ManyToManyField("Notification", blank=True) diff --git a/awx/main/models/base.py b/awx/main/models/base.py index e8c10f5fac..e39891f546 100644 --- a/awx/main/models/base.py +++ b/awx/main/models/base.py @@ -62,7 +62,7 @@ PROJECT_UPDATE_JOB_TYPE_CHOICES = [ (PERM_INVENTORY_CHECK, _('Check')), ] -CLOUD_INVENTORY_SOURCES = list(CLOUD_PROVIDERS) + ['scm', 'custom'] +CLOUD_INVENTORY_SOURCES = list(CLOUD_PROVIDERS) + ['scm'] VERBOSITY_CHOICES = [ (0, '0 (Normal)'), diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 58465ba78b..363544e0c6 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -821,7 +821,6 @@ class InventorySourceOptions(BaseModel): ('openstack', _('OpenStack')), ('rhv', _('Red Hat Virtualization')), ('tower', _('Ansible Tower')), - ('custom', _('Custom Script')), ] # From the options of the Django management base command @@ -845,14 +844,6 @@ class InventorySourceOptions(BaseModel): blank=True, default='', ) - # TODO: migrate away - source_script = models.ForeignKey( - 'CustomInventoryScript', - null=True, - default=None, - blank=True, - on_delete=models.SET_NULL, - ) source_vars = models.TextField( blank=True, default='', @@ -1329,7 +1320,6 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin, class CustomInventoryScript(CommonModelNameNotUnique, ResourceMixin): class Meta: app_label = 'main' - unique_together = [('name', 'organization')] ordering = ('name',) script = prevent_search( @@ -1339,21 +1329,6 @@ class CustomInventoryScript(CommonModelNameNotUnique, ResourceMixin): help_text=_('Inventory script contents'), ) ) - organization = models.ForeignKey( - 'Organization', - related_name='custom_inventory_scripts', - help_text=_('Organization owning this inventory script'), - blank=False, - null=True, - on_delete=models.SET_NULL, - ) - - admin_role = ImplicitRoleField( - parent_role='organization.admin_role', - ) - read_role = ImplicitRoleField( - parent_role=['organization.auditor_role', 'organization.member_role', 'admin_role'], - ) def get_absolute_url(self, request=None): return reverse('api:inventory_script_detail', kwargs={'pk': self.pk}, request=request) From f28ad90bf31699e0a38c7771aa17f97799642f95 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 15 Apr 2021 15:28:26 -0400 Subject: [PATCH 06/10] Remove source_script field from serializers Remove some other uses of source_script --- awx/api/serializers.py | 30 ------------------- awx/main/access.py | 4 +-- .../tests/functional/test_rbac_workflow.py | 2 +- 3 files changed, 3 insertions(+), 33 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 266cb01191..5b50bc999c 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -167,7 +167,6 @@ SUMMARIZABLE_FK_FIELDS = { 'current_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'license_error'), 'current_job': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'license_error'), 'inventory_source': ('id', 'name', 'source', 'last_updated', 'status'), - 'source_script': DEFAULT_SUMMARY_FIELDS, 'role': ('id', 'role_field'), 'notification_template': DEFAULT_SUMMARY_FIELDS, 'instance_group': ('id', 'name', 'controller_id', 'is_container_group'), @@ -1991,7 +1990,6 @@ class InventorySourceOptionsSerializer(BaseSerializer): '*', 'source', 'source_path', - 'source_script', 'source_vars', 'credential', 'enabled_var', @@ -2018,34 +2016,6 @@ class InventorySourceOptionsSerializer(BaseSerializer): raise serializers.ValidationError(_("`{}` is a prohibited environment variable".format(env_k))) return ret - def validate(self, attrs): - # TODO: Validate source - errors = {} - - source = attrs.get('source', self.instance and self.instance.source or '') - source_script = attrs.get('source_script', self.instance and self.instance.source_script or '') - if source == 'custom': - if source_script is None or source_script == '': - errors['source_script'] = _("If 'source' is 'custom', 'source_script' must be provided.") - else: - try: - if not self.instance: - dest_inventory = attrs.get('inventory', None) - if not dest_inventory: - errors['inventory'] = _("Must provide an inventory.") - else: - dest_inventory = self.instance.inventory - if dest_inventory and source_script.organization != dest_inventory.organization: - errors['source_script'] = _("The 'source_script' does not belong to the same organization as the inventory.") - except Exception: - errors['source_script'] = _("'source_script' doesn't exist.") - logger.exception('Problem processing source_script validation.') - - if errors: - raise serializers.ValidationError(errors) - - return super(InventorySourceOptionsSerializer, self).validate(attrs) - # TODO: remove when old 'credential' fields are removed def get_summary_fields(self, obj): summary_fields = super(InventorySourceOptionsSerializer, self).get_summary_fields(obj) diff --git a/awx/main/access.py b/awx/main/access.py index f8a5bb0cba..0647b598d2 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1030,7 +1030,7 @@ class InventorySourceAccess(NotificationAttachMixin, BaseAccess): model = InventorySource select_related = ('created_by', 'modified_by', 'inventory') - prefetch_related = ('credentials__credential_type', 'last_job', 'source_script', 'source_project') + prefetch_related = ('credentials__credential_type', 'last_job', 'source_project') def filtered_queryset(self): return self.model.objects.filter(inventory__in=Inventory.accessible_pk_qs(self.user, 'read_role')) @@ -1092,7 +1092,7 @@ class InventoryUpdateAccess(BaseAccess): 'modified_by', 'inventory_source', ) - prefetch_related = ('unified_job_template', 'instance_group', 'credentials__credential_type', 'inventory', 'source_script') + prefetch_related = ('unified_job_template', 'instance_group', 'credentials__credential_type', 'inventory') def filtered_queryset(self): return self.model.objects.filter(inventory_source__inventory__in=Inventory.accessible_pk_qs(self.user, 'read_role')) diff --git a/awx/main/tests/functional/test_rbac_workflow.py b/awx/main/tests/functional/test_rbac_workflow.py index 0195b1adf3..d48eb3f80b 100644 --- a/awx/main/tests/functional/test_rbac_workflow.py +++ b/awx/main/tests/functional/test_rbac_workflow.py @@ -198,7 +198,7 @@ class TestWFJTCopyAccess: def test_workflow_copy_no_start(self, wfjt, inventory, admin_user): # Test that un-startable resource doesn't block copy - inv_src = InventorySource.objects.create(inventory=inventory, source='custom', source_script=None) + inv_src = InventorySource.objects.create(inventory=inventory, source='file') assert not inv_src.can_update wfjt.workflow_job_template_nodes.create(unified_job_template=inv_src) access = WorkflowJobTemplateAccess(admin_user, save_messages=True) From b40e8d15c02847dc403d7d07e4d59994722ebf6d Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 15 Apr 2021 21:25:59 -0400 Subject: [PATCH 07/10] Fix migration bug Bump migration number Skip data migration for fresh migrations where ContentType for custom inventory scripts has not yet been created no scripts will exist in this case, so no-op --- ...data.py => 0137_custom_inventory_scripts_removal_data.py} | 2 +- ...s_removal.py => 0138_custom_inventory_scripts_removal.py} | 2 +- awx/main/migrations/_rbac.py | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) rename awx/main/migrations/{0136_custom_inventory_scripts_removal_data.py => 0137_custom_inventory_scripts_removal_data.py} (89%) rename awx/main/migrations/{0137_custom_inventory_scripts_removal.py => 0138_custom_inventory_scripts_removal.py} (97%) diff --git a/awx/main/migrations/0136_custom_inventory_scripts_removal_data.py b/awx/main/migrations/0137_custom_inventory_scripts_removal_data.py similarity index 89% rename from awx/main/migrations/0136_custom_inventory_scripts_removal_data.py rename to awx/main/migrations/0137_custom_inventory_scripts_removal_data.py index 7ade393641..bb965de3ab 100644 --- a/awx/main/migrations/0136_custom_inventory_scripts_removal_data.py +++ b/awx/main/migrations/0137_custom_inventory_scripts_removal_data.py @@ -10,7 +10,7 @@ from awx.main.migrations._inventory_source import delete_custom_inv_source class Migration(migrations.Migration): dependencies = [ - ('main', '0135_schedule_sort_fallback_to_id'), + ('main', '0136_scm_track_submodules'), ] operations = [ diff --git a/awx/main/migrations/0137_custom_inventory_scripts_removal.py b/awx/main/migrations/0138_custom_inventory_scripts_removal.py similarity index 97% rename from awx/main/migrations/0137_custom_inventory_scripts_removal.py rename to awx/main/migrations/0138_custom_inventory_scripts_removal.py index 10bfdc9484..fb781c8648 100644 --- a/awx/main/migrations/0137_custom_inventory_scripts_removal.py +++ b/awx/main/migrations/0138_custom_inventory_scripts_removal.py @@ -6,7 +6,7 @@ from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('main', '0136_custom_inventory_scripts_removal_data'), + ('main', '0137_custom_inventory_scripts_removal_data'), ] operations = [ diff --git a/awx/main/migrations/_rbac.py b/awx/main/migrations/_rbac.py index 480fdd8929..7f120645ef 100644 --- a/awx/main/migrations/_rbac.py +++ b/awx/main/migrations/_rbac.py @@ -50,7 +50,10 @@ def delete_all_user_roles(apps, schema_editor): def delete_all_custom_script_roles(apps, schema_editor): ContentType = apps.get_model('contenttypes', "ContentType") Role = apps.get_model('main', "Role") - cis_type = ContentType.objects.get(model='custominventoryscript') + try: + cis_type = ContentType.objects.get(model='custominventoryscript') + except ContentType.DoesNotExist: + return role_ct = 0 for role in Role.objects.filter(content_type=cis_type).iterator(): role.delete() From 8c0366675abd92769c6166a589a36dd66f283669 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 16 Apr 2021 10:25:21 -0400 Subject: [PATCH 08/10] Remove source_script from awx collection --- awx_collection/plugins/modules/tower_inventory_source.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/awx_collection/plugins/modules/tower_inventory_source.py b/awx_collection/plugins/modules/tower_inventory_source.py index 0353576b02..c1198015d7 100644 --- a/awx_collection/plugins/modules/tower_inventory_source.py +++ b/awx_collection/plugins/modules/tower_inventory_source.py @@ -48,10 +48,6 @@ options: description: - For an SCM based inventory source, the source path points to the file within the repo to use as an inventory. type: str - source_script: - description: - - Inventory script to be used when group type is C(custom). - type: str source_vars: description: - The variables or environment fields to apply to this source type. @@ -164,7 +160,6 @@ def main(): # source=dict(choices=["scm", "ec2", "gce", "azure_rm", "vmware", "satellite6", "openstack", "rhv", "tower", "custom"]), source_path=dict(), - source_script=dict(), source_vars=dict(type='dict'), enabled_var=dict(), enabled_value=dict(), @@ -194,7 +189,6 @@ def main(): new_name = module.params.get('new_name') inventory = module.params.get('inventory') organization = module.params.get('organization') - source_script = module.params.get('source_script') credential = module.params.get('credential') ee = module.params.get('execution_environment') source_project = module.params.get('source_project') From 5f39b6276de57a1597afe5663e5cc850e68bb6b2 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 16 Apr 2021 12:34:52 -0400 Subject: [PATCH 09/10] Add command to export custom inventory scripts to tar file --- .../commands/export_custom_scripts.py | 36 +++++++++++++++++++ awx/main/migrations/_inventory_source.py | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 awx/main/management/commands/export_custom_scripts.py diff --git a/awx/main/management/commands/export_custom_scripts.py b/awx/main/management/commands/export_custom_scripts.py new file mode 100644 index 0000000000..281fc886b3 --- /dev/null +++ b/awx/main/management/commands/export_custom_scripts.py @@ -0,0 +1,36 @@ +import tempfile +import tarfile +import stat +import os + +from awx.main.models.inventory import CustomInventoryScript + +from django.core.management.base import BaseCommand +from django.utils.text import slugify + + +class Command(BaseCommand): + + help = 'Export custom inventory scripts into a tarfile.' + + def add_arguments(self, parser): + parser.add_argument('--filename', dest='filename', type=str, default='custom_scripts.tar', help='Filename of the output tar file') + + def handle(self, **options): + tar_filename = options.get('filename') + + with tempfile.TemporaryDirectory() as tmpdirname: + with tarfile.open(tar_filename, "w") as tar: + + for cis in CustomInventoryScript.objects.all(): + # naming convention similar to project paths + slug_name = slugify(str(cis.name)).replace(u'-', u'_') + script_filename = u'_%d__%s' % (int(cis.pk), slug_name) + script_path = os.path.join(tmpdirname, script_filename) + + with open(script_path, 'w') as f: + f.write(cis.script) + os.chmod(script_path, stat.S_IRWXU) + tar.add(script_path, arcname=script_filename) + + print('Dump of old custom inventory scripts at {}'.format(tar_filename)) diff --git a/awx/main/migrations/_inventory_source.py b/awx/main/migrations/_inventory_source.py index b12cfa0918..e6a65a82d4 100644 --- a/awx/main/migrations/_inventory_source.py +++ b/awx/main/migrations/_inventory_source.py @@ -108,4 +108,4 @@ def delete_custom_inv_source(apps, schema_editor): src_ct = deletions['main.InventorySource'] if src_ct: logger.info('Deleted {} custom inventory script updates.'.format(src_ct)) - logger.warning('Custom inventory scripts have been removed, see awx-manage XXXXX') + logger.warning('Custom inventory scripts have been removed, see awx-manage export_custom_scripts') From 8241ebbc9a2b5e9b0f35978038d3c1524a6860dd Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Sat, 17 Apr 2021 10:54:07 -0400 Subject: [PATCH 10/10] Fix migration order of operations bug --- .../migrations/0137_custom_inventory_scripts_removal_data.py | 2 -- awx/main/migrations/0138_custom_inventory_scripts_removal.py | 3 +++ awx/main/migrations/_rbac.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/awx/main/migrations/0137_custom_inventory_scripts_removal_data.py b/awx/main/migrations/0137_custom_inventory_scripts_removal_data.py index bb965de3ab..46490d2517 100644 --- a/awx/main/migrations/0137_custom_inventory_scripts_removal_data.py +++ b/awx/main/migrations/0137_custom_inventory_scripts_removal_data.py @@ -3,7 +3,6 @@ from django.db import migrations # AWX migration utils -from awx.main.migrations._rbac import delete_all_custom_script_roles from awx.main.migrations._inventory_source import delete_custom_inv_source @@ -15,5 +14,4 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython(delete_custom_inv_source), - migrations.RunPython(delete_all_custom_script_roles), ] diff --git a/awx/main/migrations/0138_custom_inventory_scripts_removal.py b/awx/main/migrations/0138_custom_inventory_scripts_removal.py index fb781c8648..8dddf46667 100644 --- a/awx/main/migrations/0138_custom_inventory_scripts_removal.py +++ b/awx/main/migrations/0138_custom_inventory_scripts_removal.py @@ -2,6 +2,8 @@ from django.db import migrations, models +from awx.main.migrations._rbac import delete_all_custom_script_roles + class Migration(migrations.Migration): @@ -78,4 +80,5 @@ class Migration(migrations.Migration): model_name='custominventoryscript', name='read_role', ), + migrations.RunPython(delete_all_custom_script_roles), ] diff --git a/awx/main/migrations/_rbac.py b/awx/main/migrations/_rbac.py index 7f120645ef..96e6334d83 100644 --- a/awx/main/migrations/_rbac.py +++ b/awx/main/migrations/_rbac.py @@ -59,7 +59,7 @@ def delete_all_custom_script_roles(apps, schema_editor): role.delete() role_ct += 1 if role_ct: - logger.debug('Deleted roles corresponding to custom inventory sources.') + logger.debug('Deleted {} roles corresponding to custom inventory sources.'.format(role_ct)) UNIFIED_ORG_LOOKUPS = {