From 861ba8a72737923d24a99eb37140787b5f5b2abd Mon Sep 17 00:00:00 2001 From: Nikhil Date: Sun, 22 Jan 2023 13:19:46 +0530 Subject: [PATCH] add some helpers functions in validate and some other minor fixes make black changes increase the number of queries to 30 fix the flake failure add functional changes for bulk job launch and some minor fixes pull changes --- awx/api/serializers.py | 247 ++++++++++++++----------- awx/main/access.py | 8 +- awx/main/tests/functional/test_bulk.py | 112 ++++++++++- 3 files changed, 261 insertions(+), 106 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 7eec9473c6..a6c7c09e68 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2064,7 +2064,8 @@ class BulkHostCreateSerializer(serializers.Serializer): # This actually updates the cached "total_hosts" field on the inventory update_inventory_computed_fields.delay(validated_data['inventory'].id) - return {"created": len(result), "url": InventorySerializer().get_related(validated_data['inventory'])['hosts']} + ids = [item.id for item in result] + return {"created": ids, "url": InventorySerializer().get_related(validated_data['inventory'])['hosts']} class GroupTreeSerializer(GroupSerializer): @@ -4562,13 +4563,17 @@ class BulkJobNodeSerializer(serializers.Serializer): 'limit', 'labels', 'instance_groups', - 'execution_environment' 'scm_branch', - 'verbosity' 'forks' 'char_prompts', + 'execution_environment', + 'scm_branch', + 'verbosity', + 'forks', + 'char_prompts', 'diff_mode', 'extra_data', 'job_slice_count', 'job_tags', - 'job_type' 'skip_tags', + 'job_type', + 'skip_tags', 'survey_passwords', 'timeout', # these are related objects and we need to add extra validation for them in the parent BulkJobLaunchSerializer @@ -4576,11 +4581,11 @@ class BulkJobNodeSerializer(serializers.Serializer): ) -class BulkJobLaunchSerializer(serializers.Serializer): +class BulkJobLaunchSerializer(BaseSerializer): name = serializers.CharField(max_length=512, write_only=True, required=False) # limited by max name of jobs jobs = BulkJobNodeSerializer(many=True, allow_empty=False, write_only=True, max_length=1000) - description: serializers.CharField(write_only=True, required=False, allow_blank=False) - extra_vars: serializers.CharField(write_only=True, required=False, allow_blank=False) + description = serializers.CharField(write_only=True, required=False, allow_blank=False) + extra_vars = serializers.CharField(write_only=True, required=False, allow_blank=False) organization = serializers.PrimaryKeyRelatedField( queryset=Organization.objects.all(), required=False, @@ -4588,36 +4593,23 @@ class BulkJobLaunchSerializer(serializers.Serializer): allow_null=True, help_text=_('Inherit permissions from organization roles.'), ) - # inventory: "", # Here we can use PrimaryKeyRelatedField so it will automagically do rbac/turn into object - limit: serializers.CharField(write_only=True, required=False, allow_blank=False) - scm_branch: serializers.CharField(write_only=True, required=False, allow_blank=False) + inventory = serializers.PrimaryKeyRelatedField(queryset=Inventory.objects.all(), required=False, write_only=True) + limit = serializers.CharField(write_only=True, required=False, allow_blank=False) + scm_branch = serializers.CharField(write_only=True, required=False, allow_blank=False) + # not implemented yet # webhook_service: null, # Here we can use PrimaryKeyRelatedField so it will automagically do rbac/turn into object, I think, I'm actually not sure how to use this # webhook_credential: null, # Here we can use PrimaryKeyRelatedField so it will automagically do rbac/turn into object I think, I'm actually not sure how to use this - skip_tags: serializers.CharField(write_only=True, required=False, allow_blank=False) - job_tags: serializers.CharField(write_only=True, required=False, allow_blank=False) - is_bulk_job: serializers.BooleanField(default=True) + skip_tags = serializers.CharField(write_only=True, required=False, allow_blank=False) + job_tags = serializers.CharField(write_only=True, required=False, allow_blank=False) class Meta: - fields = ('name', 'jobs', 'description', 'limit') + model = WorkflowJob + fields = ('name', 'jobs', 'description', 'extra_vars', 'organization', 'inventory', 'limit', 'scm_branch', 'skip_tags', 'job_tags') read_only_fields = () def validate(self, attrs): - request = self.context.get('request', None) - # validate Organization - # - If the orgs is not set, set it to the org of the launching user - # - If the user is part of multiple orgs, throw a validation error saying user is part of multiple orgs, please provide one - - if 'organization' not in attrs or attrs['organization'] == None or attrs['oganization'] == '': - if Organization.accessible_pk_qs(request.user, 'read_role').count() == 1: - for tup in Organization.accessible_pk_qs(request.user, 'read_role').all(): - attrs['organization'] = tup[0] - elif Organization.accessible_pk_qs(request.user, 'read_role').count() > 1: - raise serializers.ValidationError(_(f"User has permission to multiple Organizations, please set one of them in the request")) - else: - raise serializers.ValidationError(_(f"User not part of any organization, please assign an organization to assign to the bulk job")) - requested_org = {attrs['organization']} - + self.check_organization_permission(attrs, request) identifiers = set() for node in attrs['jobs']: if 'identifier' in node: @@ -4646,96 +4638,39 @@ class BulkJobLaunchSerializer(serializers.Serializer): # If we are not a superuser, check we have permissions # TODO: As we add other related items, we need to add them here if request and not request.user.is_superuser: - allowed_orgs = set() - if requested_org: - [allowed_orgs.add(tup[0]) for tup in Organization.accessible_pk_qs(request.user, 'read_role').all()] - if requested_org not in allowed_orgs: - ValidationError(_(f"Organization {requested_org} not found")) - allowed_ujts = set() - [allowed_ujts.add(tup[0]) for tup in UnifiedJobTemplate.accessible_pk_qs(request.user, 'execute_role').all()] - [allowed_ujts.add(tup[0]) for tup in UnifiedJobTemplate.accessible_pk_qs(request.user, 'admin_role').all()] - [allowed_ujts.add(tup[0]) for tup in UnifiedJobTemplate.accessible_pk_qs(request.user, 'update_role').all()] - accessible_inventories_qs = Inventory.accessible_pk_qs(request.user, 'update_role') - [allowed_ujts.add(tup[0]) for tup in InventorySource.objects.filter(inventory__in=accessible_inventories_qs).values_list('id')] - - if requested_ujts - allowed_ujts: - not_allowed = requested_ujts - allowed_ujts - raise serializers.ValidationError(_(f"Unified Job Templates {not_allowed} not found.")) - + self.check_unified_job_permission(request, requested_ujts) if requested_use_inventories: - accessible_use_inventories = {tup[0] for tup in Inventory.accessible_pk_qs(request.user, 'use_role')} - if requested_use_inventories - accessible_use_inventories: - not_allowed = requested_use_inventories - accessible_use_inventories - raise serializers.ValidationError(_(f"Inventories {not_allowed} not found.")) + self.check_inventory_permission(request, requested_use_inventories) if requested_use_credentials: - accessible_use_credentials = {tup[0] for tup in Credential.accessible_pk_qs(request.user, 'use_role').all()} - if requested_use_credentials - accessible_use_credentials: - not_allowed = requested_use_credentials - accessible_use_credentials - raise serializers.ValidationError(_(f"Credentials {not_allowed} not found.")) + self.check_credential_permission(request, requested_use_credentials) if requested_use_labels: - accessible_use_labels = {tup.id for tup in Label.objects.all()} - if requested_use_labels - accessible_use_labels: - not_allowed = requested_use_labels - accessible_use_labels - raise serializers.ValidationError(_(f"Labels {not_allowed} not found")) + self.check_label_permission(requested_use_labels) if requested_use_instance_groups: - # only org admins are allowed to see instance groups - organization_admin_qs = Organization.accessible_pk_qs(request.user, 'admin_role').all() - if organization_admin_qs: - accessible_use_instance_groups = {tup.id for tup in InstanceGroup.objects.all()} - if requested_use_instance_groups - accessible_use_instance_groups: - not_allowed = requested_use_instance_groups - accessible_use_instance_groups - raise serializers.ValidationError(_(f"Instance Groups {not_allowed} not found")) + self.check_instance_group_permission(request, requested_use_instance_groups) if requested_use_execution_environments: - accessible_execution_env = { - tup.id - for tup in ExecutionEnvironment.objects.filter( - Q(organization__in=Organization.accessible_pk_qs(request.user, 'read_role')) | Q(organization__isnull=True) - ).distinct() - } - if requested_use_execution_environments - accessible_execution_env: - not_allowed = requested_use_execution_environments - accessible_execution_env - raise serializers.ValidationError(_(f"Execution Environments {not_allowed} not found")) + self.check_instance_group_permission(request, requested_use_instance_groups) # all of the unified job templates and related items have now been checked, we can now grab the objects from the DB - # TODO: As we add more related objects like Label, InstanceGroup, etc we need to add them here - objectified_jobs = [] - key_to_obj_map = { - "unified_job_template": {obj.id: obj for obj in UnifiedJobTemplate.objects.filter(id__in=requested_ujts)}, - "inventory": {obj.id: obj for obj in Inventory.objects.filter(id__in=requested_use_inventories)}, - "credentials": {obj.id: obj for obj in Credential.objects.filter(id__in=requested_use_credentials)}, - "labels": {obj.id: obj for obj in Label.objects.filter(id__in=requested_use_labels)}, - "instance_groups": {obj.id: obj for obj in InstanceGroup.objects.filter(id__in=requested_use_instance_groups)}, - "execution_environment": {obj.id: obj for obj in ExecutionEnvironment.objects.filter(id__in=requested_use_execution_environments)}, - } + jobs_object = self.get_objectified_jobs( + attrs, + requested_ujts, + requested_use_inventories, + requested_use_credentials, + requested_use_labels, + requested_use_instance_groups, + requested_use_execution_environments, + ) - # This loop is generalized so we should only have to add related items to the key_to_obj_map - for job in attrs['jobs']: - objectified_job = {} - for key, value in job.items(): - if key in key_to_obj_map: - if isinstance(value, int): - objectified_job[key] = key_to_obj_map[key][value] - elif isinstance(value, list): - objectified_job[key] = [] - for item in value: - objectified_job[key].append(key_to_obj_map[key][item]) - else: - objectified_job[key] = value - objectified_jobs.append(objectified_job) - - attrs['jobs'] = objectified_jobs - # map the organization object - for obj in Organization.objects.filter(id__in=requested_org): - attrs['organization'] = obj + attrs['jobs'] = jobs_object + attrs = super().validate(attrs) return attrs def create(self, validated_data): job_node_data = validated_data.pop('jobs') - # FIXME: Need to set organization on the WorkflowJob in order for users to be able to see it -- # normally their permission is sourced from the underlying WorkflowJobTemplate # maybe we need to add Organization to WorkflowJob @@ -4812,8 +4747,114 @@ class BulkJobLaunchSerializer(serializers.Serializer): wfj.status = 'pending' wfj.save() + return WorkflowJobSerializer().to_representation(wfj) + def check_organization_permission(self, attrs, request): + # validate Organization + # - If the orgs is not set, set it to the org of the launching user + # - If the user is part of multiple orgs, throw a validation error saying user is part of multiple orgs, please provide one + if 'organization' not in attrs or attrs['organization'] == None or attrs['organization'] == '': + if Organization.accessible_pk_qs(request.user, 'read_role').count() == 1: + for tup in Organization.accessible_pk_qs(request.user, 'read_role').all(): + attrs['organization'] = Organization.objects.filter(id__in=str(tup[0])).first() + elif Organization.accessible_pk_qs(request.user, 'read_role').count() > 1: + raise serializers.ValidationError("User has permission to multiple Organizations, please set one of them in the request") + else: + raise serializers.ValidationError("User not part of any organization, please assign an organization to assign to the bulk job") + else: + allowed_orgs = set() + requested_org = attrs['organization'] + if request and not request.user.is_superuser: + [allowed_orgs.add(tup[0]) for tup in Organization.accessible_pk_qs(request.user, 'read_role').all()] + if requested_org.id not in allowed_orgs: + raise ValidationError(_(f"Organization {requested_org.id} not found")) + + def check_unified_job_permission(self, request, requested_ujts): + allowed_ujts = set() + [allowed_ujts.add(tup[0]) for tup in UnifiedJobTemplate.accessible_pk_qs(request.user, 'execute_role').all()] + [allowed_ujts.add(tup[0]) for tup in UnifiedJobTemplate.accessible_pk_qs(request.user, 'admin_role').all()] + [allowed_ujts.add(tup[0]) for tup in UnifiedJobTemplate.accessible_pk_qs(request.user, 'update_role').all()] + accessible_inventories_qs = Inventory.accessible_pk_qs(request.user, 'update_role') + [allowed_ujts.add(tup[0]) for tup in InventorySource.objects.filter(inventory__in=accessible_inventories_qs).values_list('id')] + + if requested_ujts - allowed_ujts: + not_allowed = requested_ujts - allowed_ujts + raise serializers.ValidationError(_(f"Unified Job Templates {not_allowed} not found.")) + + def check_inventory_permission(self, request, requested_use_inventories): + accessible_use_inventories = {tup[0] for tup in Inventory.accessible_pk_qs(request.user, 'use_role')} + if requested_use_inventories - accessible_use_inventories: + not_allowed = requested_use_inventories - accessible_use_inventories + raise serializers.ValidationError(_(f"Inventories {not_allowed} not found.")) + + def check_credential_permission(self, request, requested_use_credentials): + accessible_use_credentials = {tup[0] for tup in Credential.accessible_pk_qs(request.user, 'use_role').all()} + if requested_use_credentials - accessible_use_credentials: + not_allowed = requested_use_credentials - accessible_use_credentials + raise serializers.ValidationError(_(f"Credentials {not_allowed} not found.")) + + def check_label_permission(self, requested_use_labels): + accessible_use_labels = {tup.id for tup in Label.objects.all()} + if requested_use_labels - accessible_use_labels: + not_allowed = requested_use_labels - accessible_use_labels + raise serializers.ValidationError(_(f"Labels {not_allowed} not found")) + + def check_instance_group_permission(self, request, requested_use_instance_groups): + # only org admins are allowed to see instance groups + organization_admin_qs = Organization.accessible_pk_qs(request.user, 'admin_role').all() + if organization_admin_qs: + accessible_use_instance_groups = {tup.id for tup in InstanceGroup.objects.all()} + if requested_use_instance_groups - accessible_use_instance_groups: + not_allowed = requested_use_instance_groups - accessible_use_instance_groups + raise serializers.ValidationError(_(f"Instance Groups {not_allowed} not found")) + + def check_execution_environment_permission(self, request, requested_use_execution_environments): + accessible_execution_env = { + tup.id + for tup in ExecutionEnvironment.objects.filter( + Q(organization__in=Organization.accessible_pk_qs(request.user, 'read_role')) | Q(organization__isnull=True) + ).distinct() + } + if requested_use_execution_environments - accessible_execution_env: + not_allowed = requested_use_execution_environments - accessible_execution_env + raise serializers.ValidationError(_(f"Execution Environments {not_allowed} not found")) + + def get_objectified_jobs( + self, + attrs, + requested_ujts, + requested_use_inventories, + requested_use_credentials, + requested_use_labels, + requested_use_instance_groups, + requested_use_execution_environments, + ): + objectified_jobs = [] + key_to_obj_map = { + "unified_job_template": {obj.id: obj for obj in UnifiedJobTemplate.objects.filter(id__in=requested_ujts)}, + "inventory": {obj.id: obj for obj in Inventory.objects.filter(id__in=requested_use_inventories)}, + "credentials": {obj.id: obj for obj in Credential.objects.filter(id__in=requested_use_credentials)}, + "labels": {obj.id: obj for obj in Label.objects.filter(id__in=requested_use_labels)}, + "instance_groups": {obj.id: obj for obj in InstanceGroup.objects.filter(id__in=requested_use_instance_groups)}, + "execution_environment": {obj.id: obj for obj in ExecutionEnvironment.objects.filter(id__in=requested_use_execution_environments)}, + } + # This loop is generalized so we should only have to add related items to the key_to_obj_map + for job in attrs['jobs']: + objectified_job = {} + for key, value in job.items(): + if key in key_to_obj_map: + if isinstance(value, int): + objectified_job[key] = key_to_obj_map[key][value] + elif isinstance(value, list): + objectified_job[key] = [] + for item in value: + objectified_job[key].append(key_to_obj_map[key][item]) + else: + objectified_job[key] = value + objectified_jobs.append(objectified_job) + return objectified_jobs + class NotificationTemplateSerializer(BaseSerializer): show_capabilities = ['edit', 'delete', 'copy'] diff --git a/awx/main/access.py b/awx/main/access.py index fa3d0c7677..f5eda98ad4 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1997,7 +1997,11 @@ class WorkflowJobNodeAccess(BaseAccess): ) def filtered_queryset(self): - return self.model.objects.filter(workflow_job__unified_job_template__in=UnifiedJobTemplate.accessible_pk_qs(self.user, 'read_role')) + return self.model.objects.filter( + Q(workflow_job__unified_job_template__in=UnifiedJobTemplate.accessible_pk_qs(self.user, 'read_role')) + | Q(workflow_job__created_by_id=self.user.id, workflow_job__is_bulk_job=True) + | Q(workflow_job__organization__in=Organization.objects.filter(Q(admin_role__members=self.user)), workflow_job__is_bulk_job=True) + ) @check_superuser def can_add(self, data): @@ -2125,7 +2129,7 @@ class WorkflowJobAccess(BaseAccess): def filtered_queryset(self): return WorkflowJob.objects.filter( Q(unified_job_template__in=UnifiedJobTemplate.accessible_pk_qs(self.user, 'read_role')) - | Q(created_by__in=str(self.user.id), is_bulk_job=True) + | Q(created_by_id=self.user.id, is_bulk_job=True) | Q(organization__in=Organization.objects.filter(Q(admin_role__members=self.user)), is_bulk_job=True) ) diff --git a/awx/main/tests/functional/test_bulk.py b/awx/main/tests/functional/test_bulk.py index 7c69be4335..7916e2190d 100644 --- a/awx/main/tests/functional/test_bulk.py +++ b/awx/main/tests/functional/test_bulk.py @@ -8,6 +8,8 @@ import json from contextlib import contextmanager from django.test.utils import CaptureQueriesContext from django.db import connections +from awx.main.models.jobs import JobTemplate +from awx.main.models import Organization, Inventory @contextmanager @@ -19,7 +21,7 @@ def withAssertNumQueriesLessThan(num_queries): @pytest.mark.django_db -@pytest.mark.parametrize('num_hosts, num_queries', [(9, 15), (99, 20), (999, 25)]) +@pytest.mark.parametrize('num_hosts, num_queries', [(9, 15), (99, 20), (999, 30)]) def test_bulk_host_create_num_queries(organization, inventory, post, get, user, num_hosts, num_queries): ''' If I am a... @@ -84,3 +86,111 @@ def test_bulk_host_create_rbac(organization, inventory, post, get, user): bulk_host_create_response = post( reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': [{'name': f'foobar2-{indx}'}]}, u, expect=400 ).data + + +@pytest.mark.django_db +def test_bulk_job_launch(job_template, organization, inventory, project, credential, post, get, user): + ''' + if I don't have access to the unified job templare + ... I can't launch the bulk job + ''' + normal_user = user('normal_user', False) + jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml') + jt.save() + organization.member_role.members.add(normal_user) + jt.execute_role.members.add(normal_user) + bulk_job_launch_response = post( + reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=201 + ).data + + +@pytest.mark.django_db +def test_bulk_job_launch_no_access_to_job_template(job_template, organization, inventory, project, credential, post, get, user): + ''' + if I don't have access to the unified job templare + ... I can't launch the bulk job + ''' + normal_user = user('normal_user', False) + jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml') + jt.save() + organization.member_role.members.add(normal_user) + bulk_job_launch_response = post( + reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=400 + ).data + + +@pytest.mark.django_db +def test_bulk_job_launch_no_org_assigned(job_template, organization, inventory, project, credential, post, get, user): + ''' + if I am not part of any organization... + ... I can't launch the bulk job + ''' + normal_user = user('normal_user', False) + jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml') + jt.save() + jt.execute_role.members.add(normal_user) + bulk_job_launch_response = post( + reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=400 + ).data + + +@pytest.mark.django_db +def test_bulk_job_launch_multiple_org_assigned(job_template, organization, inventory, project, credential, post, get, + user): + ''' + if I am part of multiple organization... + and if I do not provide org at the launch time + ... I can't launch the bulk job + ''' + normal_user = user('normal_user', False) + org1 = Organization.objects.create(name='foo1') + org2 = Organization.objects.create(name='foo2') + org1.member_role.members.add(normal_user) + org2.member_role.members.add(normal_user) + jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml') + jt.save() + jt.execute_role.members.add(normal_user) + bulk_job_launch_response = post( + reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=400 + ).data + + +@pytest.mark.django_db +def test_bulk_job_launch_specific_org(job_template, organization, inventory, project, credential, post, get, + user): + ''' + if I am part of multiple organization... + and if I provide org at the launch time + ... I can launch the bulk job + ''' + normal_user = user('normal_user', False) + org1 = Organization.objects.create(name='foo1') + org2 = Organization.objects.create(name='foo2') + org1.member_role.members.add(normal_user) + org2.member_role.members.add(normal_user) + jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml') + jt.save() + jt.execute_role.members.add(normal_user) + bulk_job_launch_response = post( + reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}], 'organization': org1.id}, normal_user, expect=201 + ).data + +@pytest.mark.django_db +def test_bulk_job_launch_inventory_no_access(job_template, organization, inventory, project, credential, post, get, + user): + ''' + if I don't have access to the inventory... + and if I try to use it at the launch time + ... I can't launch the bulk job + ''' + normal_user = user('normal_user', False) + org1 = Organization.objects.create(name='foo1') + org2 = Organization.objects.create(name='foo2') + jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml') + jt.save() + org1.member_role.members.add(normal_user) + inv = Inventory.objects.create(name='inv1', organization=org2) + jt.execute_role.members.add(normal_user) + bulk_job_launch_response = post( + reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id, 'inventory': inv.id}]}, normal_user, expect=400 + ).data