diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 6e23f3298f..b24c70fac3 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -109,11 +109,14 @@ from awx.main.utils import ( encrypt_dict, prefetch_page_capabilities, truncate_stdout, + get_licenser, ) from awx.main.utils.filters import SmartFilter from awx.main.utils.named_url_graph import reset_counters from awx.main.scheduler.task_manager_models import TaskManagerModels from awx.main.redact import UriCleaner, REPLACE_STR +from awx.main.signals import update_inventory_computed_fields + from awx.main.validators import vars_validate_or_raise @@ -1853,7 +1856,7 @@ class HostSerializer(BaseSerializerWithVariables): vars_dict = parse_yaml_or_json(variables) vars_dict['ansible_ssh_port'] = port attrs['variables'] = json.dumps(vars_dict) - if Group.objects.filter(name=name, inventory=inventory).exists(): + if inventory and Group.objects.filter(name=name, inventory=inventory).exists(): raise serializers.ValidationError(_('A Group with that name already exists.')) return super(HostSerializer, self).validate(attrs) @@ -1945,6 +1948,123 @@ class GroupSerializer(BaseSerializerWithVariables): return ret +class BulkHostSerializer(HostSerializer): + name = serializers.CharField(required=True, allow_blank=False, max_length=512) + instance_id = serializers.CharField(required=False, max_length=1024) + description = serializers.CharField(required=False) + enabled = serializers.BooleanField(default=True, required=False) + variables = serializers.CharField(allow_blank=True, required=False) + + class Meta: + fields = ( + 'name', + 'enabled', + 'instance_id', + 'description', + 'variables', + ) + + +class BulkHostCreateSerializer(serializers.Serializer): + inventory = serializers.PrimaryKeyRelatedField( + queryset=Inventory.objects.all(), required=True, write_only=True, help_text=_('Primary Key ID of inventory to add hosts to.') + ) + hosts = serializers.ListField(child=BulkHostSerializer(), allow_empty=False, max_length=1000, write_only=True, help_text=_('Hosts to be created.')) + + class Meta: + fields = ('inventory', 'hosts') + read_only_fields = () + + def raise_if_cannot_add_hosts(self, attrs): + validation_info = get_licenser().validate() + + org = attrs['inventory'].organization + + if org: + org_active_count = Host.objects.org_active_count(org.id) + new_hosts = [h['name'] for h in attrs['hosts']] + org_net_new_host_count = Host.objects.filter(inventory__organization=org.id).exclude(name__in=new_hosts).count() + if org.max_hosts > 0 and org_active_count + org_net_new_host_count > org.max_hosts: + raise PermissionDenied( + _( + "You have already reached the maximum number of %s hosts" + " allowed for your organization. Contact your System Administrator" + " for assistance." % org.max_hosts + ) + ) + + # Don't check license if it is open license + if validation_info.get('license_type', 'UNLICENSED') == 'open': + return True + + sys_free_instances = validation_info.get('free_instances', 0) + system_net_new_host_count = Host.objects.exclude(name__in=new_hosts).count() + + if system_net_new_host_count > sys_free_instances: + hard_error = validation_info.get('trial', False) is True or validation_info['instance_count'] == 10 + if hard_error: + # Only raise permission error for trial, otherwise just log a warning as we do in other inventory import situations + raise PermissionDenied(_("Host count exceeds available instances.")) + logger.warning(_("Number of hosts allowed by license has been exceeded.")) + + return True + + def validate(self, attrs): + request = self.context.get('request', None) + inv = attrs['inventory'] + if request and not request.user.is_superuser: + if inv.organization: + org_admin_orgs = {tup[0] for tup in Organization.accessible_pk_qs(request.user, 'admin_role')} + inv_admin_orgs = {tup[0] for tup in Organization.accessible_pk_qs(request.user, 'inventory_admin_role')} + is_org_admin = inv.organization.id in org_admin_orgs + is_org_inv_admin = inv.organization.id in inv_admin_orgs + else: + is_org_admin = False + is_org_inv_admin = False + # This may not work, need to figure out what the role is called + is_inventory_admin = inv.admin_role.members.filter(id=request.user.id).exists() + if not any([is_inventory_admin, is_org_admin, is_org_inv_admin]): + raise serializers.ValidationError(_(f'Inventory with id {inv.id} not found or lack permissions to add hosts.')) + current_hostnames = {h[0] for h in Host.objects.filter(inventory=inv).values_list('name').all()} + new_names = [host['name'] for host in attrs['hosts']] + duplicate_new_names = [n for n in new_names if n in current_hostnames or new_names.count(n) > 1] + if duplicate_new_names: + raise serializers.ValidationError(_(f'Hostnames must be unique in an inventory. Duplicates found: {duplicate_new_names}')) + + self.raise_if_cannot_add_hosts(attrs) + + _now = now() + for host in attrs['hosts']: + host['created'] = _now + host['modified'] = _now + host['inventory'] = inv + return attrs + + def create(self, validated_data): + # This assumes total_hosts is up to date, and it can get out of date if the inventory computed fields have not been updated lately. + # If we wanted to side step this we could query Hosts.objects.filter(inventory...) + old_total_hosts = validated_data['inventory'].total_hosts + result = [Host(**attrs) for attrs in validated_data['hosts']] + try: + Host.objects.bulk_create(result) + except Exception as e: + raise serializers.ValidationError({"detail": _(f"{e}")}) + new_total_hosts = old_total_hosts + len(result) + request = self.context.get('request', None) + changes = {'total_hosts': [old_total_hosts, new_total_hosts]} + activity_entry = ActivityStream.objects.create( + operation='update', + object1='inventory', + changes=json.dumps(changes), + actor=request.user, + ) + activity_entry.inventory.add(validated_data['inventory']) + + # This actually updates the cached "total_hosts" field on the inventory + update_inventory_computed_fields.delay(validated_data['inventory'].id) + return {"created": len(result), "url": InventorySerializer().get_related(validated_data['inventory'])['hosts']} + + class GroupTreeSerializer(GroupSerializer): children = serializers.SerializerMethodField() diff --git a/awx/api/templates/api/bulk_host_create_view.md b/awx/api/templates/api/bulk_host_create_view.md new file mode 100644 index 0000000000..8ac56f4d6a --- /dev/null +++ b/awx/api/templates/api/bulk_host_create_view.md @@ -0,0 +1,16 @@ +# Bulk Host Create + +This endpoint allows the client to create multiple hosts and associate them with an inventory. They may do this by providing the inventory ID and a list of json that would normally be provided to create hosts. + +Example: + +``` +{ +"inventory": 1, +"hosts": [ + {"name": "example1.com"}, + {"name": "example2.com"} +] + +} +``` diff --git a/awx/api/templates/api/bulk_view.md b/awx/api/templates/api/bulk_view.md new file mode 100644 index 0000000000..6ef5fb65ca --- /dev/null +++ b/awx/api/templates/api/bulk_view.md @@ -0,0 +1,3 @@ +# Bulk Actions + +This endpoint lists available bulk action APIs. diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index 5d0818b191..5d6121ad6c 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -31,6 +31,10 @@ from awx.api.views import ( ApplicationOAuth2TokenList, OAuth2ApplicationDetail, ) +from awx.api.views.bulk import ( + BulkView, + BulkHostCreateView, +) from awx.api.views.mesh_visualizer import MeshVisualizer from awx.api.views.metrics import MetricsView @@ -136,6 +140,8 @@ v2_urls = [ re_path(r'^activity_stream/', include(activity_stream_urls)), re_path(r'^workflow_approval_templates/', include(workflow_approval_template_urls)), re_path(r'^workflow_approvals/', include(workflow_approval_urls)), + re_path(r'^bulk/host_create/$', BulkHostCreateView.as_view(), name='bulk_host_create'), + re_path(r'^bulk/$', BulkView.as_view(), name='bulk'), ] diff --git a/awx/api/views/bulk.py b/awx/api/views/bulk.py new file mode 100644 index 0000000000..d0f4046f98 --- /dev/null +++ b/awx/api/views/bulk.py @@ -0,0 +1,49 @@ +from collections import OrderedDict + +from rest_framework.permissions import IsAuthenticated +from rest_framework.renderers import JSONRenderer +from rest_framework.reverse import reverse +from rest_framework import status +from rest_framework.response import Response + +from awx.api.generics import ( + GenericAPIView, + APIView, +) +from awx.api import ( + serializers, + renderers, +) + + +class BulkView(APIView): + _ignore_model_permissions = True + permission_classes = [IsAuthenticated] + renderer_classes = [ + renderers.BrowsableAPIRenderer, + JSONRenderer, + ] + allowed_methods = ['GET', 'OPTIONS'] + + def get(self, request, format=None): + '''List top level resources''' + data = OrderedDict() + data['bulk_host_create'] = reverse('api:bulk_host_create', request=request) + return Response(data) + + +class BulkHostCreateView(GenericAPIView): + _ignore_model_permissions = True + permission_classes = [IsAuthenticated] + serializer_class = serializers.BulkHostCreateSerializer + allowed_methods = ['GET', 'POST', 'OPTIONS'] + + def get(self, request): + return Response({"detail": "Bulk create hosts with this endpoint"}, status=status.HTTP_200_OK) + + def post(self, request): + serializer = serializers.BulkHostCreateSerializer(data=request.data, context={'request': request}) + if serializer.is_valid(): + result = serializer.create(serializer.validated_data) + return Response(result, status=status.HTTP_201_CREATED) + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) diff --git a/awx/api/views/root.py b/awx/api/views/root.py index 01a742bf74..7211600a37 100644 --- a/awx/api/views/root.py +++ b/awx/api/views/root.py @@ -121,6 +121,7 @@ class ApiVersionRootView(APIView): data['workflow_job_template_nodes'] = reverse('api:workflow_job_template_node_list', request=request) data['workflow_job_nodes'] = reverse('api:workflow_job_node_list', request=request) data['mesh_visualizer'] = reverse('api:mesh_visualizer_view', request=request) + data['bulk'] = reverse('api:bulk', request=request) return Response(data) diff --git a/awx/main/tests/functional/test_bulk.py b/awx/main/tests/functional/test_bulk.py new file mode 100644 index 0000000000..7c69be4335 --- /dev/null +++ b/awx/main/tests/functional/test_bulk.py @@ -0,0 +1,86 @@ +import pytest + +from uuid import uuid4 + +from awx.api.versioning import reverse + +import json +from contextlib import contextmanager +from django.test.utils import CaptureQueriesContext +from django.db import connections + + +@contextmanager +def withAssertNumQueriesLessThan(num_queries): + with CaptureQueriesContext(connections['default']) as context: + yield + fail_msg = f"\r\n{json.dumps(context.captured_queries, indent=4)}" + assert len(context.captured_queries) < num_queries, fail_msg + + +@pytest.mark.django_db +@pytest.mark.parametrize('num_hosts, num_queries', [(9, 15), (99, 20), (999, 25)]) +def test_bulk_host_create_num_queries(organization, inventory, post, get, user, num_hosts, num_queries): + ''' + If I am a... + org admin + inventory admin at org level + admin of a particular inventory + superuser + + Bulk Host create should take under a certain number of queries + ''' + inventory.organization = organization + inventory_admin = user('inventory_admin', False) + org_admin = user('org_admin', False) + org_inv_admin = user('org_admin', False) + superuser = user('admin', True) + for u in [org_admin, org_inv_admin, inventory_admin]: + organization.member_role.members.add(u) + organization.admin_role.members.add(org_admin) + organization.inventory_admin_role.members.add(org_inv_admin) + inventory.admin_role.members.add(inventory_admin) + + for u in [org_admin, inventory_admin, org_inv_admin, superuser]: + hosts = [{'name': uuid4()} for i in range(num_hosts)] + with withAssertNumQueriesLessThan(num_queries): + bulk_host_create_response = post(reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': hosts}, u, expect=201).data + assert bulk_host_create_response['created'] == len(hosts), f"unexpected number of hosts created for user {u}" + + +@pytest.mark.django_db +def test_bulk_host_create_rbac(organization, inventory, post, get, user): + ''' + If I am a... + org admin + inventory admin at org level + admin of a particular invenotry + ... I can bulk add hosts + + Everyone else cannot + ''' + inventory.organization = organization + inventory_admin = user('inventory_admin', False) + org_admin = user('org_admin', False) + org_inv_admin = user('org_admin', False) + auditor = user('auditor', False) + member = user('member', False) + use_inv_member = user('member', False) + for u in [org_admin, org_inv_admin, auditor, member, inventory_admin, use_inv_member]: + organization.member_role.members.add(u) + organization.admin_role.members.add(org_admin) + organization.inventory_admin_role.members.add(org_inv_admin) + inventory.admin_role.members.add(inventory_admin) + inventory.use_role.members.add(use_inv_member) + organization.auditor_role.members.add(auditor) + + for indx, u in enumerate([org_admin, inventory_admin, org_inv_admin]): + bulk_host_create_response = post( + reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': [{'name': f'foobar-{indx}'}]}, u, expect=201 + ).data + assert bulk_host_create_response['created'] == 1, f"unexpected number of hosts created for user {u}" + + for indx, u in enumerate([member, auditor, use_inv_member]): + bulk_host_create_response = post( + reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': [{'name': f'foobar2-{indx}'}]}, u, expect=400 + ).data