create new bulk host create endpoint

allow system admins, org admins, and inventory admins to bulk create
hosts.

Testing on an "open" licensed awx as system admin, I created 1000 hosts with 6 queries in ~ 0.15 seconds
Testing on an "open" licensed awx as organization admin, I created 1000 hosts with 11 queries in ~ 0.15 seconds

fix org max host check

also only do permission denied if license is a trial

add /api/v2/bulk to list bulk apis available

add api description templates

One motiviation to not take a list of hosts with mixed inventories is to
keep things simple re: RBAC and keeping a constant number of queries.

If there is great clamor for accepting list of hosts to insert into
arbitrary different inventories, we could probably make it happen - we'd
need to pop the inventory off of each of the hosts, run the
HostSerializer validate, then in top level BulkHostCreateSerializer
fetch all the inventories/check permissions/org host limits for those
inventories/etc. But that makes this that much more complicated.

add test for rbac access

test also helped me find a bug in a query, fixed that

add test to assert num queries scales as expected

also move other test to dedicated file

also test with super user like I meant to

record activity stream for the inventory

this records that a certain number of hosts were added by a certain user
we could consider if there is any other additional information we want
to include

awx/api/serializers.py

@@ -109,11 +109,14 @@ from awx.main.utils import (
     encrypt_dict,
     prefetch_page_capabilities,
     truncate_stdout,
+    get_licenser,
 )
 from awx.main.utils.filters import SmartFilter
 from awx.main.utils.named_url_graph import reset_counters
 from awx.main.scheduler.task_manager_models import TaskManagerModels
 from awx.main.redact import UriCleaner, REPLACE_STR
+from awx.main.signals import update_inventory_computed_fields
+
 
 from awx.main.validators import vars_validate_or_raise
 
@@ -1853,7 +1856,7 @@ class HostSerializer(BaseSerializerWithVariables):
             vars_dict = parse_yaml_or_json(variables)
             vars_dict['ansible_ssh_port'] = port
             attrs['variables'] = json.dumps(vars_dict)
-        if Group.objects.filter(name=name, inventory=inventory).exists():
+        if inventory and Group.objects.filter(name=name, inventory=inventory).exists():
             raise serializers.ValidationError(_('A Group with that name already exists.'))
 
         return super(HostSerializer, self).validate(attrs)
@@ -1945,6 +1948,123 @@ class GroupSerializer(BaseSerializerWithVariables):
         return ret
 
 
+class BulkHostSerializer(HostSerializer):
+    name = serializers.CharField(required=True, allow_blank=False, max_length=512)
+    instance_id = serializers.CharField(required=False, max_length=1024)
+    description = serializers.CharField(required=False)
+    enabled = serializers.BooleanField(default=True, required=False)
+    variables = serializers.CharField(allow_blank=True, required=False)
+
+    class Meta:
+        fields = (
+            'name',
+            'enabled',
+            'instance_id',
+            'description',
+            'variables',
+        )
+
+
+class BulkHostCreateSerializer(serializers.Serializer):
+    inventory = serializers.PrimaryKeyRelatedField(
+        queryset=Inventory.objects.all(), required=True, write_only=True, help_text=_('Primary Key ID of inventory to add hosts to.')
+    )
+    hosts = serializers.ListField(child=BulkHostSerializer(), allow_empty=False, max_length=1000, write_only=True, help_text=_('Hosts to be created.'))
+
+    class Meta:
+        fields = ('inventory', 'hosts')
+        read_only_fields = ()
+
+    def raise_if_cannot_add_hosts(self, attrs):
+        validation_info = get_licenser().validate()
+
+        org = attrs['inventory'].organization
+
+        if org:
+            org_active_count = Host.objects.org_active_count(org.id)
+            new_hosts = [h['name'] for h in attrs['hosts']]
+            org_net_new_host_count = Host.objects.filter(inventory__organization=org.id).exclude(name__in=new_hosts).count()
+            if org.max_hosts > 0 and org_active_count + org_net_new_host_count > org.max_hosts:
+                raise PermissionDenied(
+                    _(
+                        "You have already reached the maximum number of %s hosts"
+                        " allowed for your organization. Contact your System Administrator"
+                        " for assistance." % org.max_hosts
+                    )
+                )
+
+            # Don't check license if it is open license
+        if validation_info.get('license_type', 'UNLICENSED') == 'open':
+            return True
+
+        sys_free_instances = validation_info.get('free_instances', 0)
+        system_net_new_host_count = Host.objects.exclude(name__in=new_hosts).count()
+
+        if system_net_new_host_count > sys_free_instances:
+            hard_error = validation_info.get('trial', False) is True or validation_info['instance_count'] == 10
+            if hard_error:
+                # Only raise permission error for trial, otherwise just log a warning as we do in other inventory import situations
+                raise PermissionDenied(_("Host count exceeds available instances."))
+            logger.warning(_("Number of hosts allowed by license has been exceeded."))
+
+        return True
+
+    def validate(self, attrs):
+        request = self.context.get('request', None)
+        inv = attrs['inventory']
+        if request and not request.user.is_superuser:
+            if inv.organization:
+                org_admin_orgs = {tup[0] for tup in Organization.accessible_pk_qs(request.user, 'admin_role')}
+                inv_admin_orgs = {tup[0] for tup in Organization.accessible_pk_qs(request.user, 'inventory_admin_role')}
+                is_org_admin = inv.organization.id in org_admin_orgs
+                is_org_inv_admin = inv.organization.id in inv_admin_orgs
+            else:
+                is_org_admin = False
+                is_org_inv_admin = False
+            # This may not work, need to figure out what the role is called
+            is_inventory_admin = inv.admin_role.members.filter(id=request.user.id).exists()
+            if not any([is_inventory_admin, is_org_admin, is_org_inv_admin]):
+                raise serializers.ValidationError(_(f'Inventory with id {inv.id} not found or lack permissions to add hosts.'))
+        current_hostnames = {h[0] for h in Host.objects.filter(inventory=inv).values_list('name').all()}
+        new_names = [host['name'] for host in attrs['hosts']]
+        duplicate_new_names = [n for n in new_names if n in current_hostnames or new_names.count(n) > 1]
+        if duplicate_new_names:
+            raise serializers.ValidationError(_(f'Hostnames must be unique in an inventory. Duplicates found: {duplicate_new_names}'))
+
+        self.raise_if_cannot_add_hosts(attrs)
+
+        _now = now()
+        for host in attrs['hosts']:
+            host['created'] = _now
+            host['modified'] = _now
+            host['inventory'] = inv
+        return attrs
+
+    def create(self, validated_data):
+        # This assumes total_hosts is up to date, and it can get out of date if the inventory computed fields have not been updated lately.
+        # If we wanted to side step this we could query Hosts.objects.filter(inventory...)
+        old_total_hosts = validated_data['inventory'].total_hosts
+        result = [Host(**attrs) for attrs in validated_data['hosts']]
+        try:
+            Host.objects.bulk_create(result)
+        except Exception as e:
+            raise serializers.ValidationError({"detail": _(f"{e}")})
+        new_total_hosts = old_total_hosts + len(result)
+        request = self.context.get('request', None)
+        changes = {'total_hosts': [old_total_hosts, new_total_hosts]}
+        activity_entry = ActivityStream.objects.create(
+            operation='update',
+            object1='inventory',
+            changes=json.dumps(changes),
+            actor=request.user,
+        )
+        activity_entry.inventory.add(validated_data['inventory'])
+
+        # This actually updates the cached "total_hosts" field on the inventory
+        update_inventory_computed_fields.delay(validated_data['inventory'].id)
+        return {"created": len(result), "url": InventorySerializer().get_related(validated_data['inventory'])['hosts']}
+
+
 class GroupTreeSerializer(GroupSerializer):
     children = serializers.SerializerMethodField()
 

awx/api/templates/api/bulk_host_create_view.md

@@ -0,0 +1,16 @@
+# Bulk Host Create
+
+This endpoint allows the client to create multiple hosts and associate them with an inventory. They may do this by providing the inventory ID and a list of json that would normally be provided to create hosts.
+
+Example:
+
+```
+{
+"inventory": 1,
+"hosts": [
+    {"name": "example1.com"},
+    {"name": "example2.com"}
+]
+
+}
+```

awx/api/templates/api/bulk_view.md

@@ -0,0 +1,3 @@
+# Bulk Actions
+
+This endpoint lists available bulk action APIs. 

awx/api/urls/urls.py

@@ -31,6 +31,10 @@ from awx.api.views import (
     ApplicationOAuth2TokenList,
     OAuth2ApplicationDetail,
 )
+from awx.api.views.bulk import (
+    BulkView,
+    BulkHostCreateView,
+)
 from awx.api.views.mesh_visualizer import MeshVisualizer
 
 from awx.api.views.metrics import MetricsView
@@ -136,6 +140,8 @@ v2_urls = [
     re_path(r'^activity_stream/', include(activity_stream_urls)),
     re_path(r'^workflow_approval_templates/', include(workflow_approval_template_urls)),
     re_path(r'^workflow_approvals/', include(workflow_approval_urls)),
+    re_path(r'^bulk/host_create/$', BulkHostCreateView.as_view(), name='bulk_host_create'),
+    re_path(r'^bulk/$', BulkView.as_view(), name='bulk'),
 ]
 
 

awx/api/views/bulk.py

@@ -0,0 +1,49 @@
+from collections import OrderedDict
+
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.renderers import JSONRenderer
+from rest_framework.reverse import reverse
+from rest_framework import status
+from rest_framework.response import Response
+
+from awx.api.generics import (
+    GenericAPIView,
+    APIView,
+)
+from awx.api import (
+    serializers,
+    renderers,
+)
+
+
+class BulkView(APIView):
+    _ignore_model_permissions = True
+    permission_classes = [IsAuthenticated]
+    renderer_classes = [
+        renderers.BrowsableAPIRenderer,
+        JSONRenderer,
+    ]
+    allowed_methods = ['GET', 'OPTIONS']
+
+    def get(self, request, format=None):
+        '''List top level resources'''
+        data = OrderedDict()
+        data['bulk_host_create'] = reverse('api:bulk_host_create', request=request)
+        return Response(data)
+
+
+class BulkHostCreateView(GenericAPIView):
+    _ignore_model_permissions = True
+    permission_classes = [IsAuthenticated]
+    serializer_class = serializers.BulkHostCreateSerializer
+    allowed_methods = ['GET', 'POST', 'OPTIONS']
+
+    def get(self, request):
+        return Response({"detail": "Bulk create hosts with this endpoint"}, status=status.HTTP_200_OK)
+
+    def post(self, request):
+        serializer = serializers.BulkHostCreateSerializer(data=request.data, context={'request': request})
+        if serializer.is_valid():
+            result = serializer.create(serializer.validated_data)
+            return Response(result, status=status.HTTP_201_CREATED)
+        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

awx/api/views/root.py

@@ -121,6 +121,7 @@ class ApiVersionRootView(APIView):
         data['workflow_job_template_nodes'] = reverse('api:workflow_job_template_node_list', request=request)
         data['workflow_job_nodes'] = reverse('api:workflow_job_node_list', request=request)
         data['mesh_visualizer'] = reverse('api:mesh_visualizer_view', request=request)
+        data['bulk'] = reverse('api:bulk', request=request)
         return Response(data)
 
 

awx/main/tests/functional/test_bulk.py

@@ -0,0 +1,86 @@
+import pytest
+
+from uuid import uuid4
+
+from awx.api.versioning import reverse
+
+import json
+from contextlib import contextmanager
+from django.test.utils import CaptureQueriesContext
+from django.db import connections
+
+
+@contextmanager
+def withAssertNumQueriesLessThan(num_queries):
+    with CaptureQueriesContext(connections['default']) as context:
+        yield
+        fail_msg = f"\r\n{json.dumps(context.captured_queries, indent=4)}"
+    assert len(context.captured_queries) < num_queries, fail_msg
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize('num_hosts, num_queries', [(9, 15), (99, 20), (999, 25)])
+def test_bulk_host_create_num_queries(organization, inventory, post, get, user, num_hosts, num_queries):
+    '''
+    If I am a...
+      org admin
+      inventory admin at org level
+      admin of a particular inventory
+      superuser
+
+    Bulk Host create should take under a certain number of queries
+    '''
+    inventory.organization = organization
+    inventory_admin = user('inventory_admin', False)
+    org_admin = user('org_admin', False)
+    org_inv_admin = user('org_admin', False)
+    superuser = user('admin', True)
+    for u in [org_admin, org_inv_admin, inventory_admin]:
+        organization.member_role.members.add(u)
+    organization.admin_role.members.add(org_admin)
+    organization.inventory_admin_role.members.add(org_inv_admin)
+    inventory.admin_role.members.add(inventory_admin)
+
+    for u in [org_admin, inventory_admin, org_inv_admin, superuser]:
+        hosts = [{'name': uuid4()} for i in range(num_hosts)]
+        with withAssertNumQueriesLessThan(num_queries):
+            bulk_host_create_response = post(reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': hosts}, u, expect=201).data
+            assert bulk_host_create_response['created'] == len(hosts), f"unexpected number of hosts created for user {u}"
+
+
+@pytest.mark.django_db
+def test_bulk_host_create_rbac(organization, inventory, post, get, user):
+    '''
+    If I am a...
+      org admin
+      inventory admin at org level
+      admin of a particular invenotry
+          ... I can bulk add hosts
+
+    Everyone else cannot
+    '''
+    inventory.organization = organization
+    inventory_admin = user('inventory_admin', False)
+    org_admin = user('org_admin', False)
+    org_inv_admin = user('org_admin', False)
+    auditor = user('auditor', False)
+    member = user('member', False)
+    use_inv_member = user('member', False)
+    for u in [org_admin, org_inv_admin, auditor, member, inventory_admin, use_inv_member]:
+        organization.member_role.members.add(u)
+    organization.admin_role.members.add(org_admin)
+    organization.inventory_admin_role.members.add(org_inv_admin)
+    inventory.admin_role.members.add(inventory_admin)
+    inventory.use_role.members.add(use_inv_member)
+    organization.auditor_role.members.add(auditor)
+
+    for indx, u in enumerate([org_admin, inventory_admin, org_inv_admin]):
+        bulk_host_create_response = post(
+            reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': [{'name': f'foobar-{indx}'}]}, u, expect=201
+        ).data
+        assert bulk_host_create_response['created'] == 1, f"unexpected number of hosts created for user {u}"
+
+    for indx, u in enumerate([member, auditor, use_inv_member]):
+        bulk_host_create_response = post(
+            reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': [{'name': f'foobar2-{indx}'}]}, u, expect=400
+        ).data