remove char_prompts and survey password from bulk job

fix the api-lint

fix the api-lint

add the descrition to the bulk job launch module params

 add the description for the description field

 add the description for the description field

add docs for the bulk api

fix the models on the bulk api serializers

fix some of the issues highlighted in the code review

better use of role model

remove comments

better error message

revert the PrimaryKeyRelatedField for unified_job_template and inventory

awx/api/serializers.py

@@ -1958,6 +1958,7 @@ class BulkHostSerializer(HostSerializer):
     variables = serializers.CharField(write_only=True, required=False)
 
     class Meta:
+        model = Host
         fields = (
             'name',
             'enabled',
@@ -1977,10 +1978,11 @@ class BulkHostCreateSerializer(serializers.Serializer):
     )
 
     class Meta:
+        model = Inventory
         fields = ('inventory', 'hosts')
         read_only_fields = ()
 
-    def raise_if_cannot_add_hosts(self, attrs):
+    def raise_if_host_counts_violated(self, attrs):
         validation_info = get_licenser().validate()
 
         org = attrs['inventory'].organization
@@ -1988,7 +1990,7 @@ class BulkHostCreateSerializer(serializers.Serializer):
         if org:
             org_active_count = Host.objects.org_active_count(org.id)
             new_hosts = [h['name'] for h in attrs['hosts']]
-            org_net_new_host_count = Host.objects.filter(inventory__organization=org.id).exclude(name__in=new_hosts).count()
+            org_net_new_host_count = len(new_hosts) - Host.objects.filter(inventory__organization=1, name__in=new_hosts).values('name').distinct().count()
             if org.max_hosts > 0 and org_active_count + org_net_new_host_count > org.max_hosts:
                 raise PermissionDenied(
                     _(
@@ -2019,15 +2021,12 @@ class BulkHostCreateSerializer(serializers.Serializer):
         inv = attrs['inventory']
         if request and not request.user.is_superuser:
             if inv.organization:
-                org_admin_orgs = {tup[0] for tup in Organization.accessible_pk_qs(request.user, 'admin_role')}
+                is_org_admin = request.user in inv.organization.admin_role
-                inv_admin_orgs = {tup[0] for tup in Organization.accessible_pk_qs(request.user, 'inventory_admin_role')}
+                is_org_inv_admin = request.user in inv.organization.inventory_admin_role
-                is_org_admin = inv.organization.id in org_admin_orgs
-                is_org_inv_admin = inv.organization.id in inv_admin_orgs
             else:
                 is_org_admin = False
                 is_org_inv_admin = False
-            # This may not work, need to figure out what the role is called
+            is_inventory_admin = request.user in inv.admin_role
-            is_inventory_admin = inv.admin_role.members.filter(id=request.user.id).exists()
             if not any([is_inventory_admin, is_org_admin, is_org_inv_admin]):
                 raise serializers.ValidationError(_(f'Inventory with id {inv.id} not found or lack permissions to add hosts.'))
         current_hostnames = {h[0] for h in Host.objects.filter(inventory=inv).values_list('name').all()}
@@ -2036,7 +2035,7 @@ class BulkHostCreateSerializer(serializers.Serializer):
         if duplicate_new_names:
             raise serializers.ValidationError(_(f'Hostnames must be unique in an inventory. Duplicates found: {duplicate_new_names}'))
 
-        self.raise_if_cannot_add_hosts(attrs)
+        self.raise_if_host_counts_violated(attrs)
 
         _now = now()
         for host in attrs['hosts']:
@@ -2068,7 +2067,8 @@ class BulkHostCreateSerializer(serializers.Serializer):
         # This actually updates the cached "total_hosts" field on the inventory
         update_inventory_computed_fields.delay(validated_data['inventory'].id)
         return_keys = [k for k in BulkHostSerializer().fields.keys()] + ['id']
-        return_data = []
+        return_data = {}
+        host_data = []
         for r in result:
             item = {k: getattr(r, k) for k in return_keys}
             if not settings.IS_TESTING_MODE:
@@ -2076,7 +2076,9 @@ class BulkHostCreateSerializer(serializers.Serializer):
                 # to get it, you have to do an additional query, which is not useful for our tests
                 item['url'] = reverse('api:host_detail', kwargs={'pk': r.id})
             item['inventory'] = reverse('api:inventory_detail', kwargs={'pk': validated_data['inventory'].id})
-            return_data.append(item)
+            host_data.append(item)
+        return_data['url'] = reverse('api:inventory_hosts_list', kwargs={'pk': validated_data['inventory'].id})
+        return_data['hosts'] = host_data
         return return_data
 
 
@@ -4541,7 +4543,8 @@ class WorkflowJobLaunchSerializer(BaseSerializer):
 
 
 class BulkJobNodeSerializer(serializers.Serializer):
-    # if we can find out the user, we can filter down the UnifiedJobTemplate objects
+    # We don't do a PrimaryKeyRelatedField for unified_job_template and inventory, because that increases the number
+    # of database queries, rather we take them as integer and later convert them to objects in get_objectified_jobs
     unified_job_template = serializers.IntegerField(
         required=True,
         min_value=1,
@@ -4552,22 +4555,20 @@ class BulkJobNodeSerializer(serializers.Serializer):
     labels = serializers.ListField(child=serializers.IntegerField(min_value=1), required=False)
     instance_groups = serializers.ListField(child=serializers.IntegerField(min_value=1), required=False)
     execution_environment = serializers.IntegerField(required=False, min_value=1)
-    #
     limit = serializers.CharField(required=False, write_only=True, allow_blank=False)
     scm_branch = serializers.CharField(required=False, write_only=True, allow_blank=False)
     verbosity = serializers.IntegerField(required=False, min_value=1)
     forks = serializers.IntegerField(required=False, min_value=1)
-    char_prompts = serializers.CharField(required=False, write_only=True, allow_blank=False)
     diff_mode = serializers.CharField(required=False, write_only=True, allow_blank=False)
     job_tags = serializers.CharField(required=False, write_only=True, allow_blank=False)
     job_type = serializers.CharField(required=False, write_only=True, allow_blank=False)
     skip_tags = serializers.CharField(required=False, write_only=True, allow_blank=False)
-    survey_passwords = serializers.CharField(required=False, write_only=True, allow_blank=False)
     job_slice_count = serializers.IntegerField(required=False, min_value=1)
     timeout = serializers.IntegerField(required=False, min_value=1)
     extra_data = serializers.JSONField(write_only=True, required=False)
 
     class Meta:
+        model = WorkflowJobNode
         fields = (
             'unified_job_template',
             'identifier',
@@ -4580,17 +4581,13 @@ class BulkJobNodeSerializer(serializers.Serializer):
             'scm_branch',
             'verbosity',
             'forks',
-            'char_prompts',
             'diff_mode',
             'extra_data',
             'job_slice_count',
             'job_tags',
             'job_type',
             'skip_tags',
-            'survey_passwords',
             'timeout',
-            # these are related objects and we need to add extra validation for them in the parent BulkJobLaunchSerializer
-            #
         )
 
 
@@ -4610,9 +4607,6 @@ class BulkJobLaunchSerializer(BaseSerializer):
     inventory = serializers.PrimaryKeyRelatedField(queryset=Inventory.objects.all(), required=False, write_only=True)
     limit = serializers.CharField(write_only=True, required=False, allow_blank=False)
     scm_branch = serializers.CharField(write_only=True, required=False, allow_blank=False)
-    # not implemented yet
-    # webhook_service: null,  # Here we can use PrimaryKeyRelatedField so it will automagically do rbac/turn into object, I think, I'm actually not sure how to use this
-    # webhook_credential: null,  # Here we can use PrimaryKeyRelatedField so it will automagically do rbac/turn into object  I think, I'm actually not sure how to use this
     skip_tags = serializers.CharField(write_only=True, required=False, allow_blank=False)
     job_tags = serializers.CharField(write_only=True, required=False, allow_blank=False)
 
@@ -4632,8 +4626,6 @@ class BulkJobLaunchSerializer(BaseSerializer):
             else:
                 node['identifier'] = str(uuid4())
 
-        # Build sets of all the requested resources
-        # TODO: As we add other related items, we need to add them here
         requested_ujts = {j['unified_job_template'] for j in attrs['jobs']}
         requested_use_inventories = {job['inventory'] for job in attrs['jobs'] if 'inventory' in job}
         requested_use_execution_environments = {job['execution_environment'] for job in attrs['jobs'] if 'execution_environment' in job}
@@ -4649,15 +4641,11 @@ class BulkJobLaunchSerializer(BaseSerializer):
                 [requested_use_instance_groups.add(instance_group) for instance_group in job['instance_groups']]
 
         # If we are not a superuser, check we have permissions
-        # TODO: As we add other related items, we need to add them here
         if request and not request.user.is_superuser:
             self.check_organization_permission(attrs, request)
             self.check_unified_job_permission(request, requested_ujts)
-            if requested_use_inventories:
+            if requested_use_inventories or 'inventory' in attrs:
-                self.check_inventory_permission(request, requested_use_inventories)
+                self.check_inventory_permission(attrs, request, requested_use_inventories)
-
-            if requested_use_credentials:
-                self.check_credential_permission(request, requested_use_credentials)
 
             if requested_use_labels:
                 self.check_label_permission(requested_use_labels)
@@ -4688,9 +4676,6 @@ class BulkJobLaunchSerializer(BaseSerializer):
 
     def create(self, validated_data):
         job_node_data = validated_data.pop('jobs')
-        # FIXME: Need to set organization on the WorkflowJob in order for users to be able to see it --
-        # normally their permission is sourced from the underlying WorkflowJobTemplate
-        # maybe we need to add Organization to WorkflowJob
         wfj_deferred_attr_names = ('skip_tags', 'limit', 'job_tags')
         wfj_deferred_vals = {}
         for item in wfj_deferred_attr_names:
@@ -4703,6 +4688,7 @@ class BulkJobLaunchSerializer(BaseSerializer):
         nodes = []
         node_m2m_objects = {}
         node_m2m_object_types_to_through_model = {
+
             'credentials': WorkflowJobNode.credentials.through,
             'labels': WorkflowJobNode.labels.through,
             'instance_groups': WorkflowJobNode.instance_groups.through,
@@ -4712,12 +4698,10 @@ class BulkJobLaunchSerializer(BaseSerializer):
             'scm_branch',
             'verbosity',
             'forks',
-            'char_prompts',
             'diff_mode',
             'job_tags',
             'job_type',
             'skip_tags',
-            'survey_passwords',
             'job_slice_count',
             'timeout',
         )
@@ -4789,7 +4773,7 @@ class BulkJobLaunchSerializer(BaseSerializer):
             if request and not request.user.is_superuser:
                 [allowed_orgs.add(tup[0]) for tup in Organization.accessible_pk_qs(request.user, 'read_role').all()]
                 if requested_org.id not in allowed_orgs:
-                    raise ValidationError(_(f"Organization {requested_org.id} not found"))
+                    raise ValidationError(_(f"Organization {requested_org.id} not found or you don't have permissions to access it"))
 
     def check_unified_job_permission(self, request, requested_ujts):
         allowed_ujts = set()
@@ -4801,25 +4785,28 @@ class BulkJobLaunchSerializer(BaseSerializer):
 
         if requested_ujts - allowed_ujts:
             not_allowed = requested_ujts - allowed_ujts
-            raise serializers.ValidationError(_(f"Unified Job Templates {not_allowed} not found."))
+            raise serializers.ValidationError(_(f"Unified Job Templates {not_allowed} not found or you don't have permissions to access it"))
-
+    def check_inventory_permission(self, attrs, request, requested_use_inventories):
-    def check_inventory_permission(self, request, requested_use_inventories):
         accessible_use_inventories = {tup[0] for tup in Inventory.accessible_pk_qs(request.user, 'use_role')}
         if requested_use_inventories - accessible_use_inventories:
             not_allowed = requested_use_inventories - accessible_use_inventories
-            raise serializers.ValidationError(_(f"Inventories {not_allowed} not found."))
+            raise serializers.ValidationError(_(f"Inventories {not_allowed} not found or you don't have permissions to access it"))
+        if 'inventory' in attrs:
+            requested_workflow_inventory = attrs['inventory']
+            if requested_workflow_inventory.id not in accessible_use_inventories:
+                raise serializers.ValidationError(_(f"Inventories {requested_workflow_inventory.id} not found or you don't have permissions to access it"))
 
     def check_credential_permission(self, request, requested_use_credentials):
         accessible_use_credentials = {tup[0] for tup in Credential.accessible_pk_qs(request.user, 'use_role').all()}
         if requested_use_credentials - accessible_use_credentials:
             not_allowed = requested_use_credentials - accessible_use_credentials
-            raise serializers.ValidationError(_(f"Credentials {not_allowed} not found."))
+            raise serializers.ValidationError(_(f"Credentials {not_allowed} not found or you don't have permissions to access it"))
 
     def check_label_permission(self, requested_use_labels):
         accessible_use_labels = {tup.id for tup in Label.objects.all()}
         if requested_use_labels - accessible_use_labels:
             not_allowed = requested_use_labels - accessible_use_labels
-            raise serializers.ValidationError(_(f"Labels {not_allowed} not found"))
+            raise serializers.ValidationError(_(f"Labels {not_allowed} not found or you don't have permissions to access it"))
 
     def check_instance_group_permission(self, request, requested_use_instance_groups):
         # only org admins are allowed to see instance groups
@@ -4828,7 +4815,7 @@ class BulkJobLaunchSerializer(BaseSerializer):
             accessible_use_instance_groups = {tup.id for tup in InstanceGroup.objects.all()}
             if requested_use_instance_groups - accessible_use_instance_groups:
                 not_allowed = requested_use_instance_groups - accessible_use_instance_groups
-                raise serializers.ValidationError(_(f"Instance Groups {not_allowed} not found"))
+                raise serializers.ValidationError(_(f"Instance Groups {not_allowed} not found or you don't have permissions to access it"))
 
     def check_execution_environment_permission(self, request, requested_use_execution_environments):
         accessible_execution_env = {
@@ -4839,7 +4826,7 @@ class BulkJobLaunchSerializer(BaseSerializer):
         }
         if requested_use_execution_environments - accessible_execution_env:
             not_allowed = requested_use_execution_environments - accessible_execution_env
-            raise serializers.ValidationError(_(f"Execution Environments {not_allowed} not found"))
+            raise serializers.ValidationError(_(f"Execution Environments {not_allowed} not found or you don't have permissions to access it"))
 
     def get_objectified_jobs(
         self,

awx/api/templates/api/bulk_host_create_view.md

@@ -8,9 +8,39 @@ Example:
 {
 "inventory": 1,
 "hosts": [
-    {"name": "example1.com"},
+    {"name": "example1.com", "variables": "ansible_connection: local"},
     {"name": "example2.com"}
 ]
 
 }
 ```
+
+Return data:
+
+```commandline
+{
+    "url": "/api/v2/inventories/3/hosts/",
+    "hosts": [
+        {
+            "name": "example1.com",
+            "enabled": true,
+            "instance_id": "",
+            "description": "",
+            "variables": "ansible_connection: local",
+            "id": 1255,
+            "url": "/api/v2/hosts/1255/",
+            "inventory": "/api/v2/inventories/3/"
+        },
+        {
+            "name": "example2.com",
+            "enabled": true,
+            "instance_id": "",
+            "description": "",
+            "variables": "",
+            "id": 1256,
+            "url": "/api/v2/hosts/1256/",
+            "inventory": "/api/v2/inventories/3/"
+        }
+    ]
+}
+```

awx/main/tests/functional/test_bulk.py

@@ -47,7 +47,7 @@ def test_bulk_host_create_num_queries(organization, inventory, post, get, user,
         hosts = [{'name': uuid4()} for i in range(num_hosts)]
         with withAssertNumQueriesLessThan(num_queries):
             bulk_host_create_response = post(reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': hosts}, u, expect=201).data
-            assert len(bulk_host_create_response) == len(hosts), f"unexpected number of hosts created for user {u}"
+            assert len(bulk_host_create_response['hosts']) == len(hosts), f"unexpected number of hosts created for user {u}"
 
 
 @pytest.mark.django_db
@@ -80,7 +80,7 @@ def test_bulk_host_create_rbac(organization, inventory, post, get, user):
         bulk_host_create_response = post(
             reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': [{'name': f'foobar-{indx}'}]}, u, expect=201
         ).data
-        assert len(bulk_host_create_response) == 1, f"unexpected number of hosts created for user {u}"
+        assert len(bulk_host_create_response['hosts']) == 1, f"unexpected number of hosts created for user {u}"
 
     for indx, u in enumerate([member, auditor, use_inv_member]):
         bulk_host_create_response = post(
@@ -91,8 +91,8 @@ def test_bulk_host_create_rbac(organization, inventory, post, get, user):
 @pytest.mark.django_db
 def test_bulk_job_launch(job_template, organization, inventory, project, credential, post, get, user):
     '''
-    if I don't have access to the unified job templare
+    if I have access to the unified job template
-             ... I can't launch the bulk job
+             ... I can launch the bulk job
     '''
     normal_user = user('normal_user', False)
     jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml')
@@ -102,6 +102,7 @@ def test_bulk_job_launch(job_template, organization, inventory, project, credent
     bulk_job_launch_response = post(
         reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=201
     ).data
+    assert bulk_job_launch_response['id'] == 1
 
 
 @pytest.mark.django_db
@@ -114,9 +115,7 @@ def test_bulk_job_launch_no_access_to_job_template(job_template, organization, i
     jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml')
     jt.save()
     organization.member_role.members.add(normal_user)
-    bulk_job_launch_response = post(
+    post(reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=400)
-        reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=400
-    ).data
 
 
 @pytest.mark.django_db
@@ -129,9 +128,7 @@ def test_bulk_job_launch_no_org_assigned(job_template, organization, inventory,
     jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml')
     jt.save()
     jt.execute_role.members.add(normal_user)
-    bulk_job_launch_response = post(
+    post(reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=400)
-        reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=400
-    ).data
 
 
 @pytest.mark.django_db
@@ -149,9 +146,7 @@ def test_bulk_job_launch_multiple_org_assigned(job_template, organization, inven
     jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml')
     jt.save()
     jt.execute_role.members.add(normal_user)
-    bulk_job_launch_response = post(
+    post(reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=400)
-        reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=400
-    ).data
 
 
 @pytest.mark.django_db
@@ -172,6 +167,7 @@ def test_bulk_job_launch_specific_org(job_template, organization, inventory, pro
     bulk_job_launch_response = post(
         reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}], 'organization': org1.id}, normal_user, expect=201
     ).data
+    assert bulk_job_launch_response['id'] == 1
 
 
 @pytest.mark.django_db
@@ -189,6 +185,4 @@ def test_bulk_job_launch_inventory_no_access(job_template, organization, invento
     org1.member_role.members.add(normal_user)
     inv = Inventory.objects.create(name='inv1', organization=org2)
     jt.execute_role.members.add(normal_user)
-    bulk_job_launch_response = post(
+    post(reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id, 'inventory': inv.id}]}, normal_user, expect=400)
-        reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id, 'inventory': inv.id}]}, normal_user, expect=400
-    ).data

awx_collection/plugins/modules/bulk_host_create.py

@@ -31,8 +31,9 @@ options:
           type: str
           require: True
         description:
-          - The description to use for the host.
+          description:
-        type: str
+            - The description to use for the host.
+          type: str
         enabled:
           description:
             - If the host should be enabled.

awx_collection/plugins/modules/bulk_job_launch.py

@@ -200,7 +200,6 @@ EXAMPLES = '''
 '''
 
 from ..module_utils.controller_api import ControllerAPIModule
-import json
 
 
 def main():
@@ -208,6 +207,7 @@ def main():
     argument_spec = dict(
         jobs=dict(required=True, type='list'),
         name=dict(),
+        description=dict(),
         organization=dict(type='int'),
         inventory=dict(type='int'),
         limit=dict(),
@@ -226,6 +226,7 @@ def main():
     post_data_names = (
         'jobs',
         'name',
+        'description',
         'organization',
         'inventory',
         'limit',

awx_collection/test/awx/test_bulk.py

@@ -9,7 +9,7 @@ from awx.main.models import WorkflowJob
 @pytest.mark.django_db
 def test_bulk_job_launch(run_module, admin_user, job_template):
     jobs = [dict(unified_job_template=job_template.id)]
-    result = run_module(
+    run_module(
         'bulk_job_launch',
         {
             'name': "foo-bulk-job",
@@ -29,7 +29,7 @@ def test_bulk_job_launch(run_module, admin_user, job_template):
 @pytest.mark.django_db
 def test_bulk_host_create(run_module, admin_user, inventory):
     hosts = [dict(name="127.0.0.1"), dict(name="foo.dns.org")]
-    result = run_module(
+    run_module(
         'bulk_host_create',
         {
             'inventory': inventory.id,

docs/bulk_api.md

@@ -0,0 +1,39 @@
+# Bulk API Overview
+
+Bulk API endpoints allows to perform bulk operations in single web request. There are currently following bulk api actions:
+- /api/v2/bulk/job_launch
+- /api/v2/bulk/host_create
+
+## Bulk Job Launch
+
+Provides feature in the API that allows a single web request to achieve multiple job launches. It creates a workflow job with individual jobs as nodes within the workflow job. It also supports providing promptable fields like inventory, credential etc.
+
+Following is an example of a post request at the /api/v2/bulk/job_launch
+
+```commandline
+{
+"name": "Bulk Job Launch",
+"jobs": [
+   {"unified_job_template": 7, "identifier":"foo", "limit": "kansas", "credentials": [1]},
+   {"unified_job_template": 8, "identifier":"bar", "inventory": 1, "execution_environment": 3},
+   {"unified_job_template": 9}
+]
+}
+```
+
+The above will launch a workflow job with 3 nodes in it. 
+
+## Bulk Host Create
+
+Provides feature in the API that allows a single web request to create multiple hosts in an inventory.  
+
+Following is an example of a post request at the /api/v2/bulk/host_create:
+
+```commandline
+{
+    "inventory": 1,
+    "hosts": [{"name": "host1", "variables": "ansible_connection: local"}, {"name": "host2"}, {"name": "host3"}, {"name": "host4"}, {"name": "host5"}, {"name": "host6"}]
+}
+```
+
+The above will add 6 hosts in the inventory.