Disallow creating callback jobs for the same host under the same job

template while another one is pending/waiting.  Update unit tests to
check for this scenario

awx/api/views.py

@@ -1693,6 +1693,12 @@ class JobTemplateCallback(GenericAPIView):
             return Response(data, status=status.HTTP_400_BAD_REQUEST)
         limit = ':&'.join(filter(None, [job_template.limit, host.name]))
 
+        # NOTE: We limit this to one job waiting due to this: https://trello.com/c/yK36dGWp
+        if Job.objects.filter(status__in=['pending', 'waiting', 'running'], job_template=job_template,
+                              limit=limit).count() > 0:
+            data = dict(msg='Host callback job already pending')
+            return Response(data, status=status.HTTP_400_BAD_REQUEST)
+
         # Everything is fine; actually create the job.
         with transaction.atomic():
             job = job_template.create_job(limit=limit, launch_type='callback')

awx/main/models/jobs.py

@@ -361,6 +361,9 @@ class Job(UnifiedJob, JobOptions):
             if obj.job_template is not None and obj.job_template == self.job_template:
                 if obj.launch_type == 'callback' and self.launch_type == 'callback':
                     if obj.limit != self.limit:
+                        # NOTE: This is overriden by api/views.py.JobTemplateCallback.post() check
+                        # which limits job runs on a JT to one per host in a callback scenario
+                        # I'm leaving this here in case we change that
                         return False
                 return True
             return False

awx/main/tests/jobs.py

@@ -1642,6 +1642,14 @@ class JobTemplateCallbackTest(BaseJobTestMixin, django.test.LiveServerTestCase):
         # Try with REMOTE_ADDR set to an unknown address.
         self.post(url, data, expect=400, remote_addr='127.127.0.1')
 
+        # Create a pending job that will block creation of another job
+        j = job_template.create_job(limit=job.limit, launch_type='callback')
+        j.status = 'pending'
+        j.save()
+        # This should fail since there is already a pending/waiting callback job on that job template involving that host
+        self.post(url, data, expect=400, remote_addr=host_ip)
+        j.delete() # Remove that so it's not hanging around
+
         # Try using an alternate IP for the host (but one that also resolves
         # via reverse lookup).
         host = None