Merge pull request #13786 from AlanCoding/refresh_refresh_refresh

Fix docker-clean target, accounting for slashes

.github/workflows/devel_images.yml

@@ -7,6 +7,7 @@ on:
     branches:
       - devel
       - release_*
+      - feature_*
 jobs:
   push:
     if: endsWith(github.repository, '/awx') || startsWith(github.ref, 'refs/heads/release_')
@@ -20,6 +21,12 @@ jobs:
       - name: Get python version from Makefile
         run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
 
+      - name: Set lower case owner name
+        run: |
+          echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV}
+        env:
+          OWNER: '${{ github.repository_owner }}'
+
       - name: Install python ${{ env.py_version }}
         uses: actions/setup-python@v2
         with:
@@ -31,15 +38,18 @@ jobs:
 
       - name: Pre-pull image to warm build cache
         run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${GITHUB_REF##*/} || :
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_kube_devel:${GITHUB_REF##*/} || :
+          docker pull ghcr.io/${OWNER_LC}/awx_devel:${GITHUB_REF##*/} || :
+          docker pull ghcr.io/${OWNER_LC}/awx_kube_devel:${GITHUB_REF##*/} || :
+          docker pull ghcr.io/${OWNER_LC}/awx:${GITHUB_REF##*/} || :
 
       - name: Build images
         run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${GITHUB_REF##*/} make docker-compose-build
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-dev-build
+          DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make docker-compose-build
+          DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-dev-build
+          DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-build
 
       - name: Push image
         run: |
-          docker push ghcr.io/${{ github.repository_owner }}/awx_devel:${GITHUB_REF##*/}
-          docker push ghcr.io/${{ github.repository_owner }}/awx_kube_devel:${GITHUB_REF##*/}
+          docker push ghcr.io/${OWNER_LC}/awx_devel:${GITHUB_REF##*/}
+          docker push ghcr.io/${OWNER_LC}/awx_kube_devel:${GITHUB_REF##*/}
+          docker push ghcr.io/${OWNER_LC}/awx:${GITHUB_REF##*/}

.github/workflows/promote.yml

@@ -10,6 +10,7 @@ on:
 
 jobs:
   promote:
+    if: endsWith(github.repository, '/awx') 
     runs-on: ubuntu-latest
     steps:
       - name: Checkout awx

.github/workflows/stage.yml

@@ -21,6 +21,7 @@ on:
 
 jobs:
   stage:
+    if: endsWith(github.repository, '/awx')
     runs-on: ubuntu-latest
     permissions:
       packages: write

.gitignore

@@ -161,3 +161,6 @@ use_dev_supervisor.txt
 /_build/
 /_build_kube_dev/
 /Dockerfile.kube-dev
+
+awx/ui_next/src
+awx/ui_next/build

MANIFEST.in

@@ -6,6 +6,7 @@ recursive-include awx/templates *.html
 recursive-include awx/api/templates *.md *.html *.yml
 recursive-include awx/ui/build *.html
 recursive-include awx/ui/build *
+recursive-include awx/ui_next/build *
 recursive-include awx/playbooks *.yml
 recursive-include awx/lib/site-packages *
 recursive-include awx/plugins *.ps1

Makefile

@@ -1,4 +1,7 @@
+-include awx/ui_next/Makefile
+
 PYTHON ?= python3.9
+DOCKER_COMPOSE ?= docker-compose
 OFFICIAL ?= no
 NODE ?= node
 NPM_BIN ?= npm
@@ -83,7 +86,7 @@ clean-schema:
 
 clean-languages:
 	rm -f $(I18N_FLAG_FILE)
-	find ./awx/locale/ -type f -regex ".*\.mo$" -delete
+	find ./awx/locale/ -type f -regex '.*\.mo$$' -delete
 
 ## Remove temporary build files, compiled Python files.
 clean: clean-ui clean-api clean-awxkit clean-dist
@@ -203,19 +206,7 @@ uwsgi: collectstatic
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	uwsgi -b 32768 \
-	    --socket 127.0.0.1:8050 \
-	    --module=awx.wsgi:application \
-	    --home=/var/lib/awx/venv/awx \
-	    --chdir=/awx_devel/ \
-	    --vacuum \
-	    --processes=5 \
-	    --harakiri=120 --master \
-	    --no-orphans \
-	    --max-requests=1000 \
-	    --stats /tmp/stats.socket \
-	    --lazy-apps \
-	    --logformat "%(addr) %(method) %(uri) - %(proto) %(status)"
+	uwsgi /etc/tower/uwsgi.ini
 
 awx-autoreload:
 	@/awx_devel/tools/docker-compose/awx-autoreload /awx_devel/awx "$(DEV_RELOAD_COMMAND)"
@@ -226,12 +217,6 @@ daphne:
 	fi; \
 	daphne -b 127.0.0.1 -p 8051 awx.asgi:channel_layer
 
-wsbroadcast:
-	@if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/awx/bin/activate; \
-	fi; \
-	$(PYTHON) manage.py run_wsbroadcast
-
 ## Run to start the background task dispatcher for development.
 dispatcher:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -239,7 +224,6 @@ dispatcher:
 	fi; \
 	$(PYTHON) manage.py run_dispatcher
 
-
 ## Run to start the zeromq callback receiver
 receiver:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -256,6 +240,34 @@ jupyter:
 	fi; \
 	$(MANAGEMENT_COMMAND) shell_plus --notebook
 
+## Start the rsyslog configurer process in background in development environment.
+run-rsyslog-configurer:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/awx/bin/activate; \
+	fi; \
+	$(PYTHON) manage.py run_rsyslog_configurer
+
+## Start cache_clear process in background in development environment.
+run-cache-clear:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/awx/bin/activate; \
+	fi; \
+	$(PYTHON) manage.py run_cache_clear
+
+## Start the wsrelay process in background in development environment.
+run-wsrelay:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/awx/bin/activate; \
+	fi; \
+	$(PYTHON) manage.py run_wsrelay
+
+## Start the heartbeat process in background in development environment.
+run-heartbeet:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/awx/bin/activate; \
+	fi; \
+	$(PYTHON) manage.py run_heartbeet
+
 reports:
 	mkdir -p $@
 
@@ -422,12 +434,14 @@ ui-release: $(UI_BUILD_FLAG_FILE)
 
 ui-devel: awx/ui/node_modules
 	@$(MAKE) -B $(UI_BUILD_FLAG_FILE)
-	mkdir -p /var/lib/awx/public/static/css
-	mkdir -p /var/lib/awx/public/static/js
-	mkdir -p /var/lib/awx/public/static/media
-	cp -r awx/ui/build/static/css/* /var/lib/awx/public/static/css
-	cp -r awx/ui/build/static/js/* /var/lib/awx/public/static/js
-	cp -r awx/ui/build/static/media/* /var/lib/awx/public/static/media
+	@if [ -d "/var/lib/awx" ] ; then \
+		mkdir -p /var/lib/awx/public/static/css; \
+		mkdir -p /var/lib/awx/public/static/js; \
+		mkdir -p /var/lib/awx/public/static/media; \
+		cp -r awx/ui/build/static/css/* /var/lib/awx/public/static/css; \
+		cp -r awx/ui/build/static/js/* /var/lib/awx/public/static/js; \
+		cp -r awx/ui/build/static/media/* /var/lib/awx/public/static/media; \
+	fi
 
 ui-devel-instrumented: awx/ui/node_modules
 	$(NPM_BIN) --prefix awx/ui --loglevel warn run start-instrumented
@@ -454,11 +468,12 @@ ui-test-general:
 	$(NPM_BIN) run --prefix awx/ui pretest
 	$(NPM_BIN) run --prefix awx/ui/ test-general --runInBand
 
+# NOTE: The make target ui-next is imported from awx/ui_next/Makefile
 HEADLESS ?= no
 ifeq ($(HEADLESS), yes)
 dist/$(SDIST_TAR_FILE):
 else
-dist/$(SDIST_TAR_FILE): $(UI_BUILD_FLAG_FILE)
+dist/$(SDIST_TAR_FILE): $(UI_BUILD_FLAG_FILE) ui-next
 endif
 	$(PYTHON) -m build -s
 	ln -sf $(SDIST_TAR_FILE) dist/awx.tar.gz
@@ -506,23 +521,21 @@ docker-compose-sources: .git/hooks/pre-commit
 	    -e enable_prometheus=$(PROMETHEUS) \
 	    -e enable_grafana=$(GRAFANA) $(EXTRA_SOURCES_ANSIBLE_OPTS)
 
-
-
 docker-compose: awx/projects docker-compose-sources
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans
+	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans
 
 docker-compose-credential-plugins: awx/projects docker-compose-sources
 	echo -e "\033[0;31mTo generate a CyberArk Conjur API key: docker exec -it tools_conjur_1 conjurctl account create quick-start\033[0m"
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/docker-credential-plugins-override.yml up --no-recreate awx_1 --remove-orphans
+	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml -f tools/docker-credential-plugins-override.yml up --no-recreate awx_1 --remove-orphans
 
 docker-compose-test: awx/projects docker-compose-sources
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports awx_1 /bin/bash
+	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports awx_1 /bin/bash
 
 docker-compose-runtest: awx/projects docker-compose-sources
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports awx_1 /start_tests.sh
+	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports awx_1 /start_tests.sh
 
 docker-compose-build-swagger: awx/projects docker-compose-sources
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports --no-deps awx_1 /start_tests.sh swagger
+	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports --no-deps awx_1 /start_tests.sh swagger
 
 SCHEMA_DIFF_BASE_BRANCH ?= devel
 detect-schema-change: genschema
@@ -531,7 +544,7 @@ detect-schema-change: genschema
 	diff -u -b reference-schema.json schema.json
 
 docker-compose-clean: awx/projects
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml rm -sf
+	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml rm -sf
 
 docker-compose-container-group-clean:
 	@if [ -f "tools/docker-compose-minikube/_sources/minikube" ]; then \
@@ -547,10 +560,8 @@ docker-compose-build:
 	    --cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) .
 
 docker-clean:
-	$(foreach container_id,$(shell docker ps -f name=tools_awx -aq && docker ps -f name=tools_receptor -aq),docker stop $(container_id); docker rm -f $(container_id);)
-	if [ "$(shell docker images | grep awx_devel)" ]; then \
-	  docker images | grep awx_devel | awk '{print $$3}' | xargs docker rmi --force; \
-	fi
+	-$(foreach container_id,$(shell docker ps -f name=tools_awx -aq && docker ps -f name=tools_receptor -aq),docker stop $(container_id); docker rm -f $(container_id);)
+	-$(foreach image_id,$(shell docker images --filter=reference='*/*/*awx_devel*' --filter=reference='*/*awx_devel*' --filter=reference='*awx_devel*' -aq),docker rmi --force $(image_id);)
 
 docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean
 	docker volume rm -f tools_awx_db tools_grafana_storage tools_prometheus_storage $(docker volume ls --filter name=tools_redis_socket_ -q)
@@ -559,10 +570,10 @@ docker-refresh: docker-clean docker-compose
 
 ## Docker Development Environment with Elastic Stack Connected
 docker-compose-elk: awx/projects docker-compose-sources
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
+	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
 docker-compose-cluster-elk: awx/projects docker-compose-sources
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
+	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
 docker-compose-container-group:
 	MINIKUBE_CONTAINER_GROUP=true make docker-compose
@@ -584,6 +595,7 @@ VERSION:
 PYTHON_VERSION:
 	@echo "$(PYTHON)" | sed 's:python::'
 
+.PHONY: Dockerfile
 Dockerfile: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
 	ansible-playbook tools/ansible/dockerfile.yml -e receptor_image=$(RECEPTOR_IMAGE)
 
@@ -664,3 +676,7 @@ help/generate:
 	} \
 	{ lastLine = $$0 }' $(MAKEFILE_LIST) | sort -u
 	@printf "\n"
+
+## Display help for ui-next targets
+help/ui-next:
+	@make -s help MAKEFILE_LIST="awx/ui_next/Makefile"

awx/api/conf.py

@@ -1,5 +1,4 @@
 # Django
-from django.conf import settings
 from django.utils.translation import gettext_lazy as _
 
 # Django REST Framework
@@ -9,6 +8,7 @@ from rest_framework import serializers
 from awx.conf import fields, register, register_validate
 from awx.api.fields import OAuth2ProviderField
 from oauth2_provider.settings import oauth2_settings
+from awx.sso.common import is_remote_auth_enabled
 
 
 register(
@@ -108,19 +108,8 @@ register(
 
 
 def authentication_validate(serializer, attrs):
-    remote_auth_settings = [
-        'AUTH_LDAP_SERVER_URI',
-        'SOCIAL_AUTH_GOOGLE_OAUTH2_KEY',
-        'SOCIAL_AUTH_GITHUB_KEY',
-        'SOCIAL_AUTH_GITHUB_ORG_KEY',
-        'SOCIAL_AUTH_GITHUB_TEAM_KEY',
-        'SOCIAL_AUTH_SAML_ENABLED_IDPS',
-        'RADIUS_SERVER',
-        'TACACSPLUS_HOST',
-    ]
-    if attrs.get('DISABLE_LOCAL_AUTH', False):
-        if not any(getattr(settings, s, None) for s in remote_auth_settings):
-            raise serializers.ValidationError(_("There are no remote authentication systems configured."))
+    if attrs.get('DISABLE_LOCAL_AUTH', False) and not is_remote_auth_enabled():
+        raise serializers.ValidationError(_("There are no remote authentication systems configured."))
     return attrs
 
 

awx/api/filters.py

@@ -155,7 +155,7 @@ class FieldLookupBackend(BaseFilterBackend):
         'search',
     )
 
-    # A list of fields that we know can be filtered on without the possiblity
+    # A list of fields that we know can be filtered on without the possibility
     # of introducing duplicates
     NO_DUPLICATES_ALLOW_LIST = (CharField, IntegerField, BooleanField, TextField)
 
@@ -268,7 +268,7 @@ class FieldLookupBackend(BaseFilterBackend):
                     continue
 
                 # HACK: make `created` available via API for the Django User ORM model
-                # so it keep compatiblity with other objects which exposes the `created` attr.
+                # so it keep compatibility with other objects which exposes the `created` attr.
                 if queryset.model._meta.object_name == 'User' and key.startswith('created'):
                     key = key.replace('created', 'date_joined')
 

awx/api/generics.py

@@ -28,7 +28,7 @@ from rest_framework import generics
 from rest_framework.response import Response
 from rest_framework import status
 from rest_framework import views
-from rest_framework.permissions import AllowAny
+from rest_framework.permissions import IsAuthenticated
 from rest_framework.renderers import StaticHTMLRenderer
 from rest_framework.negotiation import DefaultContentNegotiation
 
@@ -674,7 +674,7 @@ class SubListCreateAttachDetachAPIView(SubListCreateAPIView):
                 location = None
             created = True
 
-        # Retrive the sub object (whether created or by ID).
+        # Retrieve the sub object (whether created or by ID).
         sub = get_object_or_400(self.model, pk=sub_id)
 
         # Verify we have permission to attach.
@@ -822,7 +822,7 @@ def trigger_delayed_deep_copy(*args, **kwargs):
 
 class CopyAPIView(GenericAPIView):
     serializer_class = CopySerializer
-    permission_classes = (AllowAny,)
+    permission_classes = (IsAuthenticated,)
     copy_return_serializer_class = None
     new_in_330 = True
     new_in_api_v2 = True

awx/api/renderers.py

@@ -60,7 +60,7 @@ class BrowsableAPIRenderer(renderers.BrowsableAPIRenderer):
             delattr(renderer_context['view'], '_request')
 
     def get_raw_data_form(self, data, view, method, request):
-        # Set a flag on the view to indiciate to the view/serializer that we're
+        # Set a flag on the view to indicate to the view/serializer that we're
         # creating a raw data form for the browsable API.  Store the original
         # request method to determine how to populate the raw data form.
         if request.method in {'OPTIONS', 'DELETE'}:

awx/api/serializers.py

@@ -8,6 +8,7 @@ import logging
 import re
 from collections import OrderedDict
 from datetime import timedelta
+from uuid import uuid4
 
 # OAuth2
 from oauthlib import oauth2
@@ -55,6 +56,8 @@ from awx.main.models import (
     ExecutionEnvironment,
     Group,
     Host,
+    HostMetric,
+    HostMetricSummaryMonthly,
     Instance,
     InstanceGroup,
     InstanceLink,
@@ -108,13 +111,15 @@ from awx.main.utils import (
     extract_ansible_vars,
     encrypt_dict,
     prefetch_page_capabilities,
-    get_external_account,
     truncate_stdout,
+    get_licenser,
 )
 from awx.main.utils.filters import SmartFilter
 from awx.main.utils.named_url_graph import reset_counters
 from awx.main.scheduler.task_manager_models import TaskManagerModels
 from awx.main.redact import UriCleaner, REPLACE_STR
+from awx.main.signals import update_inventory_computed_fields
+
 
 from awx.main.validators import vars_validate_or_raise
 
@@ -124,6 +129,8 @@ from awx.api.fields import BooleanNullField, CharNullField, ChoiceNullField, Ver
 # AWX Utils
 from awx.api.validators import HostnameRegexValidator
 
+from awx.sso.common import get_external_account
+
 logger = logging.getLogger('awx.api.serializers')
 
 # Fields that should be summarized regardless of object type.
@@ -151,11 +158,12 @@ SUMMARIZABLE_FK_FIELDS = {
         'kind',
     ),
     'host': DEFAULT_SUMMARY_FIELDS,
+    'constructed_host': DEFAULT_SUMMARY_FIELDS,
     'group': DEFAULT_SUMMARY_FIELDS,
     'default_environment': DEFAULT_SUMMARY_FIELDS + ('image',),
     'execution_environment': DEFAULT_SUMMARY_FIELDS + ('image',),
     'project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type', 'allow_override'),
-    'source_project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type'),
+    'source_project': DEFAULT_SUMMARY_FIELDS + ('status', 'scm_type', 'allow_override'),
     'project_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed'),
     'credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'kubernetes', 'credential_type_id'),
     'signature_validation_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'credential_type_id'),
@@ -184,6 +192,11 @@ SUMMARIZABLE_FK_FIELDS = {
 }
 
 
+# These fields can be edited on a constructed inventory's generated source (possibly by using the constructed
+# inventory's special API endpoint, but also by using the inventory sources endpoint).
+CONSTRUCTED_INVENTORY_SOURCE_EDITABLE_FIELDS = ('source_vars', 'update_cache_timeout', 'limit', 'verbosity')
+
+
 def reverse_gfk(content_object, request):
     """
     Computes a reverse for a GenericForeignKey field.
@@ -536,7 +549,7 @@ class BaseSerializer(serializers.ModelSerializer, metaclass=BaseSerializerMetacl
         #
         # This logic is to force rendering choice's on an uneditable field.
         # Note: Consider expanding this rendering for more than just choices fields
-        # Note: This logic works in conjuction with
+        # Note: This logic works in conjunction with
         if hasattr(model_field, 'choices') and model_field.choices:
             was_editable = model_field.editable
             model_field.editable = True
@@ -987,23 +1000,8 @@ class UserSerializer(BaseSerializer):
     def _update_password(self, obj, new_password):
         # For now we're not raising an error, just not saving password for
         # users managed by LDAP who already have an unusable password set.
-        if getattr(settings, 'AUTH_LDAP_SERVER_URI', None):
-            try:
-                if obj.pk and obj.profile.ldap_dn and not obj.has_usable_password():
-                    new_password = None
-            except AttributeError:
-                pass
-        if (
-            getattr(settings, 'SOCIAL_AUTH_GOOGLE_OAUTH2_KEY', None)
-            or getattr(settings, 'SOCIAL_AUTH_GITHUB_KEY', None)
-            or getattr(settings, 'SOCIAL_AUTH_GITHUB_ORG_KEY', None)
-            or getattr(settings, 'SOCIAL_AUTH_GITHUB_TEAM_KEY', None)
-            or getattr(settings, 'SOCIAL_AUTH_SAML_ENABLED_IDPS', None)
-        ) and obj.social_auth.all():
-            new_password = None
-        if (getattr(settings, 'RADIUS_SERVER', None) or getattr(settings, 'TACACSPLUS_HOST', None)) and obj.enterprise_auth.all():
-            new_password = None
-        if new_password:
+        # Get external password will return something like ldap or enterprise or None if the user isn't external. We only want to allow a password update for a None option
+        if new_password and not self.get_external_account(obj):
             obj.set_password(new_password)
             obj.save(update_fields=['password'])
 
@@ -1680,13 +1678,8 @@ class InventorySerializer(LabelsListMixin, BaseSerializerWithVariables):
         res.update(
             dict(
                 hosts=self.reverse('api:inventory_hosts_list', kwargs={'pk': obj.pk}),
-                groups=self.reverse('api:inventory_groups_list', kwargs={'pk': obj.pk}),
-                root_groups=self.reverse('api:inventory_root_groups_list', kwargs={'pk': obj.pk}),
                 variable_data=self.reverse('api:inventory_variable_data', kwargs={'pk': obj.pk}),
                 script=self.reverse('api:inventory_script_view', kwargs={'pk': obj.pk}),
-                tree=self.reverse('api:inventory_tree_view', kwargs={'pk': obj.pk}),
-                inventory_sources=self.reverse('api:inventory_inventory_sources_list', kwargs={'pk': obj.pk}),
-                update_inventory_sources=self.reverse('api:inventory_inventory_sources_update', kwargs={'pk': obj.pk}),
                 activity_stream=self.reverse('api:inventory_activity_stream_list', kwargs={'pk': obj.pk}),
                 job_templates=self.reverse('api:inventory_job_template_list', kwargs={'pk': obj.pk}),
                 ad_hoc_commands=self.reverse('api:inventory_ad_hoc_commands_list', kwargs={'pk': obj.pk}),
@@ -1697,8 +1690,18 @@ class InventorySerializer(LabelsListMixin, BaseSerializerWithVariables):
                 labels=self.reverse('api:inventory_label_list', kwargs={'pk': obj.pk}),
             )
         )
+        if obj.kind in ('', 'constructed'):
+            # links not relevant for the "old" smart inventory
+            res['groups'] = self.reverse('api:inventory_groups_list', kwargs={'pk': obj.pk})
+            res['root_groups'] = self.reverse('api:inventory_root_groups_list', kwargs={'pk': obj.pk})
+            res['update_inventory_sources'] = self.reverse('api:inventory_inventory_sources_update', kwargs={'pk': obj.pk})
+            res['inventory_sources'] = self.reverse('api:inventory_inventory_sources_list', kwargs={'pk': obj.pk})
+            res['tree'] = self.reverse('api:inventory_tree_view', kwargs={'pk': obj.pk})
         if obj.organization:
             res['organization'] = self.reverse('api:organization_detail', kwargs={'pk': obj.organization.pk})
+        if obj.kind == 'constructed':
+            res['input_inventories'] = self.reverse('api:inventory_input_inventories', kwargs={'pk': obj.pk})
+            res['constructed_url'] = self.reverse('api:constructed_inventory_detail', kwargs={'pk': obj.pk})
         return res
 
     def to_representation(self, obj):
@@ -1740,6 +1743,91 @@ class InventorySerializer(LabelsListMixin, BaseSerializerWithVariables):
         return super(InventorySerializer, self).validate(attrs)
 
 
+class ConstructedFieldMixin(serializers.Field):
+    def get_attribute(self, instance):
+        if not hasattr(instance, '_constructed_inv_src'):
+            instance._constructed_inv_src = instance.inventory_sources.first()
+        inv_src = instance._constructed_inv_src
+        return super().get_attribute(inv_src)  # yoink
+
+
+class ConstructedCharField(ConstructedFieldMixin, serializers.CharField):
+    pass
+
+
+class ConstructedIntegerField(ConstructedFieldMixin, serializers.IntegerField):
+    pass
+
+
+class ConstructedInventorySerializer(InventorySerializer):
+    source_vars = ConstructedCharField(
+        required=False,
+        default=None,
+        allow_blank=True,
+        help_text=_('The source_vars for the related auto-created inventory source, special to constructed inventory.'),
+    )
+    update_cache_timeout = ConstructedIntegerField(
+        required=False,
+        allow_null=True,
+        min_value=0,
+        default=None,
+        help_text=_('The cache timeout for the related auto-created inventory source, special to constructed inventory'),
+    )
+    limit = ConstructedCharField(
+        required=False,
+        default=None,
+        allow_blank=True,
+        help_text=_('The limit to restrict the returned hosts for the related auto-created inventory source, special to constructed inventory.'),
+    )
+    verbosity = ConstructedIntegerField(
+        required=False,
+        allow_null=True,
+        min_value=0,
+        max_value=2,
+        default=None,
+        help_text=_('The verbosity level for the related auto-created inventory source, special to constructed inventory'),
+    )
+
+    class Meta:
+        model = Inventory
+        fields = ('*', '-host_filter') + CONSTRUCTED_INVENTORY_SOURCE_EDITABLE_FIELDS
+        read_only_fields = ('*', 'kind')
+
+    def pop_inv_src_data(self, data):
+        inv_src_data = {}
+        for field in CONSTRUCTED_INVENTORY_SOURCE_EDITABLE_FIELDS:
+            if field in data:
+                # values always need to be removed, as they are not valid for Inventory model
+                value = data.pop(field)
+                # null is not valid for any of those fields, taken as not-provided
+                if value is not None:
+                    inv_src_data[field] = value
+        return inv_src_data
+
+    def apply_inv_src_data(self, inventory, inv_src_data):
+        if inv_src_data:
+            update_fields = []
+            inv_src = inventory.inventory_sources.first()
+            for field, value in inv_src_data.items():
+                setattr(inv_src, field, value)
+                update_fields.append(field)
+            if update_fields:
+                inv_src.save(update_fields=update_fields)
+
+    def create(self, validated_data):
+        validated_data['kind'] = 'constructed'
+        inv_src_data = self.pop_inv_src_data(validated_data)
+        inventory = super().create(validated_data)
+        self.apply_inv_src_data(inventory, inv_src_data)
+        return inventory
+
+    def update(self, obj, validated_data):
+        inv_src_data = self.pop_inv_src_data(validated_data)
+        obj = super().update(obj, validated_data)
+        self.apply_inv_src_data(obj, inv_src_data)
+        return obj
+
+
 class InventoryScriptSerializer(InventorySerializer):
     class Meta:
         fields = ()
@@ -1793,6 +1881,9 @@ class HostSerializer(BaseSerializerWithVariables):
                 ansible_facts=self.reverse('api:host_ansible_facts_detail', kwargs={'pk': obj.pk}),
             )
         )
+        if obj.inventory.kind == 'constructed':
+            res['original_host'] = self.reverse('api:host_detail', kwargs={'pk': obj.instance_id})
+            res['ansible_facts'] = self.reverse('api:host_ansible_facts_detail', kwargs={'pk': obj.instance_id})
         if obj.inventory:
             res['inventory'] = self.reverse('api:inventory_detail', kwargs={'pk': obj.inventory.pk})
         if obj.last_job:
@@ -1814,6 +1905,10 @@ class HostSerializer(BaseSerializerWithVariables):
             group_list = [{'id': g.id, 'name': g.name} for g in obj.groups.all().order_by('id')[:5]]
         group_cnt = obj.groups.count()
         d.setdefault('groups', {'count': group_cnt, 'results': group_list})
+        if obj.inventory.kind == 'constructed':
+            summaries_qs = obj.constructed_host_summaries
+        else:
+            summaries_qs = obj.job_host_summaries
         d.setdefault(
             'recent_jobs',
             [
@@ -1824,7 +1919,7 @@ class HostSerializer(BaseSerializerWithVariables):
                     'status': j.job.status,
                     'finished': j.job.finished,
                 }
-                for j in obj.job_host_summaries.select_related('job__job_template').order_by('-created').defer('job__extra_vars', 'job__artifacts')[:5]
+                for j in summaries_qs.select_related('job__job_template').order_by('-created').defer('job__extra_vars', 'job__artifacts')[:5]
             ],
         )
         return d
@@ -1849,8 +1944,8 @@ class HostSerializer(BaseSerializerWithVariables):
         return value
 
     def validate_inventory(self, value):
-        if value.kind == 'smart':
-            raise serializers.ValidationError({"detail": _("Cannot create Host for Smart Inventory")})
+        if value.kind in ('constructed', 'smart'):
+            raise serializers.ValidationError({"detail": _("Cannot create Host for Smart or Constructed Inventories")})
         return value
 
     def validate_variables(self, value):
@@ -1867,7 +1962,7 @@ class HostSerializer(BaseSerializerWithVariables):
             vars_dict = parse_yaml_or_json(variables)
             vars_dict['ansible_ssh_port'] = port
             attrs['variables'] = json.dumps(vars_dict)
-        if Group.objects.filter(name=name, inventory=inventory).exists():
+        if inventory and Group.objects.filter(name=name, inventory=inventory).exists():
             raise serializers.ValidationError(_('A Group with that name already exists.'))
 
         return super(HostSerializer, self).validate(attrs)
@@ -1948,8 +2043,8 @@ class GroupSerializer(BaseSerializerWithVariables):
         return value
 
     def validate_inventory(self, value):
-        if value.kind == 'smart':
-            raise serializers.ValidationError({"detail": _("Cannot create Group for Smart Inventory")})
+        if value.kind in ('constructed', 'smart'):
+            raise serializers.ValidationError({"detail": _("Cannot create Group for Smart or Constructed Inventories")})
         return value
 
     def to_representation(self, obj):
@@ -1959,6 +2054,130 @@ class GroupSerializer(BaseSerializerWithVariables):
         return ret
 
 
+class BulkHostSerializer(HostSerializer):
+    class Meta:
+        model = Host
+        fields = (
+            'name',
+            'enabled',
+            'instance_id',
+            'description',
+            'variables',
+        )
+
+
+class BulkHostCreateSerializer(serializers.Serializer):
+    inventory = serializers.PrimaryKeyRelatedField(
+        queryset=Inventory.objects.all(), required=True, write_only=True, help_text=_('Primary Key ID of inventory to add hosts to.')
+    )
+    hosts = serializers.ListField(
+        child=BulkHostSerializer(),
+        allow_empty=False,
+        max_length=100000,
+        write_only=True,
+        help_text=_('List of hosts to be created, JSON. e.g. [{"name": "example.com"}, {"name": "127.0.0.1"}]'),
+    )
+
+    class Meta:
+        model = Inventory
+        fields = ('inventory', 'hosts')
+        read_only_fields = ()
+
+    def raise_if_host_counts_violated(self, attrs):
+        validation_info = get_licenser().validate()
+
+        org = attrs['inventory'].organization
+
+        if org:
+            org_active_count = Host.objects.org_active_count(org.id)
+            new_hosts = [h['name'] for h in attrs['hosts']]
+            org_net_new_host_count = len(new_hosts) - Host.objects.filter(inventory__organization=1, name__in=new_hosts).values('name').distinct().count()
+            if org.max_hosts > 0 and org_active_count + org_net_new_host_count > org.max_hosts:
+                raise PermissionDenied(
+                    _(
+                        "You have already reached the maximum number of %s hosts"
+                        " allowed for your organization. Contact your System Administrator"
+                        " for assistance." % org.max_hosts
+                    )
+                )
+
+            # Don't check license if it is open license
+        if validation_info.get('license_type', 'UNLICENSED') == 'open':
+            return
+
+        sys_free_instances = validation_info.get('free_instances', 0)
+        system_net_new_host_count = Host.objects.exclude(name__in=new_hosts).count()
+
+        if system_net_new_host_count > sys_free_instances:
+            hard_error = validation_info.get('trial', False) is True or validation_info['instance_count'] == 10
+            if hard_error:
+                # Only raise permission error for trial, otherwise just log a warning as we do in other inventory import situations
+                raise PermissionDenied(_("Host count exceeds available instances."))
+            logger.warning(_("Number of hosts allowed by license has been exceeded."))
+
+    def validate(self, attrs):
+        request = self.context.get('request', None)
+        inv = attrs['inventory']
+        if inv.kind != '':
+            raise serializers.ValidationError(_('Hosts can only be created in manual inventories (not smart or constructed types).'))
+        if len(attrs['hosts']) > settings.BULK_HOST_MAX_CREATE:
+            raise serializers.ValidationError(_('Number of hosts exceeds system setting BULK_HOST_MAX_CREATE'))
+        if request and not request.user.is_superuser:
+            if request.user not in inv.admin_role:
+                raise serializers.ValidationError(_(f'Inventory with id {inv.id} not found or lack permissions to add hosts.'))
+        current_hostnames = set(inv.hosts.values_list('name', flat=True))
+        new_names = [host['name'] for host in attrs['hosts']]
+        duplicate_new_names = [n for n in new_names if n in current_hostnames or new_names.count(n) > 1]
+        if duplicate_new_names:
+            raise serializers.ValidationError(_(f'Hostnames must be unique in an inventory. Duplicates found: {duplicate_new_names}'))
+
+        self.raise_if_host_counts_violated(attrs)
+
+        _now = now()
+        for host in attrs['hosts']:
+            host['created'] = _now
+            host['modified'] = _now
+            host['inventory'] = inv
+        return attrs
+
+    def create(self, validated_data):
+        # This assumes total_hosts is up to date, and it can get out of date if the inventory computed fields have not been updated lately.
+        # If we wanted to side step this we could query Hosts.objects.filter(inventory...)
+        old_total_hosts = validated_data['inventory'].total_hosts
+        result = [Host(**attrs) for attrs in validated_data['hosts']]
+        try:
+            Host.objects.bulk_create(result)
+        except Exception as e:
+            raise serializers.ValidationError({"detail": _(f"cannot create host, host creation error {e}")})
+        new_total_hosts = old_total_hosts + len(result)
+        request = self.context.get('request', None)
+        changes = {'total_hosts': [old_total_hosts, new_total_hosts]}
+        activity_entry = ActivityStream.objects.create(
+            operation='update',
+            object1='inventory',
+            changes=json.dumps(changes),
+            actor=request.user,
+        )
+        activity_entry.inventory.add(validated_data['inventory'])
+
+        # This actually updates the cached "total_hosts" field on the inventory
+        update_inventory_computed_fields.delay(validated_data['inventory'].id)
+        return_keys = [k for k in BulkHostSerializer().fields.keys()] + ['id']
+        return_data = {}
+        host_data = []
+        for r in result:
+            item = {k: getattr(r, k) for k in return_keys}
+            if not settings.IS_TESTING_MODE:
+                # sqlite acts different with bulk_create -- it doesn't return the id of the objects
+                # to get it, you have to do an additional query, which is not useful for our tests
+                item['url'] = reverse('api:host_detail', kwargs={'pk': r.id})
+            item['inventory'] = reverse('api:inventory_detail', kwargs={'pk': validated_data['inventory'].id})
+            host_data.append(item)
+        return_data['url'] = reverse('api:inventory_detail', kwargs={'pk': validated_data['inventory'].id})
+        return_data['hosts'] = host_data
+        return return_data
+
+
 class GroupTreeSerializer(GroupSerializer):
     children = serializers.SerializerMethodField()
 
@@ -2014,6 +2233,7 @@ class InventorySourceOptionsSerializer(BaseSerializer):
             'source',
             'source_path',
             'source_vars',
+            'scm_branch',
             'credential',
             'enabled_var',
             'enabled_value',
@@ -2023,6 +2243,7 @@ class InventorySourceOptionsSerializer(BaseSerializer):
             'custom_virtualenv',
             'timeout',
             'verbosity',
+            'limit',
         )
         read_only_fields = ('*', 'custom_virtualenv')
 
@@ -2129,8 +2350,8 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
         return value
 
     def validate_inventory(self, value):
-        if value and value.kind == 'smart':
-            raise serializers.ValidationError({"detail": _("Cannot create Inventory Source for Smart Inventory")})
+        if value and value.kind in ('constructed', 'smart'):
+            raise serializers.ValidationError({"detail": _("Cannot create Inventory Source for Smart or Constructed Inventories")})
         return value
 
     # TODO: remove when old 'credential' fields are removed
@@ -2174,14 +2395,25 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt
         def get_field_from_model_or_attrs(fd):
             return attrs.get(fd, self.instance and getattr(self.instance, fd) or None)
 
-        if get_field_from_model_or_attrs('source') == 'scm':
+        if self.instance and self.instance.source == 'constructed':
+            allowed_fields = CONSTRUCTED_INVENTORY_SOURCE_EDITABLE_FIELDS
+            for field in attrs:
+                if attrs[field] != getattr(self.instance, field) and field not in allowed_fields:
+                    raise serializers.ValidationError({"error": _("Cannot change field '{}' on a constructed inventory source.").format(field)})
+        elif get_field_from_model_or_attrs('source') == 'scm':
             if ('source' in attrs or 'source_project' in attrs) and get_field_from_model_or_attrs('source_project') is None:
                 raise serializers.ValidationError({"source_project": _("Project required for scm type sources.")})
+        elif get_field_from_model_or_attrs('source') == 'constructed':
+            raise serializers.ValidationError({"error": _('constructed not a valid source for inventory')})
         else:
-            redundant_scm_fields = list(filter(lambda x: attrs.get(x, None), ['source_project', 'source_path']))
+            redundant_scm_fields = list(filter(lambda x: attrs.get(x, None), ['source_project', 'source_path', 'scm_branch']))
             if redundant_scm_fields:
                 raise serializers.ValidationError({"detail": _("Cannot set %s if not SCM type." % ' '.join(redundant_scm_fields))})
 
+        project = get_field_from_model_or_attrs('source_project')
+        if get_field_from_model_or_attrs('scm_branch') and not project.allow_override:
+            raise serializers.ValidationError({'scm_branch': _('Project does not allow overriding branch.')})
+
         attrs = super(InventorySourceSerializer, self).validate(attrs)
 
         # Check type consistency of source and cloud credential, if provided
@@ -3914,6 +4146,7 @@ class JobHostSummarySerializer(BaseSerializer):
             '-description',
             'job',
             'host',
+            'constructed_host',
             'host_name',
             'changed',
             'dark',
@@ -3997,7 +4230,7 @@ class JobEventSerializer(BaseSerializer):
         # Show full stdout for playbook_on_* events.
         if obj and obj.event.startswith('playbook_on'):
             return data
-        # If the view logic says to not trunctate (request was to the detail view or a param was used)
+        # If the view logic says to not truncate (request was to the detail view or a param was used)
         if self.context.get('no_truncate', False):
             return data
         max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY
@@ -4028,7 +4261,7 @@ class ProjectUpdateEventSerializer(JobEventSerializer):
         # raw SCM URLs in their stdout (which *could* contain passwords)
         # attempt to detect and filter HTTP basic auth passwords in the stdout
         # of these types of events
-        if obj.event_data.get('task_action') in ('git', 'svn'):
+        if obj.event_data.get('task_action') in ('git', 'svn', 'ansible.builtin.git', 'ansible.builtin.svn'):
             try:
                 return json.loads(UriCleaner.remove_sensitive(json.dumps(obj.event_data)))
             except Exception:
@@ -4072,7 +4305,7 @@ class AdHocCommandEventSerializer(BaseSerializer):
 
     def to_representation(self, obj):
         data = super(AdHocCommandEventSerializer, self).to_representation(obj)
-        # If the view logic says to not trunctate (request was to the detail view or a param was used)
+        # If the view logic says to not truncate (request was to the detail view or a param was used)
         if self.context.get('no_truncate', False):
             return data
         max_bytes = settings.EVENT_STDOUT_MAX_BYTES_DISPLAY
@@ -4419,6 +4652,271 @@ class WorkflowJobLaunchSerializer(BaseSerializer):
         return accepted
 
 
+class BulkJobNodeSerializer(WorkflowJobNodeSerializer):
+    # We don't do a PrimaryKeyRelatedField for unified_job_template and others, because that increases the number
+    # of database queries, rather we take them as integer and later convert them to objects in get_objectified_jobs
+    unified_job_template = serializers.IntegerField(
+        required=True, min_value=1, help_text=_('Primary key of the template for this job, can be a job template or inventory source.')
+    )
+    inventory = serializers.IntegerField(required=False, min_value=1)
+    execution_environment = serializers.IntegerField(required=False, min_value=1)
+    # many-to-many fields
+    credentials = serializers.ListField(child=serializers.IntegerField(min_value=1), required=False)
+    labels = serializers.ListField(child=serializers.IntegerField(min_value=1), required=False)
+    # TODO: Use instance group role added via PR 13584(once merged), for now everything related to instance group is commented
+    # instance_groups = serializers.ListField(child=serializers.IntegerField(min_value=1), required=False)
+
+    class Meta:
+        model = WorkflowJobNode
+        fields = ('*', 'credentials', 'labels')  # m2m fields are not canonical for WJ nodes, TODO: add instance_groups once supported
+
+    def validate(self, attrs):
+        return super(LaunchConfigurationBaseSerializer, self).validate(attrs)
+
+    def get_validation_exclusions(self, obj=None):
+        ret = super().get_validation_exclusions(obj)
+        ret.extend(['unified_job_template', 'inventory', 'execution_environment'])
+        return ret
+
+
+class BulkJobLaunchSerializer(serializers.Serializer):
+    name = serializers.CharField(default='Bulk Job Launch', max_length=512, write_only=True, required=False, allow_blank=True)  # limited by max name of jobs
+    jobs = BulkJobNodeSerializer(
+        many=True,
+        allow_empty=False,
+        write_only=True,
+        max_length=100000,
+        help_text=_('List of jobs to be launched, JSON. e.g. [{"unified_job_template": 7}, {"unified_job_template": 10}]'),
+    )
+    description = serializers.CharField(write_only=True, required=False, allow_blank=False)
+    extra_vars = serializers.JSONField(write_only=True, required=False)
+    organization = serializers.PrimaryKeyRelatedField(
+        queryset=Organization.objects.all(),
+        required=False,
+        default=None,
+        allow_null=True,
+        write_only=True,
+        help_text=_('Inherit permissions from this organization. If not provided, a organization the user is a member of will be selected automatically.'),
+    )
+    inventory = serializers.PrimaryKeyRelatedField(queryset=Inventory.objects.all(), required=False, write_only=True)
+    limit = serializers.CharField(write_only=True, required=False, allow_blank=False)
+    scm_branch = serializers.CharField(write_only=True, required=False, allow_blank=False)
+    skip_tags = serializers.CharField(write_only=True, required=False, allow_blank=False)
+    job_tags = serializers.CharField(write_only=True, required=False, allow_blank=False)
+
+    class Meta:
+        model = WorkflowJob
+        fields = ('name', 'jobs', 'description', 'extra_vars', 'organization', 'inventory', 'limit', 'scm_branch', 'skip_tags', 'job_tags')
+        read_only_fields = ()
+
+    def validate(self, attrs):
+        request = self.context.get('request', None)
+        identifiers = set()
+        if len(attrs['jobs']) > settings.BULK_JOB_MAX_LAUNCH:
+            raise serializers.ValidationError(_('Number of requested jobs exceeds system setting BULK_JOB_MAX_LAUNCH'))
+
+        for node in attrs['jobs']:
+            if 'identifier' in node:
+                if node['identifier'] in identifiers:
+                    raise serializers.ValidationError(_(f"Identifier {node['identifier']} not unique"))
+                identifiers.add(node['identifier'])
+            else:
+                node['identifier'] = str(uuid4())
+
+        requested_ujts = {j['unified_job_template'] for j in attrs['jobs']}
+        requested_use_inventories = {job['inventory'] for job in attrs['jobs'] if 'inventory' in job}
+        requested_use_execution_environments = {job['execution_environment'] for job in attrs['jobs'] if 'execution_environment' in job}
+        requested_use_credentials = set()
+        requested_use_labels = set()
+        # requested_use_instance_groups = set()
+        for job in attrs['jobs']:
+            for cred in job.get('credentials', []):
+                requested_use_credentials.add(cred)
+            for label in job.get('labels', []):
+                requested_use_labels.add(label)
+            # for instance_group in job.get('instance_groups', []):
+            #     requested_use_instance_groups.add(instance_group)
+
+        key_to_obj_map = {
+            "unified_job_template": {obj.id: obj for obj in UnifiedJobTemplate.objects.filter(id__in=requested_ujts)},
+            "inventory": {obj.id: obj for obj in Inventory.objects.filter(id__in=requested_use_inventories)},
+            "credentials": {obj.id: obj for obj in Credential.objects.filter(id__in=requested_use_credentials)},
+            "labels": {obj.id: obj for obj in Label.objects.filter(id__in=requested_use_labels)},
+            # "instance_groups": {obj.id: obj for obj in InstanceGroup.objects.filter(id__in=requested_use_instance_groups)},
+            "execution_environment": {obj.id: obj for obj in ExecutionEnvironment.objects.filter(id__in=requested_use_execution_environments)},
+        }
+
+        ujts = {}
+        for ujt in key_to_obj_map['unified_job_template'].values():
+            ujts.setdefault(type(ujt), [])
+            ujts[type(ujt)].append(ujt)
+
+        unallowed_types = set(ujts.keys()) - set([JobTemplate, Project, InventorySource, WorkflowJobTemplate])
+        if unallowed_types:
+            type_names = ' '.join([cls._meta.verbose_name.title() for cls in unallowed_types])
+            raise serializers.ValidationError(_("Template types {type_names} not allowed in bulk jobs").format(type_names=type_names))
+
+        for model, obj_list in ujts.items():
+            role_field = 'execute_role' if issubclass(model, (JobTemplate, WorkflowJobTemplate)) else 'update_role'
+            self.check_list_permission(model, set([obj.id for obj in obj_list]), role_field)
+
+        self.check_organization_permission(attrs, request)
+
+        if 'inventory' in attrs:
+            requested_use_inventories.add(attrs['inventory'].id)
+
+        self.check_list_permission(Inventory, requested_use_inventories, 'use_role')
+
+        self.check_list_permission(Credential, requested_use_credentials, 'use_role')
+        self.check_list_permission(Label, requested_use_labels)
+        # self.check_list_permission(InstanceGroup, requested_use_instance_groups)  # TODO: change to use_role for conflict
+        self.check_list_permission(ExecutionEnvironment, requested_use_execution_environments)  # TODO: change if roles introduced
+
+        jobs_object = self.get_objectified_jobs(attrs, key_to_obj_map)
+
+        attrs['jobs'] = jobs_object
+        if 'extra_vars' in attrs:
+            extra_vars_dict = parse_yaml_or_json(attrs['extra_vars'])
+            attrs['extra_vars'] = json.dumps(extra_vars_dict)
+        attrs = super().validate(attrs)
+        return attrs
+
+    def check_list_permission(self, model, id_list, role_field=None):
+        if not id_list:
+            return
+        user = self.context['request'].user
+        if role_field is None:  # implies "read" level permission is required
+            access_qs = user.get_queryset(model)
+        else:
+            access_qs = model.accessible_objects(user, role_field)
+
+        not_allowed = set(id_list) - set(access_qs.filter(id__in=id_list).values_list('id', flat=True))
+        if not_allowed:
+            raise serializers.ValidationError(
+                _("{model_name} {not_allowed} not found or you don't have permissions to access it").format(
+                    model_name=model._meta.verbose_name_plural.title(), not_allowed=not_allowed
+                )
+            )
+
+    def create(self, validated_data):
+        request = self.context.get('request', None)
+        launch_user = request.user if request else None
+        job_node_data = validated_data.pop('jobs')
+        wfj_deferred_attr_names = ('skip_tags', 'limit', 'job_tags')
+        wfj_deferred_vals = {}
+        for item in wfj_deferred_attr_names:
+            wfj_deferred_vals[item] = validated_data.pop(item, None)
+
+        wfj = WorkflowJob.objects.create(**validated_data, is_bulk_job=True, launch_type='manual', created_by=launch_user)
+        for key, val in wfj_deferred_vals.items():
+            if val:
+                setattr(wfj, key, val)
+        nodes = []
+        node_m2m_objects = {}
+        node_m2m_object_types_to_through_model = {
+            'credentials': WorkflowJobNode.credentials.through,
+            'labels': WorkflowJobNode.labels.through,
+            # 'instance_groups': WorkflowJobNode.instance_groups.through,
+        }
+        node_deferred_attr_names = (
+            'limit',
+            'scm_branch',
+            'verbosity',
+            'forks',
+            'diff_mode',
+            'job_tags',
+            'job_type',
+            'skip_tags',
+            'job_slice_count',
+            'timeout',
+        )
+        node_deferred_attrs = {}
+        for node_attrs in job_node_data:
+            # we need to add any m2m objects after creation via the through model
+            node_m2m_objects[node_attrs['identifier']] = {}
+            node_deferred_attrs[node_attrs['identifier']] = {}
+            for item in node_m2m_object_types_to_through_model.keys():
+                if item in node_attrs:
+                    node_m2m_objects[node_attrs['identifier']][item] = node_attrs.pop(item)
+
+            # Some attributes are not accepted by WorkflowJobNode __init__, we have to set them after
+            for item in node_deferred_attr_names:
+                if item in node_attrs:
+                    node_deferred_attrs[node_attrs['identifier']][item] = node_attrs.pop(item)
+
+            # Create the node objects
+            node_obj = WorkflowJobNode(workflow_job=wfj, created=wfj.created, modified=wfj.modified, **node_attrs)
+
+            # we can set the deferred attrs now
+            for item, value in node_deferred_attrs[node_attrs['identifier']].items():
+                setattr(node_obj, item, value)
+
+            # the node is now ready to be bulk created
+            nodes.append(node_obj)
+
+            # we'll need this later when we do the m2m through model bulk create
+            node_m2m_objects[node_attrs['identifier']]['node'] = node_obj
+
+        WorkflowJobNode.objects.bulk_create(nodes)
+
+        # Deal with the m2m objects we have to create once the node exists
+        for field_name, through_model in node_m2m_object_types_to_through_model.items():
+            through_model_objects = []
+            for node_identifier in node_m2m_objects.keys():
+                if field_name in node_m2m_objects[node_identifier] and field_name == 'credentials':
+                    for cred in node_m2m_objects[node_identifier][field_name]:
+                        through_model_objects.append(through_model(credential=cred, workflowjobnode=node_m2m_objects[node_identifier]['node']))
+                if field_name in node_m2m_objects[node_identifier] and field_name == 'labels':
+                    for label in node_m2m_objects[node_identifier][field_name]:
+                        through_model_objects.append(through_model(label=label, workflowjobnode=node_m2m_objects[node_identifier]['node']))
+                # if obj_type in node_m2m_objects[node_identifier] and obj_type == 'instance_groups':
+                #     for instance_group in node_m2m_objects[node_identifier][obj_type]:
+                #         through_model_objects.append(through_model(instancegroup=instance_group, workflowjobnode=node_m2m_objects[node_identifier]['node']))
+            if through_model_objects:
+                through_model.objects.bulk_create(through_model_objects)
+
+        wfj.save()
+        wfj.signal_start()
+
+        return WorkflowJobSerializer().to_representation(wfj)
+
+    def check_organization_permission(self, attrs, request):
+        # validate Organization
+        # - If the orgs is not set, set it to the org of the launching user
+        # - If the user is part of multiple orgs, throw a validation error saying user is part of multiple orgs, please provide one
+        if not request.user.is_superuser:
+            read_org_qs = Organization.accessible_objects(request.user, 'member_role')
+            if 'organization' not in attrs or attrs['organization'] == None or attrs['organization'] == '':
+                read_org_ct = read_org_qs.count()
+                if read_org_ct == 1:
+                    attrs['organization'] = read_org_qs.first()
+                elif read_org_ct > 1:
+                    raise serializers.ValidationError("User has permission to multiple Organizations, please set one of them in the request")
+                else:
+                    raise serializers.ValidationError("User not part of any organization, please assign an organization to assign to the bulk job")
+            else:
+                allowed_orgs = set(read_org_qs.values_list('id', flat=True))
+                requested_org = attrs['organization']
+                if requested_org.id not in allowed_orgs:
+                    raise ValidationError(_(f"Organization {requested_org.id} not found or you don't have permissions to access it"))
+
+    def get_objectified_jobs(self, attrs, key_to_obj_map):
+        objectified_jobs = []
+        # This loop is generalized so we should only have to add related items to the key_to_obj_map
+        for job in attrs['jobs']:
+            objectified_job = {}
+            for key, value in job.items():
+                if key in key_to_obj_map:
+                    if isinstance(value, int):
+                        objectified_job[key] = key_to_obj_map[key][value]
+                    elif isinstance(value, list):
+                        objectified_job[key] = [key_to_obj_map[key][item] for item in value]
+                else:
+                    objectified_job[key] = value
+            objectified_jobs.append(objectified_job)
+        return objectified_jobs
+
+
 class NotificationTemplateSerializer(BaseSerializer):
     show_capabilities = ['edit', 'delete', 'copy']
     capabilities_prefetch = [{'copy': 'organization.admin'}]
@@ -4765,7 +5263,7 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria
         ),
     )
     until = serializers.SerializerMethodField(
-        help_text=_('The date this schedule will end. This field is computed from the RRULE. If the schedule does not end an emptry string will be returned'),
+        help_text=_('The date this schedule will end. This field is computed from the RRULE. If the schedule does not end an empty string will be returned'),
     )
 
     class Meta:
@@ -5002,6 +5500,32 @@ class InstanceHealthCheckSerializer(BaseSerializer):
         fields = read_only_fields
 
 
+class HostMetricSerializer(BaseSerializer):
+    show_capabilities = ['delete']
+
+    class Meta:
+        model = HostMetric
+        fields = (
+            "id",
+            "hostname",
+            "url",
+            "first_automation",
+            "last_automation",
+            "last_deleted",
+            "automated_counter",
+            "deleted_counter",
+            "deleted",
+            "used_in_inventories",
+        )
+
+
+class HostMetricSummaryMonthlySerializer(BaseSerializer):
+    class Meta:
+        model = HostMetricSummaryMonthly
+        read_only_fields = ("id", "date", "license_consumed", "license_capacity", "hosts_added", "hosts_deleted", "indirectly_managed_hosts")
+        fields = read_only_fields
+
+
 class InstanceGroupSerializer(BaseSerializer):
     show_capabilities = ['edit', 'delete']
     capacity = serializers.SerializerMethodField()
@@ -5087,6 +5611,8 @@ class InstanceGroupSerializer(BaseSerializer):
         res = super(InstanceGroupSerializer, self).get_related(obj)
         res['jobs'] = self.reverse('api:instance_group_unified_jobs_list', kwargs={'pk': obj.pk})
         res['instances'] = self.reverse('api:instance_group_instance_list', kwargs={'pk': obj.pk})
+        res['access_list'] = self.reverse('api:instance_group_access_list', kwargs={'pk': obj.pk})
+        res['object_roles'] = self.reverse('api:instance_group_object_role_list', kwargs={'pk': obj.pk})
         if obj.credential:
             res['credential'] = self.reverse('api:credential_detail', kwargs={'pk': obj.credential_id})
 

awx/api/templates/api/api_v1_root_view.md

@@ -1,4 +0,0 @@
-Version 1 of the Ansible Tower REST API.
-
-Make a GET request to this resource to obtain a list of all child resources
-available via the API.

awx/api/templates/api/api_v2_config_view.md

@@ -7,10 +7,12 @@ the following fields (some fields may not be visible to all users):
 * `project_base_dir`: Path on the server where projects and playbooks are \
   stored.
 * `project_local_paths`: List of directories beneath `project_base_dir` to
-  use when creating/editing a project.
+  use when creating/editing a manual project.
 * `time_zone`: The configured time zone for the server.
 * `license_info`: Information about the current license.
 * `version`: Version of Ansible Tower package installed.
+* `custom_virtualenvs`: Deprecated venv locations from before migration to
+  execution environments. Export tooling is in `awx-manage` commands.
 * `eula`: The current End-User License Agreement
 {% endifmeth %}
 

awx/api/templates/api/bulk_host_create_view.md

@@ -0,0 +1,41 @@
+# Bulk Host Create
+
+This endpoint allows the client to create multiple hosts and associate them with an inventory. They may do this by providing the inventory ID and a list of json that would normally be provided to create hosts.
+
+Example:
+
+    {
+        "inventory": 1,
+        "hosts": [
+            {"name": "example1.com", "variables": "ansible_connection: local"},
+            {"name": "example2.com"}
+        ]
+    }
+
+Return data:
+
+    {
+        "url": "/api/v2/inventories/3/hosts/",
+        "hosts": [
+            {
+                "name": "example1.com",
+                "enabled": true,
+                "instance_id": "",
+                "description": "",
+                "variables": "ansible_connection: local",
+                "id": 1255,
+                "url": "/api/v2/hosts/1255/",
+                "inventory": "/api/v2/inventories/3/"
+            },
+            {
+                "name": "example2.com",
+                "enabled": true,
+                "instance_id": "",
+                "description": "",
+                "variables": "",
+                "id": 1256,
+                "url": "/api/v2/hosts/1256/",
+                "inventory": "/api/v2/inventories/3/"
+            }
+        ]
+    }

awx/api/templates/api/bulk_job_launch_view.md

@@ -0,0 +1,13 @@
+# Bulk Job Launch
+
+This endpoint allows the client to launch multiple UnifiedJobTemplates at a time, along side any launch time parameters that they would normally set at launch time.
+
+Example:
+
+    {
+        "name": "my bulk job",
+        "jobs": [
+            {"unified_job_template": 7, "inventory": 2},
+            {"unified_job_template": 7, "credentials": [3]}
+        ]
+    }

awx/api/templates/api/bulk_view.md

@@ -0,0 +1,3 @@
+# Bulk Actions
+
+This endpoint lists available bulk action APIs. 

awx/api/templates/api/dashboard_inventory_graph_view.md

@@ -3,7 +3,7 @@ Make a GET request to this resource to retrieve aggregate statistics about inven
 Including fetching the number of total hosts tracked by Tower over an amount of time and the current success or
 failed status of hosts which have run jobs within an Inventory.
 
-## Parmeters and Filtering
+## Parameters and Filtering
 
 The `period` of the data can be adjusted with:
 
@@ -24,7 +24,7 @@ Data about the number of hosts will be returned in the following format:
 Each element contains an epoch timestamp represented in seconds and a numerical value indicating
 the number of hosts that exist at a given moment
 
-Data about failed and successfull hosts by inventory will be given as:
+Data about failed and successful hosts by inventory will be given as:
 
     {
             "sources": [

awx/api/templates/api/dashboard_jobs_graph_view.md

@@ -2,7 +2,7 @@
 
 Make a GET request to this resource to retrieve aggregate statistics about job runs suitable for graphing.
 
-## Parmeters and Filtering
+## Parameters and Filtering
 
 The `period` of the data can be adjusted with:
 

awx/api/templates/api/host_fact_compare_view.md

@@ -1,11 +0,0 @@
-# List Fact Scans for a Host Specific Host Scan
-
-Make a GET request to this resource to retrieve system tracking data for a particular scan
-
-You may filter by datetime:
-
-`?datetime=2015-06-01`
-
-and module
-
-`?datetime=2015-06-01&module=ansible`

awx/api/templates/api/host_fact_versions_list.md

@@ -1,11 +0,0 @@
-# List Fact Scans for a Host by Module and Date
-
-Make a GET request to this resource to retrieve system tracking scans by module and date/time
-
-You may filter scan runs using the `from` and `to` properties:
-
-`?from=2015-06-01%2012:00:00&to=2015-06-03`
-
-You may also filter by module
-
-`?module=packages`

awx/api/templates/api/host_insights.md

@@ -1 +0,0 @@
-# List Red Hat Insights for a Host

awx/api/templates/api/host_metric_detail.md

@@ -0,0 +1,18 @@
+{% ifmeth GET %}
+# Retrieve {{ model_verbose_name|title|anora }}:
+
+Make GET request to this resource to retrieve a single {{ model_verbose_name }}
+record containing the following fields:
+
+{% include "api/_result_fields_common.md" %}
+{% endifmeth %}
+
+{% ifmeth DELETE %}
+# Delete {{ model_verbose_name|title|anora }}:
+
+Make a DELETE request to this resource to soft-delete this {{ model_verbose_name }}.
+
+A soft deletion will mark the `deleted` field as true and exclude the host
+metric from license calculations.
+This may be undone later if the same hostname is automated again afterwards.
+{% endifmeth %}

awx/api/templates/api/inventory_inventory_sources_update.md

@@ -18,7 +18,7 @@ inventory sources:
 * `inventory_update`: ID of the inventory update job that was started.
   (integer, read-only)
 * `project_update`: ID of the project update job that was started if this inventory source is an SCM source.
-  (interger, read-only, optional)
+  (integer, read-only, optional)
 
 Note: All manual inventory sources (source="") will be ignored by the update_inventory_sources endpoint.  This endpoint will not update inventory sources for Smart Inventories.  
 

awx/api/templates/api/job_start.md

@@ -1,21 +0,0 @@
-{% ifmeth GET %}
-# Determine if a Job can be started
-
-Make a GET request to this resource to determine if the job can be started and
-whether any passwords are required to start the job.  The response will include
-the following fields:
-
-* `can_start`: Flag indicating if this job can be started (boolean, read-only)
-* `passwords_needed_to_start`: Password names required to start the job (array,
-  read-only)
-{% endifmeth %}
-
-{% ifmeth POST %}
-# Start a Job
-Make a POST request to this resource to start the job.  If any passwords are
-required, they must be passed via POST data.
-
-If successful, the response status code will be 202.  If any required passwords
-are not provided, a 400 status code will be returned.  If the job cannot be
-started, a 405 status code will be returned.
-{% endifmeth %}

awx/api/templates/instance_install_bundle/group_vars/all.yml

@@ -2,6 +2,7 @@ receptor_user: awx
 receptor_group: awx
 receptor_verify: true
 receptor_tls: true
+receptor_mintls13: false
 receptor_work_commands:
   ansible-runner:
     command: ansible-runner

awx/api/urls/host_metric.py

@@ -0,0 +1,10 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.urls import re_path
+
+from awx.api.views import HostMetricList, HostMetricDetail
+
+urls = [re_path(r'^$', HostMetricList.as_view(), name='host_metric_list'), re_path(r'^(?P<pk>[0-9]+)/$', HostMetricDetail.as_view(), name='host_metric_detail')]
+
+__all__ = ['urls']

awx/api/urls/instance_group.py

@@ -3,7 +3,14 @@
 
 from django.urls import re_path
 
-from awx.api.views import InstanceGroupList, InstanceGroupDetail, InstanceGroupUnifiedJobsList, InstanceGroupInstanceList
+from awx.api.views import (
+    InstanceGroupList,
+    InstanceGroupDetail,
+    InstanceGroupUnifiedJobsList,
+    InstanceGroupInstanceList,
+    InstanceGroupAccessList,
+    InstanceGroupObjectRolesList,
+)
 
 
 urls = [
@@ -11,6 +18,8 @@ urls = [
     re_path(r'^(?P<pk>[0-9]+)/$', InstanceGroupDetail.as_view(), name='instance_group_detail'),
     re_path(r'^(?P<pk>[0-9]+)/jobs/$', InstanceGroupUnifiedJobsList.as_view(), name='instance_group_unified_jobs_list'),
     re_path(r'^(?P<pk>[0-9]+)/instances/$', InstanceGroupInstanceList.as_view(), name='instance_group_instance_list'),
+    re_path(r'^(?P<pk>[0-9]+)/access_list/$', InstanceGroupAccessList.as_view(), name='instance_group_access_list'),
+    re_path(r'^(?P<pk>[0-9]+)/object_roles/$', InstanceGroupObjectRolesList.as_view(), name='instance_group_object_role_list'),
 ]
 
 __all__ = ['urls']

awx/api/urls/inventory.py

@@ -6,7 +6,10 @@ from django.urls import re_path
 from awx.api.views.inventory import (
     InventoryList,
     InventoryDetail,
+    ConstructedInventoryDetail,
+    ConstructedInventoryList,
     InventoryActivityStreamList,
+    InventoryInputInventoriesList,
     InventoryJobTemplateList,
     InventoryAccessList,
     InventoryObjectRolesList,
@@ -37,6 +40,7 @@ urls = [
     re_path(r'^(?P<pk>[0-9]+)/script/$', InventoryScriptView.as_view(), name='inventory_script_view'),
     re_path(r'^(?P<pk>[0-9]+)/tree/$', InventoryTreeView.as_view(), name='inventory_tree_view'),
     re_path(r'^(?P<pk>[0-9]+)/inventory_sources/$', InventoryInventorySourcesList.as_view(), name='inventory_inventory_sources_list'),
+    re_path(r'^(?P<pk>[0-9]+)/input_inventories/$', InventoryInputInventoriesList.as_view(), name='inventory_input_inventories'),
     re_path(r'^(?P<pk>[0-9]+)/update_inventory_sources/$', InventoryInventorySourcesUpdate.as_view(), name='inventory_inventory_sources_update'),
     re_path(r'^(?P<pk>[0-9]+)/activity_stream/$', InventoryActivityStreamList.as_view(), name='inventory_activity_stream_list'),
     re_path(r'^(?P<pk>[0-9]+)/job_templates/$', InventoryJobTemplateList.as_view(), name='inventory_job_template_list'),
@@ -48,4 +52,10 @@ urls = [
     re_path(r'^(?P<pk>[0-9]+)/copy/$', InventoryCopy.as_view(), name='inventory_copy'),
 ]
 
-__all__ = ['urls']
+# Constructed inventory special views
+constructed_inventory_urls = [
+    re_path(r'^$', ConstructedInventoryList.as_view(), name='constructed_inventory_list'),
+    re_path(r'^(?P<pk>[0-9]+)/$', ConstructedInventoryDetail.as_view(), name='constructed_inventory_detail'),
+]
+
+__all__ = ['urls', 'constructed_inventory_urls']

awx/api/urls/urls.py

@@ -30,7 +30,15 @@ from awx.api.views import (
     OAuth2TokenList,
     ApplicationOAuth2TokenList,
     OAuth2ApplicationDetail,
+    # HostMetricSummaryMonthlyList, # It will be enabled in future version of the AWX
 )
+
+from awx.api.views.bulk import (
+    BulkView,
+    BulkHostCreateView,
+    BulkJobLaunchView,
+)
+
 from awx.api.views.mesh_visualizer import MeshVisualizer
 
 from awx.api.views.metrics import MetricsView
@@ -39,10 +47,11 @@ from .organization import urls as organization_urls
 from .user import urls as user_urls
 from .project import urls as project_urls
 from .project_update import urls as project_update_urls
-from .inventory import urls as inventory_urls
+from .inventory import urls as inventory_urls, constructed_inventory_urls
 from .execution_environments import urls as execution_environment_urls
 from .team import urls as team_urls
 from .host import urls as host_urls
+from .host_metric import urls as host_metric_urls
 from .group import urls as group_urls
 from .inventory_source import urls as inventory_source_urls
 from .inventory_update import urls as inventory_update_urls
@@ -110,7 +119,11 @@ v2_urls = [
     re_path(r'^project_updates/', include(project_update_urls)),
     re_path(r'^teams/', include(team_urls)),
     re_path(r'^inventories/', include(inventory_urls)),
+    re_path(r'^constructed_inventories/', include(constructed_inventory_urls)),
     re_path(r'^hosts/', include(host_urls)),
+    re_path(r'^host_metrics/', include(host_metric_urls)),
+    # It will be enabled in future version of the AWX
+    # re_path(r'^host_metric_summary_monthly/$', HostMetricSummaryMonthlyList.as_view(), name='host_metric_summary_monthly_list'),
     re_path(r'^groups/', include(group_urls)),
     re_path(r'^inventory_sources/', include(inventory_source_urls)),
     re_path(r'^inventory_updates/', include(inventory_update_urls)),
@@ -136,6 +149,9 @@ v2_urls = [
     re_path(r'^activity_stream/', include(activity_stream_urls)),
     re_path(r'^workflow_approval_templates/', include(workflow_approval_template_urls)),
     re_path(r'^workflow_approvals/', include(workflow_approval_urls)),
+    re_path(r'^bulk/$', BulkView.as_view(), name='bulk'),
+    re_path(r'^bulk/host_create/$', BulkHostCreateView.as_view(), name='bulk_host_create'),
+    re_path(r'^bulk/job_launch/$', BulkJobLaunchView.as_view(), name='bulk_job_launch'),
 ]
 
 

awx/api/views/__init__.py

@@ -17,7 +17,6 @@ from collections import OrderedDict
 
 from urllib3.exceptions import ConnectTimeoutError
 
-
 # Django
 from django.conf import settings
 from django.core.exceptions import FieldError, ObjectDoesNotExist
@@ -30,7 +29,7 @@ from django.utils.safestring import mark_safe
 from django.utils.timezone import now
 from django.views.decorators.csrf import csrf_exempt
 from django.template.loader import render_to_string
-from django.http import HttpResponse
+from django.http import HttpResponse, HttpResponseRedirect
 from django.contrib.contenttypes.models import ContentType
 from django.utils.translation import gettext_lazy as _
 
@@ -152,7 +151,7 @@ def api_exception_handler(exc, context):
         if 'awx.named_url_rewritten' in req.environ and not str(getattr(exc, 'status_code', 0)).startswith('2'):
             # if the URL was rewritten, and it's not a 2xx level status code,
             # revert the request.path to its original value to avoid leaking
-            # any context about the existance of resources
+            # any context about the existence of resources
             req.path = req.environ['awx.named_url_rewritten']
             if exc.status_code == 403:
                 exc = NotFound(detail=_('Not found.'))
@@ -172,7 +171,7 @@ class DashboardView(APIView):
         user_inventory = get_user_queryset(request.user, models.Inventory)
         inventory_with_failed_hosts = user_inventory.filter(hosts_with_active_failures__gt=0)
         user_inventory_external = user_inventory.filter(has_inventory_sources=True)
-        # if there are *zero* inventories, this aggregrate query will be None, fall back to 0
+        # if there are *zero* inventories, this aggregate query will be None, fall back to 0
         failed_inventory = user_inventory.aggregate(Sum('inventory_sources_with_failures'))['inventory_sources_with_failures__sum'] or 0
         data['inventories'] = {
             'url': reverse('api:inventory_list', request=request),
@@ -466,6 +465,23 @@ class InstanceGroupUnifiedJobsList(SubListAPIView):
     relationship = "unifiedjob_set"
 
 
+class InstanceGroupAccessList(ResourceAccessList):
+    model = models.User  # needs to be User for AccessLists
+    parent_model = models.InstanceGroup
+
+
+class InstanceGroupObjectRolesList(SubListAPIView):
+    model = models.Role
+    serializer_class = serializers.RoleSerializer
+    parent_model = models.InstanceGroup
+    search_fields = ('role_field', 'content_type__model')
+
+    def get_queryset(self):
+        po = self.get_parent_object()
+        content_type = ContentType.objects.get_for_model(self.parent_model)
+        return models.Role.objects.filter(content_type=content_type, object_id=po.pk)
+
+
 class InstanceGroupInstanceList(InstanceGroupMembershipMixin, SubListAttachDetachAPIView):
     name = _("Instance Group's Instances")
     model = models.Instance
@@ -1531,6 +1547,41 @@ class HostRelatedSearchMixin(object):
         return ret
 
 
+class HostMetricList(ListAPIView):
+    name = _("Host Metrics List")
+    model = models.HostMetric
+    serializer_class = serializers.HostMetricSerializer
+    permission_classes = (IsSystemAdminOrAuditor,)
+    search_fields = ('hostname', 'deleted')
+
+    def get_queryset(self):
+        return self.model.objects.all()
+
+
+class HostMetricDetail(RetrieveDestroyAPIView):
+    name = _("Host Metric Detail")
+    model = models.HostMetric
+    serializer_class = serializers.HostMetricSerializer
+    permission_classes = (IsSystemAdminOrAuditor,)
+
+    def delete(self, request, *args, **kwargs):
+        self.get_object().soft_delete()
+
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
+
+# It will be enabled in future version of the AWX
+# class HostMetricSummaryMonthlyList(ListAPIView):
+#     name = _("Host Metrics Summary Monthly")
+#     model = models.HostMetricSummaryMonthly
+#     serializer_class = serializers.HostMetricSummaryMonthlySerializer
+#     permission_classes = (IsSystemAdminOrAuditor,)
+#     search_fields = ('date',)
+#
+#     def get_queryset(self):
+#         return self.model.objects.all()
+
+
 class HostList(HostRelatedSearchMixin, ListCreateAPIView):
     always_allow_superuser = False
     model = models.Host
@@ -1559,6 +1610,8 @@ class HostDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIView):
     def delete(self, request, *args, **kwargs):
         if self.get_object().inventory.pending_deletion:
             return Response({"error": _("The inventory for this host is already being deleted.")}, status=status.HTTP_400_BAD_REQUEST)
+        if self.get_object().inventory.kind == 'constructed':
+            return Response({"error": _("Delete constructed inventory hosts from input inventory.")}, status=status.HTTP_400_BAD_REQUEST)
         return super(HostDetail, self).delete(request, *args, **kwargs)
 
 
@@ -1566,6 +1619,14 @@ class HostAnsibleFactsDetail(RetrieveAPIView):
     model = models.Host
     serializer_class = serializers.AnsibleFactsSerializer
 
+    def get(self, request, *args, **kwargs):
+        obj = self.get_object()
+        if obj.inventory.kind == 'constructed':
+            # If this is a constructed inventory host, it is not the source of truth about facts
+            # redirect to the original input inventory host instead
+            return HttpResponseRedirect(reverse('api:host_ansible_facts_detail', kwargs={'pk': obj.instance_id}, request=self.request))
+        return super().get(request, *args, **kwargs)
+
 
 class InventoryHostsList(HostRelatedSearchMixin, SubListCreateAttachDetachAPIView):
     model = models.Host
@@ -1667,7 +1728,7 @@ class GroupList(ListCreateAPIView):
 
 class EnforceParentRelationshipMixin(object):
     """
-    Useful when you have a self-refering ManyToManyRelationship.
+    Useful when you have a self-referring ManyToManyRelationship.
     * Tower uses a shallow (2-deep only) url pattern. For example:
 
     When an object hangs off of a parent object you would have the url of the
@@ -2415,7 +2476,7 @@ class JobTemplateSurveySpec(GenericAPIView):
                             status=status.HTTP_400_BAD_REQUEST,
                         )
             # if it's a multiselect or multiple choice, it must have coices listed
-            # choices and defualts must come in as strings seperated by /n characters.
+            # choices and defaults must come in as strings separated by /n characters.
             if qtype == 'multiselect' or qtype == 'multiplechoice':
                 if 'choices' in survey_item:
                     if isinstance(survey_item['choices'], str):
@@ -3078,7 +3139,9 @@ class WorkflowJobTemplateWorkflowNodesList(SubListCreateAPIView):
     search_fields = ('unified_job_template__name', 'unified_job_template__description')
 
     def get_queryset(self):
-        return super(WorkflowJobTemplateWorkflowNodesList, self).get_queryset().order_by('id')
+        parent = self.get_parent_object()
+        self.check_parent_access(parent)
+        return getattr(parent, self.relationship).order_by('id')
 
 
 class WorkflowJobTemplateJobsList(SubListAPIView):
@@ -3172,7 +3235,9 @@ class WorkflowJobWorkflowNodesList(SubListAPIView):
     search_fields = ('unified_job_template__name', 'unified_job_template__description')
 
     def get_queryset(self):
-        return super(WorkflowJobWorkflowNodesList, self).get_queryset().order_by('id')
+        parent = self.get_parent_object()
+        self.check_parent_access(parent)
+        return getattr(parent, self.relationship).order_by('id')
 
 
 class WorkflowJobCancel(GenericCancelView):
@@ -3430,7 +3495,7 @@ class JobCreateSchedule(RetrieveAPIView):
 
         config = obj.launch_config
 
-        # Make up a name for the schedule, guarentee that it is unique
+        # Make up a name for the schedule, guarantee that it is unique
         name = 'Auto-generated schedule from job {}'.format(obj.id)
         existing_names = models.Schedule.objects.filter(name__startswith=name).values_list('name', flat=True)
         if name in existing_names:
@@ -3621,7 +3686,7 @@ class JobJobEventsChildrenSummary(APIView):
         # key is counter of meta events (i.e. verbose), value is uuid of the assigned parent
         map_meta_counter_nested_uuid = {}
 
-        # collapsable tree view in the UI only makes sense for tree-like
+        # collapsible tree view in the UI only makes sense for tree-like
         # hierarchy. If ansible is ran with a strategy like free or host_pinned, then
         # events can be out of sequential order, and no longer follow a tree structure
         # E1
@@ -4288,7 +4353,7 @@ class WorkflowApprovalTemplateJobsList(SubListAPIView):
     parent_key = 'workflow_approval_template'
 
 
-class WorkflowApprovalList(ListCreateAPIView):
+class WorkflowApprovalList(ListAPIView):
     model = models.WorkflowApproval
     serializer_class = serializers.WorkflowApprovalListSerializer
 

awx/api/views/bulk.py

@@ -0,0 +1,69 @@
+from collections import OrderedDict
+
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.renderers import JSONRenderer
+from rest_framework.reverse import reverse
+from rest_framework import status
+from rest_framework.response import Response
+
+from awx.main.models import UnifiedJob, Host
+from awx.api.generics import (
+    GenericAPIView,
+    APIView,
+)
+from awx.api import (
+    serializers,
+    renderers,
+)
+
+
+class BulkView(APIView):
+    permission_classes = [IsAuthenticated]
+    renderer_classes = [
+        renderers.BrowsableAPIRenderer,
+        JSONRenderer,
+    ]
+    allowed_methods = ['GET', 'OPTIONS']
+
+    def get(self, request, format=None):
+        '''List top level resources'''
+        data = OrderedDict()
+        data['host_create'] = reverse('api:bulk_host_create', request=request)
+        data['job_launch'] = reverse('api:bulk_job_launch', request=request)
+        return Response(data)
+
+
+class BulkJobLaunchView(GenericAPIView):
+    permission_classes = [IsAuthenticated]
+    model = UnifiedJob
+    serializer_class = serializers.BulkJobLaunchSerializer
+    allowed_methods = ['GET', 'POST', 'OPTIONS']
+
+    def get(self, request):
+        data = OrderedDict()
+        data['detail'] = "Specify a list of unified job templates to launch alongside their launchtime parameters"
+        return Response(data, status=status.HTTP_200_OK)
+
+    def post(self, request):
+        bulkjob_serializer = serializers.BulkJobLaunchSerializer(data=request.data, context={'request': request})
+        if bulkjob_serializer.is_valid():
+            result = bulkjob_serializer.create(bulkjob_serializer.validated_data)
+            return Response(result, status=status.HTTP_201_CREATED)
+        return Response(bulkjob_serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+
+class BulkHostCreateView(GenericAPIView):
+    permission_classes = [IsAuthenticated]
+    model = Host
+    serializer_class = serializers.BulkHostCreateSerializer
+    allowed_methods = ['GET', 'POST', 'OPTIONS']
+
+    def get(self, request):
+        return Response({"detail": "Bulk create hosts with this endpoint"}, status=status.HTTP_200_OK)
+
+    def post(self, request):
+        serializer = serializers.BulkHostCreateSerializer(data=request.data, context={'request': request})
+        if serializer.is_valid():
+            result = serializer.create(serializer.validated_data)
+            return Response(result, status=status.HTTP_201_CREATED)
+        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

awx/api/views/inventory.py

@@ -14,6 +14,7 @@ from django.utils.translation import gettext_lazy as _
 from rest_framework.exceptions import PermissionDenied
 from rest_framework.response import Response
 from rest_framework import status
+from rest_framework import serializers
 
 # AWX
 from awx.main.models import ActivityStream, Inventory, JobTemplate, Role, User, InstanceGroup, InventoryUpdateEvent, InventoryUpdate
@@ -31,6 +32,7 @@ from awx.api.views.labels import LabelSubListCreateAttachDetachView
 
 from awx.api.serializers import (
     InventorySerializer,
+    ConstructedInventorySerializer,
     ActivityStreamSerializer,
     RoleSerializer,
     InstanceGroupSerializer,
@@ -79,7 +81,9 @@ class InventoryDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIVie
 
         # Do not allow changes to an Inventory kind.
         if kind is not None and obj.kind != kind:
-            return Response(dict(error=_('You cannot turn a regular inventory into a "smart" inventory.')), status=status.HTTP_405_METHOD_NOT_ALLOWED)
+            return Response(
+                dict(error=_('You cannot turn a regular inventory into a "smart" or "constructed" inventory.')), status=status.HTTP_405_METHOD_NOT_ALLOWED
+            )
         return super(InventoryDetail, self).update(request, *args, **kwargs)
 
     def destroy(self, request, *args, **kwargs):
@@ -94,6 +98,29 @@ class InventoryDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIVie
             return Response(dict(error=_("{0}".format(e))), status=status.HTTP_400_BAD_REQUEST)
 
 
+class ConstructedInventoryDetail(InventoryDetail):
+    serializer_class = ConstructedInventorySerializer
+
+
+class ConstructedInventoryList(InventoryList):
+    serializer_class = ConstructedInventorySerializer
+
+    def get_queryset(self):
+        r = super().get_queryset()
+        return r.filter(kind='constructed')
+
+
+class InventoryInputInventoriesList(SubListAttachDetachAPIView):
+    model = Inventory
+    serializer_class = InventorySerializer
+    parent_model = Inventory
+    relationship = 'input_inventories'
+
+    def is_valid_relation(self, parent, sub, created=False):
+        if sub.kind == 'constructed':
+            raise serializers.ValidationError({'error': 'You cannot add a constructed inventory to another constructed inventory.'})
+
+
 class InventoryActivityStreamList(SubListAPIView):
     model = ActivityStream
     serializer_class = ActivityStreamSerializer

awx/api/views/root.py

@@ -98,10 +98,14 @@ class ApiVersionRootView(APIView):
         data['tokens'] = reverse('api:o_auth2_token_list', request=request)
         data['metrics'] = reverse('api:metrics_view', request=request)
         data['inventory'] = reverse('api:inventory_list', request=request)
+        data['constructed_inventory'] = reverse('api:constructed_inventory_list', request=request)
         data['inventory_sources'] = reverse('api:inventory_source_list', request=request)
         data['inventory_updates'] = reverse('api:inventory_update_list', request=request)
         data['groups'] = reverse('api:group_list', request=request)
         data['hosts'] = reverse('api:host_list', request=request)
+        data['host_metrics'] = reverse('api:host_metric_list', request=request)
+        # It will be enabled in future version of the AWX
+        # data['host_metric_summary_monthly'] = reverse('api:host_metric_summary_monthly_list', request=request)
         data['job_templates'] = reverse('api:job_template_list', request=request)
         data['jobs'] = reverse('api:job_list', request=request)
         data['ad_hoc_commands'] = reverse('api:ad_hoc_command_list', request=request)
@@ -121,6 +125,7 @@ class ApiVersionRootView(APIView):
         data['workflow_job_template_nodes'] = reverse('api:workflow_job_template_node_list', request=request)
         data['workflow_job_nodes'] = reverse('api:workflow_job_node_list', request=request)
         data['mesh_visualizer'] = reverse('api:mesh_visualizer_view', request=request)
+        data['bulk'] = reverse('api:bulk', request=request)
         return Response(data)
 
 
@@ -271,6 +276,9 @@ class ApiV2ConfigView(APIView):
 
         pendo_state = settings.PENDO_TRACKING_STATE if settings.PENDO_TRACKING_STATE in ('off', 'anonymous', 'detailed') else 'off'
 
+        # Guarding against settings.UI_NEXT being set to a non-boolean value
+        ui_next_state = settings.UI_NEXT if settings.UI_NEXT in (True, False) else False
+
         data = dict(
             time_zone=settings.TIME_ZONE,
             license_info=license_data,
@@ -279,6 +287,7 @@ class ApiV2ConfigView(APIView):
             analytics_status=pendo_state,
             analytics_collectors=all_collectors(),
             become_methods=PRIVILEGE_ESCALATION_METHODS,
+            ui_next=ui_next_state,
         )
 
         # If LDAP is enabled, user_ldap_fields will return a list of field

awx/conf/fields.py

@@ -21,7 +21,7 @@ logger = logging.getLogger('awx.conf.fields')
 # Use DRF fields to convert/validate settings:
 # - to_representation(obj) should convert a native Python object to a primitive
 #   serializable type. This primitive type will be what is presented in the API
-#   and stored in the JSON field in the datbase.
+#   and stored in the JSON field in the database.
 # - to_internal_value(data) should convert the primitive type back into the
 #   appropriate Python type to be used in settings.
 

awx/conf/migrations/_ldap_group_type.py

@@ -1,7 +1,11 @@
 import inspect
 
 from django.conf import settings
-from django.utils.timezone import now
+
+import logging
+
+
+logger = logging.getLogger('awx.conf.migrations')
 
 
 def fill_ldap_group_type_params(apps, schema_editor):
@@ -15,7 +19,7 @@ def fill_ldap_group_type_params(apps, schema_editor):
         entry = qs[0]
         group_type_params = entry.value
     else:
-        entry = Setting(key='AUTH_LDAP_GROUP_TYPE_PARAMS', value=group_type_params, created=now(), modified=now())
+        return  # for new installs we prefer to use the default value
 
     init_attrs = set(inspect.getfullargspec(group_type.__init__).args[1:])
     for k in list(group_type_params.keys()):
@@ -23,4 +27,5 @@ def fill_ldap_group_type_params(apps, schema_editor):
             del group_type_params[k]
 
     entry.value = group_type_params
+    logger.warning(f'Migration updating AUTH_LDAP_GROUP_TYPE_PARAMS with value {entry.value}')
     entry.save()

awx/conf/settings.py

@@ -5,11 +5,13 @@ import threading
 import time
 import os
 
+from concurrent.futures import ThreadPoolExecutor
+
 # Django
 from django.conf import LazySettings
 from django.conf import settings, UserSettingsHolder
 from django.core.cache import cache as django_cache
-from django.core.exceptions import ImproperlyConfigured
+from django.core.exceptions import ImproperlyConfigured, SynchronousOnlyOperation
 from django.db import transaction, connection
 from django.db.utils import Error as DBError, ProgrammingError
 from django.utils.functional import cached_property
@@ -157,7 +159,7 @@ class EncryptedCacheProxy(object):
             obj_id = self.cache.get(Setting.get_cache_id_key(key), default=empty)
             if obj_id is empty:
                 logger.info('Efficiency notice: Corresponding id not stored in cache %s', Setting.get_cache_id_key(key))
-                obj_id = getattr(self._get_setting_from_db(key), 'pk', None)
+                obj_id = getattr(_get_setting_from_db(self.registry, key), 'pk', None)
             elif obj_id == SETTING_CACHE_NONE:
                 obj_id = None
             return method(TransientSetting(pk=obj_id, value=value), 'value')
@@ -166,11 +168,6 @@ class EncryptedCacheProxy(object):
         # a no-op; it just returns the provided value
         return value
 
-    def _get_setting_from_db(self, key):
-        field = self.registry.get_setting_field(key)
-        if not field.read_only:
-            return Setting.objects.filter(key=key, user__isnull=True).order_by('pk').first()
-
     def __getattr__(self, name):
         return getattr(self.cache, name)
 
@@ -186,6 +183,22 @@ def get_settings_to_cache(registry):
     return dict([(key, SETTING_CACHE_NOTSET) for key in get_writeable_settings(registry)])
 
 
+# Will first attempt to get the setting from the database in synchronous mode.
+# If call from async context, it will attempt to get the setting from the database in a thread.
+def _get_setting_from_db(registry, key):
+    def get_settings_from_db_sync(registry, key):
+        field = registry.get_setting_field(key)
+        if not field.read_only or key == 'INSTALL_UUID':
+            return Setting.objects.filter(key=key, user__isnull=True).order_by('pk').first()
+
+    try:
+        return get_settings_from_db_sync(registry, key)
+    except SynchronousOnlyOperation:
+        with ThreadPoolExecutor(max_workers=1) as executor:
+            future = executor.submit(get_settings_from_db_sync, registry, key)
+            return future.result()
+
+
 def get_cache_value(value):
     """Returns the proper special cache setting for a value
     based on instance type.
@@ -345,7 +358,7 @@ class SettingsWrapper(UserSettingsHolder):
             setting_id = None
             # this value is read-only, however we *do* want to fetch its value from the database
             if not field.read_only or name == 'INSTALL_UUID':
-                setting = Setting.objects.filter(key=name, user__isnull=True).order_by('pk').first()
+                setting = _get_setting_from_db(self.registry, name)
             if setting:
                 if getattr(field, 'encrypted', False):
                     value = decrypt_field(setting, 'value')

awx/conf/tests/functional/test_api.py

@@ -94,9 +94,7 @@ def test_setting_singleton_retrieve_readonly(api_request, dummy_setting):
 
 @pytest.mark.django_db
 def test_setting_singleton_update(api_request, dummy_setting):
-    with dummy_setting('FOO_BAR', field_class=fields.IntegerField, category='FooBar', category_slug='foobar'), mock.patch(
-        'awx.conf.views.handle_setting_changes'
-    ):
+    with dummy_setting('FOO_BAR', field_class=fields.IntegerField, category='FooBar', category_slug='foobar'), mock.patch('awx.conf.views.clear_setting_cache'):
         api_request('patch', reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}), data={'FOO_BAR': 3})
         response = api_request('get', reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}))
         assert response.data['FOO_BAR'] == 3
@@ -112,7 +110,7 @@ def test_setting_singleton_update_hybriddictfield_with_forbidden(api_request, du
     # sure that the _Forbidden validator doesn't get used for the
     # fields.  See also https://github.com/ansible/awx/issues/4099.
     with dummy_setting('FOO_BAR', field_class=sso_fields.SAMLOrgAttrField, category='FooBar', category_slug='foobar'), mock.patch(
-        'awx.conf.views.handle_setting_changes'
+        'awx.conf.views.clear_setting_cache'
     ):
         api_request(
             'patch',
@@ -126,7 +124,7 @@ def test_setting_singleton_update_hybriddictfield_with_forbidden(api_request, du
 @pytest.mark.django_db
 def test_setting_singleton_update_dont_change_readonly_fields(api_request, dummy_setting):
     with dummy_setting('FOO_BAR', field_class=fields.IntegerField, read_only=True, default=4, category='FooBar', category_slug='foobar'), mock.patch(
-        'awx.conf.views.handle_setting_changes'
+        'awx.conf.views.clear_setting_cache'
     ):
         api_request('patch', reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}), data={'FOO_BAR': 5})
         response = api_request('get', reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}))
@@ -136,7 +134,7 @@ def test_setting_singleton_update_dont_change_readonly_fields(api_request, dummy
 @pytest.mark.django_db
 def test_setting_singleton_update_dont_change_encrypted_mark(api_request, dummy_setting):
     with dummy_setting('FOO_BAR', field_class=fields.CharField, encrypted=True, category='FooBar', category_slug='foobar'), mock.patch(
-        'awx.conf.views.handle_setting_changes'
+        'awx.conf.views.clear_setting_cache'
     ):
         api_request('patch', reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}), data={'FOO_BAR': 'password'})
         assert Setting.objects.get(key='FOO_BAR').value.startswith('$encrypted$')
@@ -155,16 +153,14 @@ def test_setting_singleton_update_runs_custom_validate(api_request, dummy_settin
 
     with dummy_setting('FOO_BAR', field_class=fields.IntegerField, category='FooBar', category_slug='foobar'), dummy_validate(
         'foobar', func_raising_exception
-    ), mock.patch('awx.conf.views.handle_setting_changes'):
+    ), mock.patch('awx.conf.views.clear_setting_cache'):
         response = api_request('patch', reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}), data={'FOO_BAR': 23})
         assert response.status_code == 400
 
 
 @pytest.mark.django_db
 def test_setting_singleton_delete(api_request, dummy_setting):
-    with dummy_setting('FOO_BAR', field_class=fields.IntegerField, category='FooBar', category_slug='foobar'), mock.patch(
-        'awx.conf.views.handle_setting_changes'
-    ):
+    with dummy_setting('FOO_BAR', field_class=fields.IntegerField, category='FooBar', category_slug='foobar'), mock.patch('awx.conf.views.clear_setting_cache'):
         api_request('delete', reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}))
         response = api_request('get', reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}))
         assert not response.data['FOO_BAR']
@@ -173,7 +169,7 @@ def test_setting_singleton_delete(api_request, dummy_setting):
 @pytest.mark.django_db
 def test_setting_singleton_delete_no_read_only_fields(api_request, dummy_setting):
     with dummy_setting('FOO_BAR', field_class=fields.IntegerField, read_only=True, default=23, category='FooBar', category_slug='foobar'), mock.patch(
-        'awx.conf.views.handle_setting_changes'
+        'awx.conf.views.clear_setting_cache'
     ):
         api_request('delete', reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}))
         response = api_request('get', reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}))

awx/conf/tests/functional/test_migrations.py

@@ -0,0 +1,25 @@
+import pytest
+
+from awx.conf.migrations._ldap_group_type import fill_ldap_group_type_params
+from awx.conf.models import Setting
+
+from django.apps import apps
+
+
+@pytest.mark.django_db
+def test_fill_group_type_params_no_op():
+    fill_ldap_group_type_params(apps, 'dont-use-me')
+    assert Setting.objects.count() == 0
+
+
+@pytest.mark.django_db
+def test_keep_old_setting_with_default_value():
+    Setting.objects.create(key='AUTH_LDAP_GROUP_TYPE', value={'name_attr': 'cn', 'member_attr': 'member'})
+    fill_ldap_group_type_params(apps, 'dont-use-me')
+    assert Setting.objects.count() == 1
+    s = Setting.objects.first()
+    assert s.value == {'name_attr': 'cn', 'member_attr': 'member'}
+
+
+# NOTE: would be good to test the removal of attributes by migration
+# but this requires fighting with the validator and is not done here

awx/conf/views.py

@@ -26,10 +26,11 @@ from awx.api.generics import APIView, GenericAPIView, ListAPIView, RetrieveUpdat
 from awx.api.permissions import IsSystemAdminOrAuditor
 from awx.api.versioning import reverse
 from awx.main.utils import camelcase_to_underscore
-from awx.main.tasks.system import handle_setting_changes
+from awx.main.tasks.system import clear_setting_cache
 from awx.conf.models import Setting
 from awx.conf.serializers import SettingCategorySerializer, SettingSingletonSerializer
 from awx.conf import settings_registry
+from awx.main.utils.external_logging import reconfigure_rsyslog
 
 
 SettingCategory = collections.namedtuple('SettingCategory', ('url', 'slug', 'name'))
@@ -118,7 +119,10 @@ class SettingSingletonDetail(RetrieveUpdateDestroyAPIView):
                 setting.save(update_fields=['value'])
                 settings_change_list.append(key)
         if settings_change_list:
-            connection.on_commit(lambda: handle_setting_changes.delay(settings_change_list))
+            connection.on_commit(lambda: clear_setting_cache.delay(settings_change_list))
+            if any([setting.startswith('LOG_AGGREGATOR') for setting in settings_change_list]):
+                # call notify to rsyslog. no data is need so payload is empty
+                reconfigure_rsyslog.delay()
 
     def destroy(self, request, *args, **kwargs):
         instance = self.get_object()
@@ -133,7 +137,10 @@ class SettingSingletonDetail(RetrieveUpdateDestroyAPIView):
             setting.delete()
             settings_change_list.append(setting.key)
         if settings_change_list:
-            connection.on_commit(lambda: handle_setting_changes.delay(settings_change_list))
+            connection.on_commit(lambda: clear_setting_cache.delay(settings_change_list))
+            if any([setting.startswith('LOG_AGGREGATOR') for setting in settings_change_list]):
+                # call notify to rsyslog. no data is need so payload is empty
+                reconfigure_rsyslog.delay()
 
         # When TOWER_URL_BASE is deleted from the API, reset it to the hostname
         # used to make the request as a default.
@@ -180,7 +187,7 @@ class SettingLoggingTest(GenericAPIView):
             if not port:
                 return Response({'error': 'Port required for ' + protocol}, status=status.HTTP_400_BAD_REQUEST)
         else:
-            # if http/https by this point, domain is reacheable
+            # if http/https by this point, domain is reachable
             return Response(status=status.HTTP_202_ACCEPTED)
 
         if protocol == 'udp':

awx/locale/en-us/LC_MESSAGES/django.po

@@ -1972,7 +1972,7 @@ msgid ""
 "HTTP headers and meta keys to search to determine remote host name or IP. "
 "Add additional items to this list, such as \"HTTP_X_FORWARDED_FOR\", if "
 "behind a reverse proxy. See the \"Proxy Support\" section of the "
-"Adminstrator guide for more details."
+"Administrator guide for more details."
 msgstr ""
 
 #: awx/main/conf.py:85
@@ -2457,7 +2457,7 @@ msgid ""
 msgstr ""
 
 #: awx/main/conf.py:631
-msgid "Maximum disk persistance for external log aggregation (in GB)"
+msgid "Maximum disk persistence for external log aggregation (in GB)"
 msgstr ""
 
 #: awx/main/conf.py:633
@@ -2548,7 +2548,7 @@ msgid "Enable"
 msgstr ""
 
 #: awx/main/constants.py:27
-msgid "Doas"
+msgid "Does"
 msgstr ""
 
 #: awx/main/constants.py:28
@@ -4801,7 +4801,7 @@ msgstr ""
 
 #: awx/main/models/workflow.py:251
 msgid ""
-"An identifier coresponding to the workflow job template node that this node "
+"An identifier corresponding to the workflow job template node that this node "
 "was created from."
 msgstr ""
 
@@ -5521,7 +5521,7 @@ msgstr ""
 #: awx/sso/conf.py:606
 msgid ""
 "Extra arguments for Google OAuth2 login. You can restrict it to only allow a "
-"single domain to authenticate, even if the user is logged in with multple "
+"single domain to authenticate, even if the user is logged in with multiple "
 "Google accounts. Refer to the documentation for more detail."
 msgstr ""
 
@@ -5905,7 +5905,7 @@ msgstr ""
 
 #: awx/sso/conf.py:1290
 msgid ""
-"Create a keypair to use as a service provider (SP) and include the "
+"Create a key pair to use as a service provider (SP) and include the "
 "certificate content here."
 msgstr ""
 
@@ -5915,7 +5915,7 @@ msgstr ""
 
 #: awx/sso/conf.py:1302
 msgid ""
-"Create a keypair to use as a service provider (SP) and include the private "
+"Create a key pair to use as a service provider (SP) and include the private "
 "key content here."
 msgstr ""
 

awx/locale/es/LC_MESSAGES/django.po

@@ -1971,7 +1971,7 @@ msgid ""
 "HTTP headers and meta keys to search to determine remote host name or IP. "
 "Add additional items to this list, such as \"HTTP_X_FORWARDED_FOR\", if "
 "behind a reverse proxy. See the \"Proxy Support\" section of the "
-"Adminstrator guide for more details."
+"Administrator guide for more details."
 msgstr "Los encabezados HTTP y las llaves de activación para buscar y determinar el nombre de host remoto o IP. Añada elementos adicionales a esta lista, como \"HTTP_X_FORWARDED_FOR\", si está detrás de un proxy inverso. Consulte la sección \"Soporte de proxy\" de la guía del adminstrador para obtener más información."
 
 #: awx/main/conf.py:85
@@ -4804,7 +4804,7 @@ msgstr "Indica que un trabajo no se creará cuando es sea True. La semántica de
 
 #: awx/main/models/workflow.py:251
 msgid ""
-"An identifier coresponding to the workflow job template node that this node "
+"An identifier corresponding to the workflow job template node that this node "
 "was created from."
 msgstr "Un identificador que corresponde al nodo de plantilla de tarea del flujo de trabajo a partir del cual se creó este nodo."
 
@@ -5526,7 +5526,7 @@ msgstr "Argumentos adicionales para Google OAuth2"
 #: awx/sso/conf.py:606
 msgid ""
 "Extra arguments for Google OAuth2 login. You can restrict it to only allow a "
-"single domain to authenticate, even if the user is logged in with multple "
+"single domain to authenticate, even if the user is logged in with multiple "
 "Google accounts. Refer to the documentation for more detail."
 msgstr "Argumentos adicionales para el inicio de sesión en Google OAuth2. Puede limitarlo para permitir la autenticación de un solo dominio, incluso si el usuario ha iniciado sesión con varias cuentas de Google. Consulte la documentación para obtener información detallada."
 
@@ -5910,7 +5910,7 @@ msgstr "Certificado público del proveedor de servicio SAML"
 
 #: awx/sso/conf.py:1290
 msgid ""
-"Create a keypair to use as a service provider (SP) and include the "
+"Create a key pair to use as a service provider (SP) and include the "
 "certificate content here."
 msgstr "Crear un par de claves para usar como proveedor de servicio (SP) e incluir el contenido del certificado aquí."
 
@@ -5920,7 +5920,7 @@ msgstr "Clave privada del proveedor de servicio SAML"
 
 #: awx/sso/conf.py:1302
 msgid ""
-"Create a keypair to use as a service provider (SP) and include the private "
+"Create a key pair to use as a service provider (SP) and include the private "
 "key content here."
 msgstr "Crear un par de claves para usar como proveedor de servicio (SP) e incluir el contenido de la clave privada aquí."
 

awx/main/access.py

@@ -588,17 +588,39 @@ class InstanceAccess(BaseAccess):
 
 
 class InstanceGroupAccess(BaseAccess):
+    """
+    I can see Instance Groups when I am:
+       - a superuser(system administrator)
+       - at least read_role on the instance group
+    I can edit Instance Groups when I am:
+       - a superuser
+       - admin role on the Instance group
+    I can add/delete Instance Groups:
+       - a superuser(system administrator)
+    I can use Instance Groups when I have:
+       - use_role on the instance group
+    """
+
     model = InstanceGroup
     prefetch_related = ('instances',)
 
     def filtered_queryset(self):
-        return InstanceGroup.objects.filter(organization__in=Organization.accessible_pk_qs(self.user, 'admin_role')).distinct()
+        return self.model.accessible_objects(self.user, 'read_role')
+
+    @check_superuser
+    def can_use(self, obj):
+        return self.user in obj.use_role
 
     def can_add(self, data):
         return self.user.is_superuser
 
+    @check_superuser
     def can_change(self, obj, data):
-        return self.user.is_superuser
+        return self.can_admin(obj)
+
+    @check_superuser
+    def can_admin(self, obj):
+        return self.user in obj.admin_role
 
     def can_delete(self, obj):
         if obj.name in [settings.DEFAULT_EXECUTION_QUEUE_NAME, settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME]:
@@ -845,7 +867,7 @@ class OrganizationAccess(NotificationAttachMixin, BaseAccess):
             return RoleAccess(self.user).can_attach(rel_role, sub_obj, 'members', *args, **kwargs)
 
         if relationship == "instance_groups":
-            if self.user.is_superuser:
+            if self.user in obj.admin_role and self.user in sub_obj.use_role:
                 return True
             return False
         return super(OrganizationAccess, self).can_attach(obj, sub_obj, relationship, *args, **kwargs)
@@ -934,7 +956,7 @@ class InventoryAccess(BaseAccess):
 
     def can_attach(self, obj, sub_obj, relationship, *args, **kwargs):
         if relationship == "instance_groups":
-            if self.user.can_access(type(sub_obj), "read", sub_obj) and self.user in obj.organization.admin_role:
+            if self.user in sub_obj.use_role and self.user in obj.admin_role:
                 return True
             return False
         return super(InventoryAccess, self).can_attach(obj, sub_obj, relationship, *args, **kwargs)
@@ -1671,11 +1693,12 @@ class JobTemplateAccess(NotificationAttachMixin, UnifiedCredentialsMixin, BaseAc
         return self.user.is_superuser or self.user in obj.admin_role
 
     @check_superuser
+    # object here is the job template. sub_object here is what is being attached
     def can_attach(self, obj, sub_obj, relationship, data, skip_sub_obj_read_check=False):
         if relationship == "instance_groups":
             if not obj.organization:
                 return False
-            return self.user.can_access(type(sub_obj), "read", sub_obj) and self.user in obj.organization.admin_role
+            return self.user in sub_obj.use_role and self.user in obj.admin_role
         return super(JobTemplateAccess, self).can_attach(obj, sub_obj, relationship, data, skip_sub_obj_read_check=skip_sub_obj_read_check)
 
     @check_superuser
@@ -1852,8 +1875,6 @@ class JobLaunchConfigAccess(UnifiedCredentialsMixin, BaseAccess):
     def _related_filtered_queryset(self, cls):
         if cls is Label:
             return LabelAccess(self.user).filtered_queryset()
-        elif cls is InstanceGroup:
-            return InstanceGroupAccess(self.user).filtered_queryset()
         else:
             return cls._accessible_pk_qs(cls, self.user, 'use_role')
 
@@ -1865,6 +1886,7 @@ class JobLaunchConfigAccess(UnifiedCredentialsMixin, BaseAccess):
 
     @check_superuser
     def can_add(self, data, template=None):
+        # WARNING: duplicated with BulkJobLaunchSerializer, check when changing permission levels
         # This is a special case, we don't check related many-to-many elsewhere
         # launch RBAC checks use this
         if 'reference_obj' in data:
@@ -1997,7 +2019,16 @@ class WorkflowJobNodeAccess(BaseAccess):
     )
 
     def filtered_queryset(self):
-        return self.model.objects.filter(workflow_job__unified_job_template__in=UnifiedJobTemplate.accessible_pk_qs(self.user, 'read_role'))
+        return self.model.objects.filter(
+            Q(workflow_job__unified_job_template__in=UnifiedJobTemplate.accessible_pk_qs(self.user, 'read_role'))
+            | Q(workflow_job__organization__in=Organization.objects.filter(Q(admin_role__members=self.user)))
+        )
+
+    def can_read(self, obj):
+        """Overriding this opens up detail view access for bulk jobs, where the workflow job has no associated workflow job template."""
+        if obj.workflow_job.is_bulk_job and obj.workflow_job.created_by_id == self.user.id:
+            return True
+        return super().can_read(obj)
 
     @check_superuser
     def can_add(self, data):
@@ -2123,7 +2154,16 @@ class WorkflowJobAccess(BaseAccess):
     )
 
     def filtered_queryset(self):
-        return WorkflowJob.objects.filter(unified_job_template__in=UnifiedJobTemplate.accessible_pk_qs(self.user, 'read_role'))
+        return WorkflowJob.objects.filter(
+            Q(unified_job_template__in=UnifiedJobTemplate.accessible_pk_qs(self.user, 'read_role'))
+            | Q(organization__in=Organization.objects.filter(Q(admin_role__members=self.user)), is_bulk_job=True)
+        )
+
+    def can_read(self, obj):
+        """Overriding this opens up detail view access for bulk jobs, where the workflow job has no associated workflow job template."""
+        if obj.is_bulk_job and obj.created_by_id == self.user.id:
+            return True
+        return super().can_read(obj)
 
     def can_add(self, data):
         # Old add-start system for launching jobs is being depreciated, and

awx/main/analytics/analytics_tasks.py

@@ -4,11 +4,11 @@ import logging
 # AWX
 from awx.main.analytics.subsystem_metrics import Metrics
 from awx.main.dispatch.publish import task
-from awx.main.dispatch import get_local_queuename
+from awx.main.dispatch import get_task_queuename
 
 logger = logging.getLogger('awx.main.scheduler')
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def send_subsystem_metrics():
     Metrics().send_metrics()

awx/main/analytics/broadcast_websocket.py

@@ -65,7 +65,7 @@ class FixedSlidingWindow:
         return sum(self.buckets.values()) or 0
 
 
-class BroadcastWebsocketStatsManager:
+class RelayWebsocketStatsManager:
     def __init__(self, event_loop, local_hostname):
         self._local_hostname = local_hostname
 
@@ -74,7 +74,7 @@ class BroadcastWebsocketStatsManager:
         self._redis_key = BROADCAST_WEBSOCKET_REDIS_KEY_NAME
 
     def new_remote_host_stats(self, remote_hostname):
-        self._stats[remote_hostname] = BroadcastWebsocketStats(self._local_hostname, remote_hostname)
+        self._stats[remote_hostname] = RelayWebsocketStats(self._local_hostname, remote_hostname)
         return self._stats[remote_hostname]
 
     def delete_remote_host_stats(self, remote_hostname):
@@ -107,7 +107,7 @@ class BroadcastWebsocketStatsManager:
         return parser.text_string_to_metric_families(stats_str.decode('UTF-8'))
 
 
-class BroadcastWebsocketStats:
+class RelayWebsocketStats:
     def __init__(self, local_hostname, remote_hostname):
         self._local_hostname = local_hostname
         self._remote_hostname = remote_hostname

awx/main/analytics/collectors.py

@@ -83,7 +83,7 @@ def _identify_lower(key, since, until, last_gather):
     return lower, last_entries
 
 
-@register('config', '1.4', description=_('General platform configuration.'))
+@register('config', '1.5', description=_('General platform configuration.'))
 def config(since, **kwargs):
     license_info = get_license()
     install_type = 'traditional'
@@ -119,6 +119,7 @@ def config(since, **kwargs):
         'compliant': license_info.get('compliant'),
         'date_warning': license_info.get('date_warning'),
         'date_expired': license_info.get('date_expired'),
+        'subscription_usage_model': getattr(settings, 'SUBSCRIPTION_USAGE_MODEL', ''),  # 1.5+
         'free_instances': license_info.get('free_instances', 0),
         'total_licensed_instances': license_info.get('instance_count', 0),
         'license_expiry': license_info.get('time_remaining', 0),
@@ -233,11 +234,13 @@ def projects_by_scm_type(since, **kwargs):
     return counts
 
 
-@register('instance_info', '1.2', description=_('Cluster topology and capacity'))
+@register('instance_info', '1.3', description=_('Cluster topology and capacity'))
 def instance_info(since, include_hostnames=False, **kwargs):
     info = {}
     # Use same method that the TaskManager does to compute consumed capacity without querying all running jobs for each Instance
-    tm_models = TaskManagerModels.init_with_consumed_capacity(instance_fields=['uuid', 'version', 'capacity', 'cpu', 'memory', 'managed_by_policy', 'enabled'])
+    tm_models = TaskManagerModels.init_with_consumed_capacity(
+        instance_fields=['uuid', 'version', 'capacity', 'cpu', 'memory', 'managed_by_policy', 'enabled', 'node_type']
+    )
     for tm_instance in tm_models.instances.instances_by_hostname.values():
         instance = tm_instance.obj
         instance_info = {

awx/main/analytics/subsystem_metrics.py

@@ -9,7 +9,7 @@ from django.apps import apps
 from awx.main.consumers import emit_channel_notification
 from awx.main.utils import is_testing
 
-root_key = 'awx_metrics'
+root_key = settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX
 logger = logging.getLogger('awx.main.analytics')
 
 
@@ -264,13 +264,6 @@ class Metrics:
             data[field] = self.METRICS[field].decode(self.conn)
         return data
 
-    def store_metrics(self, data_json):
-        # called when receiving metrics from other instances
-        data = json.loads(data_json)
-        if self.instance_name != data['instance']:
-            logger.debug(f"{self.instance_name} received subsystem metrics from {data['instance']}")
-        self.conn.set(root_key + "_instance_" + data['instance'], data['metrics'])
-
     def should_pipe_execute(self):
         if self.metrics_have_changed is False:
             return False
@@ -309,9 +302,9 @@ class Metrics:
                     'instance': self.instance_name,
                     'metrics': self.serialize_local_metrics(),
                 }
-                # store a local copy as well
-                self.store_metrics(json.dumps(payload))
+
                 emit_channel_notification("metrics", payload)
+
                 self.previous_send_metrics.set(current_time)
                 self.previous_send_metrics.store_value(self.conn)
         finally:

awx/main/conf.py

@@ -10,7 +10,7 @@ from rest_framework import serializers
 # AWX
 from awx.conf import fields, register, register_validate
 from awx.main.models import ExecutionEnvironment
-
+from awx.main.constants import SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS
 
 logger = logging.getLogger('awx.main.conf')
 
@@ -282,6 +282,16 @@ register(
     placeholder={'HTTP_PROXY': 'myproxy.local:8080'},
 )
 
+register(
+    'AWX_RUNNER_KEEPALIVE_SECONDS',
+    field_class=fields.IntegerField,
+    label=_('K8S Ansible Runner Keep-Alive Message Interval'),
+    help_text=_('Only applies to jobs running in a Container Group. If not 0, send a message every so-many seconds to keep connection open.'),
+    category=_('Jobs'),
+    category_slug='jobs',
+    placeholder=240,  # intended to be under common 5 minute idle timeout
+)
+
 register(
     'GALAXY_TASK_ENV',
     field_class=fields.KeyValueField,
@@ -765,6 +775,53 @@ register(
     help_text=_('Indicates whether the instance is part of a kubernetes-based deployment.'),
 )
 
+register(
+    'BULK_JOB_MAX_LAUNCH',
+    field_class=fields.IntegerField,
+    default=100,
+    label=_('Max jobs to allow bulk jobs to launch'),
+    help_text=_('Max jobs to allow bulk jobs to launch'),
+    category=_('Bulk Actions'),
+    category_slug='bulk',
+)
+
+register(
+    'BULK_HOST_MAX_CREATE',
+    field_class=fields.IntegerField,
+    default=100,
+    label=_('Max number of hosts to allow to be created in a single bulk action'),
+    help_text=_('Max number of hosts to allow to be created in a single bulk action'),
+    category=_('Bulk Actions'),
+    category_slug='bulk',
+)
+
+register(
+    'UI_NEXT',
+    field_class=fields.BooleanField,
+    default=False,
+    label=_('Enable Preview of New User Interface'),
+    help_text=_('Enable preview of new user interface.'),
+    category=_('System'),
+    category_slug='system',
+)
+
+register(
+    'SUBSCRIPTION_USAGE_MODEL',
+    field_class=fields.ChoiceField,
+    choices=[
+        ('', _('Default model for AWX - no subscription. Deletion of host_metrics will not be considered for purposes of managed host counting')),
+        (
+            SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS,
+            _('Usage based on unique managed nodes in a large historical time frame and delete functionality for no longer used managed nodes'),
+        ),
+    ],
+    default='',
+    allow_blank=True,
+    label=_('Defines subscription usage model and shows Host Metrics'),
+    category=_('System'),
+    category_slug='system',
+)
+
 
 def logging_validate(serializer, attrs):
     if not serializer.instance or not hasattr(serializer.instance, 'LOG_AGGREGATOR_HOST') or not hasattr(serializer.instance, 'LOG_AGGREGATOR_TYPE'):

awx/main/constants.py

@@ -38,6 +38,8 @@ STANDARD_INVENTORY_UPDATE_ENV = {
     'ANSIBLE_INVENTORY_EXPORT': 'True',
     # Redirecting output to stderr allows JSON parsing to still work with -vvv
     'ANSIBLE_VERBOSE_TO_STDERR': 'True',
+    # if ansible-inventory --limit is used for an inventory import, unmatched should be a failure
+    'ANSIBLE_HOST_PATTERN_MISMATCH': 'error',
 }
 CAN_CANCEL = ('new', 'pending', 'waiting', 'running')
 ACTIVE_STATES = CAN_CANCEL
@@ -63,7 +65,7 @@ ENV_BLOCKLIST = frozenset(
         'INVENTORY_HOSTVARS',
         'AWX_HOST',
         'PROJECT_REVISION',
-        'SUPERVISOR_WEB_CONFIG_PATH',
+        'SUPERVISOR_CONFIG_PATH',
     )
 )
 
@@ -106,3 +108,9 @@ JOB_VARIABLE_PREFIXES = [
 ANSIBLE_RUNNER_NEEDS_UPDATE_MESSAGE = (
     '\u001b[31m \u001b[1m This can be caused if the version of ansible-runner in your execution environment is out of date.\u001b[0m'
 )
+
+# Values for setting SUBSCRIPTION_USAGE_MODEL
+SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS = 'unique_managed_hosts'
+
+# Shared prefetch to use for creating a queryset for the purpose of writing or saving facts
+HOST_FACTS_FIELDS = ('name', 'ansible_facts', 'ansible_facts_modified', 'modified', 'inventory_id')

awx/main/consumers.py

@@ -3,6 +3,7 @@ import logging
 import time
 import hmac
 import asyncio
+import redis
 
 from django.core.serializers.json import DjangoJSONEncoder
 from django.conf import settings
@@ -80,7 +81,7 @@ class WebsocketSecretAuthHelper:
         WebsocketSecretAuthHelper.verify_secret(secret)
 
 
-class BroadcastConsumer(AsyncJsonWebsocketConsumer):
+class RelayConsumer(AsyncJsonWebsocketConsumer):
     async def connect(self):
         try:
             WebsocketSecretAuthHelper.is_authorized(self.scope)
@@ -100,6 +101,21 @@ class BroadcastConsumer(AsyncJsonWebsocketConsumer):
     async def internal_message(self, event):
         await self.send(event['text'])
 
+    async def receive_json(self, data):
+        (group, message) = unwrap_broadcast_msg(data)
+        if group == "metrics":
+            message = json.loads(message['text'])
+            conn = redis.Redis.from_url(settings.BROKER_URL)
+            conn.set(settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX + "_instance_" + message['instance'], message['metrics'])
+        else:
+            await self.channel_layer.group_send(group, message)
+
+    async def consumer_subscribe(self, event):
+        await self.send_json(event)
+
+    async def consumer_unsubscribe(self, event):
+        await self.send_json(event)
+
 
 class EventConsumer(AsyncJsonWebsocketConsumer):
     async def connect(self):
@@ -128,6 +144,11 @@ class EventConsumer(AsyncJsonWebsocketConsumer):
                 self.channel_name,
             )
 
+        await self.channel_layer.group_send(
+            settings.BROADCAST_WEBSOCKET_GROUP_NAME,
+            {"type": "consumer.unsubscribe", "groups": list(current_groups), "origin_channel": self.channel_name},
+        )
+
     @database_sync_to_async
     def user_can_see_object_id(self, user_access, oid):
         # At this point user is a channels.auth.UserLazyObject object
@@ -176,9 +197,20 @@ class EventConsumer(AsyncJsonWebsocketConsumer):
                     self.channel_name,
                 )
 
+            if len(old_groups):
+                await self.channel_layer.group_send(
+                    settings.BROADCAST_WEBSOCKET_GROUP_NAME,
+                    {"type": "consumer.unsubscribe", "groups": list(old_groups), "origin_channel": self.channel_name},
+                )
+
             new_groups_exclusive = new_groups - current_groups
             for group_name in new_groups_exclusive:
                 await self.channel_layer.group_add(group_name, self.channel_name)
+
+            await self.channel_layer.group_send(
+                settings.BROADCAST_WEBSOCKET_GROUP_NAME,
+                {"type": "consumer.subscribe", "groups": list(new_groups), "origin_channel": self.channel_name},
+            )
             self.scope['session']['groups'] = new_groups
             await self.send_json({"groups_current": list(new_groups), "groups_left": list(old_groups), "groups_joined": list(new_groups_exclusive)})
 
@@ -200,9 +232,11 @@ def _dump_payload(payload):
         return None
 
 
-def emit_channel_notification(group, payload):
-    from awx.main.wsbroadcast import wrap_broadcast_msg  # noqa
+def unwrap_broadcast_msg(payload: dict):
+    return (payload['group'], payload['message'])
 
+
+def emit_channel_notification(group, payload):
     payload_dumped = _dump_payload(payload)
     if payload_dumped is None:
         return
@@ -212,16 +246,6 @@ def emit_channel_notification(group, payload):
     run_sync(
         channel_layer.group_send(
             group,
-            {"type": "internal.message", "text": payload_dumped},
-        )
-    )
-
-    run_sync(
-        channel_layer.group_send(
-            settings.BROADCAST_WEBSOCKET_GROUP_NAME,
-            {
-                "type": "internal.message",
-                "text": wrap_broadcast_msg(group, payload_dumped),
-            },
+            {"type": "internal.message", "text": payload_dumped, "needs_relay": True},
         )
     )

awx/main/credential_plugins/aim.py

@@ -70,7 +70,7 @@ def aim_backend(**kwargs):
     client_cert = kwargs.get('client_cert', None)
     client_key = kwargs.get('client_key', None)
     verify = kwargs['verify']
-    webservice_id = kwargs['webservice_id']
+    webservice_id = kwargs.get('webservice_id', '')
     app_id = kwargs['app_id']
     object_query = kwargs['object_query']
     object_query_format = kwargs['object_query_format']

awx/main/credential_plugins/tss.py

@@ -49,7 +49,10 @@ def tss_backend(**kwargs):
     secret_dict = secret_server.get_secret(kwargs['secret_id'])
     secret = ServerSecret(**secret_dict)
 
-    return secret.fields[kwargs['secret_field']].value
+    if isinstance(secret.fields[kwargs['secret_field']].value, str) == False:
+        return secret.fields[kwargs['secret_field']].value.text
+    else:
+        return secret.fields[kwargs['secret_field']].value
 
 
 tss_plugin = CredentialPlugin(

awx/main/db/profiled_pg/base.py

@@ -63,7 +63,7 @@ class RecordedQueryLog(object):
             if not os.path.isdir(self.dest):
                 os.makedirs(self.dest)
             progname = ' '.join(sys.argv)
-            for match in ('uwsgi', 'dispatcher', 'callback_receiver', 'wsbroadcast'):
+            for match in ('uwsgi', 'dispatcher', 'callback_receiver', 'wsrelay'):
                 if match in progname:
                     progname = match
                     break

awx/main/dispatch/__init__.py

@@ -1,3 +1,4 @@
+import os
 import psycopg2
 import select
 
@@ -6,7 +7,6 @@ from contextlib import contextmanager
 from django.conf import settings
 from django.db import connection as pg_connection
 
-
 NOT_READY = ([], [], [])
 
 
@@ -14,6 +14,29 @@ def get_local_queuename():
     return settings.CLUSTER_HOST_ID
 
 
+def get_task_queuename():
+    if os.getenv('AWX_COMPONENT') != 'web':
+        return settings.CLUSTER_HOST_ID
+
+    from awx.main.models.ha import Instance
+
+    random_task_instance = (
+        Instance.objects.filter(
+            node_type__in=(Instance.Types.CONTROL, Instance.Types.HYBRID),
+            node_state=Instance.States.READY,
+            enabled=True,
+        )
+        .only('hostname')
+        .order_by('?')
+        .first()
+    )
+
+    if random_task_instance is None:
+        raise ValueError('No task instances are READY and Enabled.')
+
+    return random_task_instance.hostname
+
+
 class PubSub(object):
     def __init__(self, conn):
         self.conn = conn

awx/main/dispatch/control.py

@@ -6,7 +6,7 @@ from django.conf import settings
 from django.db import connection
 import redis
 
-from awx.main.dispatch import get_local_queuename
+from awx.main.dispatch import get_task_queuename
 
 from . import pg_bus_conn
 
@@ -21,7 +21,7 @@ class Control(object):
         if service not in self.services:
             raise RuntimeError('{} must be in {}'.format(service, self.services))
         self.service = service
-        self.queuename = host or get_local_queuename()
+        self.queuename = host or get_task_queuename()
 
     def status(self, *args, **kwargs):
         r = redis.Redis.from_url(settings.BROKER_URL)

awx/main/dispatch/reaper.py

@@ -70,7 +70,7 @@ def reap_waiting(instance=None, status='failed', job_explanation=None, grace_per
         reap_job(j, status, job_explanation=job_explanation)
 
 
-def reap(instance=None, status='failed', job_explanation=None, excluded_uuids=None):
+def reap(instance=None, status='failed', job_explanation=None, excluded_uuids=None, ref_time=None):
     """
     Reap all jobs in running for this instance.
     """
@@ -79,9 +79,11 @@ def reap(instance=None, status='failed', job_explanation=None, excluded_uuids=No
     else:
         hostname = instance.hostname
     workflow_ctype_id = ContentType.objects.get_for_model(WorkflowJob).id
-    jobs = UnifiedJob.objects.filter(
-        Q(status='running') & (Q(execution_node=hostname) | Q(controller_node=hostname)) & ~Q(polymorphic_ctype_id=workflow_ctype_id)
-    )
+    base_Q = Q(status='running') & (Q(execution_node=hostname) | Q(controller_node=hostname)) & ~Q(polymorphic_ctype_id=workflow_ctype_id)
+    if ref_time:
+        jobs = UnifiedJob.objects.filter(base_Q & Q(started__lte=ref_time))
+    else:
+        jobs = UnifiedJob.objects.filter(base_Q)
     if excluded_uuids:
         jobs = jobs.exclude(celery_task_id__in=excluded_uuids)
     for j in jobs:

awx/main/dispatch/worker/task.py

@@ -26,8 +26,8 @@ class TaskWorker(BaseWorker):
     `awx.main.dispatch.publish`.
     """
 
-    @classmethod
-    def resolve_callable(cls, task):
+    @staticmethod
+    def resolve_callable(task):
         """
         Transform a dotted notation task into an imported, callable function, e.g.,
 
@@ -46,7 +46,8 @@ class TaskWorker(BaseWorker):
 
         return _call
 
-    def run_callable(self, body):
+    @staticmethod
+    def run_callable(body):
         """
         Given some AMQP message, import the correct Python code and run it.
         """

awx/main/fields.py

@@ -954,6 +954,16 @@ class OrderedManyToManyDescriptor(ManyToManyDescriptor):
                 def get_queryset(self):
                     return super(OrderedManyRelatedManager, self).get_queryset().order_by('%s__position' % self.through._meta.model_name)
 
+                def add(self, *objects):
+                    if len(objects) > 1:
+                        raise RuntimeError('Ordered many-to-many fields do not support multiple objects')
+                    return super().add(*objects)
+
+                def remove(self, *objects):
+                    if len(objects) > 1:
+                        raise RuntimeError('Ordered many-to-many fields do not support multiple objects')
+                    return super().remove(*objects)
+
             return OrderedManyRelatedManager
 
         return add_custom_queryset_to_many_related_manager(
@@ -971,13 +981,12 @@ class OrderedManyToManyField(models.ManyToManyField):
     by a special `position` column on the M2M table
     """
 
-    def _update_m2m_position(self, sender, **kwargs):
-        if kwargs.get('action') in ('post_add', 'post_remove'):
-            order_with_respect_to = None
-            for field in sender._meta.local_fields:
-                if isinstance(field, models.ForeignKey) and isinstance(kwargs['instance'], field.related_model):
-                    order_with_respect_to = field.name
-            for i, ig in enumerate(sender.objects.filter(**{order_with_respect_to: kwargs['instance'].pk})):
+    def _update_m2m_position(self, sender, instance, action, **kwargs):
+        if action in ('post_add', 'post_remove'):
+            descriptor = getattr(instance, self.name)
+            order_with_respect_to = descriptor.source_field_name
+
+            for i, ig in enumerate(sender.objects.filter(**{order_with_respect_to: instance.pk})):
                 if ig.position != i:
                     ig.position = i
                     ig.save()

awx/main/management/commands/disable_instance.py

@@ -0,0 +1,143 @@
+import time
+from urllib.parse import urljoin
+
+from argparse import ArgumentTypeError
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.db.models import Q
+from django.utils.timezone import now
+
+from awx.main.models import Instance, UnifiedJob
+
+
+class AWXInstance:
+    def __init__(self, **filter):
+        self.filter = filter
+        self.get_instance()
+
+    def get_instance(self):
+        filter = self.filter if self.filter is not None else dict(hostname=settings.CLUSTER_HOST_ID)
+        qs = Instance.objects.filter(**filter)
+        if not qs.exists():
+            raise ValueError(f"No AWX instance found with {filter} parameters")
+        self.instance = qs.first()
+
+    def disable(self):
+        if self.instance.enabled:
+            self.instance.enabled = False
+            self.instance.save()
+            return True
+
+    def enable(self):
+        if not self.instance.enabled:
+            self.instance.enabled = True
+            self.instance.save()
+            return True
+
+    def jobs(self):
+        return UnifiedJob.objects.filter(
+            Q(controller_node=self.instance.hostname) | Q(execution_node=self.instance.hostname), status__in=("running", "waiting")
+        )
+
+    def jobs_pretty(self):
+        jobs = []
+        for j in self.jobs():
+            job_started = j.started if j.started else now()
+            # similar calculation of `elapsed` as the corresponding serializer
+            # does
+            td = now() - job_started
+            elapsed = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / (10**6 * 1.0)
+            elapsed = float(elapsed)
+            details = dict(
+                name=j.name,
+                url=j.get_ui_url(),
+                elapsed=elapsed,
+            )
+            jobs.append(details)
+
+        jobs = sorted(jobs, reverse=True, key=lambda j: j["elapsed"])
+
+        return ", ".join([f"[\"{j['name']}\"]({j['url']})" for j in jobs])
+
+    def instance_pretty(self):
+        instance = (
+            self.instance.hostname,
+            urljoin(settings.TOWER_URL_BASE, f"/#/instances/{self.instance.pk}/details"),
+        )
+        return f"[\"{instance[0]}\"]({instance[1]})"
+
+
+class Command(BaseCommand):
+    help = "Disable instance, optionally waiting for all its managed jobs to finish."
+
+    @staticmethod
+    def ge_1(arg):
+        if arg == "inf":
+            return float("inf")
+
+        int_arg = int(arg)
+        if int_arg < 1:
+            raise ArgumentTypeError(f"The value must be a positive number >= 1. Provided: \"{arg}\"")
+        return int_arg
+
+    def add_arguments(self, parser):
+        filter_group = parser.add_mutually_exclusive_group()
+
+        filter_group.add_argument(
+            "--hostname",
+            type=str,
+            default=settings.CLUSTER_HOST_ID,
+            help=f"{Instance.hostname.field.help_text} Defaults to the hostname of the machine where the Python interpreter is currently executing".strip(),
+        )
+        filter_group.add_argument("--id", type=self.ge_1, help=Instance.id.field.help_text)
+
+        parser.add_argument(
+            "--wait",
+            action="store_true",
+            help="Wait for jobs managed by the instance to finish. With default retry arguments waits ~1h",
+        )
+
+        parser.add_argument(
+            "--retry",
+            type=self.ge_1,
+            default=120,
+            help="Number of retries when waiting for jobs to finish. Default: 120. Also accepts \"inf\" to wait indefinitely",
+        )
+
+        parser.add_argument(
+            "--retry_sleep",
+            type=self.ge_1,
+            default=30,
+            help="Number of seconds to sleep before consequtive retries when waiting. Default: 30",
+        )
+
+    def handle(self, *args, **options):
+        try:
+            filter = dict(id=options["id"]) if options["id"] is not None else dict(hostname=options["hostname"])
+            instance = AWXInstance(**filter)
+        except ValueError as e:
+            raise CommandError(e)
+
+        if instance.disable():
+            self.stdout.write(self.style.SUCCESS(f"Instance {instance.instance_pretty()} has been disabled"))
+        else:
+            self.stdout.write(f"Instance {instance.instance_pretty()} has already been disabled")
+
+        if not options["wait"]:
+            return
+
+        rc = 1
+        while instance.jobs().count() > 0:
+            if rc < options["retry"]:
+                self.stdout.write(
+                    f"{rc}/{options['retry']}: Waiting {options['retry_sleep']}s before the next attempt to see if the following instance' managed jobs have finished: {instance.jobs_pretty()}"
+                )
+                rc += 1
+                time.sleep(options["retry_sleep"])
+            else:
+                raise CommandError(
+                    f"{rc}/{options['retry']}: No more retry attempts left, but the instance still has associated managed jobs: {instance.jobs_pretty()}"
+                )
+        else:
+            self.stdout.write(self.style.SUCCESS("Done waiting for instance' managed jobs to finish!"))

awx/main/management/commands/enable_local_authentication.py

@@ -1,5 +1,6 @@
-from django.core.management.base import BaseCommand, CommandError
+from awx.main.tasks.system import clear_setting_cache
 from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
 
 
 class Command(BaseCommand):
@@ -31,5 +32,7 @@ class Command(BaseCommand):
         else:
             raise CommandError('Please pass --enable flag to allow local auth or --disable flag to disable local auth')
 
+        clear_setting_cache.delay(['DISABLE_LOCAL_AUTH'])
+
     def handle(self, **options):
         self._enable_disable_auth(options.get('enable'), options.get('disable'))

awx/main/management/commands/host_metric.py

@@ -1,53 +1,230 @@
 from django.core.management.base import BaseCommand
 import datetime
 from django.core.serializers.json import DjangoJSONEncoder
-from awx.main.models.inventory import HostMetric
+from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly
+from awx.main.analytics.collectors import config
 import json
+import sys
+import tempfile
+import tarfile
+import csv
+
+CSV_PREFERRED_ROW_COUNT = 500000
+BATCHED_FETCH_COUNT = 10000
 
 
 class Command(BaseCommand):
     help = 'This is for offline licensing usage'
 
+    def host_metric_queryset(self, result, offset=0, limit=BATCHED_FETCH_COUNT):
+        list_of_queryset = list(
+            result.values(
+                'id',
+                'hostname',
+                'first_automation',
+                'last_automation',
+                'last_deleted',
+                'automated_counter',
+                'deleted_counter',
+                'deleted',
+                'used_in_inventories',
+            ).order_by('first_automation')[offset : offset + limit]
+        )
+
+        return list_of_queryset
+
+    def host_metric_summary_monthly_queryset(self, result, offset=0, limit=BATCHED_FETCH_COUNT):
+        list_of_queryset = list(
+            result.values(
+                'id',
+                'date',
+                'license_consumed',
+                'license_capacity',
+                'hosts_added',
+                'hosts_deleted',
+                'indirectly_managed_hosts',
+            ).order_by(
+                'date'
+            )[offset : offset + limit]
+        )
+
+        return list_of_queryset
+
+    def paginated_db_retrieval(self, type, filter_kwargs, rows_per_file):
+        offset = 0
+        list_of_queryset = []
+        while True:
+            if type == 'host_metric':
+                result = HostMetric.objects.filter(**filter_kwargs)
+                list_of_queryset = self.host_metric_queryset(result, offset, rows_per_file)
+            elif type == 'host_metric_summary_monthly':
+                result = HostMetricSummaryMonthly.objects.filter(**filter_kwargs)
+                list_of_queryset = self.host_metric_summary_monthly_queryset(result, offset, rows_per_file)
+
+            if not list_of_queryset:
+                break
+            else:
+                yield list_of_queryset
+
+            offset += len(list_of_queryset)
+
+    def controlled_db_retrieval(self, type, filter_kwargs, offset=0, fetch_count=BATCHED_FETCH_COUNT):
+        if type == 'host_metric':
+            result = HostMetric.objects.filter(**filter_kwargs)
+            return self.host_metric_queryset(result, offset, fetch_count)
+        elif type == 'host_metric_summary_monthly':
+            result = HostMetricSummaryMonthly.objects.filter(**filter_kwargs)
+            return self.host_metric_summary_monthly_queryset(result, offset, fetch_count)
+
+    def write_to_csv(self, csv_file, list_of_queryset, always_header, first_write=False, mode='a'):
+        with open(csv_file, mode, newline='') as output_file:
+            try:
+                keys = list_of_queryset[0].keys() if list_of_queryset else []
+                dict_writer = csv.DictWriter(output_file, keys)
+                if always_header or first_write:
+                    dict_writer.writeheader()
+                dict_writer.writerows(list_of_queryset)
+
+            except Exception as e:
+                print(e)
+
+    def csv_for_tar(self, temp_dir, type, filter_kwargs, rows_per_file, always_header=True):
+        for index, list_of_queryset in enumerate(self.paginated_db_retrieval(type, filter_kwargs, rows_per_file)):
+            csv_file = f'{temp_dir}/{type}{index+1}.csv'
+            arcname_file = f'{type}{index+1}.csv'
+
+            first_write = True if index == 0 else False
+
+            self.write_to_csv(csv_file, list_of_queryset, always_header, first_write, 'w')
+            yield csv_file, arcname_file
+
+    def csv_for_tar_batched_fetch(self, temp_dir, type, filter_kwargs, rows_per_file, always_header=True):
+        csv_iteration = 1
+
+        offset = 0
+        rows_written_per_csv = 0
+        to_fetch = BATCHED_FETCH_COUNT
+
+        while True:
+            list_of_queryset = self.controlled_db_retrieval(type, filter_kwargs, offset, to_fetch)
+
+            if not list_of_queryset:
+                break
+
+            csv_file = f'{temp_dir}/{type}{csv_iteration}.csv'
+            arcname_file = f'{type}{csv_iteration}.csv'
+            self.write_to_csv(csv_file, list_of_queryset, always_header)
+
+            offset += to_fetch
+            rows_written_per_csv += to_fetch
+            always_header = False
+
+            remaining_rows_per_csv = rows_per_file - rows_written_per_csv
+
+            if not remaining_rows_per_csv:
+                yield csv_file, arcname_file
+
+                rows_written_per_csv = 0
+                always_header = True
+                to_fetch = BATCHED_FETCH_COUNT
+                csv_iteration += 1
+            elif remaining_rows_per_csv < BATCHED_FETCH_COUNT:
+                to_fetch = remaining_rows_per_csv
+
+        if rows_written_per_csv:
+            yield csv_file, arcname_file
+
+    def config_for_tar(self, options, temp_dir):
+        config_json = json.dumps(config(options.get('since')))
+        config_file = f'{temp_dir}/config.json'
+        arcname_file = 'config.json'
+        with open(config_file, 'w') as f:
+            f.write(config_json)
+        return config_file, arcname_file
+
+    def output_json(self, options, filter_kwargs):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            for csv_detail in self.csv_for_tar(temp_dir, options.get('json', 'host_metric'), filter_kwargs, BATCHED_FETCH_COUNT, True):
+                csv_file = csv_detail[0]
+
+                with open(csv_file) as f:
+                    reader = csv.DictReader(f)
+                    rows = list(reader)
+                    json_result = json.dumps(rows, cls=DjangoJSONEncoder)
+                    print(json_result)
+
+    def output_csv(self, options, filter_kwargs):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            for csv_detail in self.csv_for_tar(temp_dir, options.get('csv', 'host_metric'), filter_kwargs, BATCHED_FETCH_COUNT, False):
+                csv_file = csv_detail[0]
+                with open(csv_file) as f:
+                    sys.stdout.write(f.read())
+
+    def output_tarball(self, options, filter_kwargs):
+        always_header = True
+        rows_per_file = options['rows_per_file'] or CSV_PREFERRED_ROW_COUNT
+
+        tar = tarfile.open("./host_metrics.tar.gz", "w:gz")
+
+        if rows_per_file <= BATCHED_FETCH_COUNT:
+            csv_function = self.csv_for_tar
+        else:
+            csv_function = self.csv_for_tar_batched_fetch
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            for csv_detail in csv_function(temp_dir, 'host_metric', filter_kwargs, rows_per_file, always_header):
+                tar.add(csv_detail[0], arcname=csv_detail[1])
+
+            for csv_detail in csv_function(temp_dir, 'host_metric_summary_monthly', filter_kwargs, rows_per_file, always_header):
+                tar.add(csv_detail[0], arcname=csv_detail[1])
+
+            config_file, arcname_file = self.config_for_tar(options, temp_dir)
+            tar.add(config_file, arcname=arcname_file)
+
+        tar.close()
+
     def add_arguments(self, parser):
         parser.add_argument('--since', type=datetime.datetime.fromisoformat, help='Start Date in ISO format YYYY-MM-DD')
-        parser.add_argument('--until', type=datetime.datetime.fromisoformat, help='End Date in ISO format YYYY-MM-DD')
-        parser.add_argument('--json', action='store_true', help='Select output as JSON')
+        parser.add_argument('--json', type=str, const='host_metric', nargs='?', help='Select output as JSON for host_metric or host_metric_summary_monthly')
+        parser.add_argument('--csv', type=str, const='host_metric', nargs='?', help='Select output as CSV for host_metric or host_metric_summary_monthly')
+        parser.add_argument('--tarball', action='store_true', help=f'Package CSV files into a tar with upto {CSV_PREFERRED_ROW_COUNT} rows')
+        parser.add_argument('--rows_per_file', type=int, help=f'Split rows in chunks of {CSV_PREFERRED_ROW_COUNT}')
 
     def handle(self, *args, **options):
         since = options.get('since')
-        until = options.get('until')
-
-        if since is None and until is None:
-            print("No Arguments received")
-            return None
 
         if since is not None and since.tzinfo is None:
             since = since.replace(tzinfo=datetime.timezone.utc)
 
-        if until is not None and until.tzinfo is None:
-            until = until.replace(tzinfo=datetime.timezone.utc)
-
         filter_kwargs = {}
         if since is not None:
             filter_kwargs['last_automation__gte'] = since
-        if until is not None:
-            filter_kwargs['last_automation__lte'] = until
 
-        result = HostMetric.objects.filter(**filter_kwargs)
+        filter_kwargs_host_metrics_summary = {}
+        if since is not None:
+            filter_kwargs_host_metrics_summary['date__gte'] = since
+
+        if options['rows_per_file'] and options.get('rows_per_file') > CSV_PREFERRED_ROW_COUNT:
+            print(f"rows_per_file exceeds the allowable limit of {CSV_PREFERRED_ROW_COUNT}.")
+            return
 
         # if --json flag is set, output the result in json format
         if options['json']:
-            list_of_queryset = list(result.values('hostname', 'first_automation', 'last_automation'))
-            json_result = json.dumps(list_of_queryset, cls=DjangoJSONEncoder)
-            print(json_result)
+            self.output_json(options, filter_kwargs)
+        elif options['csv']:
+            self.output_csv(options, filter_kwargs)
+        elif options['tarball']:
+            self.output_tarball(options, filter_kwargs)
 
         # --json flag is not set, output in plain text
         else:
-            print(f"Total Number of hosts automated: {len(result)}")
-            for item in result:
+            print(f"Printing up to {BATCHED_FETCH_COUNT} automated hosts:")
+            result = HostMetric.objects.filter(**filter_kwargs)
+            list_of_queryset = self.host_metric_queryset(result, 0, BATCHED_FETCH_COUNT)
+            for item in list_of_queryset:
                 print(
                     "Hostname : {hostname} | first_automation : {first_automation} | last_automation : {last_automation}".format(
-                        hostname=item.hostname, first_automation=item.first_automation, last_automation=item.last_automation
+                        hostname=item['hostname'], first_automation=item['first_automation'], last_automation=item['last_automation']
                     )
                 )
         return

awx/main/management/commands/inventory_import.py

@@ -458,12 +458,19 @@ class Command(BaseCommand):
         # TODO: We disable variable overwrite here in case user-defined inventory variables get
         # mangled. But we still need to figure out a better way of processing multiple inventory
         # update variables mixing with each other.
-        all_obj = self.inventory
-        db_variables = all_obj.variables_dict
-        db_variables.update(self.all_group.variables)
-        if db_variables != all_obj.variables_dict:
-            all_obj.variables = json.dumps(db_variables)
-            all_obj.save(update_fields=['variables'])
+        # issue for this: https://github.com/ansible/awx/issues/11623
+
+        if self.inventory.kind == 'constructed' and self.inventory_source.overwrite_vars:
+            # NOTE: we had to add a exception case to not merge variables
+            # to make constructed inventory coherent
+            db_variables = self.all_group.variables
+        else:
+            db_variables = self.inventory.variables_dict
+            db_variables.update(self.all_group.variables)
+
+        if db_variables != self.inventory.variables_dict:
+            self.inventory.variables = json.dumps(db_variables)
+            self.inventory.save(update_fields=['variables'])
             logger.debug('Inventory variables updated from "all" group')
         else:
             logger.debug('Inventory variables unmodified')
@@ -522,16 +529,32 @@ class Command(BaseCommand):
     def _update_db_host_from_mem_host(self, db_host, mem_host):
         # Update host variables.
         db_variables = db_host.variables_dict
-        if self.overwrite_vars:
-            db_variables = mem_host.variables
-        else:
-            db_variables.update(mem_host.variables)
+        mem_variables = mem_host.variables
         update_fields = []
+
+        # Update host instance_id.
+        instance_id = self._get_instance_id(mem_variables)
+        if instance_id != db_host.instance_id:
+            old_instance_id = db_host.instance_id
+            db_host.instance_id = instance_id
+            update_fields.append('instance_id')
+
+        if self.inventory.kind == 'constructed':
+            # remote towervars so the constructed hosts do not have extra variables
+            for prefix in ('host', 'tower'):
+                for var in ('remote_{}_enabled', 'remote_{}_id'):
+                    mem_variables.pop(var.format(prefix), None)
+
+        if self.overwrite_vars:
+            db_variables = mem_variables
+        else:
+            db_variables.update(mem_variables)
+
         if db_variables != db_host.variables_dict:
             db_host.variables = json.dumps(db_variables)
             update_fields.append('variables')
         # Update host enabled flag.
-        enabled = self._get_enabled(mem_host.variables)
+        enabled = self._get_enabled(mem_variables)
         if enabled is not None and db_host.enabled != enabled:
             db_host.enabled = enabled
             update_fields.append('enabled')
@@ -540,12 +563,6 @@ class Command(BaseCommand):
             old_name = db_host.name
             db_host.name = mem_host.name
             update_fields.append('name')
-        # Update host instance_id.
-        instance_id = self._get_instance_id(mem_host.variables)
-        if instance_id != db_host.instance_id:
-            old_instance_id = db_host.instance_id
-            db_host.instance_id = instance_id
-            update_fields.append('instance_id')
         # Update host and display message(s) on what changed.
         if update_fields:
             db_host.save(update_fields=update_fields)
@@ -654,13 +671,19 @@ class Command(BaseCommand):
             mem_host = self.all_group.all_hosts[mem_host_name]
             import_vars = mem_host.variables
             host_desc = import_vars.pop('_awx_description', 'imported')
-            host_attrs = dict(variables=json.dumps(import_vars), description=host_desc)
+            host_attrs = dict(description=host_desc)
             enabled = self._get_enabled(mem_host.variables)
             if enabled is not None:
                 host_attrs['enabled'] = enabled
             if self.instance_id_var:
                 instance_id = self._get_instance_id(mem_host.variables)
                 host_attrs['instance_id'] = instance_id
+            if self.inventory.kind == 'constructed':
+                # remote towervars so the constructed hosts do not have extra variables
+                for prefix in ('host', 'tower'):
+                    for var in ('remote_{}_enabled', 'remote_{}_id'):
+                        import_vars.pop(var.format(prefix), None)
+            host_attrs['variables'] = json.dumps(import_vars)
             try:
                 sanitize_jinja(mem_host_name)
             except ValueError as e:
@@ -851,6 +874,7 @@ class Command(BaseCommand):
             logger.info('Updating inventory %d: %s' % (inventory.pk, inventory.name))
 
             # Create ad-hoc inventory source and inventory update objects
+            ee = get_default_execution_environment()
             with ignore_inventory_computed_fields():
                 source = Command.get_source_absolute_path(raw_source)
 
@@ -860,14 +884,22 @@ class Command(BaseCommand):
                     source_path=os.path.abspath(source),
                     overwrite=bool(options.get('overwrite', False)),
                     overwrite_vars=bool(options.get('overwrite_vars', False)),
+                    execution_environment=ee,
                 )
                 inventory_update = inventory_source.create_inventory_update(
-                    _eager_fields=dict(status='running', job_args=json.dumps(sys.argv), job_env=dict(os.environ.items()), job_cwd=os.getcwd())
+                    _eager_fields=dict(
+                        status='running', job_args=json.dumps(sys.argv), job_env=dict(os.environ.items()), job_cwd=os.getcwd(), execution_environment=ee
+                    )
                 )
 
-            data = AnsibleInventoryLoader(source=source, verbosity=verbosity).load()
+            try:
+                data = AnsibleInventoryLoader(source=source, verbosity=verbosity).load()
+                logger.debug('Finished loading from source: %s', source)
 
-            logger.debug('Finished loading from source: %s', source)
+            except SystemExit:
+                logger.debug("Error occurred while running ansible-inventory")
+                inventory_update.cancel()
+                sys.exit(1)
 
             status, tb, exc = 'error', '', None
             try:

awx/main/management/commands/run_cache_clear.py

@@ -0,0 +1,32 @@
+import logging
+import json
+
+from django.core.management.base import BaseCommand
+from awx.main.dispatch import pg_bus_conn
+from awx.main.dispatch.worker.task import TaskWorker
+
+logger = logging.getLogger('awx.main.cache_clear')
+
+
+class Command(BaseCommand):
+    """
+    Cache Clear
+    Runs as a management command and starts a daemon that listens for a pg_notify message to clear the cache.
+    """
+
+    help = 'Launch the cache clear daemon'
+
+    def handle(self, *arg, **options):
+        try:
+            with pg_bus_conn(new_connection=True) as conn:
+                conn.listen("tower_settings_change")
+                for e in conn.events(yield_timeouts=True):
+                    if e is not None:
+                        body = json.loads(e.payload)
+                        logger.info(f"Cache clear request received. Clearing now, payload: {e.payload}")
+                        TaskWorker.run_callable(body)
+
+        except Exception:
+            # Log unanticipated exception in addition to writing to stderr to get timestamps and other metadata
+            logger.exception('Encountered unhandled error in cache clear main loop')
+            raise

awx/main/management/commands/run_dispatcher.py

@@ -8,7 +8,7 @@ from django.core.cache import cache as django_cache
 from django.core.management.base import BaseCommand
 from django.db import connection as django_connection
 
-from awx.main.dispatch import get_local_queuename
+from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.control import Control
 from awx.main.dispatch.pool import AutoscalePool
 from awx.main.dispatch.worker import AWXConsumerPG, TaskWorker
@@ -76,7 +76,7 @@ class Command(BaseCommand):
         consumer = None
 
         try:
-            queues = ['tower_broadcast_all', get_local_queuename()]
+            queues = ['tower_broadcast_all', 'tower_settings_change', get_task_queuename()]
             consumer = AWXConsumerPG('dispatcher', TaskWorker(), queues, AutoscalePool(min_workers=4))
             consumer.run()
         except KeyboardInterrupt:

awx/main/management/commands/run_heartbeet.py

@@ -0,0 +1,67 @@
+import json
+import logging
+import os
+import time
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+from awx.main.dispatch import pg_bus_conn
+
+logger = logging.getLogger('awx.main.commands.run_heartbeet')
+
+
+class Command(BaseCommand):
+    help = 'Launch the web server beacon (heartbeet)'
+
+    def print_banner(self):
+        heartbeet = r"""
+   **********   **********
+ ************* *************
+*****************************
+ ***********HEART***********
+  *************************
+     *******************
+       *************** _._
+         *********** /`._ `'.      __
+           *******   \  .\| \   _'`  `)
+             ***  (``_)  \| ).'` /`- /
+              *   `\ `;\_ `\\//`-'` /
+                    \ `'.'.| /  __/`
+                     `'--v_|/`'`
+                         __||-._
+                      /'` `-``  `'\\
+                     /         .'` )
+                     \  BEET  '    )
+                      \.          /
+                        '.     /'`
+                           `) |
+                            //
+                            '(.
+                             `\`.
+                               ``"""
+        print(heartbeet)
+
+    def construct_payload(self, action='online'):
+        payload = {
+            'hostname': settings.CLUSTER_HOST_ID,
+            'ip': os.environ.get('MY_POD_IP'),
+            'action': action,
+        }
+        return json.dumps(payload)
+
+    def do_hearbeat_loop(self):
+        with pg_bus_conn(new_connection=True) as conn:
+            while True:
+                logger.debug('Sending heartbeat')
+                conn.notify('web_heartbeet', self.construct_payload())
+                time.sleep(settings.BROADCAST_WEBSOCKET_BEACON_FROM_WEB_RATE_SECONDS)
+
+    # TODO: Send a message with action=offline if we notice a SIGTERM or SIGINT
+    # (wsrelay can use this to remove the node quicker)
+    def handle(self, *arg, **options):
+        self.print_banner()
+
+        # Note: We don't really try any reconnect logic to pg_notify here,
+        # just let supervisor restart if we fail.
+        self.do_hearbeat_loop()

awx/main/management/commands/run_rsyslog_configurer.py

@@ -0,0 +1,41 @@
+import logging
+import json
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from django.core.cache import cache
+from awx.main.dispatch import pg_bus_conn
+from awx.main.dispatch.worker.task import TaskWorker
+from awx.main.utils.external_logging import reconfigure_rsyslog
+
+logger = logging.getLogger('awx.main.rsyslog_configurer')
+
+
+class Command(BaseCommand):
+    """
+    Rsyslog Configurer
+    Runs as a management command and starts rsyslog configurer daemon. Daemon listens
+    for pg_notify then calls reconfigure_rsyslog
+    """
+
+    help = 'Launch the rsyslog_configurer daemon'
+
+    def handle(self, *arg, **options):
+        try:
+            with pg_bus_conn(new_connection=True) as conn:
+                conn.listen("rsyslog_configurer")
+                # reconfigure rsyslog on start up
+                reconfigure_rsyslog()
+                for e in conn.events(yield_timeouts=True):
+                    if e is not None:
+                        logger.info("Change in logging settings found. Restarting rsyslogd")
+                        # clear the cache of relevant settings then restart
+                        setting_keys = [k for k in dir(settings) if k.startswith('LOG_AGGREGATOR')]
+                        cache.delete_many(setting_keys)
+                        settings._awx_conf_memoizedcache.clear()
+                        body = json.loads(e.payload)
+                        TaskWorker.run_callable(body)
+        except Exception:
+            # Log unanticipated exception in addition to writing to stderr to get timestamps and other metadata
+            logger.exception('Encountered unhandled error in rsyslog_configurer main loop')
+            raise

awx/main/management/commands/run_wsrelay.py

@@ -13,13 +13,13 @@ from django.db import connection
 from django.db.migrations.executor import MigrationExecutor
 
 from awx.main.analytics.broadcast_websocket import (
-    BroadcastWebsocketStatsManager,
+    RelayWebsocketStatsManager,
     safe_name,
 )
-from awx.main.wsbroadcast import BroadcastWebsocketManager
+from awx.main.wsrelay import WebSocketRelayManager
 
 
-logger = logging.getLogger('awx.main.wsbroadcast')
+logger = logging.getLogger('awx.main.wsrelay')
 
 
 class Command(BaseCommand):
@@ -99,7 +99,7 @@ class Command(BaseCommand):
             executor = MigrationExecutor(connection)
             migrating = bool(executor.migration_plan(executor.loader.graph.leaf_nodes()))
         except Exception as exc:
-            logger.info(f'Error on startup of run_wsbroadcast (error: {exc}), retry in 10s...')
+            logger.warning(f'Error on startup of run_wsrelay (error: {exc}), retry in 10s...')
             time.sleep(10)
             return
 
@@ -130,9 +130,9 @@ class Command(BaseCommand):
 
         if options.get('status'):
             try:
-                stats_all = BroadcastWebsocketStatsManager.get_stats_sync()
+                stats_all = RelayWebsocketStatsManager.get_stats_sync()
             except redis.exceptions.ConnectionError as e:
-                print(f"Unable to get Broadcast Websocket Status. Failed to connect to redis {e}")
+                print(f"Unable to get Relay Websocket Status. Failed to connect to redis {e}")
                 return
 
             data = {}
@@ -151,22 +151,19 @@ class Command(BaseCommand):
             host_stats = Command.get_connection_status(hostnames, data)
             lines = Command._format_lines(host_stats)
 
-            print(f'Broadcast websocket connection status from "{my_hostname}" to:')
+            print(f'Relay websocket connection status from "{my_hostname}" to:')
             print('\n'.join(lines))
 
             host_stats = Command.get_connection_stats(hostnames, data)
             lines = Command._format_lines(host_stats)
 
-            print(f'\nBroadcast websocket connection stats from "{my_hostname}" to:')
+            print(f'\nRelay websocket connection stats from "{my_hostname}" to:')
             print('\n'.join(lines))
 
             return
 
         try:
-            broadcast_websocket_mgr = BroadcastWebsocketManager()
-            task = broadcast_websocket_mgr.start()
-
-            loop = asyncio.get_event_loop()
-            loop.run_until_complete(task)
+            websocket_relay_manager = WebSocketRelayManager()
+            asyncio.run(websocket_relay_manager.run())
         except KeyboardInterrupt:
-            logger.debug('Terminating Websocket Broadcaster')
+            logger.info('Terminating Websocket Relayer')

awx/main/managers.py

@@ -79,6 +79,11 @@ class HostManager(models.Manager):
         return qs
 
 
+class HostMetricActiveManager(models.Manager):
+    def get_queryset(self):
+        return super().get_queryset().filter(deleted=False)
+
+
 def get_ig_ig_mapping(ig_instance_mapping, instance_ig_mapping):
     # Create IG mapping by union of all groups their instances are members of
     ig_ig_mapping = {}

awx/main/migrations/0175_workflowjob_is_bulk_job.py

@@ -0,0 +1,17 @@
+# Generated by Django 3.2.16 on 2023-01-05 15:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('main', '0174_ensure_org_ee_admin_roles'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='workflowjob',
+            name='is_bulk_job',
+            field=models.BooleanField(default=False),
+        ),
+    ]

awx/main/migrations/0176_inventorysource_scm_branch.py

@@ -0,0 +1,32 @@
+# Generated by Django 3.2.16 on 2023-03-03 20:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('main', '0175_workflowjob_is_bulk_job'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='inventorysource',
+            name='scm_branch',
+            field=models.CharField(
+                blank=True,
+                default='',
+                help_text='Inventory source SCM branch. Project default used if blank. Only allowed if project allow_override field is set to true.',
+                max_length=1024,
+            ),
+        ),
+        migrations.AddField(
+            model_name='inventoryupdate',
+            name='scm_branch',
+            field=models.CharField(
+                blank=True,
+                default='',
+                help_text='Inventory source SCM branch. Project default used if blank. Only allowed if project allow_override field is set to true.',
+                max_length=1024,
+            ),
+        ),
+    ]

awx/main/migrations/0177_instance_group_role_addition.py

@@ -0,0 +1,48 @@
+# Generated by Django 3.2.16 on 2023-02-17 02:45
+
+import awx.main.fields
+from django.db import migrations
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('main', '0176_inventorysource_scm_branch'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='instancegroup',
+            name='admin_role',
+            field=awx.main.fields.ImplicitRoleField(
+                editable=False,
+                null='True',
+                on_delete=django.db.models.deletion.CASCADE,
+                parent_role=['singleton:system_administrator'],
+                related_name='+',
+                to='main.role',
+            ),
+            preserve_default='True',
+        ),
+        migrations.AddField(
+            model_name='instancegroup',
+            name='read_role',
+            field=awx.main.fields.ImplicitRoleField(
+                editable=False,
+                null='True',
+                on_delete=django.db.models.deletion.CASCADE,
+                parent_role=['singleton:system_auditor', 'use_role', 'admin_role'],
+                related_name='+',
+                to='main.role',
+            ),
+            preserve_default='True',
+        ),
+        migrations.AddField(
+            model_name='instancegroup',
+            name='use_role',
+            field=awx.main.fields.ImplicitRoleField(
+                editable=False, null='True', on_delete=django.db.models.deletion.CASCADE, parent_role=['admin_role'], related_name='+', to='main.role'
+            ),
+            preserve_default='True',
+        ),
+    ]

awx/main/migrations/0178_instance_group_admin_migration.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2023-02-17 02:45
+
+from django.db import migrations
+from awx.main.migrations import _rbac as rbac
+from awx.main.migrations import _migration_utils as migration_utils
+from awx.main.migrations import _OrgAdmin_to_use_ig as oamigrate
+from awx.main.migrations import ActivityStreamDisabledMigration
+
+
+class Migration(ActivityStreamDisabledMigration):
+    dependencies = [
+        ('main', '0177_instance_group_role_addition'),
+    ]
+    operations = [
+        migrations.RunPython(migration_utils.set_current_apps_for_migrations),
+        migrations.RunPython(rbac.create_roles),
+        migrations.RunPython(oamigrate.migrate_org_admin_to_use),
+    ]

awx/main/migrations/0179_change_cyberark_plugin_names.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2023-03-16 15:16
+from django.db import migrations
+
+from awx.main.migrations._credentialtypes import migrate_credential_type
+from awx.main.models import CredentialType
+
+
+class Migration(migrations.Migration):
+    def update_cyberark_plugin_names(apps, schema_editor):
+        CredentialType.setup_tower_managed_defaults(apps)
+        migrate_credential_type(apps, 'aim')
+        migrate_credential_type(apps, 'conjur')
+
+    dependencies = [
+        ('main', '0178_instance_group_admin_migration'),
+    ]
+
+    operations = [migrations.RunPython(update_cyberark_plugin_names)]

awx/main/migrations/0180_add_hostmetric_fields.py

@@ -0,0 +1,43 @@
+# Generated by Django 3.2.16 on 2023-02-03 09:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('main', '0179_change_cyberark_plugin_names'),
+    ]
+
+    operations = [
+        migrations.AlterField(model_name='hostmetric', name='hostname', field=models.CharField(max_length=512, primary_key=False, serialize=True, unique=True)),
+        migrations.AddField(
+            model_name='hostmetric',
+            name='last_deleted',
+            field=models.DateTimeField(db_index=True, null=True, help_text='When the host was last deleted'),
+        ),
+        migrations.AddField(
+            model_name='hostmetric',
+            name='automated_counter',
+            field=models.BigIntegerField(default=0, help_text='How many times was the host automated'),
+        ),
+        migrations.AddField(
+            model_name='hostmetric',
+            name='deleted_counter',
+            field=models.IntegerField(default=0, help_text='How many times was the host deleted'),
+        ),
+        migrations.AddField(
+            model_name='hostmetric',
+            name='deleted',
+            field=models.BooleanField(
+                default=False, help_text='Boolean flag saying whether the host is deleted and therefore not counted into the subscription consumption'
+            ),
+        ),
+        migrations.AddField(
+            model_name='hostmetric',
+            name='used_in_inventories',
+            field=models.IntegerField(null=True, help_text='How many inventories contain this host'),
+        ),
+        migrations.AddField(
+            model_name='hostmetric', name='id', field=models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')
+        ),
+    ]

awx/main/migrations/0181_hostmetricsummarymonthly.py

@@ -0,0 +1,33 @@
+# Generated by Django 3.2.16 on 2023-02-10 12:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('main', '0180_add_hostmetric_fields'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='HostMetricSummaryMonthly',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('date', models.DateField(unique=True)),
+                ('license_consumed', models.BigIntegerField(default=0, help_text='How many unique hosts are consumed from the license')),
+                ('license_capacity', models.BigIntegerField(default=0, help_text="'License capacity as max. number of unique hosts")),
+                (
+                    'hosts_added',
+                    models.IntegerField(default=0, help_text='How many hosts were added in the associated month, consuming more license capacity'),
+                ),
+                (
+                    'hosts_deleted',
+                    models.IntegerField(default=0, help_text='How many hosts were deleted in the associated month, freeing the license capacity'),
+                ),
+                (
+                    'indirectly_managed_hosts',
+                    models.IntegerField(default=0, help_text='Manually entered number indirectly managed hosts for a certain month'),
+                ),
+            ],
+        ),
+    ]

awx/main/migrations/0182_constructed_inventory.py

@@ -0,0 +1,138 @@
+# Generated by Django 3.2.16 on 2022-12-07 14:20
+
+import awx.main.fields
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('main', '0181_hostmetricsummarymonthly'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='InventoryConstructedInventoryMembership',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('position', models.PositiveIntegerField(db_index=True, default=None, null=True)),
+                (
+                    'constructed_inventory',
+                    models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.inventory', related_name='constructed_inventory_memberships'),
+                ),
+                ('input_inventory', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.inventory')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='inventory',
+            name='input_inventories',
+            field=awx.main.fields.OrderedManyToManyField(
+                blank=True,
+                through_fields=('constructed_inventory', 'input_inventory'),
+                help_text='Only valid for constructed inventories, this links to the inventories that will be used.',
+                related_name='destination_inventories',
+                through='main.InventoryConstructedInventoryMembership',
+                to='main.Inventory',
+            ),
+        ),
+        migrations.AlterField(
+            model_name='inventory',
+            name='kind',
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ('', 'Hosts have a direct link to this inventory.'),
+                    ('smart', 'Hosts for inventory generated using the host_filter property.'),
+                    ('constructed', 'Parse list of source inventories with the constructed inventory plugin.'),
+                ],
+                default='',
+                help_text='Kind of inventory being represented.',
+                max_length=32,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='inventorysource',
+            name='source',
+            field=models.CharField(
+                choices=[
+                    ('file', 'File, Directory or Script'),
+                    ('constructed', 'Template additional groups and hostvars at runtime'),
+                    ('scm', 'Sourced from a Project'),
+                    ('ec2', 'Amazon EC2'),
+                    ('gce', 'Google Compute Engine'),
+                    ('azure_rm', 'Microsoft Azure Resource Manager'),
+                    ('vmware', 'VMware vCenter'),
+                    ('satellite6', 'Red Hat Satellite 6'),
+                    ('openstack', 'OpenStack'),
+                    ('rhv', 'Red Hat Virtualization'),
+                    ('controller', 'Red Hat Ansible Automation Platform'),
+                    ('insights', 'Red Hat Insights'),
+                ],
+                default=None,
+                max_length=32,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='inventoryupdate',
+            name='source',
+            field=models.CharField(
+                choices=[
+                    ('file', 'File, Directory or Script'),
+                    ('constructed', 'Template additional groups and hostvars at runtime'),
+                    ('scm', 'Sourced from a Project'),
+                    ('ec2', 'Amazon EC2'),
+                    ('gce', 'Google Compute Engine'),
+                    ('azure_rm', 'Microsoft Azure Resource Manager'),
+                    ('vmware', 'VMware vCenter'),
+                    ('satellite6', 'Red Hat Satellite 6'),
+                    ('openstack', 'OpenStack'),
+                    ('rhv', 'Red Hat Virtualization'),
+                    ('controller', 'Red Hat Ansible Automation Platform'),
+                    ('insights', 'Red Hat Insights'),
+                ],
+                default=None,
+                max_length=32,
+            ),
+        ),
+        migrations.AddField(
+            model_name='inventorysource',
+            name='limit',
+            field=models.TextField(blank=True, default='', help_text='Enter host, group or pattern match'),
+        ),
+        migrations.AddField(
+            model_name='inventoryupdate',
+            name='limit',
+            field=models.TextField(blank=True, default='', help_text='Enter host, group or pattern match'),
+        ),
+        migrations.AlterField(
+            model_name='inventorysource',
+            name='host_filter',
+            field=models.TextField(
+                blank=True,
+                default='',
+                help_text='This field is deprecated and will be removed in a future release. Regex where only matching hosts will be imported.',
+            ),
+        ),
+        migrations.AlterField(
+            model_name='inventoryupdate',
+            name='host_filter',
+            field=models.TextField(
+                blank=True,
+                default='',
+                help_text='This field is deprecated and will be removed in a future release. Regex where only matching hosts will be imported.',
+            ),
+        ),
+        migrations.AddField(
+            model_name='jobhostsummary',
+            name='constructed_host',
+            field=models.ForeignKey(
+                default=None,
+                editable=False,
+                help_text='Only for jobs run against constructed inventories, this links to the host inside the constructed inventory.',
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name='constructed_host_summaries',
+                to='main.host',
+            ),
+        ),
+    ]

awx/main/migrations/_OrgAdmin_to_use_ig.py

@@ -0,0 +1,20 @@
+import logging
+
+from awx.main.models import Organization
+
+logger = logging.getLogger('awx.main.migrations')
+
+
+def migrate_org_admin_to_use(apps, schema_editor):
+    logger.info('Initiated migration from Org admin to use role')
+    roles_added = 0
+    for org in Organization.objects.prefetch_related('admin_role__members').iterator():
+        igs = list(org.instance_groups.all())
+        if not igs:
+            continue
+        for admin in org.admin_role.members.filter(is_superuser=False):
+            for ig in igs:
+                ig.use_role.members.add(admin)
+                roles_added += 1
+    if roles_added:
+        logger.info(f'Migration converted {roles_added} from organization admin to use role')

awx/main/migrations/_credentialtypes.py

@@ -1,6 +1,9 @@
+import logging
+
 from awx.main.models import CredentialType
 from django.db.models import Q
 
+logger = logging.getLogger('awx.main.migrations')
 
 DEPRECATED_CRED_KIND = {
     'rax': {
@@ -76,3 +79,14 @@ def add_tower_verify_field(apps, schema_editor):
 def remove_become_methods(apps, schema_editor):
     # this is no longer necessary; schemas are defined in code
     pass
+
+
+def migrate_credential_type(apps, namespace):
+    ns_types = apps.get_model('main', 'CredentialType').objects.filter(namespace=namespace).order_by('created')
+    if ns_types.count() == 2:
+        original, renamed = ns_types.all()
+        logger.info(f'There are credential types to migrate in the "{namespace}" namespace: {original.name}')
+        apps.get_model('main', 'Credential').objects.filter(credential_type_id=original.id).update(credential_type_id=renamed.id)
+
+        logger.info(f'Removing old credential type: {renamed.name}')
+        original.delete()

awx/main/migrations/_rbac.py

@@ -29,6 +29,7 @@ def create_roles(apps, schema_editor):
             'Project',
             'Credential',
             'JobTemplate',
+            'InstanceGroup',
         ]
     ]
 

awx/main/models/__init__.py

@@ -16,7 +16,9 @@ from awx.main.models.inventory import (  # noqa
     Group,
     Host,
     HostMetric,
+    HostMetricSummaryMonthly,
     Inventory,
+    InventoryConstructedInventoryMembership,
     InventorySource,
     InventoryUpdate,
     SmartInventoryMembership,

awx/main/models/events.py

@@ -7,6 +7,7 @@ from collections import defaultdict
 from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import models, DatabaseError
+from django.db.models.functions import Cast
 from django.utils.dateparse import parse_datetime
 from django.utils.text import Truncator
 from django.utils.timezone import utc, now
@@ -536,25 +537,38 @@ class JobEvent(BasePlaybookEvent):
                 return
             job = self.job
 
-            from awx.main.models import Host, JobHostSummary, HostMetric  # circular import
+            from awx.main.models import Host, JobHostSummary  # circular import
+
+            if self.job.inventory.kind == 'constructed':
+                all_hosts = Host.objects.filter(id__in=self.job.inventory.hosts.values_list(Cast('instance_id', output_field=models.IntegerField()))).only(
+                    'id', 'name'
+                )
+                constructed_host_map = self.host_map
+                host_map = {host.name: host.id for host in all_hosts}
+            else:
+                all_hosts = Host.objects.filter(pk__in=self.host_map.values()).only('id', 'name')
+                constructed_host_map = {}
+                host_map = self.host_map
 
-            all_hosts = Host.objects.filter(pk__in=self.host_map.values()).only('id', 'name')
             existing_host_ids = set(h.id for h in all_hosts)
 
             summaries = dict()
             updated_hosts_list = list()
             for host in hostnames:
                 updated_hosts_list.append(host.lower())
-                host_id = self.host_map.get(host, None)
+                host_id = host_map.get(host)
                 if host_id not in existing_host_ids:
                     host_id = None
+                constructed_host_id = constructed_host_map.get(host)
                 host_stats = {}
                 for stat in ('changed', 'dark', 'failures', 'ignored', 'ok', 'processed', 'rescued', 'skipped'):
                     try:
                         host_stats[stat] = self.event_data.get(stat, {}).get(host, 0)
                     except AttributeError:  # in case event_data[stat] isn't a dict.
                         pass
-                summary = JobHostSummary(created=now(), modified=now(), job_id=job.id, host_id=host_id, host_name=host, **host_stats)
+                summary = JobHostSummary(
+                    created=now(), modified=now(), job_id=job.id, host_id=host_id, constructed_host_id=constructed_host_id, host_name=host, **host_stats
+                )
                 summary.failed = bool(summary.dark or summary.failures)
                 summaries[(host_id, host)] = summary
 
@@ -575,12 +589,26 @@ class JobEvent(BasePlaybookEvent):
 
             Host.objects.bulk_update(list(updated_hosts), ['last_job_id', 'last_job_host_summary_id'], batch_size=100)
 
-            # bulk-create
-            current_time = now()
-            HostMetric.objects.bulk_create(
-                [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=100
+            # Create/update Host Metrics
+            self._update_host_metrics(updated_hosts_list)
+
+    @staticmethod
+    def _update_host_metrics(updated_hosts_list):
+        from awx.main.models import HostMetric  # circular import
+
+        # bulk-create
+        current_time = now()
+        HostMetric.objects.bulk_create(
+            [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=100
+        )
+        # bulk-update
+        batch_start, batch_size = 0, 1000
+        while batch_start <= len(updated_hosts_list):
+            batched_host_list = updated_hosts_list[batch_start : (batch_start + batch_size)]
+            HostMetric.objects.filter(hostname__in=batched_host_list).update(
+                last_automation=current_time, automated_counter=models.F('automated_counter') + 1, deleted=False
             )
-            HostMetric.objects.filter(hostname__in=updated_hosts_list).update(last_automation=current_time)
+            batch_start += batch_size
 
     @property
     def job_verbosity(self):

awx/main/models/ha.py

@@ -17,15 +17,20 @@ from django.db.models import Sum
 import redis
 from solo.models import SingletonModel
 
+# AWX
 from awx import __version__ as awx_application_version
 from awx.api.versioning import reverse
-from awx.main.fields import JSONBlob
+from awx.main.fields import JSONBlob, ImplicitRoleField
 from awx.main.managers import InstanceManager, UUID_DEFAULT
 from awx.main.constants import JOB_FOLDER_PREFIX
 from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search
+from awx.main.models.rbac import (
+    ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
+    ROLE_SINGLETON_SYSTEM_AUDITOR,
+)
 from awx.main.models.unified_jobs import UnifiedJob
 from awx.main.utils.common import get_corrected_cpu, get_cpu_effective_capacity, get_corrected_memory, get_mem_effective_capacity
-from awx.main.models.mixins import RelatedJobsMixin
+from awx.main.models.mixins import RelatedJobsMixin, ResourceMixin
 
 # ansible-runner
 from ansible_runner.utils.capacity import get_cpu_count, get_mem_in_bytes
@@ -352,7 +357,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         self.save_health_data(awx_application_version, get_cpu_count(), get_mem_in_bytes(), update_last_seen=True, errors=errors)
 
 
-class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
+class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin, ResourceMixin):
     """A model representing a Queue/Group of AWX Instances."""
 
     name = models.CharField(max_length=250, unique=True)
@@ -379,6 +384,24 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
             default='',
         )
     )
+    admin_role = ImplicitRoleField(
+        parent_role=[
+            'singleton:' + ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
+        ]
+    )
+    use_role = ImplicitRoleField(
+        parent_role=[
+            'admin_role',
+        ]
+    )
+    read_role = ImplicitRoleField(
+        parent_role=[
+            'singleton:' + ROLE_SINGLETON_SYSTEM_AUDITOR,
+            'use_role',
+            'admin_role',
+        ]
+    )
+
     max_concurrent_jobs = models.IntegerField(default=0, help_text=_("Maximum number of concurrent jobs to run on this group. Zero means no limit."))
     max_forks = models.IntegerField(default=0, help_text=_("Max forks to execute on this group. Zero means no limit."))
     policy_instance_percentage = models.IntegerField(default=0, help_text=_("Percentage of Instances to automatically assign to this group"))

awx/main/models/inventory.py

@@ -32,7 +32,7 @@ from awx.main.fields import (
     SmartFilterField,
     OrderedManyToManyField,
 )
-from awx.main.managers import HostManager
+from awx.main.managers import HostManager, HostMetricActiveManager
 from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, prevent_search, accepts_json
 from awx.main.models.events import InventoryUpdateEvent, UnpartitionedInventoryUpdateEvent
 from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate
@@ -49,15 +49,25 @@ from awx.main.models.notifications import (
 from awx.main.models.credential.injectors import _openstack_data
 from awx.main.utils import _inventory_updates
 from awx.main.utils.safe_yaml import sanitize_jinja
-from awx.main.utils.execution_environments import to_container_path
+from awx.main.utils.execution_environments import to_container_path, get_control_plane_execution_environment
 from awx.main.utils.licensing import server_product_name
 
 
-__all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'SmartInventoryMembership']
+__all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'SmartInventoryMembership', 'HostMetric', 'HostMetricSummaryMonthly']
 
 logger = logging.getLogger('awx.main.models.inventory')
 
 
+class InventoryConstructedInventoryMembership(models.Model):
+    constructed_inventory = models.ForeignKey('Inventory', on_delete=models.CASCADE, related_name='constructed_inventory_memberships')
+    input_inventory = models.ForeignKey('Inventory', on_delete=models.CASCADE)
+    position = models.PositiveIntegerField(
+        null=True,
+        default=None,
+        db_index=True,
+    )
+
+
 class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
     """
     an inventory source contains lists and hosts.
@@ -67,6 +77,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
     KIND_CHOICES = [
         ('', _('Hosts have a direct link to this inventory.')),
         ('smart', _('Hosts for inventory generated using the host_filter property.')),
+        ('constructed', _('Parse list of source inventories with the constructed inventory plugin.')),
     ]
 
     class Meta:
@@ -139,6 +150,14 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
         default=None,
         help_text=_('Filter that will be applied to the hosts of this inventory.'),
     )
+    input_inventories = OrderedManyToManyField(
+        'Inventory',
+        blank=True,
+        through_fields=('constructed_inventory', 'input_inventory'),
+        related_name='destination_inventories',
+        help_text=_('Only valid for constructed inventories, this links to the inventories that will be used.'),
+        through='InventoryConstructedInventoryMembership',
+    )
     instance_groups = OrderedManyToManyField(
         'InstanceGroup',
         blank=True,
@@ -187,6 +206,8 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
     )
 
     def get_absolute_url(self, request=None):
+        if self.kind == 'constructed':
+            return reverse('api:constructed_inventory_detail', kwargs={'pk': self.pk}, request=request)
         return reverse('api:inventory_detail', kwargs={'pk': self.pk}, request=request)
 
     variables_dict = VarsDictProperty('variables')
@@ -338,13 +359,12 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
             for host in hosts:
                 data['_meta']['hostvars'][host.name] = host.variables_dict
                 if towervars:
-                    tower_dict = dict(
-                        remote_tower_enabled=str(host.enabled).lower(),
-                        remote_tower_id=host.id,
-                        remote_host_enabled=str(host.enabled).lower(),
-                        remote_host_id=host.id,
-                    )
-                    data['_meta']['hostvars'][host.name].update(tower_dict)
+                    for prefix in ('host', 'tower'):
+                        tower_dict = {
+                            f'remote_{prefix}_enabled': str(host.enabled).lower(),
+                            f'remote_{prefix}_id': host.id,
+                        }
+                        data['_meta']['hostvars'][host.name].update(tower_dict)
 
         return data
 
@@ -431,12 +451,24 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
 
             connection.on_commit(on_commit)
 
+    def _enforce_constructed_source(self):
+        """
+        Constructed inventory should always have exactly 1 inventory source, constructed type
+        this enforces that requirement
+        """
+        if self.kind == 'constructed':
+            if not self.inventory_sources.exists():
+                self.inventory_sources.create(
+                    source='constructed', name=f'Auto-created source for: {self.name}'[:512], overwrite=True, overwrite_vars=True, update_on_launch=True
+                )
+
     def save(self, *args, **kwargs):
         self._update_host_smart_inventory_memeberships()
         super(Inventory, self).save(*args, **kwargs)
         if self.kind == 'smart' and 'host_filter' in kwargs.get('update_fields', ['host_filter']) and connection.vendor != 'sqlite':
             # Minimal update of host_count for smart inventory host filter changes
             self.update_computed_fields()
+        self._enforce_constructed_source()
 
     def delete(self, *args, **kwargs):
         self._update_host_smart_inventory_memeberships()
@@ -820,9 +852,47 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
 
 
 class HostMetric(models.Model):
-    hostname = models.CharField(primary_key=True, max_length=512)
+    hostname = models.CharField(unique=True, max_length=512)
     first_automation = models.DateTimeField(auto_now_add=True, null=False, db_index=True, help_text=_('When the host was first automated against'))
     last_automation = models.DateTimeField(db_index=True, help_text=_('When the host was last automated against'))
+    last_deleted = models.DateTimeField(null=True, db_index=True, help_text=_('When the host was last deleted'))
+    automated_counter = models.BigIntegerField(default=0, help_text=_('How many times was the host automated'))
+    deleted_counter = models.IntegerField(default=0, help_text=_('How many times was the host deleted'))
+    deleted = models.BooleanField(
+        default=False, help_text=_('Boolean flag saying whether the host is deleted and therefore not counted into the subscription consumption')
+    )
+    used_in_inventories = models.IntegerField(null=True, help_text=_('How many inventories contain this host'))
+
+    objects = models.Manager()
+    active_objects = HostMetricActiveManager()
+
+    def get_absolute_url(self, request=None):
+        return reverse('api:host_metric_detail', kwargs={'pk': self.pk}, request=request)
+
+    def soft_delete(self):
+        if not self.deleted:
+            self.deleted_counter = (self.deleted_counter or 0) + 1
+            self.last_deleted = now()
+            self.deleted = True
+            self.save(update_fields=['deleted', 'deleted_counter', 'last_deleted'])
+
+    def soft_restore(self):
+        if self.deleted:
+            self.deleted = False
+            self.save(update_fields=['deleted'])
+
+
+class HostMetricSummaryMonthly(models.Model):
+    """
+    HostMetric summaries computed by scheduled task <TODO> monthly
+    """
+
+    date = models.DateField(unique=True)
+    license_consumed = models.BigIntegerField(default=0, help_text=_("How many unique hosts are consumed from the license"))
+    license_capacity = models.BigIntegerField(default=0, help_text=_("'License capacity as max. number of unique hosts"))
+    hosts_added = models.IntegerField(default=0, help_text=_("How many hosts were added in the associated month, consuming more license capacity"))
+    hosts_deleted = models.IntegerField(default=0, help_text=_("How many hosts were deleted in the associated month, freeing the license capacity"))
+    indirectly_managed_hosts = models.IntegerField(default=0, help_text=("Manually entered number indirectly managed hosts for a certain month"))
 
 
 class InventorySourceOptions(BaseModel):
@@ -834,6 +904,7 @@ class InventorySourceOptions(BaseModel):
 
     SOURCE_CHOICES = [
         ('file', _('File, Directory or Script')),
+        ('constructed', _('Template additional groups and hostvars at runtime')),
         ('scm', _('Sourced from a Project')),
         ('ec2', _('Amazon EC2')),
         ('gce', _('Google Compute Engine')),
@@ -872,6 +943,12 @@ class InventorySourceOptions(BaseModel):
         default='',
         help_text=_('Inventory source variables in YAML or JSON format.'),
     )
+    scm_branch = models.CharField(
+        max_length=1024,
+        default='',
+        blank=True,
+        help_text=_('Inventory source SCM branch. Project default used if blank. Only allowed if project allow_override field is set to true.'),
+    )
     enabled_var = models.TextField(
         blank=True,
         default='',
@@ -907,7 +984,7 @@ class InventorySourceOptions(BaseModel):
     host_filter = models.TextField(
         blank=True,
         default='',
-        help_text=_('Regex where only matching hosts will be imported.'),
+        help_text=_('This field is deprecated and will be removed in a future release. Regex where only matching hosts will be imported.'),
     )
     overwrite = models.BooleanField(
         default=False,
@@ -927,6 +1004,21 @@ class InventorySourceOptions(BaseModel):
         blank=True,
         default=1,
     )
+    limit = models.TextField(
+        blank=True,
+        default='',
+        help_text=_("Enter host, group or pattern match"),
+    )
+
+    def resolve_execution_environment(self):
+        """
+        Project updates, themselves, will use the control plane execution environment.
+        Jobs using the project can use the default_environment, but the project updates
+        are not flexible enough to allow customizing the image they use.
+        """
+        if self.inventory.kind == 'constructed':
+            return get_control_plane_execution_environment()
+        return super().resolve_execution_environment()
 
     @staticmethod
     def cloud_credential_validation(source, cred):
@@ -1363,6 +1455,8 @@ class PluginFileInjector(object):
         env.update(injector_env)
         # Preserves current behavior for Ansible change in default planned for 2.10
         env['ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS'] = 'never'
+        # All CLOUD_PROVIDERS sources implement as inventory plugin from collection
+        env['ANSIBLE_INVENTORY_ENABLED'] = 'auto'
         return env
 
     def _get_shared_env(self, inventory_update, private_data_dir, private_data_files):
@@ -1546,5 +1640,18 @@ class insights(PluginFileInjector):
     use_fqcn = True
 
 
+class constructed(PluginFileInjector):
+    plugin_name = 'constructed'
+    namespace = 'ansible'
+    collection = 'builtin'
+
+    def build_env(self, *args, **kwargs):
+        env = super().build_env(*args, **kwargs)
+        # Enable script inventory plugin so we pick up the script files from source inventories
+        env['ANSIBLE_INVENTORY_ENABLED'] += ',script'
+        env['ANSIBLE_INVENTORY_ANY_UNPARSED_IS_FAILED'] = 'True'
+        return env
+
+
 for cls in PluginFileInjector.__subclasses__():
     InventorySourceOptions.injectors[cls.__name__] = cls

awx/main/models/jobs.py

@@ -2,12 +2,8 @@
 # All Rights Reserved.
 
 # Python
-import codecs
-import datetime
 import logging
-import os
 import time
-import json
 from urllib.parse import urljoin
 
 
@@ -15,11 +11,9 @@ from urllib.parse import urljoin
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.db import models
-from django.db.models.query import QuerySet
+from django.db.models.functions import Cast
 
 # from django.core.cache import cache
-from django.utils.encoding import smart_str
-from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _
 from django.core.exceptions import FieldDoesNotExist
 
@@ -28,6 +22,7 @@ from rest_framework.exceptions import ParseError
 
 # AWX
 from awx.api.versioning import reverse
+from awx.main.constants import HOST_FACTS_FIELDS
 from awx.main.models.base import (
     BaseModel,
     CreatedModifiedModel,
@@ -44,7 +39,7 @@ from awx.main.models.notifications import (
     NotificationTemplate,
     JobNotificationMixin,
 )
-from awx.main.utils import parse_yaml_or_json, getattr_dne, NullablePromptPseudoField, polymorphic, log_excess_runtime
+from awx.main.utils import parse_yaml_or_json, getattr_dne, NullablePromptPseudoField, polymorphic
 from awx.main.fields import ImplicitRoleField, AskForField, JSONBlob, OrderedManyToManyField
 from awx.main.models.mixins import (
     ResourceMixin,
@@ -60,8 +55,6 @@ from awx.main.constants import JOB_VARIABLE_PREFIXES
 
 
 logger = logging.getLogger('awx.main.models.jobs')
-analytics_logger = logging.getLogger('awx.analytics.job_events')
-system_tracking_logger = logging.getLogger('awx.analytics.system_tracking')
 
 __all__ = ['JobTemplate', 'JobLaunchConfig', 'Job', 'JobHostSummary', 'SystemJobTemplate', 'SystemJob']
 
@@ -578,12 +571,7 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
         default=None,
         on_delete=models.SET_NULL,
     )
-    hosts = models.ManyToManyField(
-        'Host',
-        related_name='jobs',
-        editable=False,
-        through='JobHostSummary',
-    )
+    hosts = models.ManyToManyField('Host', related_name='jobs', editable=False, through='JobHostSummary', through_fields=('job', 'host'))
     artifacts = JSONBlob(
         default=dict,
         blank=True,
@@ -831,6 +819,9 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
             for name in JOB_VARIABLE_PREFIXES:
                 r['{}_job_template_id'.format(name)] = self.job_template.pk
                 r['{}_job_template_name'.format(name)] = self.job_template.name
+        if self.execution_node:
+            for name in JOB_VARIABLE_PREFIXES:
+                r['{}_execution_node'.format(name)] = self.execution_node
         return r
 
     '''
@@ -845,109 +836,26 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
     def get_notification_friendly_name(self):
         return "Job"
 
-    def _get_inventory_hosts(self, only=('name', 'ansible_facts', 'ansible_facts_modified', 'modified', 'inventory_id'), **filters):
-        """Return value is an iterable for the relevant hosts for this job"""
-        if not self.inventory:
-            return []
-        host_queryset = self.inventory.hosts.only(*only)
-        if filters:
-            host_queryset = host_queryset.filter(**filters)
-        host_queryset = self.inventory.get_sliced_hosts(host_queryset, self.job_slice_number, self.job_slice_count)
-        if isinstance(host_queryset, QuerySet):
-            return host_queryset.iterator()
-        return host_queryset
+    def get_hosts_for_fact_cache(self):
+        """
+        Builds the queryset to use for writing or finalizing the fact cache
+        these need to be the 'real' hosts associated with the job.
+        For constructed inventories, that means the original (input inventory) hosts
+        when slicing, that means only returning hosts in that slice
+        """
+        Host = JobHostSummary._meta.get_field('host').related_model
+        if not self.inventory_id:
+            return Host.objects.none()
 
-    @log_excess_runtime(logger, debug_cutoff=0.01, msg='Job {job_id} host facts prepared for {written_ct} hosts, took {delta:.3f} s', add_log_data=True)
-    def start_job_fact_cache(self, destination, log_data, timeout=None):
-        self.log_lifecycle("start_job_fact_cache")
-        log_data['job_id'] = self.id
-        log_data['written_ct'] = 0
-        os.makedirs(destination, mode=0o700)
-
-        if timeout is None:
-            timeout = settings.ANSIBLE_FACT_CACHE_TIMEOUT
-        if timeout > 0:
-            # exclude hosts with fact data older than `settings.ANSIBLE_FACT_CACHE_TIMEOUT seconds`
-            timeout = now() - datetime.timedelta(seconds=timeout)
-            hosts = self._get_inventory_hosts(ansible_facts_modified__gte=timeout)
+        if self.inventory.kind == 'constructed':
+            id_field = Host._meta.get_field('id')
+            host_qs = Host.objects.filter(id__in=self.inventory.hosts.exclude(instance_id='').values_list(Cast('instance_id', output_field=id_field)))
         else:
-            hosts = self._get_inventory_hosts()
+            host_qs = self.inventory.hosts
 
-        last_filepath_written = None
-        for host in hosts:
-            filepath = os.sep.join(map(str, [destination, host.name]))
-            if not os.path.realpath(filepath).startswith(destination):
-                system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
-                continue
-            try:
-                with codecs.open(filepath, 'w', encoding='utf-8') as f:
-                    os.chmod(f.name, 0o600)
-                    json.dump(host.ansible_facts, f)
-                    log_data['written_ct'] += 1
-                    last_filepath_written = filepath
-            except IOError:
-                system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
-                continue
-        # make note of the time we wrote the last file so we can check if any file changed later
-        if last_filepath_written:
-            return os.path.getmtime(last_filepath_written)
-        return None
-
-    @log_excess_runtime(
-        logger,
-        debug_cutoff=0.01,
-        msg='Job {job_id} host facts: updated {updated_ct}, cleared {cleared_ct}, unchanged {unmodified_ct}, took {delta:.3f} s',
-        add_log_data=True,
-    )
-    def finish_job_fact_cache(self, destination, facts_write_time, log_data):
-        self.log_lifecycle("finish_job_fact_cache")
-        log_data['job_id'] = self.id
-        log_data['updated_ct'] = 0
-        log_data['unmodified_ct'] = 0
-        log_data['cleared_ct'] = 0
-        hosts_to_update = []
-        for host in self._get_inventory_hosts():
-            filepath = os.sep.join(map(str, [destination, host.name]))
-            if not os.path.realpath(filepath).startswith(destination):
-                system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
-                continue
-            if os.path.exists(filepath):
-                # If the file changed since we wrote the last facts file, pre-playbook run...
-                modified = os.path.getmtime(filepath)
-                if (not facts_write_time) or modified > facts_write_time:
-                    with codecs.open(filepath, 'r', encoding='utf-8') as f:
-                        try:
-                            ansible_facts = json.load(f)
-                        except ValueError:
-                            continue
-                        host.ansible_facts = ansible_facts
-                        host.ansible_facts_modified = now()
-                        hosts_to_update.append(host)
-                        system_tracking_logger.info(
-                            'New fact for inventory {} host {}'.format(smart_str(host.inventory.name), smart_str(host.name)),
-                            extra=dict(
-                                inventory_id=host.inventory.id,
-                                host_name=host.name,
-                                ansible_facts=host.ansible_facts,
-                                ansible_facts_modified=host.ansible_facts_modified.isoformat(),
-                                job_id=self.id,
-                            ),
-                        )
-                        log_data['updated_ct'] += 1
-                else:
-                    log_data['unmodified_ct'] += 1
-            else:
-                # if the file goes missing, ansible removed it (likely via clear_facts)
-                host.ansible_facts = {}
-                host.ansible_facts_modified = now()
-                hosts_to_update.append(host)
-                system_tracking_logger.info('Facts cleared for inventory {} host {}'.format(smart_str(host.inventory.name), smart_str(host.name)))
-                log_data['cleared_ct'] += 1
-            if len(hosts_to_update) > 100:
-                self.inventory.hosts.bulk_update(hosts_to_update, ['ansible_facts', 'ansible_facts_modified'])
-                hosts_to_update = []
-        if hosts_to_update:
-            self.inventory.hosts.bulk_update(hosts_to_update, ['ansible_facts', 'ansible_facts_modified'])
+        host_qs = host_qs.only(*HOST_FACTS_FIELDS)
+        host_qs = self.inventory.get_sliced_hosts(host_qs, self.job_slice_number, self.job_slice_count)
+        return host_qs
 
 
 class LaunchTimeConfigBase(BaseModel):
@@ -1169,6 +1077,15 @@ class JobHostSummary(CreatedModifiedModel):
         editable=False,
     )
     host = models.ForeignKey('Host', related_name='job_host_summaries', null=True, default=None, on_delete=models.SET_NULL, editable=False)
+    constructed_host = models.ForeignKey(
+        'Host',
+        related_name='constructed_host_summaries',
+        null=True,
+        default=None,
+        on_delete=models.SET_NULL,
+        editable=False,
+        help_text='Only for jobs run against constructed inventories, this links to the host inside the constructed inventory.',
+    )
 
     host_name = models.CharField(
         max_length=1024,

awx/main/models/oauth.py

@@ -14,7 +14,7 @@ from oauth2_provider.models import AbstractApplication, AbstractAccessToken
 from oauth2_provider.generators import generate_client_secret
 from oauthlib import oauth2
 
-from awx.main.utils import get_external_account
+from awx.sso.common import get_external_account
 from awx.main.fields import OAuth2ClientSecretField
 
 

awx/main/models/unified_jobs.py

@@ -32,7 +32,7 @@ from polymorphic.models import PolymorphicModel
 
 # AWX
 from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel, NotificationFieldsModel, prevent_search
-from awx.main.dispatch import get_local_queuename
+from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.control import Control as ControlDispatcher
 from awx.main.registrar import activity_stream_registrar
 from awx.main.models.mixins import ResourceMixin, TaskManagerUnifiedJobMixin, ExecutionEnvironmentMixin
@@ -1567,7 +1567,7 @@ class UnifiedJob(
         return r
 
     def get_queue_name(self):
-        return self.controller_node or self.execution_node or get_local_queuename()
+        return self.controller_node or self.execution_node or get_task_queuename()
 
     @property
     def is_container_group_task(self):

awx/main/models/workflow.py

@@ -650,6 +650,7 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificatio
         help_text=_("If automatically created for a sliced job run, the job template " "the workflow job was created from."),
     )
     is_sliced_job = models.BooleanField(default=False)
+    is_bulk_job = models.BooleanField(default=False)
 
     def _set_default_dependencies_processed(self):
         self.dependencies_processed = True

awx/main/routing.py

@@ -28,7 +28,7 @@ class AWXProtocolTypeRouter(ProtocolTypeRouter):
 
 websocket_urlpatterns = [
     re_path(r'websocket/$', consumers.EventConsumer.as_asgi()),
-    re_path(r'websocket/broadcast/$', consumers.BroadcastConsumer.as_asgi()),
+    re_path(r'websocket/relay/$', consumers.RelayConsumer.as_asgi()),
 ]
 
 application = AWXProtocolTypeRouter(

awx/main/scheduler/tasks.py

@@ -8,7 +8,7 @@ from django.conf import settings
 from awx import MODE
 from awx.main.scheduler import TaskManager, DependencyManager, WorkflowManager
 from awx.main.dispatch.publish import task
-from awx.main.dispatch import get_local_queuename
+from awx.main.dispatch import get_task_queuename
 
 logger = logging.getLogger('awx.main.scheduler')
 
@@ -20,16 +20,16 @@ def run_manager(manager, prefix):
     manager().schedule()
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def task_manager():
     run_manager(TaskManager, "task")
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def dependency_manager():
     run_manager(DependencyManager, "dependency")
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def workflow_manager():
     run_manager(WorkflowManager, "workflow")

awx/main/tasks/callback.py

@@ -85,6 +85,8 @@ class RunnerCallback:
         # which generate job events from two 'streams':
         # ansible-inventory and the awx.main.commands.inventory_import
         # logger
+        if event_data.get('event') == 'keepalive':
+            return
 
         if event_data.get(self.event_data_key, None):
             if self.event_data_key != 'job_id':
@@ -116,7 +118,7 @@ class RunnerCallback:
             # so it *should* have a negligible performance impact
             task = event_data.get('event_data', {}).get('task_action')
             try:
-                if task in ('git', 'svn'):
+                if task in ('git', 'svn', 'ansible.builtin.git', 'ansible.builtin.svn'):
                     event_data_json = json.dumps(event_data)
                     event_data_json = UriCleaner.remove_sensitive(event_data_json)
                     event_data = json.loads(event_data_json)
@@ -219,7 +221,7 @@ class RunnerCallbackForProjectUpdate(RunnerCallback):
     def event_handler(self, event_data):
         super_return_value = super(RunnerCallbackForProjectUpdate, self).event_handler(event_data)
         returned_data = event_data.get('event_data', {})
-        if returned_data.get('task_action', '') == 'set_fact':
+        if returned_data.get('task_action', '') in ('set_fact', 'ansible.builtin.set_fact'):
             returned_facts = returned_data.get('res', {}).get('ansible_facts', {})
             if 'scm_version' in returned_facts:
                 self.playbook_new_revision = returned_facts['scm_version']

awx/main/tasks/facts.py

@@ -0,0 +1,117 @@
+import codecs
+import datetime
+import os
+import json
+import logging
+
+# Django
+from django.conf import settings
+from django.db.models.query import QuerySet
+from django.utils.encoding import smart_str
+from django.utils.timezone import now
+
+# AWX
+from awx.main.utils.common import log_excess_runtime
+from awx.main.models.inventory import Host
+
+
+logger = logging.getLogger('awx.main.tasks.facts')
+system_tracking_logger = logging.getLogger('awx.analytics.system_tracking')
+
+
+@log_excess_runtime(logger, debug_cutoff=0.01, msg='Inventory {inventory_id} host facts prepared for {written_ct} hosts, took {delta:.3f} s', add_log_data=True)
+def start_fact_cache(hosts, destination, log_data, timeout=None, inventory_id=None):
+    log_data['inventory_id'] = inventory_id
+    log_data['written_ct'] = 0
+    try:
+        os.makedirs(destination, mode=0o700)
+    except FileExistsError:
+        pass
+
+    if timeout is None:
+        timeout = settings.ANSIBLE_FACT_CACHE_TIMEOUT
+
+    if isinstance(hosts, QuerySet):
+        hosts = hosts.iterator()
+
+    last_filepath_written = None
+    for host in hosts:
+        if (not host.ansible_facts_modified) or (timeout and host.ansible_facts_modified < now() - datetime.timedelta(seconds=timeout)):
+            continue  # facts are expired - do not write them
+        filepath = os.sep.join(map(str, [destination, host.name]))
+        if not os.path.realpath(filepath).startswith(destination):
+            system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
+            continue
+        try:
+            with codecs.open(filepath, 'w', encoding='utf-8') as f:
+                os.chmod(f.name, 0o600)
+                json.dump(host.ansible_facts, f)
+                log_data['written_ct'] += 1
+                last_filepath_written = filepath
+        except IOError:
+            system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
+            continue
+    # make note of the time we wrote the last file so we can check if any file changed later
+    if last_filepath_written:
+        return os.path.getmtime(last_filepath_written)
+    return None
+
+
+@log_excess_runtime(
+    logger,
+    debug_cutoff=0.01,
+    msg='Inventory {inventory_id} host facts: updated {updated_ct}, cleared {cleared_ct}, unchanged {unmodified_ct}, took {delta:.3f} s',
+    add_log_data=True,
+)
+def finish_fact_cache(hosts, destination, facts_write_time, log_data, job_id=None, inventory_id=None):
+    log_data['inventory_id'] = inventory_id
+    log_data['updated_ct'] = 0
+    log_data['unmodified_ct'] = 0
+    log_data['cleared_ct'] = 0
+
+    if isinstance(hosts, QuerySet):
+        hosts = hosts.iterator()
+
+    hosts_to_update = []
+    for host in hosts:
+        filepath = os.sep.join(map(str, [destination, host.name]))
+        if not os.path.realpath(filepath).startswith(destination):
+            system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
+            continue
+        if os.path.exists(filepath):
+            # If the file changed since we wrote the last facts file, pre-playbook run...
+            modified = os.path.getmtime(filepath)
+            if (not facts_write_time) or modified > facts_write_time:
+                with codecs.open(filepath, 'r', encoding='utf-8') as f:
+                    try:
+                        ansible_facts = json.load(f)
+                    except ValueError:
+                        continue
+                    host.ansible_facts = ansible_facts
+                    host.ansible_facts_modified = now()
+                    hosts_to_update.append(host)
+                    system_tracking_logger.info(
+                        'New fact for inventory {} host {}'.format(smart_str(host.inventory.name), smart_str(host.name)),
+                        extra=dict(
+                            inventory_id=host.inventory.id,
+                            host_name=host.name,
+                            ansible_facts=host.ansible_facts,
+                            ansible_facts_modified=host.ansible_facts_modified.isoformat(),
+                            job_id=job_id,
+                        ),
+                    )
+                    log_data['updated_ct'] += 1
+            else:
+                log_data['unmodified_ct'] += 1
+        else:
+            # if the file goes missing, ansible removed it (likely via clear_facts)
+            host.ansible_facts = {}
+            host.ansible_facts_modified = now()
+            hosts_to_update.append(host)
+            system_tracking_logger.info('Facts cleared for inventory {} host {}'.format(smart_str(host.inventory.name), smart_str(host.name)))
+            log_data['cleared_ct'] += 1
+        if len(hosts_to_update) > 100:
+            Host.objects.bulk_update(hosts_to_update, ['ansible_facts', 'ansible_facts_modified'])
+            hosts_to_update = []
+    if hosts_to_update:
+        Host.objects.bulk_update(hosts_to_update, ['ansible_facts', 'ansible_facts_modified'])

awx/main/tasks/jobs.py

@@ -29,7 +29,7 @@ from gitdb.exc import BadName as BadGitName
 
 # AWX
 from awx.main.dispatch.publish import task
-from awx.main.dispatch import get_local_queuename
+from awx.main.dispatch import get_task_queuename
 from awx.main.constants import (
     PRIVILEGE_ESCALATION_METHODS,
     STANDARD_INVENTORY_UPDATE_ENV,
@@ -37,6 +37,7 @@ from awx.main.constants import (
     MAX_ISOLATED_PATH_COLON_DELIMITER,
     CONTAINER_VOLUMES_MOUNT_TYPES,
     ACTIVE_STATES,
+    HOST_FACTS_FIELDS,
 )
 from awx.main.models import (
     Instance,
@@ -63,6 +64,7 @@ from awx.main.tasks.callback import (
 )
 from awx.main.tasks.signals import with_signal_handling, signal_callback
 from awx.main.tasks.receptor import AWXReceptorJob
+from awx.main.tasks.facts import start_fact_cache, finish_fact_cache
 from awx.main.exceptions import AwxTaskError, PostRunError, ReceptorNodeNotFound
 from awx.main.utils.ansible import read_ansible_config
 from awx.main.utils.execution_environments import CONTAINER_ROOT, to_container_path
@@ -311,21 +313,26 @@ class BaseTask(object):
         env['AWX_PRIVATE_DATA_DIR'] = private_data_dir
 
         if self.instance.execution_environment is None:
-            raise RuntimeError('The project could not sync because there is no Execution Environment.')
+            raise RuntimeError(f'The {self.model.__name__} could not run because there is no Execution Environment.')
 
         return env
 
+    def write_inventory_file(self, inventory, private_data_dir, file_name, script_params):
+        script_data = inventory.get_script_data(**script_params)
+        for hostname, hv in script_data.get('_meta', {}).get('hostvars', {}).items():
+            # maintain a list of host_name --> host_id
+            # so we can associate emitted events to Host objects
+            self.runner_callback.host_map[hostname] = hv.get('remote_tower_id', '')
+        file_content = '#! /usr/bin/env python3\n# -*- coding: utf-8 -*-\nprint(%r)\n' % json.dumps(script_data)
+        return self.write_private_data_file(private_data_dir, file_name, file_content, sub_dir='inventory', file_permissions=0o700)
+
     def build_inventory(self, instance, private_data_dir):
         script_params = dict(hostvars=True, towervars=True)
         if hasattr(instance, 'job_slice_number'):
             script_params['slice_number'] = instance.job_slice_number
             script_params['slice_count'] = instance.job_slice_count
-        script_data = instance.inventory.get_script_data(**script_params)
-        # maintain a list of host_name --> host_id
-        # so we can associate emitted events to Host objects
-        self.runner_callback.host_map = {hostname: hv.pop('remote_tower_id', '') for hostname, hv in script_data.get('_meta', {}).get('hostvars', {}).items()}
-        file_content = '#! /usr/bin/env python3\n# -*- coding: utf-8 -*-\nprint(%r)\n' % json.dumps(script_data)
-        return self.write_private_data_file(private_data_dir, 'hosts', file_content, sub_dir='inventory', file_permissions=0o700)
+
+        return self.write_inventory_file(instance.inventory, private_data_dir, 'hosts', script_params)
 
     def build_args(self, instance, private_data_dir, passwords):
         raise NotImplementedError
@@ -450,6 +457,9 @@ class BaseTask(object):
                 instance.ansible_version = ansible_version_info
                 instance.save(update_fields=['ansible_version'])
 
+    def should_use_fact_cache(self):
+        return False
+
     @with_path_cleanup
     @with_signal_handling
     def run(self, pk, **kwargs):
@@ -548,7 +558,8 @@ class BaseTask(object):
                 params['module'] = self.build_module_name(self.instance)
                 params['module_args'] = self.build_module_args(self.instance)
 
-            if getattr(self.instance, 'use_fact_cache', False):
+            # TODO: refactor into a better BasTask method
+            if self.should_use_fact_cache():
                 # Enable Ansible fact cache.
                 params['fact_cache_type'] = 'jsonfile'
             else:
@@ -759,7 +770,7 @@ class SourceControlMixin(BaseTask):
 
     def sync_and_copy(self, project, private_data_dir, scm_branch=None):
         self.acquire_lock(project, self.instance.id)
-
+        is_commit = False
         try:
             original_branch = None
             failed_reason = project.get_reason_if_failed()
@@ -771,6 +782,7 @@ class SourceControlMixin(BaseTask):
                 if os.path.exists(project_path):
                     git_repo = git.Repo(project_path)
                     if git_repo.head.is_detached:
+                        is_commit = True
                         original_branch = git_repo.head.commit
                     else:
                         original_branch = git_repo.active_branch
@@ -782,7 +794,11 @@ class SourceControlMixin(BaseTask):
                 # for git project syncs, non-default branches can be problems
                 # restore to branch the repo was on before this run
                 try:
-                    original_branch.checkout()
+                    if is_commit:
+                        git_repo.head.set_commit(original_branch)
+                        git_repo.head.reset(index=True, working_tree=True)
+                    else:
+                        original_branch.checkout()
                 except Exception:
                     # this could have failed due to dirty tree, but difficult to predict all cases
                     logger.exception(f'Failed to restore project repo to prior state after {self.instance.id}')
@@ -790,7 +806,7 @@ class SourceControlMixin(BaseTask):
             self.release_lock(project)
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 class RunJob(SourceControlMixin, BaseTask):
     """
     Run a job using ansible-playbook.
@@ -998,6 +1014,9 @@ class RunJob(SourceControlMixin, BaseTask):
 
         return args
 
+    def should_use_fact_cache(self):
+        return self.instance.use_fact_cache
+
     def build_playbook_path_relative_to_cwd(self, job, private_data_dir):
         return job.playbook
 
@@ -1063,8 +1082,11 @@ class RunJob(SourceControlMixin, BaseTask):
 
         # Fetch "cached" fact data from prior runs and put on the disk
         # where ansible expects to find it
-        if job.use_fact_cache:
-            self.facts_write_time = self.instance.start_job_fact_cache(os.path.join(private_data_dir, 'artifacts', str(job.id), 'fact_cache'))
+        if self.should_use_fact_cache():
+            job.log_lifecycle("start_job_fact_cache")
+            self.facts_write_time = start_fact_cache(
+                job.get_hosts_for_fact_cache(), os.path.join(private_data_dir, 'artifacts', str(job.id), 'fact_cache'), inventory_id=job.inventory_id
+            )
 
     def build_project_dir(self, job, private_data_dir):
         self.sync_and_copy(job.project, private_data_dir, scm_branch=job.scm_branch)
@@ -1078,10 +1100,14 @@ class RunJob(SourceControlMixin, BaseTask):
             # actual `run()` call; this _usually_ means something failed in
             # the pre_run_hook method
             return
-        if job.use_fact_cache:
-            job.finish_job_fact_cache(
+        if self.should_use_fact_cache():
+            job.log_lifecycle("finish_job_fact_cache")
+            finish_fact_cache(
+                job.get_hosts_for_fact_cache(),
                 os.path.join(private_data_dir, 'artifacts', str(job.id), 'fact_cache'),
-                self.facts_write_time,
+                facts_write_time=self.facts_write_time,
+                job_id=job.id,
+                inventory_id=job.inventory_id,
             )
 
     def final_run_hook(self, job, status, private_data_dir):
@@ -1095,7 +1121,7 @@ class RunJob(SourceControlMixin, BaseTask):
                 update_inventory_computed_fields.delay(inventory.id)
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 class RunProjectUpdate(BaseTask):
     model = ProjectUpdate
     event_model = ProjectUpdateEvent
@@ -1417,7 +1443,7 @@ class RunProjectUpdate(BaseTask):
         return params
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 class RunInventoryUpdate(SourceControlMixin, BaseTask):
     model = InventoryUpdate
     event_model = InventoryUpdateEvent
@@ -1464,8 +1490,6 @@ class RunInventoryUpdate(SourceControlMixin, BaseTask):
 
         if injector is not None:
             env = injector.build_env(inventory_update, env, private_data_dir, private_data_files)
-            # All CLOUD_PROVIDERS sources implement as inventory plugin from collection
-            env['ANSIBLE_INVENTORY_ENABLED'] = 'auto'
 
         if inventory_update.source == 'scm':
             for env_k in inventory_update.source_vars_dict:
@@ -1518,6 +1542,22 @@ class RunInventoryUpdate(SourceControlMixin, BaseTask):
 
         args = ['ansible-inventory', '--list', '--export']
 
+        # special case for constructed inventories, we pass source inventories from database
+        # these must come in order, and in order _before_ the constructed inventory itself
+        if inventory_update.inventory.kind == 'constructed':
+            inventory_update.log_lifecycle("start_job_fact_cache")
+            for input_inventory in inventory_update.inventory.input_inventories.all():
+                args.append('-i')
+                script_params = dict(hostvars=True, towervars=True)
+                source_inv_path = self.write_inventory_file(input_inventory, private_data_dir, f'hosts_{input_inventory.id}', script_params)
+                args.append(to_container_path(source_inv_path, private_data_dir))
+                # Include any facts from input inventories so they can be used in filters
+                start_fact_cache(
+                    input_inventory.hosts.only(*HOST_FACTS_FIELDS),
+                    os.path.join(private_data_dir, 'artifacts', str(inventory_update.id), 'fact_cache'),
+                    inventory_id=input_inventory.id,
+                )
+
         # Add arguments for the source inventory file/script/thing
         rel_path = self.pseudo_build_inventory(inventory_update, private_data_dir)
         container_location = os.path.join(CONTAINER_ROOT, rel_path)
@@ -1525,6 +1565,11 @@ class RunInventoryUpdate(SourceControlMixin, BaseTask):
 
         args.append('-i')
         args.append(container_location)
+        # Added this in order to allow older versions of ansible-inventory https://github.com/ansible/ansible/pull/79596
+        # limit should be usable in ansible-inventory 2.15+
+        if inventory_update.limit:
+            args.append('--limit')
+            args.append(inventory_update.limit)
 
         args.append('--output')
         args.append(os.path.join(CONTAINER_ROOT, 'artifacts', str(inventory_update.id), 'output.json'))
@@ -1540,6 +1585,9 @@ class RunInventoryUpdate(SourceControlMixin, BaseTask):
 
         return args
 
+    def should_use_fact_cache(self):
+        return bool(self.instance.source == 'constructed')
+
     def build_inventory(self, inventory_update, private_data_dir):
         return None  # what runner expects in order to not deal with inventory
 
@@ -1581,7 +1629,7 @@ class RunInventoryUpdate(SourceControlMixin, BaseTask):
         if inventory_update.source == 'scm':
             if not source_project:
                 raise RuntimeError('Could not find project to run SCM inventory update from.')
-            self.sync_and_copy(source_project, private_data_dir)
+            self.sync_and_copy(source_project, private_data_dir, scm_branch=inventory_update.inventory_source.scm_branch)
         else:
             # If source is not SCM make an empty project directory, content is built inside inventory folder
             super(RunInventoryUpdate, self).build_project_dir(inventory_update, private_data_dir)
@@ -1658,7 +1706,7 @@ class RunInventoryUpdate(SourceControlMixin, BaseTask):
             raise PostRunError('Error occured while saving inventory data, see traceback or server logs', status='error', tb=traceback.format_exc())
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 class RunAdHocCommand(BaseTask):
     """
     Run an ad hoc command using ansible.
@@ -1811,7 +1859,7 @@ class RunAdHocCommand(BaseTask):
         return d
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 class RunSystemJob(BaseTask):
     model = SystemJob
     event_model = SystemJobEvent

awx/main/tasks/receptor.py

@@ -28,7 +28,7 @@ from awx.main.utils.common import (
 from awx.main.constants import MAX_ISOLATED_PATH_COLON_DELIMITER
 from awx.main.tasks.signals import signal_state, signal_callback, SignalExit
 from awx.main.models import Instance, InstanceLink, UnifiedJob
-from awx.main.dispatch import get_local_queuename
+from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.publish import task
 
 # Receptorctl
@@ -526,6 +526,10 @@ class AWXReceptorJob:
         pod_spec['spec']['containers'][0]['image'] = ee.image
         pod_spec['spec']['containers'][0]['args'] = ['ansible-runner', 'worker', '--private-data-dir=/runner']
 
+        if settings.AWX_RUNNER_KEEPALIVE_SECONDS:
+            pod_spec['spec']['containers'][0].setdefault('env', [])
+            pod_spec['spec']['containers'][0]['env'].append({'name': 'ANSIBLE_RUNNER_KEEPALIVE_SECONDS', 'value': str(settings.AWX_RUNNER_KEEPALIVE_SECONDS)})
+
         # Enforce EE Pull Policy
         pull_options = {"always": "Always", "missing": "IfNotPresent", "never": "Never"}
         if self.task and self.task.instance.execution_environment:
@@ -664,6 +668,7 @@ RECEPTOR_CONFIG_STARTER = (
             'rootcas': '/etc/receptor/tls/ca/receptor-ca.crt',
             'cert': '/etc/receptor/tls/receptor.crt',
             'key': '/etc/receptor/tls/receptor.key',
+            'mintls13': False,
         }
     },
 )
@@ -708,7 +713,7 @@ def write_receptor_config():
     links.update(link_state=InstanceLink.States.ESTABLISHED)
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def remove_deprovisioned_node(hostname):
     InstanceLink.objects.filter(source__hostname=hostname).update(link_state=InstanceLink.States.REMOVING)
     InstanceLink.objects.filter(target__hostname=hostname).update(link_state=InstanceLink.States.REMOVING)

awx/main/tasks/system.py

@@ -50,7 +50,7 @@ from awx.main.models import (
 )
 from awx.main.constants import ACTIVE_STATES
 from awx.main.dispatch.publish import task
-from awx.main.dispatch import get_local_queuename, reaper
+from awx.main.dispatch import get_task_queuename, reaper
 from awx.main.utils.common import (
     get_type_for_model,
     ignore_inventory_computed_fields,
@@ -59,7 +59,6 @@ from awx.main.utils.common import (
     ScheduleTaskManager,
 )
 
-from awx.main.utils.external_logging import reconfigure_rsyslog
 from awx.main.utils.reload import stop_local_services
 from awx.main.utils.pglock import advisory_lock
 from awx.main.tasks.receptor import get_receptor_ctl, worker_info, worker_cleanup, administrative_workunit_reaper, write_receptor_config
@@ -115,9 +114,6 @@ def dispatch_startup():
     m = Metrics()
     m.reset_values()
 
-    # Update Tower's rsyslog.conf file based on loggins settings in the db
-    reconfigure_rsyslog()
-
 
 def inform_cluster_of_shutdown():
     try:
@@ -132,7 +128,7 @@ def inform_cluster_of_shutdown():
         logger.exception('Encountered problem with normal shutdown signal.')
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def apply_cluster_membership_policies():
     from awx.main.signals import disable_activity_stream
 
@@ -244,8 +240,10 @@ def apply_cluster_membership_policies():
         logger.debug('Cluster policy computation finished in {} seconds'.format(time.time() - started_compute))
 
 
-@task(queue='tower_broadcast_all')
-def handle_setting_changes(setting_keys):
+@task(queue='tower_settings_change')
+def clear_setting_cache(setting_keys):
+    # log that cache is being cleared
+    logger.info(f"clear_setting_cache of keys {setting_keys}")
     orig_len = len(setting_keys)
     for i in range(orig_len):
         for dependent_key in settings_registry.get_dependent_settings(setting_keys[i]):
@@ -254,9 +252,6 @@ def handle_setting_changes(setting_keys):
     logger.debug('cache delete_many(%r)', cache_keys)
     cache.delete_many(cache_keys)
 
-    if any([setting.startswith('LOG_AGGREGATOR') for setting in setting_keys]):
-        reconfigure_rsyslog()
-
 
 @task(queue='tower_broadcast_all')
 def delete_project_files(project_path):
@@ -286,7 +281,7 @@ def profile_sql(threshold=1, minutes=1):
         logger.error('SQL QUERIES >={}s ENABLED FOR {} MINUTE(S)'.format(threshold, minutes))
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def send_notifications(notification_list, job_id=None):
     if not isinstance(notification_list, list):
         raise TypeError("notification_list should be of type list")
@@ -317,7 +312,7 @@ def send_notifications(notification_list, job_id=None):
                 logger.exception('Error saving notification {} result.'.format(notification.id))
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def gather_analytics():
     from awx.conf.models import Setting
     from rest_framework.fields import DateTimeField
@@ -330,7 +325,7 @@ def gather_analytics():
         analytics.gather()
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def purge_old_stdout_files():
     nowtime = time.time()
     for f in os.listdir(settings.JOBOUTPUT_ROOT):
@@ -378,12 +373,12 @@ def handle_removed_image(remove_images=None):
     _cleanup_images_and_files(remove_images=remove_images, file_pattern='')
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def cleanup_images_and_files():
     _cleanup_images_and_files()
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def cluster_node_health_check(node):
     """
     Used for the health check endpoint, refreshes the status of the instance, but must be ran on target node
@@ -402,7 +397,7 @@ def cluster_node_health_check(node):
     this_inst.local_health_check()
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def execution_node_health_check(node):
     if node == '':
         logger.warning('Remote health check incorrectly called with blank string')
@@ -496,7 +491,7 @@ def inspect_execution_nodes(instance_list):
                     execution_node_health_check.apply_async([hostname])
 
 
-@task(queue=get_local_queuename, bind_kwargs=['dispatch_time', 'worker_tasks'])
+@task(queue=get_task_queuename, bind_kwargs=['dispatch_time', 'worker_tasks'])
 def cluster_node_heartbeat(dispatch_time=None, worker_tasks=None):
     logger.debug("Cluster node heartbeat task.")
     nowtime = now()
@@ -581,12 +576,12 @@ def cluster_node_heartbeat(dispatch_time=None, worker_tasks=None):
         active_task_ids = []
         for task_list in worker_tasks.values():
             active_task_ids.extend(task_list)
-        reaper.reap(instance=this_inst, excluded_uuids=active_task_ids)
+        reaper.reap(instance=this_inst, excluded_uuids=active_task_ids, ref_time=datetime.fromisoformat(dispatch_time))
         if max(len(task_list) for task_list in worker_tasks.values()) <= 1:
             reaper.reap_waiting(instance=this_inst, excluded_uuids=active_task_ids, ref_time=datetime.fromisoformat(dispatch_time))
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def awx_receptor_workunit_reaper():
     """
     When an AWX job is launched via receptor, files such as status, stdin, and stdout are created
@@ -622,7 +617,7 @@ def awx_receptor_workunit_reaper():
     administrative_workunit_reaper(receptor_work_list)
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def awx_k8s_reaper():
     if not settings.RECEPTOR_RELEASE_WORK:
         return
@@ -642,7 +637,7 @@ def awx_k8s_reaper():
                 logger.exception("Failed to delete orphaned pod {} from {}".format(job.log_format, group))
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def awx_periodic_scheduler():
     with advisory_lock('awx_periodic_scheduler_lock', wait=False) as acquired:
         if acquired is False:
@@ -708,7 +703,7 @@ def schedule_manager_success_or_error(instance):
         ScheduleWorkflowManager().schedule()
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def handle_work_success(task_actual):
     try:
         instance = UnifiedJob.get_instance_by_type(task_actual['type'], task_actual['id'])
@@ -720,7 +715,7 @@ def handle_work_success(task_actual):
     schedule_manager_success_or_error(instance)
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def handle_work_error(task_actual):
     try:
         instance = UnifiedJob.get_instance_by_type(task_actual['type'], task_actual['id'])
@@ -760,7 +755,7 @@ def handle_work_error(task_actual):
     schedule_manager_success_or_error(instance)
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def update_inventory_computed_fields(inventory_id):
     """
     Signal handler and wrapper around inventory.update_computed_fields to
@@ -801,7 +796,7 @@ def update_smart_memberships_for_inventory(smart_inventory):
     return False
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def update_host_smart_inventory_memberships():
     smart_inventories = Inventory.objects.filter(kind='smart', host_filter__isnull=False, pending_deletion=False)
     changed_inventories = set([])
@@ -817,7 +812,7 @@ def update_host_smart_inventory_memberships():
         smart_inventory.update_computed_fields()
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def delete_inventory(inventory_id, user_id, retries=5):
     # Delete inventory as user
     if user_id is None:
@@ -882,7 +877,7 @@ def _reconstruct_relationships(copy_mapping):
         new_obj.save()
 
 
-@task(queue=get_local_queuename)
+@task(queue=get_task_queuename)
 def deep_copy_model_obj(model_module, model_name, obj_pk, new_obj_pk, user_pk, uuid, permission_check_func=None):
     sub_obj_list = cache.get(uuid)
     if sub_obj_list is None:

awx/main/tests/functional/api/test_inventory.py

@@ -594,3 +594,89 @@ class TestControlledBySCM:
             rando,
             expect=403,
         )
+
+
+@pytest.mark.django_db
+class TestConstructedInventory:
+    @pytest.fixture
+    def constructed_inventory(self, organization):
+        return Inventory.objects.create(name='constructed-test-inventory', kind='constructed', organization=organization)
+
+    def test_get_constructed_inventory(self, constructed_inventory, admin_user, get):
+        inv_src = constructed_inventory.inventory_sources.first()
+        inv_src.update_cache_timeout = 53
+        inv_src.save(update_fields=['update_cache_timeout'])
+        r = get(url=reverse('api:constructed_inventory_detail', kwargs={'pk': constructed_inventory.pk}), user=admin_user, expect=200)
+        assert r.data['update_cache_timeout'] == 53
+
+    def test_patch_constructed_inventory(self, constructed_inventory, admin_user, patch):
+        inv_src = constructed_inventory.inventory_sources.first()
+        assert inv_src.update_cache_timeout == 0
+        assert inv_src.limit == ''
+        r = patch(
+            url=reverse('api:constructed_inventory_detail', kwargs={'pk': constructed_inventory.pk}),
+            data=dict(update_cache_timeout=54, limit='foobar'),
+            user=admin_user,
+            expect=200,
+        )
+        assert r.data['update_cache_timeout'] == 54
+        inv_src = constructed_inventory.inventory_sources.first()
+        assert inv_src.update_cache_timeout == 54
+        assert inv_src.limit == 'foobar'
+
+    def test_patch_constructed_inventory_generated_source_limits_editable_fields(self, constructed_inventory, admin_user, project, patch):
+        inv_src = constructed_inventory.inventory_sources.first()
+        r = patch(
+            url=inv_src.get_absolute_url(),
+            data={
+                'source': 'scm',
+                'source_project': project.pk,
+                'source_path': '',
+                'source_vars': 'plugin: a.b.c',
+            },
+            expect=400,
+            user=admin_user,
+        )
+        assert str(r.data['error'][0]) == "Cannot change field 'source' on a constructed inventory source."
+
+        # Make sure it didn't get updated before we got the error
+        inv_src_after_err = constructed_inventory.inventory_sources.first()
+        assert inv_src.id == inv_src_after_err.id
+        assert inv_src.source == inv_src_after_err.source
+        assert inv_src.source_project == inv_src_after_err.source_project
+        assert inv_src.source_path == inv_src_after_err.source_path
+        assert inv_src.source_vars == inv_src_after_err.source_vars
+
+    def test_patch_constructed_inventory_generated_source_allows_source_vars_edit(self, constructed_inventory, admin_user, patch):
+        inv_src = constructed_inventory.inventory_sources.first()
+        patch(
+            url=inv_src.get_absolute_url(),
+            data={
+                'source_vars': 'plugin: a.b.c',
+            },
+            expect=200,
+            user=admin_user,
+        )
+
+        inv_src_after_patch = constructed_inventory.inventory_sources.first()
+
+        # sanity checks
+        assert inv_src.id == inv_src_after_patch.id
+        assert inv_src.source == 'constructed'
+        assert inv_src_after_patch.source == 'constructed'
+        assert inv_src.source_vars == ''
+
+        assert inv_src_after_patch.source_vars == 'plugin: a.b.c'
+
+    def test_create_constructed_inventory(self, constructed_inventory, admin_user, post, organization):
+        r = post(
+            url=reverse('api:constructed_inventory_list'),
+            data=dict(name='constructed-inventory-just-created', kind='constructed', organization=organization.id, update_cache_timeout=55, limit='foobar'),
+            user=admin_user,
+            expect=201,
+        )
+        pk = r.data['id']
+        constructed_inventory = Inventory.objects.get(pk=pk)
+        inv_src = constructed_inventory.inventory_sources.first()
+        assert inv_src.update_cache_timeout == 55
+        assert inv_src.limit == 'foobar'

awx/main/tests/functional/conftest.py

@@ -511,6 +511,14 @@ def group(inventory):
     return inventory.groups.create(name='single-group')
 
 
+@pytest.fixture
+def constructed_inventory(organization):
+    """
+    creates a new constructed inventory source
+    """
+    return Inventory.objects.create(name='dummy1', kind='constructed', organization=organization)
+
+
 @pytest.fixture
 def inventory_source(inventory):
     # by making it ec2, the credential is not required

awx/main/tests/functional/models/test_events.py

@@ -3,178 +3,209 @@ import pytest
 
 from django.utils.timezone import now
 
-from awx.main.models import Job, JobEvent, Inventory, Host, JobHostSummary
+from django.db.models import Q
+
+from awx.main.models import Job, JobEvent, Inventory, Host, JobHostSummary, HostMetric
 
 
 @pytest.mark.django_db
-@mock.patch('awx.main.models.events.emit_event_detail')
-def test_parent_changed(emit):
-    j = Job()
-    j.save()
-    JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save()
-    assert JobEvent.objects.count() == 1
-    for e in JobEvent.objects.all():
-        assert e.changed is False
+class TestEvents:
+    def setup_method(self):
+        self.hostnames = []
+        self.host_map = dict()
+        self.inventory = None
+        self.job = None
 
-    JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='runner_on_ok', event_data={'res': {'changed': ['localhost']}}).save()
-    # the `playbook_on_stats` event is where we update the parent changed linkage
-    JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save()
-    events = JobEvent.objects.filter(event__in=['playbook_on_task_start', 'runner_on_ok'])
-    assert events.count() == 2
-    for e in events.all():
-        assert e.changed is True
+    @mock.patch('awx.main.models.events.emit_event_detail')
+    def test_parent_changed(self, emit):
+        j = Job()
+        j.save()
+        JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save()
+        assert JobEvent.objects.count() == 1
+        for e in JobEvent.objects.all():
+            assert e.changed is False
 
+        JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='runner_on_ok', event_data={'res': {'changed': ['localhost']}}).save()
+        # the `playbook_on_stats` event is where we update the parent changed linkage
+        JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save()
+        events = JobEvent.objects.filter(event__in=['playbook_on_task_start', 'runner_on_ok'])
+        assert events.count() == 2
+        for e in events.all():
+            assert e.changed is True
 
-@pytest.mark.django_db
-@pytest.mark.parametrize('event', JobEvent.FAILED_EVENTS)
-@mock.patch('awx.main.models.events.emit_event_detail')
-def test_parent_failed(emit, event):
-    j = Job()
-    j.save()
-    JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save()
-    assert JobEvent.objects.count() == 1
-    for e in JobEvent.objects.all():
-        assert e.failed is False
+    @pytest.mark.parametrize('event', JobEvent.FAILED_EVENTS)
+    @mock.patch('awx.main.models.events.emit_event_detail')
+    def test_parent_failed(self, emit, event):
+        j = Job()
+        j.save()
+        JobEvent.create_from_data(job_id=j.pk, uuid='abc123', event='playbook_on_task_start').save()
+        assert JobEvent.objects.count() == 1
+        for e in JobEvent.objects.all():
+            assert e.failed is False
 
-    JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event=event).save()
+        JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event=event).save()
 
-    # the `playbook_on_stats` event is where we update the parent failed linkage
-    JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save()
-    events = JobEvent.objects.filter(event__in=['playbook_on_task_start', event])
-    assert events.count() == 2
-    for e in events.all():
-        assert e.failed is True
+        # the `playbook_on_stats` event is where we update the parent failed linkage
+        JobEvent.create_from_data(job_id=j.pk, parent_uuid='abc123', event='playbook_on_stats').save()
+        events = JobEvent.objects.filter(event__in=['playbook_on_task_start', event])
+        assert events.count() == 2
+        for e in events.all():
+            assert e.failed is True
 
+    def test_host_summary_generation(self):
+        self._generate_hosts(100)
+        self._create_job_event(ok=dict((hostname, len(hostname)) for hostname in self.hostnames))
 
-@pytest.mark.django_db
-def test_host_summary_generation():
-    hostnames = [f'Host {i}' for i in range(100)]
-    inv = Inventory()
-    inv.save()
-    Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames])
-    j = Job(inventory=inv)
-    j.save()
-    host_map = dict((host.name, host.id) for host in inv.hosts.all())
-    JobEvent.create_from_data(
-        job_id=j.pk,
+        assert self.job.job_host_summaries.count() == len(self.hostnames)
+        assert sorted([s.host_name for s in self.job.job_host_summaries.all()]) == sorted(self.hostnames)
+
+        for s in self.job.job_host_summaries.all():
+            assert self.host_map[s.host_name] == s.host_id
+            assert s.ok == len(s.host_name)
+            assert s.changed == 0
+            assert s.dark == 0
+            assert s.failures == 0
+            assert s.ignored == 0
+            assert s.processed == 0
+            assert s.rescued == 0
+            assert s.skipped == 0
+
+        for host in Host.objects.all():
+            assert host.last_job_id == self.job.id
+            assert host.last_job_host_summary.host == host
+
+    def test_host_summary_generation_with_deleted_hosts(self):
+        self._generate_hosts(10)
+
+        # delete half of the hosts during the playbook run
+        for h in self.inventory.hosts.all()[:5]:
+            h.delete()
+
+        self._create_job_event(ok=dict((hostname, len(hostname)) for hostname in self.hostnames))
+
+        ids = sorted([s.host_id or -1 for s in self.job.job_host_summaries.order_by('id').all()])
+        names = sorted([s.host_name for s in self.job.job_host_summaries.all()])
+        assert ids == [-1, -1, -1, -1, -1, 6, 7, 8, 9, 10]
+        assert names == ['Host 0', 'Host 1', 'Host 2', 'Host 3', 'Host 4', 'Host 5', 'Host 6', 'Host 7', 'Host 8', 'Host 9']
+
+    def test_host_summary_generation_with_limit(self):
+        # Make an inventory with 10 hosts, run a playbook with a --limit
+        # pointed at *one* host,
+        # Verify that *only* that host has an associated JobHostSummary and that
+        # *only* that host has an updated value for .last_job.
+        self._generate_hosts(10)
+
+        # by making the playbook_on_stats *only* include Host 1, we're emulating
+        # the behavior of a `--limit=Host 1`
+        matching_host = Host.objects.get(name='Host 1')
+        self._create_job_event(ok={matching_host.name: len(matching_host.name)})  # effectively, limit=Host 1
+
+        # since the playbook_on_stats only references one host,
+        # there should *only* be on JobHostSummary record (and it should
+        # be related to the appropriate Host)
+        assert JobHostSummary.objects.count() == 1
+        for h in Host.objects.all():
+            if h.name == 'Host 1':
+                assert h.last_job_id == self.job.id
+                assert h.last_job_host_summary_id == JobHostSummary.objects.first().id
+            else:
+                # all other hosts in the inventory should remain untouched
+                assert h.last_job_id is None
+                assert h.last_job_host_summary_id is None
+
+    def test_host_metrics_insert(self):
+        self._generate_hosts(10)
+
+        self._create_job_event(
+            ok=dict((hostname, len(hostname)) for hostname in self.hostnames[0:3]),
+            failures=dict((hostname, len(hostname)) for hostname in self.hostnames[3:6]),
+            processed=dict((hostname, len(hostname)) for hostname in self.hostnames[6:9]),
+            skipped=dict((hostname, len(hostname)) for hostname in [self.hostnames[9]]),
+        )
+
+        metrics = HostMetric.objects.all()
+        assert len(metrics) == 10
+        for hm in metrics:
+            assert hm.automated_counter == 1
+            assert hm.last_automation is not None
+            assert hm.deleted is False
+
+    def test_host_metrics_update(self):
+        self._generate_hosts(12)
+
+        self._create_job_event(ok=dict((hostname, len(hostname)) for hostname in self.hostnames))
+
+        # Soft delete 6 host metrics
+        for hm in HostMetric.objects.filter(id__in=[1, 3, 5, 7, 9, 11]):
+            hm.soft_delete()
+
+        assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=0) & Q(last_deleted__isnull=True))) == 6
+        assert len(HostMetric.objects.filter(Q(deleted=True) & Q(deleted_counter=1) & Q(last_deleted__isnull=False))) == 6
+
+        # hostnames in 'ignored' and 'rescued' stats are ignored
+        self.job = Job(inventory=self.inventory)
+        self.job.save()
+        self._create_job_event(
+            ignored=dict((hostname, len(hostname)) for hostname in self.hostnames[0:6]),
+            rescued=dict((hostname, len(hostname)) for hostname in self.hostnames[6:11]),
+        )
+
+        assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=0) & Q(last_deleted__isnull=True))) == 6
+        assert len(HostMetric.objects.filter(Q(deleted=True) & Q(deleted_counter=1) & Q(last_deleted__isnull=False))) == 6
+
+        # hostnames in 'changed', 'dark', 'failures', 'ok', 'processed', 'skipped' are processed
+        self.job = Job(inventory=self.inventory)
+        self.job.save()
+        self._create_job_event(
+            changed=dict((hostname, len(hostname)) for hostname in self.hostnames[0:2]),
+            dark=dict((hostname, len(hostname)) for hostname in self.hostnames[2:4]),
+            failures=dict((hostname, len(hostname)) for hostname in self.hostnames[4:6]),
+            ok=dict((hostname, len(hostname)) for hostname in self.hostnames[6:8]),
+            processed=dict((hostname, len(hostname)) for hostname in self.hostnames[8:10]),
+            skipped=dict((hostname, len(hostname)) for hostname in self.hostnames[10:12]),
+        )
+        assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=0) & Q(last_deleted__isnull=True))) == 6
+        assert len(HostMetric.objects.filter(Q(deleted=False) & Q(deleted_counter=1) & Q(last_deleted__isnull=False))) == 6
+
+    def _generate_hosts(self, cnt, id_from=0):
+        self.hostnames = [f'Host {i}' for i in range(id_from, id_from + cnt)]
+        self.inventory = Inventory()
+        self.inventory.save()
+        Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=self.inventory.id) for h in self.hostnames])
+        self.job = Job(inventory=self.inventory)
+        self.job.save()
+
+        # host map is a data structure that tracks a mapping of host name --> ID
+        # for the inventory, _regardless_ of whether or not there's a limit
+        # applied to the actual playbook run
+        self.host_map = dict((host.name, host.id) for host in self.inventory.hosts.all())
+
+    def _create_job_event(
+        self,
         parent_uuid='abc123',
         event='playbook_on_stats',
-        event_data={
-            'ok': dict((hostname, len(hostname)) for hostname in hostnames),
-            'changed': {},
-            'dark': {},
-            'failures': {},
-            'ignored': {},
-            'processed': {},
-            'rescued': {},
-            'skipped': {},
-        },
-        host_map=host_map,
-    ).save()
-
-    assert j.job_host_summaries.count() == len(hostnames)
-    assert sorted([s.host_name for s in j.job_host_summaries.all()]) == sorted(hostnames)
-
-    for s in j.job_host_summaries.all():
-        assert host_map[s.host_name] == s.host_id
-        assert s.ok == len(s.host_name)
-        assert s.changed == 0
-        assert s.dark == 0
-        assert s.failures == 0
-        assert s.ignored == 0
-        assert s.processed == 0
-        assert s.rescued == 0
-        assert s.skipped == 0
-
-    for host in Host.objects.all():
-        assert host.last_job_id == j.id
-        assert host.last_job_host_summary.host == host
-
-
-@pytest.mark.django_db
-def test_host_summary_generation_with_deleted_hosts():
-    hostnames = [f'Host {i}' for i in range(10)]
-    inv = Inventory()
-    inv.save()
-    Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames])
-    j = Job(inventory=inv)
-    j.save()
-    host_map = dict((host.name, host.id) for host in inv.hosts.all())
-
-    # delete half of the hosts during the playbook run
-    for h in inv.hosts.all()[:5]:
-        h.delete()
-
-    JobEvent.create_from_data(
-        job_id=j.pk,
-        parent_uuid='abc123',
-        event='playbook_on_stats',
-        event_data={
-            'ok': dict((hostname, len(hostname)) for hostname in hostnames),
-            'changed': {},
-            'dark': {},
-            'failures': {},
-            'ignored': {},
-            'processed': {},
-            'rescued': {},
-            'skipped': {},
-        },
-        host_map=host_map,
-    ).save()
-
-    ids = sorted([s.host_id or -1 for s in j.job_host_summaries.order_by('id').all()])
-    names = sorted([s.host_name for s in j.job_host_summaries.all()])
-    assert ids == [-1, -1, -1, -1, -1, 6, 7, 8, 9, 10]
-    assert names == ['Host 0', 'Host 1', 'Host 2', 'Host 3', 'Host 4', 'Host 5', 'Host 6', 'Host 7', 'Host 8', 'Host 9']
-
-
-@pytest.mark.django_db
-def test_host_summary_generation_with_limit():
-    # Make an inventory with 10 hosts, run a playbook with a --limit
-    # pointed at *one* host,
-    # Verify that *only* that host has an associated JobHostSummary and that
-    # *only* that host has an updated value for .last_job.
-    hostnames = [f'Host {i}' for i in range(10)]
-    inv = Inventory()
-    inv.save()
-    Host.objects.bulk_create([Host(created=now(), modified=now(), name=h, inventory_id=inv.id) for h in hostnames])
-    j = Job(inventory=inv)
-    j.save()
-
-    # host map is a data structure that tracks a mapping of host name --> ID
-    # for the inventory, _regardless_ of whether or not there's a limit
-    # applied to the actual playbook run
-    host_map = dict((host.name, host.id) for host in inv.hosts.all())
-
-    # by making the playbook_on_stats *only* include Host 1, we're emulating
-    # the behavior of a `--limit=Host 1`
-    matching_host = Host.objects.get(name='Host 1')
-    JobEvent.create_from_data(
-        job_id=j.pk,
-        parent_uuid='abc123',
-        event='playbook_on_stats',
-        event_data={
-            'ok': {matching_host.name: len(matching_host.name)},  # effectively, limit=Host 1
-            'changed': {},
-            'dark': {},
-            'failures': {},
-            'ignored': {},
-            'processed': {},
-            'rescued': {},
-            'skipped': {},
-        },
-        host_map=host_map,
-    ).save()
-
-    # since the playbook_on_stats only references one host,
-    # there should *only* be on JobHostSummary record (and it should
-    # be related to the appropriate Host)
-    assert JobHostSummary.objects.count() == 1
-    for h in Host.objects.all():
-        if h.name == 'Host 1':
-            assert h.last_job_id == j.id
-            assert h.last_job_host_summary_id == JobHostSummary.objects.first().id
-        else:
-            # all other hosts in the inventory should remain untouched
-            assert h.last_job_id is None
-            assert h.last_job_host_summary_id is None
+        ok=None,
+        changed=None,
+        dark=None,
+        failures=None,
+        ignored=None,
+        processed=None,
+        rescued=None,
+        skipped=None,
+    ):
+        JobEvent.create_from_data(
+            job_id=self.job.pk,
+            parent_uuid=parent_uuid,
+            event=event,
+            event_data={
+                'ok': ok or {},
+                'changed': changed or {},
+                'dark': dark or {},
+                'failures': failures or {},
+                'ignored': ignored or {},
+                'processed': processed or {},
+                'rescued': rescued or {},
+                'skipped': skipped or {},
+            },
+            host_map=self.host_map,
+        ).save()

awx/main/tests/functional/models/test_host_metric.py

@@ -20,3 +20,53 @@ def test_host_metrics_generation():
     date_today = now().strftime('%Y-%m-%d')
     result = HostMetric.objects.filter(first_automation__startswith=date_today).count()
     assert result == len(hostnames)
+
+
+@pytest.mark.django_db
+def test_soft_delete():
+    hostnames = [f'Host to delete {i}' for i in range(2)]
+    current_time = now()
+    HostMetric.objects.bulk_create([HostMetric(hostname=h, last_automation=current_time, automated_counter=42) for h in hostnames])
+
+    hm = HostMetric.objects.get(hostname="Host to delete 0")
+    assert hm.last_deleted is None
+
+    last_deleted = None
+    for _ in range(3):
+        # soft delete 1st
+        # 2nd/3rd delete don't have an effect
+        hm.soft_delete()
+        if last_deleted is None:
+            last_deleted = hm.last_deleted
+
+        assert hm.deleted is True
+        assert hm.deleted_counter == 1
+        assert hm.last_deleted == last_deleted
+        assert hm.automated_counter == 42
+
+    # 2nd record is not touched
+    hm = HostMetric.objects.get(hostname="Host to delete 1")
+    assert hm.deleted is False
+    assert hm.deleted_counter == 0
+    assert hm.last_deleted is None
+    assert hm.automated_counter == 42
+
+
+@pytest.mark.django_db
+def test_soft_restore():
+    current_time = now()
+    HostMetric.objects.create(hostname="Host 1", last_automation=current_time, deleted=True)
+    HostMetric.objects.create(hostname="Host 2", last_automation=current_time, deleted=True, last_deleted=current_time)
+    HostMetric.objects.create(hostname="Host 3", last_automation=current_time, deleted=False, last_deleted=current_time)
+    HostMetric.objects.all().update(automated_counter=42, deleted_counter=10)
+
+    # 1. deleted, last_deleted not null
+    for hm in HostMetric.objects.all():
+        for _ in range(3):
+            hm.soft_restore()
+            assert hm.deleted is False
+            assert hm.automated_counter == 42 and hm.deleted_counter == 10
+            if hm.hostname == "Host 1":
+                assert hm.last_deleted is None
+            else:
+                assert hm.last_deleted == current_time

awx/main/tests/functional/models/test_inventory.py

@@ -169,7 +169,8 @@ class TestInventorySourceInjectors:
         CLOUD_PROVIDERS constant contains the same names as what are
         defined within the injectors
         """
-        assert set(CLOUD_PROVIDERS) == set(InventorySource.injectors.keys())
+        # slight exception case for constructed, because it has a FQCN but is not a cloud source
+        assert set(CLOUD_PROVIDERS) | set(['constructed']) == set(InventorySource.injectors.keys())
 
     @pytest.mark.parametrize('source,filename', [('ec2', 'aws_ec2.yml'), ('openstack', 'openstack.yml'), ('gce', 'gcp_compute.yml')])
     def test_plugin_filenames(self, source, filename):

awx/main/tests/functional/models/test_unified_job.py

@@ -14,7 +14,7 @@ from awx.main.constants import JOB_VARIABLE_PREFIXES
 
 
 @pytest.mark.django_db
-def test_subclass_types(rando):
+def test_subclass_types():
     assert set(UnifiedJobTemplate._submodels_with_roles()) == set(
         [
             ContentType.objects.get_for_model(JobTemplate).id,

awx/main/tests/functional/test_bulk.py

@@ -0,0 +1,311 @@
+import pytest
+
+from uuid import uuid4
+
+from awx.api.versioning import reverse
+
+from awx.main.models.jobs import JobTemplate
+from awx.main.models import Organization, Inventory, WorkflowJob, ExecutionEnvironment, Host
+from awx.main.scheduler import TaskManager
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize('num_hosts, num_queries', [(1, 15), (10, 15)])
+def test_bulk_host_create_num_queries(organization, inventory, post, get, user, num_hosts, num_queries, django_assert_max_num_queries):
+    '''
+    If I am a...
+      org admin
+      inventory admin at org level
+      admin of a particular inventory
+      superuser
+
+    Bulk Host create should take under a certain number of queries
+    '''
+    inventory.organization = organization
+    inventory_admin = user('inventory_admin', False)
+    org_admin = user('org_admin', False)
+    org_inv_admin = user('org_admin', False)
+    superuser = user('admin', True)
+    for u in [org_admin, org_inv_admin, inventory_admin]:
+        organization.member_role.members.add(u)
+    organization.admin_role.members.add(org_admin)
+    organization.inventory_admin_role.members.add(org_inv_admin)
+    inventory.admin_role.members.add(inventory_admin)
+
+    for u in [org_admin, inventory_admin, org_inv_admin, superuser]:
+        hosts = [{'name': uuid4()} for i in range(num_hosts)]
+        with django_assert_max_num_queries(num_queries):
+            bulk_host_create_response = post(reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': hosts}, u, expect=201).data
+            assert len(bulk_host_create_response['hosts']) == len(hosts), f"unexpected number of hosts created for user {u}"
+
+
+@pytest.mark.django_db
+def test_bulk_host_create_rbac(organization, inventory, post, get, user):
+    '''
+    If I am a...
+      org admin
+      inventory admin at org level
+      admin of a particular invenotry
+          ... I can bulk add hosts
+
+    Everyone else cannot
+    '''
+    inventory.organization = organization
+    inventory_admin = user('inventory_admin', False)
+    org_admin = user('org_admin', False)
+    org_inv_admin = user('org_admin', False)
+    auditor = user('auditor', False)
+    member = user('member', False)
+    use_inv_member = user('member', False)
+    for u in [org_admin, org_inv_admin, auditor, member, inventory_admin, use_inv_member]:
+        organization.member_role.members.add(u)
+    organization.admin_role.members.add(org_admin)
+    organization.inventory_admin_role.members.add(org_inv_admin)
+    inventory.admin_role.members.add(inventory_admin)
+    inventory.use_role.members.add(use_inv_member)
+    organization.auditor_role.members.add(auditor)
+
+    for indx, u in enumerate([org_admin, inventory_admin, org_inv_admin]):
+        bulk_host_create_response = post(
+            reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': [{'name': f'foobar-{indx}'}]}, u, expect=201
+        ).data
+        assert len(bulk_host_create_response['hosts']) == 1, f"unexpected number of hosts created for user {u}"
+        assert Host.objects.filter(inventory__id=inventory.id)[0].name == 'foobar-0'
+
+    for indx, u in enumerate([member, auditor, use_inv_member]):
+        bulk_host_create_response = post(
+            reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': [{'name': f'foobar2-{indx}'}]}, u, expect=400
+        ).data
+        assert bulk_host_create_response['__all__'][0] == f'Inventory with id {inventory.id} not found or lack permissions to add hosts.'
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize('num_jobs, num_queries', [(1, 25), (10, 25)])
+def test_bulk_job_launch_queries(job_template, organization, inventory, project, post, get, user, num_jobs, num_queries, django_assert_max_num_queries):
+    '''
+    if I have access to the unified job template
+             ... I can launch the bulk job
+             ... and the number of queries should NOT scale with the number of jobs
+    '''
+    normal_user = user('normal_user', False)
+    org_admin = user('org_admin', False)
+    jt = JobTemplate.objects.create(name='my-jt', ask_inventory_on_launch=True, project=project, playbook='helloworld.yml')
+    organization.member_role.members.add(normal_user)
+    organization.admin_role.members.add(org_admin)
+    jt.execute_role.members.add(normal_user)
+    inventory.use_role.members.add(normal_user)
+    jt.save()
+    inventory.save()
+    jobs = [{'unified_job_template': jt.id, 'inventory': inventory.id} for _ in range(num_jobs)]
+
+    # This is not working, we need to figure that out if we want to include tests for more jobs
+    # with mock.patch('awx.api.serializers.settings.BULK_JOB_MAX_LAUNCH', num_jobs + 1):
+    with django_assert_max_num_queries(num_queries):
+        bulk_job_launch_response = post(reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': jobs}, normal_user, expect=201).data
+
+    # Run task manager so the workflow job nodes actually spawn
+    TaskManager().schedule()
+
+    for u in (org_admin, normal_user):
+        bulk_job = get(bulk_job_launch_response['url'], u, expect=200).data
+        assert organization.id == bulk_job['summary_fields']['organization']['id']
+        resp = get(bulk_job_launch_response['related']['workflow_nodes'], u)
+        assert resp.data['count'] == num_jobs
+        for item in resp.data['results']:
+            assert item["unified_job_template"] == jt.id
+            assert item["inventory"] == inventory.id
+
+
+@pytest.mark.django_db
+def test_bulk_job_launch_no_access_to_job_template(job_template, organization, inventory, project, credential, post, get, user):
+    '''
+    if I don't have access to the unified job templare
+             ... I can't launch the bulk job
+    '''
+    normal_user = user('normal_user', False)
+    jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml')
+    jt.save()
+    organization.member_role.members.add(normal_user)
+    bulk_job_launch_response = post(
+        reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=400
+    ).data
+    assert bulk_job_launch_response['__all__'][0] == f'Job Templates {{{jt.id}}} not found or you don\'t have permissions to access it'
+
+
+@pytest.mark.django_db
+def test_bulk_job_launch_no_org_assigned(job_template, organization, inventory, project, credential, post, get, user):
+    '''
+    if I am not part of any organization...
+             ... I can't launch the bulk job
+    '''
+    normal_user = user('normal_user', False)
+    jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml')
+    jt.save()
+    jt.execute_role.members.add(normal_user)
+    bulk_job_launch_response = post(
+        reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=400
+    ).data
+    assert bulk_job_launch_response['__all__'][0] == 'User not part of any organization, please assign an organization to assign to the bulk job'
+
+
+@pytest.mark.django_db
+def test_bulk_job_launch_multiple_org_assigned(job_template, organization, inventory, project, credential, post, get, user):
+    '''
+    if I am part of multiple organization...
+        and if I do not provide org at the launch time
+             ... I can't launch the bulk job
+    '''
+    normal_user = user('normal_user', False)
+    org1 = Organization.objects.create(name='foo1')
+    org2 = Organization.objects.create(name='foo2')
+    org1.member_role.members.add(normal_user)
+    org2.member_role.members.add(normal_user)
+    jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml')
+    jt.save()
+    jt.execute_role.members.add(normal_user)
+    bulk_job_launch_response = post(
+        reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}]}, normal_user, expect=400
+    ).data
+    assert bulk_job_launch_response['__all__'][0] == 'User has permission to multiple Organizations, please set one of them in the request'
+
+
+@pytest.mark.django_db
+def test_bulk_job_launch_specific_org(job_template, organization, inventory, project, credential, post, get, user):
+    '''
+    if I am part of multiple organization...
+        and if I provide org at the launch time
+             ... I can launch the bulk job
+    '''
+    normal_user = user('normal_user', False)
+    org1 = Organization.objects.create(name='foo1')
+    org2 = Organization.objects.create(name='foo2')
+    org1.member_role.members.add(normal_user)
+    org2.member_role.members.add(normal_user)
+    jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml')
+    jt.save()
+    jt.execute_role.members.add(normal_user)
+    bulk_job_launch_response = post(
+        reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id}], 'organization': org1.id}, normal_user, expect=201
+    ).data
+    bulk_job_id = bulk_job_launch_response['id']
+    bulk_job_obj = WorkflowJob.objects.filter(id=bulk_job_id, is_bulk_job=True).first()
+    assert org1.id == bulk_job_obj.organization.id
+
+
+@pytest.mark.django_db
+def test_bulk_job_launch_inventory_no_access(job_template, organization, inventory, project, credential, post, get, user):
+    '''
+    if I don't have access to the inventory...
+        and if I try to use it at the launch time
+             ... I can't launch the bulk job
+    '''
+    normal_user = user('normal_user', False)
+    org1 = Organization.objects.create(name='foo1')
+    org2 = Organization.objects.create(name='foo2')
+    jt = JobTemplate.objects.create(name='my-jt', inventory=inventory, project=project, playbook='helloworld.yml')
+    jt.save()
+    org1.member_role.members.add(normal_user)
+    inv = Inventory.objects.create(name='inv1', organization=org2)
+    jt.execute_role.members.add(normal_user)
+    bulk_job_launch_response = post(
+        reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id, 'inventory': inv.id}]}, normal_user, expect=400
+    ).data
+    assert bulk_job_launch_response['__all__'][0] == f'Inventories {{{inv.id}}} not found or you don\'t have permissions to access it'
+
+
+@pytest.mark.django_db
+def test_bulk_job_inventory_prompt(job_template, organization, inventory, project, credential, post, get, user):
+    '''
+    Job template has an inventory set as prompt_on_launch
+        and if I provide the inventory as a parameter in bulk job
+        ... job uses that inventory
+    '''
+    normal_user = user('normal_user', False)
+    org1 = Organization.objects.create(name='foo1')
+    jt = JobTemplate.objects.create(name='my-jt', ask_inventory_on_launch=True, project=project, playbook='helloworld.yml')
+    jt.save()
+    org1.member_role.members.add(normal_user)
+    inv = Inventory.objects.create(name='inv1', organization=org1)
+    jt.execute_role.members.add(normal_user)
+    inv.use_role.members.add(normal_user)
+    bulk_job_launch_response = post(
+        reverse('api:bulk_job_launch'), {'name': 'Bulk Job Launch', 'jobs': [{'unified_job_template': jt.id, 'inventory': inv.id}]}, normal_user, expect=201
+    ).data
+    bulk_job_id = bulk_job_launch_response['id']
+    node = WorkflowJob.objects.get(id=bulk_job_id).workflow_job_nodes.all().order_by('created')
+    assert inv.id == node[0].inventory.id
+
+
+@pytest.mark.django_db
+def test_bulk_job_set_all_prompt(job_template, organization, inventory, project, credentialtype_ssh, post, get, user):
+    '''
+    Job template has many fields set as prompt_on_launch
+        and if I provide all those fields as a parameter in bulk job
+        ... job uses them
+    '''
+    normal_user = user('normal_user', False)
+    jt = JobTemplate.objects.create(
+        name='my-jt',
+        ask_inventory_on_launch=True,
+        ask_diff_mode_on_launch=True,
+        ask_job_type_on_launch=True,
+        ask_verbosity_on_launch=True,
+        ask_execution_environment_on_launch=True,
+        ask_forks_on_launch=True,
+        ask_job_slice_count_on_launch=True,
+        ask_timeout_on_launch=True,
+        ask_variables_on_launch=True,
+        ask_scm_branch_on_launch=True,
+        ask_limit_on_launch=True,
+        ask_skip_tags_on_launch=True,
+        ask_tags_on_launch=True,
+        project=project,
+        playbook='helloworld.yml',
+    )
+    jt.save()
+    organization.member_role.members.add(normal_user)
+    inv = Inventory.objects.create(name='inv1', organization=organization)
+    ee = ExecutionEnvironment.objects.create(name='test-ee', image='quay.io/foo/bar')
+    jt.execute_role.members.add(normal_user)
+    inv.use_role.members.add(normal_user)
+    bulk_job_launch_response = post(
+        reverse('api:bulk_job_launch'),
+        {
+            'name': 'Bulk Job Launch',
+            'jobs': [
+                {
+                    'unified_job_template': jt.id,
+                    'inventory': inv.id,
+                    'diff_mode': True,
+                    'job_type': 'check',
+                    'verbosity': 3,
+                    'execution_environment': ee.id,
+                    'forks': 1,
+                    'job_slice_count': 1,
+                    'timeout': 200,
+                    'extra_data': {'prompted_key': 'prompted_val'},
+                    'scm_branch': 'non_dev',
+                    'limit': 'kansas',
+                    'skip_tags': 'foobar',
+                    'job_tags': 'untagged',
+                }
+            ],
+        },
+        normal_user,
+        expect=201,
+    ).data
+    bulk_job_id = bulk_job_launch_response['id']
+    node = WorkflowJob.objects.get(id=bulk_job_id).workflow_job_nodes.all().order_by('created')
+    assert node[0].inventory.id == inv.id
+    assert node[0].diff_mode == True
+    assert node[0].job_type == 'check'
+    assert node[0].verbosity == 3
+    assert node[0].execution_environment.id == ee.id
+    assert node[0].forks == 1
+    assert node[0].job_slice_count == 1
+    assert node[0].timeout == 200
+    assert node[0].extra_data == {'prompted_key': 'prompted_val'}
+    assert node[0].scm_branch == 'non_dev'
+    assert node[0].limit == 'kansas'
+    assert node[0].skip_tags == 'foobar'
+    assert node[0].job_tags == 'untagged'