Fix cartesian product in organization user/admin count queries

The organizations list and detail endpoints annotated each org with user and admin counts using two Count() calls that traverse the Role.members M2M. Django generated two LEFT JOINs on the same through table, crossing every member row with every admin row before COUNT(DISTINCT) reduced the product.

At scale (2,617 members × 46,233 admins) this produced 120M intermediate rows and 96-second query times, causing 504 timeouts.

Replace with independent Subquery expressions that each query main_rbac_roles_members separately - no cross product.

Fixes: AAP-72817
Fixes: AAP-72480

.github/PULL_REQUEST_TEMPLATE.md

@@ -24,7 +24,7 @@ in as the first entry for your PR title.
 
 
 
-##### ADDITIONAL INFORMATION
+##### STEPS TO REPRODUCE AND EXTRA INFO
 <!---
 Include additional information to help people understand the change here.
 For bugs that don't have a linked bug report, a step-by-step reproduction

.github/workflows/api_schema_check.yml

@@ -45,15 +45,45 @@ jobs:
           make docker-runner 2>&1 | tee schema-diff.txt
           exit ${PIPESTATUS[0]}
 
-      - name: Add schema diff to job summary
+      - name: Validate OpenAPI schema
+        id: schema-validation
+        continue-on-error: true
+        run: |
+          AWX_DOCKER_ARGS='-e GITHUB_ACTIONS' \
+          AWX_DOCKER_CMD='make validate-openapi-schema' \
+          make docker-runner 2>&1 | tee schema-validation.txt
+          exit ${PIPESTATUS[0]}
+
+      - name: Add schema validation and diff to job summary
         if: always()
         # show text and if for some reason, it can't be generated, state that it can't be.
         run: |
-          echo "## API Schema Change Detection Results" >> $GITHUB_STEP_SUMMARY
+          echo "## API Schema Check Results" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
+
+          # Show validation status
+          echo "### OpenAPI Validation" >> $GITHUB_STEP_SUMMARY
+          if [ -f schema-validation.txt ] && grep -q "✓ Schema is valid" schema-validation.txt; then
+            echo "✅ **Status:** PASSED - Schema is valid OpenAPI 3.0.3" >> $GITHUB_STEP_SUMMARY
+          else
+            echo "❌ **Status:** FAILED - Schema validation failed" >> $GITHUB_STEP_SUMMARY
+            if [ -f schema-validation.txt ]; then
+              echo "" >> $GITHUB_STEP_SUMMARY
+              echo "<details><summary>Validation errors</summary>" >> $GITHUB_STEP_SUMMARY
+              echo "" >> $GITHUB_STEP_SUMMARY
+              echo '```' >> $GITHUB_STEP_SUMMARY
+              cat schema-validation.txt >> $GITHUB_STEP_SUMMARY
+              echo '```' >> $GITHUB_STEP_SUMMARY
+              echo "</details>" >> $GITHUB_STEP_SUMMARY
+            fi
+          fi
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          # Show schema changes
+          echo "### Schema Changes" >> $GITHUB_STEP_SUMMARY
           if [ -f schema-diff.txt ]; then
             if grep -q "^+" schema-diff.txt || grep -q "^-" schema-diff.txt; then
-              echo "### Schema changes detected" >> $GITHUB_STEP_SUMMARY
+              echo "**Changes detected** between this PR and the base branch" >> $GITHUB_STEP_SUMMARY
               echo "" >> $GITHUB_STEP_SUMMARY
               # Truncate to first 1000 lines to stay under GitHub's 1MB summary limit
               TOTAL_LINES=$(wc -l < schema-diff.txt)
@@ -65,8 +95,8 @@ jobs:
               head -n 1000 schema-diff.txt >> $GITHUB_STEP_SUMMARY
               echo '```' >> $GITHUB_STEP_SUMMARY
             else
-              echo "### No schema changes detected" >> $GITHUB_STEP_SUMMARY
+              echo "No schema changes detected" >> $GITHUB_STEP_SUMMARY
             fi
           else
-            echo "### Unable to generate schema diff" >> $GITHUB_STEP_SUMMARY
+            echo "Unable to generate schema diff" >> $GITHUB_STEP_SUMMARY
           fi

.github/workflows/ci.yml

@@ -4,14 +4,46 @@ env:
   LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
   CI_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   DEV_DOCKER_OWNER: ${{ github.repository_owner }}
-  COMPOSE_TAG: ${{ github.base_ref || 'devel' }}
+  COMPOSE_TAG: ${{ github.base_ref || github.ref_name || 'devel' }}
   UPSTREAM_REPOSITORY_ID: 91594105
 on:
   pull_request:
   push:
     branches:
       - devel  # needed to publish code coverage post-merge
+  schedule:
+    - cron: '0 12,18 * * 1-5'
+  workflow_dispatch: {}
 jobs:
+  trigger-release-branches:
+    name: "Dispatch CI to release branches"
+    if: github.event_name == 'schedule'
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+    steps:
+      - name: Trigger CI on release_4.6
+        id: dispatch_release_46
+        continue-on-error: true
+        run: gh workflow run ci.yml --ref release_4.6
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+      - name: Trigger CI on stable-2.6
+        id: dispatch_stable_26
+        continue-on-error: true
+        run: gh workflow run ci.yml --ref stable-2.6
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+      - name: Check dispatch results
+        if: steps.dispatch_release_46.outcome == 'failure' || steps.dispatch_stable_26.outcome == 'failure'
+        run: |
+          echo "One or more dispatches failed:"
+          echo "  release_4.6: ${{ steps.dispatch_release_46.outcome }}"
+          echo "  stable-2.6: ${{ steps.dispatch_stable_26.outcome }}"
+          exit 1
+
   common-tests:
     name: ${{ matrix.tests.name }}
     runs-on: ubuntu-latest
@@ -62,7 +94,11 @@ jobs:
         run: |
           if [ -f "reports/coverage.xml" ]; then
             sed -i '2i<!-- PR ${{ github.event.pull_request.number }} -->' reports/coverage.xml
-            echo "Injected PR number ${{ github.event.pull_request.number }} into coverage.xml"
+            echo "Injected PR number ${{ github.event.pull_request.number }} into reports/coverage.xml"
+          fi
+          if [ -f "awxkit/coverage.xml" ]; then
+            sed -i '2i<!-- PR ${{ github.event.pull_request.number }} -->' awxkit/coverage.xml
+            echo "Injected PR number ${{ github.event.pull_request.number }} into awxkit/coverage.xml"
           fi
 
       - name: Upload test coverage to Codecov
@@ -109,28 +145,32 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.tests.name }}-artifacts
-          path: reports/coverage.xml
+          path: |
+            reports/coverage.xml
+            awxkit/coverage.xml
           retention-days: 5
 
-      - name: Upload awx jUnit test reports
+      - name: >-
+          Upload ${{
+            matrix.tests.coverage-upload-name || 'awx'
+          }} jUnit test reports to the unified dashboard
         if: >-
           !cancelled()
           && steps.make-run.outputs.test-result-files != ''
           && github.event_name == 'push'
           && env.UPSTREAM_REPOSITORY_ID == github.repository_id
           && github.ref_name == github.event.repository.default_branch
-        run: |
-          for junit_file in $(echo '${{ steps.make-run.outputs.test-result-files }}' | sed 's/,/ /')
-          do
-              curl \
-                -v \
-                --user "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_USER }}:${{ secrets.PDE_ORG_RESULTS_UPLOAD_PASSWORD }}" \
-                --form "xunit_xml=@${junit_file}" \
-                --form "component_name=${{ matrix.tests.coverage-upload-name || 'awx' }}" \
-                --form "git_commit_sha=${{ github.sha }}" \
-                --form "git_repository_url=https://github.com/${{ github.repository }}" \
-                "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}/api/results/upload/"
-          done
+        uses: ansible/gh-action-record-test-results@3784db66a1b7fb3809999a7251c8a7203a7ffbe8
+        with:
+          aggregation-server-url: ${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}
+          http-auth-password: >-
+            ${{ secrets.PDE_ORG_RESULTS_UPLOAD_PASSWORD }}
+          http-auth-username: >-
+            ${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_USER }}
+          project-component-name: >-
+            ${{ matrix.tests.coverage-upload-name || 'awx' }}
+          test-result-files: >-
+            ${{ steps.make-run.outputs.test-result-files }}
 
   dev-env:
     runs-on: ubuntu-latest
@@ -294,18 +334,16 @@ jobs:
           && github.event_name == 'push'
           && env.UPSTREAM_REPOSITORY_ID == github.repository_id
           && github.ref_name == github.event.repository.default_branch
-        run: |
-          for junit_file in $(echo '${{ steps.make-run.outputs.test-result-files }}' | sed 's/,/ /')
-          do
-              curl \
-                -v \
-                --user "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_USER }}:${{ secrets.PDE_ORG_RESULTS_UPLOAD_PASSWORD }}" \
-                --form "xunit_xml=@${junit_file}" \
-                --form "component_name=awx" \
-                --form "git_commit_sha=${{ github.sha }}" \
-                --form "git_repository_url=https://github.com/${{ github.repository }}" \
-                "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}/api/results/upload/"
-          done
+        uses: ansible/gh-action-record-test-results@3784db66a1b7fb3809999a7251c8a7203a7ffbe8
+        with:
+          aggregation-server-url: ${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}
+          http-auth-password: >-
+            ${{ secrets.PDE_ORG_RESULTS_UPLOAD_PASSWORD }}
+          http-auth-username: >-
+            ${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_USER }}
+          project-component-name: awx
+          test-result-files: >-
+            ${{ steps.make-run.outputs.test-result-files }}
 
   collection-integration:
     name: awx_collection integration

.github/workflows/devel_images.yml

@@ -13,6 +13,10 @@ on:
       - stable-*
 jobs:
   push-development-images:
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      (github.repository == 'ansible/awx' && (github.ref_name == 'devel' || startsWith(github.ref_name, 'feature_'))) ||
+      (github.repository == 'ansible/tower' && (startsWith(github.ref_name, 'stable-') || startsWith(github.ref_name, 'release_')))
     runs-on: ubuntu-latest
     timeout-minutes: 120
     permissions:
@@ -30,12 +34,6 @@ jobs:
             make-target: awx-kube-buildx
     steps:
 
-      - name: Skipping build of awx image for non-awx repository
-        run: |
-          echo "Skipping build of awx image for non-awx repository"
-          exit 0
-        if: matrix.build-targets.image-name == 'awx' && !endsWith(github.repository, '/awx')
-
       - uses: actions/checkout@v4
         with:
           show-progress: false

.github/workflows/pr_body_check.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     permissions:
-      packages: write
+      packages: read
       contents: read
     steps:
       - name: Check for each of the lines

.github/workflows/spec-sync-on-merge.yml

@@ -0,0 +1,182 @@
+# Sync OpenAPI Spec on Merge
+#
+# This workflow runs when code is merged to the devel branch.
+# It runs the dev environment to generate the OpenAPI spec, then syncs it to
+# the central spec repository.
+#
+# FLOW: PR merged → push to branch → dev environment runs → spec synced to central repo
+#
+# NOTE: This is an inlined version for testing with private forks.
+# Production version will use a reusable workflow from the org repos.
+name: Sync OpenAPI Spec on Merge
+env:
+  LC_ALL: "C.UTF-8"
+  DEV_DOCKER_OWNER: ${{ github.repository_owner }}
+on:
+  push:
+    branches:
+      - devel
+      - 'stable-2.[6-9]'
+      - 'stable-2.[1-9][0-9]'
+  workflow_dispatch: # Allow manual triggering for testing
+jobs:
+  sync-openapi-spec:
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      (github.repository == 'ansible/awx' && (github.ref_name == 'devel' || startsWith(github.ref_name, 'feature_'))) ||
+      (github.repository == 'ansible/tower' && (startsWith(github.ref_name, 'stable-') || startsWith(github.ref_name, 'release_')))
+    name: Sync OpenAPI spec to central repo
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Checkout Controller repository
+        uses: actions/checkout@v4
+        with:
+          show-progress: false
+
+      - name: Build awx_devel image to use for schema gen
+        uses: ./.github/actions/awx_devel_image
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          private-github-key: ${{ secrets.PRIVATE_GITHUB_KEY }}
+
+      - name: Generate API Schema
+        run: |
+          DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} \
+          COMPOSE_TAG=${{ github.base_ref || github.ref_name }} \
+          docker run -u $(id -u) --rm -v ${{ github.workspace }}:/awx_devel/:Z \
+            --workdir=/awx_devel `make print-DEVEL_IMAGE_NAME` /start_tests.sh genschema
+
+      - name: Verify spec file exists
+        run: |
+          SPEC_FILE="./schema.json"
+          if [ ! -f "$SPEC_FILE" ]; then
+            echo "❌ Spec file not found at $SPEC_FILE"
+            echo "Contents of workspace:"
+            ls -la .
+            exit 1
+          fi
+          echo "✅ Found spec file at $SPEC_FILE"
+
+      - name: Checkout spec repo
+        id: checkout_spec_repo
+        continue-on-error: true
+        uses: actions/checkout@v4
+        with:
+          repository: ansible-automation-platform/aap-openapi-specs
+          ref: ${{ github.ref_name }}
+          path: spec-repo
+          token: ${{ secrets.OPENAPI_SPEC_SYNC_TOKEN }}
+
+      - name: Fail if branch doesn't exist
+        if: steps.checkout_spec_repo.outcome == 'failure'
+        run: |
+          echo "##[error]❌ Branch '${{ github.ref_name }}' does not exist in the central spec repository."
+          echo "##[error]Expected branch: ${{ github.ref_name }}"
+          echo "##[error]This branch must be created in the spec repo before specs can be synced."
+          exit 1
+
+      - name: Compare specs
+        id: compare
+        run: |
+          COMPONENT_SPEC="./schema.json"
+          SPEC_REPO_FILE="spec-repo/controller.json"
+
+          # Check if spec file exists in spec repo
+          if [ ! -f "$SPEC_REPO_FILE" ]; then
+            echo "Spec file doesn't exist in spec repo - will create new file"
+            echo "has_diff=true" >> $GITHUB_OUTPUT
+            echo "is_new_file=true" >> $GITHUB_OUTPUT
+          else
+            # Compare files
+            if diff -q "$COMPONENT_SPEC" "$SPEC_REPO_FILE" > /dev/null; then
+              echo "✅ No differences found - specs are identical"
+              echo "has_diff=false" >> $GITHUB_OUTPUT
+            else
+              echo "📝 Differences found - spec has changed"
+              echo "has_diff=true" >> $GITHUB_OUTPUT
+              echo "is_new_file=false" >> $GITHUB_OUTPUT
+            fi
+          fi
+
+      - name: Update spec file
+        if: steps.compare.outputs.has_diff == 'true'
+        run: |
+          cp "./schema.json" "spec-repo/controller.json"
+          echo "✅ Updated spec-repo/controller.json"
+
+      - name: Create PR in spec repo
+        if: steps.compare.outputs.has_diff == 'true'
+        working-directory: spec-repo
+        env:
+          GH_TOKEN: ${{ secrets.OPENAPI_SPEC_SYNC_TOKEN }}
+          COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
+        run: |
+          # Configure git
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Create branch for PR
+          SHORT_SHA="${{ github.sha }}"
+          SHORT_SHA="${SHORT_SHA:0:7}"
+          BRANCH_NAME="update-Controller-${{ github.ref_name }}-${SHORT_SHA}"
+          git checkout -b "$BRANCH_NAME"
+
+          # Add and commit changes
+          git add "controller.json"
+
+          if [ "${{ steps.compare.outputs.is_new_file }}" == "true" ]; then
+            COMMIT_MSG="Add Controller OpenAPI spec for ${{ github.ref_name }}"
+          else
+            COMMIT_MSG="Update Controller OpenAPI spec for ${{ github.ref_name }}"
+          fi
+
+          git commit -m "$COMMIT_MSG
+
+          Synced from ${{ github.repository }}@${{ github.sha }}
+          Source branch: ${{ github.ref_name }}
+
+          Co-Authored-By: github-actions[bot] <github-actions[bot]@users.noreply.github.com>"
+
+          # Push branch
+          git push origin "$BRANCH_NAME"
+
+          # Create PR
+          PR_TITLE="[${{ github.ref_name }}] Update Controller spec from merged commit"
+          PR_BODY="## Summary
+          Automated OpenAPI spec sync from component repository merge.
+
+          **Source:** ${{ github.repository }}@${{ github.sha }}
+          **Branch:** \`${{ github.ref_name }}\`
+          **Component:** \`Controller\`
+          **Spec File:** \`controller.json\`
+
+          ## Changes
+          $(if [ "${{ steps.compare.outputs.is_new_file }}" == "true" ]; then echo "- 🆕 New spec file created"; else echo "- 📝 Spec file updated with latest changes"; fi)
+
+          ## Source Commit
+          \`\`\`
+          ${COMMIT_MESSAGE}
+          \`\`\`
+
+          ---
+          🤖 This PR was automatically generated by the OpenAPI spec sync workflow."
+
+          gh pr create \
+            --title "$PR_TITLE" \
+            --body "$PR_BODY" \
+            --base "${{ github.ref_name }}" \
+            --head "$BRANCH_NAME"
+
+          echo "✅ Created PR in spec repo"
+
+      - name: Report results
+        if: always()
+        run: |
+          if [ "${{ steps.compare.outputs.has_diff }}" == "true" ]; then
+            echo "📝 Spec sync completed - PR created in spec repo"
+          else
+            echo "✅ Spec sync completed - no changes needed"
+          fi

.github/workflows/upload_schema.yml

@@ -14,6 +14,10 @@ on:
       - stable-**
 jobs:
   push:
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      (github.repository == 'ansible/awx' && (github.ref_name == 'devel' || startsWith(github.ref_name, 'feature_'))) ||
+      (github.repository == 'ansible/tower' && (startsWith(github.ref_name, 'stable-') || startsWith(github.ref_name, 'release_')))
     runs-on: ubuntu-latest
     timeout-minutes: 60
     permissions:

.tekton/run-atf-tests-pull-request.yaml

@@ -0,0 +1,65 @@
+---
+apiVersion: tekton.dev/v1
+kind: PipelineRun
+metadata:
+  name: awx-atf-tests-pull-request
+  annotations:
+    build.appstudio.openshift.io/repo: https://github.com/{{repo_owner}}/{{repo_name}}?rev={{revision}}
+    build.appstudio.redhat.com/commit_sha: '{{revision}}'
+    build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}'
+    build.appstudio.redhat.com/target_branch: '{{target_branch}}'
+    pipelinesascode.tekton.dev/cancel-in-progress: 'true'
+    pipelinesascode.tekton.dev/max-keep-runs: "3"
+    pipelinesascode.tekton.dev/on-comment: "^/run-atf-tests$"
+    pipelinesascode.tekton.dev/target-namespace: ansible-ci-tenant
+  labels:
+    appstudio.openshift.io/application: '{{repo_owner}}'
+    appstudio.openshift.io/component: '{{repo_owner}}-{{repo_name}}'
+    pipelines.appstudio.openshift.io/type: build
+spec:
+  timeouts:
+    pipeline: "8h"
+    tasks: "7h"
+    finally: "1h"
+  pipelineRef:
+    resolver: bundles
+    params:
+      - name: name
+        value: aap-api-tests
+      - name: bundle
+        value: quay.io/aap-ci/tekton-catalog/pipeline/test/aap-api-tests:0.1@sha256:50aadd6725a239ab53247deb7cf601d1163ceb1792792fd239a3f37d21a490d7
+      - name: kind
+        value: pipeline
+      - name: secret
+        value: quay-aap-ci-viewer
+
+  taskRunTemplate:
+    serviceAccountName: konflux-integration-runner
+
+  params:
+    - name: git-url
+      value: "{{source_url}}"
+    - name: pipeline-github-org
+      value: "{{repo_owner}}"
+    - name: pipeline-github-repo
+      value: "{{repo_name}}"
+    - name: pipeline-github-target-branch
+      value: '{{target_branch}}'
+    - name: pipeline-github-pr-revision
+      value: "{{revision}}"
+    - name: pipeline-github-pr-number
+      value: "{{pull_request_number}}"
+    - name: aap-dev-component-source-name
+      value: "controller"
+    - name: pytest-number-of-parallel-processes
+      value: "6"
+
+  workspaces:
+    - name: workspace
+      volumeClaimTemplate:
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 1Gi

CONTRIBUTING.md

@@ -103,6 +103,12 @@ When necessary, remove any AWX containers and images by running the following:
 
 ### Pre commit hooks
 
+Install the pre-commit hook before contributing:
+
+```
+make pre-commit
+```
+
 When you attempt to perform a `git commit` there will be a pre-commit hook that gets run before the commit is allowed to your local repository. For example, python's [black](https://pypi.org/project/black/) will be run to test the formatting of any python files.
 
 While you can use environment variables to skip the pre-commit hooks GitHub will run similar tests and prevent merging of PRs if the tests do not pass.

Makefile

@@ -1,6 +1,6 @@
 -include awx/ui/Makefile
 
-PYTHON := $(notdir $(shell for i in python3.12 python3; do command -v $$i; done|sed 1q))
+PYTHON := $(notdir $(shell for i in python3.12 python3.11 python3; do command -v $$i; done|sed 1q))
 SHELL := bash
 DOCKER_COMPOSE ?= docker compose
 OFFICIAL ?= no
@@ -10,6 +10,7 @@ KIND_BIN ?= $(shell which kind)
 CHROMIUM_BIN=/tmp/chrome-linux/chrome
 GIT_REPO_NAME ?= $(shell basename `git rev-parse --show-toplevel`)
 GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
+GIT_IS_WORKTREE := $(shell test -f .git && echo yes)
 MANAGEMENT_COMMAND ?= awx-manage
 VERSION ?= $(shell $(PYTHON) tools/scripts/scm_version.py 2> /dev/null)
 
@@ -79,7 +80,7 @@ RECEPTOR_IMAGE ?= quay.io/ansible/receptor:devel
 SRC_ONLY_PKGS ?= cffi,pycparser,psycopg,twilio
 # These should be upgraded in the AWX and Ansible venv before attempting
 # to install the actual requirements
-VENV_BOOTSTRAP ?= pip==25.3 setuptools==80.9.0 setuptools_scm[toml]==9.2.2 wheel==0.45.1 cython==3.1.3
+VENV_BOOTSTRAP ?= pip==25.3 setuptools==80.9.0 setuptools_scm[toml]==9.2.2 wheel==0.46.3 cython==3.1.3
 
 NAME ?= awx
 
@@ -106,6 +107,15 @@ else
  DOCKER_KUBE_CACHE_FLAG=$(DOCKER_CACHE)
 endif
 
+# AWX TUI variables
+AWX_HOST ?= https://localhost:8043
+AWX_USER ?= admin
+AWX_PASSWORD ?= $$(awk -F"'" '/^admin_password:/{print $$2}' tools/docker-compose/_sources/secrets/admin_password.yml 2>/dev/null || echo "admin")
+AWX_VERIFY_SSL ?= false
+
+# For git worktree to find the referenced git dir
+GIT_COMMON_DIR := $(shell git rev-parse --git-common-dir 2>/dev/null || echo .git)
+
 .PHONY: awx-link clean clean-tmp clean-venv requirements requirements_dev \
 	update_requirements upgrade_requirements update_requirements_dev \
 	docker_update_requirements docker_upgrade_requirements docker_update_requirements_dev \
@@ -113,7 +123,7 @@ endif
 	receiver test test_unit test_coverage coverage_html \
 	sdist \
 	VERSION PYTHON_VERSION docker-compose-sources \
-	.git/hooks/pre-commit
+	pre-commit
 
 clean-tmp:
 	rm -rf tmp/
@@ -289,7 +299,7 @@ dispatcher:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	$(PYTHON) manage.py run_dispatcher
+	$(PYTHON) manage.py dispatcherd
 
 ## Run to start the zeromq callback receiver
 receiver:
@@ -342,11 +352,10 @@ black: reports
 	@command -v black >/dev/null 2>&1 || { echo "could not find black on your PATH, you may need to \`pip install black\`, or set AWX_IGNORE_BLACK=1" && exit 1; }
 	@(set -o pipefail && $@ $(BLACK_ARGS) awx awxkit awx_collection | tee reports/$@.report)
 
-.git/hooks/pre-commit:
-	@echo "if [ -x pre-commit.sh ]; then" > .git/hooks/pre-commit
-	@echo "    ./pre-commit.sh;" >> .git/hooks/pre-commit
-	@echo "fi" >> .git/hooks/pre-commit
-	@chmod +x .git/hooks/pre-commit
+$(GIT_COMMON_DIR)/hooks/pre-commit:
+	ln -sf ../../pre-commit.sh $(GIT_COMMON_DIR)/hooks/pre-commit
+
+pre-commit: $(GIT_COMMON_DIR)/hooks/pre-commit
 
 genschema: awx-link reports
 	@if [ "$(VENV_BASE)" ]; then \
@@ -521,7 +530,7 @@ ifneq ($(ADMIN_PASSWORD),)
 	EXTRA_SOURCES_ANSIBLE_OPTS := -e admin_password=$(ADMIN_PASSWORD) $(EXTRA_SOURCES_ANSIBLE_OPTS)
 endif
 
-docker-compose-sources: .git/hooks/pre-commit
+docker-compose-sources:
 	@if [ $(MINIKUBE_CONTAINER_GROUP) = true ]; then\
 	    $(ANSIBLE_PLAYBOOK) -i tools/docker-compose/inventory -e minikube_setup=$(MINIKUBE_SETUP) tools/docker-compose-minikube/deploy.yml; \
 	fi;
@@ -553,7 +562,7 @@ docker-compose: awx/projects docker-compose-sources
 	$(MAKE) docker-compose-up
 
 docker-compose-up:
-	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans
+	$(if $(GIT_IS_WORKTREE),SETUPTOOLS_SCM_PRETEND_VERSION="$(VERSION)") $(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans
 
 docker-compose-down:
 	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) down --remove-orphans
@@ -571,6 +580,20 @@ docker-compose-runtest: awx/projects docker-compose-sources
 docker-compose-build-schema: awx/projects docker-compose-sources
 	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports --no-deps awx_1 make genschema
 
+awx-tui:
+	@if ! command -v awx-tui > /dev/null 2>&1; then \
+		$(PYTHON) -m pip install awx-tui; \
+	fi
+	@if [ -f "$(HOME)/.config/awx-tui/config.yaml" ]; then \
+		$(PYTHON) -m awx_tui.main; \
+	else \
+		AWX_HOST=$(AWX_HOST) \
+		AWX_USER=$(AWX_USER) \
+		AWX_PASSWORD=$(AWX_PASSWORD) \
+		AWX_VERIFY_SSL=$(AWX_VERIFY_SSL) \
+		$(PYTHON) -m awx_tui.main --host $(AWX_HOST); \
+	fi
+
 SCHEMA_DIFF_BASE_FOLDER ?= awx
 SCHEMA_DIFF_BASE_BRANCH ?= devel
 detect-schema-change: genschema
@@ -579,6 +602,10 @@ detect-schema-change: genschema
 	# diff exits with 1 when files differ - capture but don't fail
 	-diff -u -b reference-schema.json schema.json
 
+validate-openapi-schema: genschema
+	@echo "Validating OpenAPI schema from schema.json..."
+	@python3 -c "from openapi_spec_validator import validate; import json; spec = json.load(open('schema.json')); validate(spec); print('✓ Schema is valid')"
+
 docker-compose-clean: awx/projects
 	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml rm -sf
 

awx/__init__.py

@@ -52,14 +52,6 @@ except ImportError:  # pragma: no cover
     MODE = 'production'
 
 
-try:
-    import django  # noqa: F401
-except ImportError:
-    pass
-else:
-    from django.db import connection
-
-
 def prepare_env():
     # Update the default settings environment variable based on current mode.
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings')
@@ -79,14 +71,6 @@ def manage():
     from django.conf import settings
     from django.core.management import execute_from_command_line
 
-    # enforce the postgres version is a minimum of 12 (we need this for partitioning); if not, then terminate program with exit code of 1
-    # In the future if we require a feature of a version of postgres > 12 this should be updated to reflect that.
-    # The return of connection.pg_version is something like 12013
-    if not os.getenv('SKIP_PG_VERSION_CHECK', False) and not MODE == 'development':
-        if (connection.pg_version // 10000) < 12:
-            sys.stderr.write("At a minimum, postgres version 12 is required\n")
-            sys.exit(1)
-
     if len(sys.argv) >= 2 and sys.argv[1] in ('version', '--version'):  # pragma: no cover
         sys.stdout.write('%s\n' % __version__)
     # If running as a user without permission to read settings, display an

awx/api/fields.py

@@ -89,7 +89,7 @@ class DeprecatedCredentialField(serializers.IntegerField):
     def to_internal_value(self, pk):
         try:
             pk = int(pk)
-        except ValueError:
+        except (ValueError, TypeError):
             self.fail('invalid')
         try:
             Credential.objects.get(pk=pk)

awx/api/generics.py

@@ -131,8 +131,14 @@ class LoggedLoginView(auth_views.LoginView):
 
 
 class LoggedLogoutView(auth_views.LogoutView):
+    # Override http_method_names to allow GET requests (Django 5.2+ defaults to POST only)
+    http_method_names = ["get", "post", "options"]
     success_url_allowed_hosts = set(settings.LOGOUT_ALLOWED_HOSTS.split(",")) if settings.LOGOUT_ALLOWED_HOSTS else set()
 
+    def get(self, request, *args, **kwargs):
+        """Handle GET requests for logout (for backward compatibility)."""
+        return self.post(request, *args, **kwargs)
+
     def dispatch(self, request, *args, **kwargs):
         if is_proxied_request():
             # 1) We intentionally don't obey ?next= here, just always redirect to platform login
@@ -266,7 +272,10 @@ class APIView(views.APIView):
                 response = self.handle_exception(self.__init_request_error__)
             if response.status_code == 401:
                 if response.data and 'detail' in response.data:
-                    response.data['detail'] += _(' To establish a login session, visit') + ' /api/login/.'
+                    if getattr(settings, 'RESOURCE_SERVER__URL', None):
+                        response.data['detail'] += _(' Direct access is not allowed, authenticate via the platform gateway.')
+                    else:
+                        response.data['detail'] += _(' To establish a login session, visit') + ' /api/login/.'
                 logger.info(status_msg)
             else:
                 logger.warning(status_msg)

awx/api/pagination.py

@@ -111,7 +111,7 @@ class UnifiedJobEventPagination(Pagination):
     def __init__(self, *args, **kwargs):
         self.use_limit_paginator = False
         self.limit_pagination = LimitPagination()
-        return super().__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
     def paginate_queryset(self, queryset, request, view=None):
         if 'limit' in request.query_params:

awx/api/schema.py

@@ -9,6 +9,50 @@ from drf_spectacular.views import (
 )
 
 
+def filter_credential_type_schema(
+    result,
+    generator,  # NOSONAR
+    request,  # NOSONAR
+    public,  # NOSONAR
+):
+    """
+    Postprocessing hook to filter CredentialType kind enum values.
+
+    For CredentialTypeRequest and PatchedCredentialTypeRequest schemas (POST/PUT/PATCH),
+    filter the 'kind' enum to only show 'cloud' and 'net' values.
+
+    This ensures the OpenAPI schema accurately reflects that only 'cloud' and 'net'
+    credential types can be created or modified via the API, matching the validation
+    in CredentialTypeSerializer.validate().
+
+    Args:
+        result: The OpenAPI schema dict to be modified
+        generator, request, public: Required by drf-spectacular interface (unused)
+
+    Returns:
+        The modified OpenAPI schema dict
+    """
+    schemas = result.get('components', {}).get('schemas', {})
+
+    # Filter CredentialTypeRequest (POST/PUT) - field is required
+    if 'CredentialTypeRequest' in schemas:
+        kind_prop = schemas['CredentialTypeRequest'].get('properties', {}).get('kind', {})
+        if 'enum' in kind_prop:
+            # Filter to only cloud and net (no None - field is required)
+            kind_prop['enum'] = ['cloud', 'net']
+            kind_prop['description'] = "* `cloud` - Cloud\\n* `net` - Network"
+
+    # Filter PatchedCredentialTypeRequest (PATCH) - field is optional
+    if 'PatchedCredentialTypeRequest' in schemas:
+        kind_prop = schemas['PatchedCredentialTypeRequest'].get('properties', {}).get('kind', {})
+        if 'enum' in kind_prop:
+            # Filter to only cloud and net (None allowed - field can be omitted in PATCH)
+            kind_prop['enum'] = ['cloud', 'net', None]
+            kind_prop['description'] = "* `cloud` - Cloud\\n* `net` - Network"
+
+    return result
+
+
 class CustomAutoSchema(AutoSchema):
     """Custom AutoSchema to add swagger_topic to tags and handle deprecated endpoints."""
 

awx/api/serializers.py

@@ -120,8 +120,7 @@ from awx.main.utils.named_url_graph import reset_counters
 from awx.main.utils.inventory_vars import update_group_variables
 from awx.main.scheduler.task_manager_models import TaskManagerModels
 from awx.main.redact import UriCleaner, REPLACE_STR
-from awx.main.signals import update_inventory_computed_fields
-
+from awx.main.tasks.system import update_inventory_computed_fields
 
 from awx.main.validators import vars_validate_or_raise
 
@@ -175,8 +174,8 @@ SUMMARIZABLE_FK_FIELDS = {
     'workflow_approval': DEFAULT_SUMMARY_FIELDS + ('timeout',),
     'schedule': DEFAULT_SUMMARY_FIELDS + ('next_run',),
     'unified_job_template': DEFAULT_SUMMARY_FIELDS + ('unified_job_type',),
-    'last_job': DEFAULT_SUMMARY_FIELDS + ('finished', 'status', 'failed', 'license_error', 'canceled_on'),
-    'last_job_host_summary': DEFAULT_SUMMARY_FIELDS + ('failed',),
+    # last_job and last_job_host_summary are derived from JobHostSummary in HostSerializer,
+    # not from the stale FK fields on Host.
     'last_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'license_error'),
     'current_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'license_error'),
     'current_job': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'license_error'),
@@ -1022,7 +1021,7 @@ class UnifiedJobStdoutSerializer(UnifiedJobSerializer):
 
 
 class UserSerializer(BaseSerializer):
-    password = serializers.CharField(required=False, default='', help_text=_('Field used to change the password.'))
+    password = serializers.CharField(required=False, default='', allow_blank=True, help_text=_('Field used to change the password.'))
     is_system_auditor = serializers.BooleanField(default=False)
     show_capabilities = ['edit', 'delete']
 
@@ -1230,7 +1229,7 @@ class OrganizationSerializer(BaseSerializer, OpaQueryPathMixin):
         # to a team. This provides a hint to the ui so it can know to not
         # display these roles for team role selection.
         for key in ('admin_role', 'member_role'):
-            if key in summary_dict.get('object_roles', {}):
+            if summary_dict and key in summary_dict.get('object_roles', {}):
                 summary_dict['object_roles'][key]['user_only'] = True
 
         return summary_dict
@@ -1838,19 +1837,35 @@ class HostSerializer(BaseSerializerWithVariables):
             res['ansible_facts'] = self.reverse('api:host_ansible_facts_detail', kwargs={'pk': obj.instance_id})
         if obj.inventory:
             res['inventory'] = self.reverse('api:inventory_detail', kwargs={'pk': obj.inventory.pk})
-        if obj.last_job:
-            res['last_job'] = self.reverse('api:job_detail', kwargs={'pk': obj.last_job.pk})
-        if obj.last_job_host_summary:
-            res['last_job_host_summary'] = self.reverse('api:job_host_summary_detail', kwargs={'pk': obj.last_job_host_summary.pk})
+        last_summary = obj.latest_summary
+        if last_summary:
+            res['last_job_host_summary'] = self.reverse('api:job_host_summary_detail', kwargs={'pk': last_summary.pk})
+            if last_summary.job_id:
+                res['last_job'] = self.reverse('api:job_detail', kwargs={'pk': last_summary.job_id})
         return res
 
     def get_summary_fields(self, obj):
         d = super(HostSerializer, self).get_summary_fields(obj)
-        try:
-            d['last_job']['job_template_id'] = obj.last_job.job_template.id
-            d['last_job']['job_template_name'] = obj.last_job.job_template.name
-        except (KeyError, AttributeError):
-            pass
+        last_summary = obj.latest_summary
+        if last_summary:
+            d['last_job_host_summary'] = OrderedDict()
+            d['last_job_host_summary']['id'] = last_summary.id
+            d['last_job_host_summary']['failed'] = last_summary.failed
+            try:
+                last_job = last_summary.job
+                d['last_job'] = OrderedDict()
+                for field in DEFAULT_SUMMARY_FIELDS + ('finished', 'status', 'failed', 'canceled_on'):
+                    fval = getattr(last_job, field, None)
+                    if fval is not None:
+                        d['last_job'][field] = fval
+                if last_job.job_template:
+                    d['last_job']['job_template_id'] = last_job.job_template.id
+                    d['last_job']['job_template_name'] = last_job.job_template.name
+            except ObjectDoesNotExist:
+                pass
+        else:
+            d.pop('last_job', None)
+            d.pop('last_job_host_summary', None)
         if has_model_field_prefetched(obj, 'groups'):
             group_list = sorted([{'id': g.id, 'name': g.name} for g in obj.groups.all()], key=lambda x: x['id'])[:5]
         else:
@@ -1925,14 +1940,16 @@ class HostSerializer(BaseSerializerWithVariables):
             return ret
         if 'inventory' in ret and not obj.inventory:
             ret['inventory'] = None
-        if 'last_job' in ret and not obj.last_job:
-            ret['last_job'] = None
-        if 'last_job_host_summary' in ret and not obj.last_job_host_summary:
-            ret['last_job_host_summary'] = None
+        last_summary = obj.latest_summary
+        if 'last_job' in ret:
+            ret['last_job'] = last_summary.job_id if last_summary else None
+        if 'last_job_host_summary' in ret:
+            ret['last_job_host_summary'] = last_summary.pk if last_summary else None
         return ret
 
     def get_has_active_failures(self, obj):
-        return bool(obj.last_job_host_summary and obj.last_job_host_summary.failed)
+        last_summary = obj.latest_summary
+        return bool(last_summary and last_summary.failed)
 
     def get_has_inventory_sources(self, obj):
         return obj.inventory_sources.exists()
@@ -2079,9 +2096,17 @@ class BulkHostCreateSerializer(serializers.Serializer):
         if request and not request.user.is_superuser:
             if request.user not in inv.admin_role:
                 raise serializers.ValidationError(_(f'Inventory with id {inv.id} not found or lack permissions to add hosts.'))
-        current_hostnames = set(inv.hosts.values_list('name', flat=True))
+
+        # Performance optimization (AAP-67978): Instead of loading ALL host names from
+        # the inventory, only check if the specific new names already exist in the database.
         new_names = [host['name'] for host in attrs['hosts']]
-        duplicate_new_names = [n for n in new_names if n in current_hostnames or new_names.count(n) > 1]
+
+        new_name_counts = Counter(new_names)
+        duplicates_in_new = [name for name, count in new_name_counts.items() if count > 1]
+        unique_new_names = list(new_name_counts.keys())
+        existing_duplicates = list(Host.objects.filter(inventory=inv, name__in=unique_new_names).values_list('name', flat=True))
+        duplicate_new_names = list(set(duplicates_in_new + existing_duplicates))
+
         if duplicate_new_names:
             raise serializers.ValidationError(_(f'Hostnames must be unique in an inventory. Duplicates found: {duplicate_new_names}'))
 
@@ -2165,13 +2190,13 @@ class BulkHostDeleteSerializer(serializers.Serializer):
         attrs['hosts_data'] = attrs['host_qs'].values()
 
         if len(attrs['host_qs']) == 0:
-            error_hosts = {host: "Hosts do not exist or you lack permission to delete it" for host in attrs['hosts']}
+            error_hosts = dict.fromkeys(attrs['hosts'], "Hosts do not exist or you lack permission to delete it")
             raise serializers.ValidationError({'hosts': error_hosts})
 
         if len(attrs['host_qs']) < len(attrs['hosts']):
             hosts_exists = [host['id'] for host in attrs['hosts_data']]
             failed_hosts = list(set(attrs['hosts']).difference(hosts_exists))
-            error_hosts = {host: "Hosts do not exist or you lack permission to delete it" for host in failed_hosts}
+            error_hosts = dict.fromkeys(failed_hosts, "Hosts do not exist or you lack permission to delete it")
             raise serializers.ValidationError({'hosts': error_hosts})
 
         # Getting all inventories that the hosts can be in
@@ -2932,6 +2957,19 @@ class CredentialTypeSerializer(BaseSerializer):
                 field['label'] = _(field['label'])
                 if 'help_text' in field:
                     field['help_text'] = _(field['help_text'])
+
+        # Deep copy inputs to avoid modifying the original model data
+        inputs = value.get('inputs')
+        if not isinstance(inputs, dict):
+            inputs = {}
+        value['inputs'] = copy.deepcopy(inputs)
+        fields = value['inputs'].get('fields', [])
+        if not isinstance(fields, list):
+            fields = []
+
+        # Normalize fields and filter out internal fields
+        value['inputs']['fields'] = [f for f in fields if not f.get('internal')]
+
         return value
 
     def filter_field_metadata(self, fields, method):
@@ -3527,7 +3565,7 @@ class JobRelaunchSerializer(BaseSerializer):
         choices=NEW_JOB_TYPE_CHOICES,
         write_only=True,
     )
-    credential_passwords = VerbatimField(required=True, write_only=True)
+    credential_passwords = VerbatimField(required=False, write_only=True)
 
     class Meta:
         model = Job
@@ -4122,9 +4160,28 @@ class LaunchConfigurationBaseSerializer(BaseSerializer):
                             attrs['extra_data'][key] = db_extra_data[key]
 
         # Build unsaved version of this config, use it to detect prompts errors
+        # Capture keys before _build_mock_obj pops pseudo-fields from attrs
+        incoming_attr_keys = set(attrs.keys())
         mock_obj = self._build_mock_obj(attrs)
-        if set(list(ujt.get_ask_mapping().keys()) + ['extra_data']) & set(attrs.keys()):
-            accepted, rejected, errors = ujt._accept_or_ignore_job_kwargs(_exclude_errors=self.exclude_errors, **mock_obj.prompts_dict())
+        ask_mapping_keys = set(ujt.get_ask_mapping().keys())
+        requested_prompt_fields = incoming_attr_keys & ask_mapping_keys
+        if 'extra_data' in incoming_attr_keys:
+            requested_prompt_fields.add('extra_vars')
+            requested_prompt_fields.add('survey_passwords')
+
+        # prompts_dict() pulls persisted M2M state (labels, credentials,
+        # instance_groups) via the instance pk.  Only re-validate the full prompt
+        # state when the caller is switching the underlying template; otherwise
+        # restrict validation to the fields the request explicitly provided.
+        if 'unified_job_template' in attrs:
+            prompts_to_validate = mock_obj.prompts_dict()
+        elif requested_prompt_fields:
+            prompts_to_validate = {k: v for k, v in mock_obj.prompts_dict().items() if k in requested_prompt_fields}
+        else:
+            prompts_to_validate = None
+
+        if prompts_to_validate is not None:
+            accepted, rejected, errors = ujt._accept_or_ignore_job_kwargs(_exclude_errors=self.exclude_errors, **prompts_to_validate)
         else:
             # Only perform validation of prompts if prompts fields are provided
             errors = {}
@@ -5393,7 +5450,11 @@ class SchedulePreviewSerializer(BaseSerializer):
         for a_rule in match_multiple_rrule:
             if 'interval' not in a_rule.lower():
                 errors.append("{0}: {1}".format(_('INTERVAL required in rrule'), a_rule))
-            elif 'secondly' in a_rule.lower():
+            else:
+                match_interval = re.match(r".*?INTERVAL=([0-9]+)", a_rule)
+                if match_interval and int(match_interval.group(1)) < 1:
+                    errors.append("{0}: {1}".format(_("INTERVAL must be a positive integer"), a_rule))
+            if 'secondly' in a_rule.lower():
                 errors.append("{0}: {1}".format(_('SECONDLY is not supported'), a_rule))
             if re.match(by_day_with_numeric_prefix, a_rule):
                 errors.append("{0}: {1}".format(_("BYDAY with numeric prefix not supported"), a_rule))

awx/api/templates/api/stdout.html

@@ -1,6 +1,6 @@
 {% if content_only %}<div class="nocode ansi_fore ansi_back{% if dark %} ansi_dark{% endif %}">{% else %}
 <!DOCTYPE HTML>
-<html>
+<html lang="en">
   <head>
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
     <title>{{ title }}</title>

awx/api/templates/instance_install_bundle/requirements.yml

@@ -1,4 +1,4 @@
 ---
 collections:
   - name: ansible.receptor
-    version: 2.0.6
+    version: 2.0.8

awx/api/views/__init__.py

@@ -14,13 +14,14 @@ import sys
 import time
 from base64 import b64encode
 from collections import OrderedDict
+from jwt import decode as _jwt_decode
 
 from urllib3.exceptions import ConnectTimeoutError
 
 # Django
 from django.conf import settings
 from django.core.exceptions import FieldError, ObjectDoesNotExist
-from django.db.models import Q, Sum, Count
+from django.db.models import Q, Sum, Count, Subquery, OuterRef
 from django.db import IntegrityError, ProgrammingError, transaction, connection
 from django.db.models.fields.related import ManyToManyField, ForeignKey
 from django.db.models.functions import Trunc
@@ -52,13 +53,19 @@ from ansi2html import Ansi2HTMLConverter
 
 from datetime import timezone as dt_timezone
 from wsgiref.util import FileWrapper
+from drf_spectacular.utils import extend_schema_view, extend_schema
 
 # django-ansible-base
 from ansible_base.lib.utils.requests import get_remote_hosts
 from ansible_base.rbac.models import RoleEvaluation
 from ansible_base.lib.utils.schema import extend_schema_if_available
+from ansible_base.lib.workload_identity.controller import AutomationControllerJobScope
+
+# flags
+from flags.state import flag_enabled
 
 # AWX
+from awx.main.utils.workload_identity import retrieve_workload_identity_jwt_with_claims
 from awx.main.tasks.system import send_notifications, update_inventory_computed_fields
 from awx.main.access import get_user_queryset
 from awx.api.generics import (
@@ -202,11 +209,12 @@ class DashboardView(APIView):
         groups_inventory_failed = models.Group.objects.filter(inventory_sources__last_job_failed=True).count()
         data['groups'] = {'url': reverse('api:group_list', request=request), 'total': user_groups.count(), 'inventory_failed': groups_inventory_failed}
 
-        user_hosts = get_user_queryset(request.user, models.Host)
-        user_hosts_failed = user_hosts.filter(last_job_host_summary__failed=True)
+        user_hosts = get_user_queryset(request.user, models.Host).exclude(inventory__kind='constructed')
+        latest_summary_failed = Subquery(models.JobHostSummary.objects.filter(host_id=OuterRef('pk')).order_by('-id').values('failed')[:1])
+        user_hosts_failed = user_hosts.annotate(_latest_failed=latest_summary_failed).filter(_latest_failed=True)
+
         data['hosts'] = {
             'url': reverse('api:host_list', request=request),
-            'failures_url': reverse('api:host_list', request=request) + "?last_job_host_summary__failed=True",
             'total': user_hosts.count(),
             'failed': user_hosts_failed.count(),
         }
@@ -378,6 +386,10 @@ class DashboardJobsGraphView(APIView):
 
 
 class InstanceList(ListCreateAPIView):
+    """
+    Creates an instance if used on a Kubernetes or OpenShift deployment of Ansible Automation Platform.
+    """
+
     name = _("Instances")
     model = models.Instance
     serializer_class = serializers.InstanceSerializer
@@ -789,22 +801,11 @@ class TeamRolesList(SubListAttachDetachAPIView):
             data = dict(msg=_("You cannot grant system-level permissions to a team."))
             return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-        team = get_object_or_404(models.Team, pk=self.kwargs['pk'])
-        credential_content_type = ContentType.objects.get_for_model(models.Credential)
-        if role.content_type == credential_content_type:
-            if not role.content_object.organization:
-                data = dict(
-                    msg=_("You cannot grant access to a credential that is not assigned to an organization (private credentials cannot be assigned to teams)")
-                )
-                return Response(data, status=status.HTTP_400_BAD_REQUEST)
-            elif role.content_object.organization.id != team.organization.id:
-                if not request.user.is_superuser:
-                    data = dict(
-                        msg=_(
-                            "You cannot grant a team access to a credential in a different organization. Only superusers can grant cross-organization credential access to teams"
-                        )
-                    )
-                    return Response(data, status=status.HTTP_400_BAD_REQUEST)
+        if not request.data.get('disassociate'):
+            team = get_object_or_404(models.Team, pk=self.kwargs['pk'])
+            content_object = role.content_object
+            if hasattr(content_object, 'validate_role_assignment'):
+                content_object.validate_role_assignment(team, role_definition=None, requesting_user=request.user)
 
         return super(TeamRolesList, self).post(request, *args, **kwargs)
 
@@ -1263,19 +1264,12 @@ class UserRolesList(SubListAttachDetachAPIView):
         if not sub_id:
             return super(UserRolesList, self).post(request)
 
-        user = get_object_or_400(models.User, pk=self.kwargs['pk'])
-        role = get_object_or_400(models.Role, pk=sub_id)
-
-        content_types = ContentType.objects.get_for_models(models.Organization, models.Team, models.Credential)  # dict of {model: content_type}
-        credential_content_type = content_types[models.Credential]
-        if role.content_type == credential_content_type:
-            if 'disassociate' not in request.data and role.content_object.organization and user not in role.content_object.organization.member_role:
-                data = dict(msg=_("You cannot grant credential access to a user not in the credentials' organization"))
-                return Response(data, status=status.HTTP_400_BAD_REQUEST)
-
-            if not role.content_object.organization and not request.user.is_superuser:
-                data = dict(msg=_("You cannot grant private credential access to another user"))
-                return Response(data, status=status.HTTP_400_BAD_REQUEST)
+        if not request.data.get('disassociate'):
+            role = get_object_or_400(models.Role, pk=sub_id)
+            user = get_object_or_400(models.User, pk=self.kwargs['pk'])
+            content_object = role.content_object
+            if hasattr(content_object, 'validate_role_assignment'):
+                content_object.validate_role_assignment(user, role_definition=None, requesting_user=request.user)
 
         return super(UserRolesList, self).post(request, *args, **kwargs)
 
@@ -1454,7 +1448,7 @@ class CredentialList(ListCreateAPIView):
 
     @extend_schema_if_available(
         extensions={
-            "x-ai-description": "Create a new credential. The `inputs` field contain type-specific input fields. The required fields depend on related `credential_type`. Use GET /v2/credential_types/{id}/ (tool name: controller.credential_types_retrieve) and inspect `inputs` field for the specific credential type's expected schema."
+            "x-ai-description": "Create a new credential. The `inputs` field contain type-specific input fields. The required fields depend on related `credential_type`. Use GET /v2/credential_types/{id}/ (tool name: controller.credential_types_retrieve) and inspect `inputs` field for the specific credential type's expected schema.  The fields `user` and `team` are deprecated and should not be included in the payload."
         }
     )
     def post(self, request, *args, **kwargs):
@@ -1590,7 +1584,175 @@ class CredentialCopy(CopyAPIView):
     resource_purpose = 'copy of a credential'
 
 
-class CredentialExternalTest(SubDetailAPIView):
+class OIDCCredentialTestMixin:
+    """
+    Mixin to add OIDC workload identity token support to credential test endpoints.
+
+    This mixin provides methods to handle OIDC-enabled external credentials that use
+    workload identity tokens for authentication.
+    """
+
+    @staticmethod
+    def _get_workload_identity_token(job_template: models.JobTemplate, audience: str) -> str:
+        """Generate a workload identity token for a job template.
+
+        Args:
+            job_template: The JobTemplate instance to generate claims for
+            audience: The JWT audience claim value
+
+        Returns:
+            str: The generated JWT token
+        """
+        claims = {
+            AutomationControllerJobScope.CLAIM_ORGANIZATION_NAME: job_template.organization.name,
+            AutomationControllerJobScope.CLAIM_ORGANIZATION_ID: job_template.organization.id,
+            AutomationControllerJobScope.CLAIM_PROJECT_NAME: job_template.project.name,
+            AutomationControllerJobScope.CLAIM_PROJECT_ID: job_template.project.id,
+            AutomationControllerJobScope.CLAIM_JOB_TEMPLATE_NAME: job_template.name,
+            AutomationControllerJobScope.CLAIM_JOB_TEMPLATE_ID: job_template.id,
+            AutomationControllerJobScope.CLAIM_PLAYBOOK_NAME: job_template.playbook,
+        }
+        return retrieve_workload_identity_jwt_with_claims(
+            claims=claims,
+            audience=audience,
+            scope=AutomationControllerJobScope.name,
+        )
+
+    @staticmethod
+    def _decode_jwt_payload_for_display(jwt_token):
+        """Decode JWT payload for display purposes only (signature not verified).
+
+        This is safe because the JWT was just created by AWX and is only decoded
+        to show the user what claims are being sent to the external system.
+        The external system will perform proper signature verification.
+
+        Args:
+            jwt_token: The JWT token to decode
+
+        Returns:
+            dict: The decoded JWT payload
+        """
+        return _jwt_decode(jwt_token, algorithms=["RS256"], options={"verify_signature": False})  # NOSONAR python:S5659
+
+    def _has_workload_identity_token(self, credential_type_inputs):
+        """Check if credential type has an internal workload_identity_token field.
+
+        Args:
+            credential_type_inputs: The inputs dict from a credential type
+
+        Returns:
+            bool: True if the credential type has a workload_identity_token field marked as internal
+        """
+        fields = credential_type_inputs.get('fields', []) if isinstance(credential_type_inputs, dict) else []
+        return any(field.get('internal') and field.get('id') == 'workload_identity_token' for field in fields)
+
+    def _validate_and_get_job_template(self, job_template_id):
+        """Validate job template ID and return the JobTemplate instance.
+
+        Args:
+            job_template_id: The job template ID from metadata
+
+        Returns:
+            JobTemplate instance
+
+        Raises:
+            ParseError: If job_template_id is invalid or not found
+        """
+        if job_template_id is None:
+            raise ParseError(_('Job template ID is required.'))
+
+        try:
+            return models.JobTemplate.objects.get(id=int(job_template_id))
+        except ValueError:
+            raise ParseError(_('Job template ID must be an integer.'))
+        except models.JobTemplate.DoesNotExist:
+            raise ParseError(_('Job template with ID %(id)s does not exist.') % {'id': job_template_id})
+
+    def _handle_oidc_credential_test(self, backend_kwargs):
+        """
+        Handle OIDC workload identity token generation for external credential test endpoints.
+
+        This method should only be called when FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED is enabled
+        and the credential type has a workload_identity_token field.
+
+        Args:
+            backend_kwargs: The kwargs dict to pass to the backend (will be modified in place)
+
+        Returns:
+            dict: Response body containing details with the sent JWT payload
+
+        Raises:
+            PermissionDenied: If user lacks access to the job template (re-raised for 403 response)
+
+            All other exceptions are caught and converted to 400 responses with error details.
+
+            Modifies backend_kwargs in place to add workload_identity_token.
+        """
+        # Validate job template
+        job_template_id = backend_kwargs.pop('job_template_id', None)
+        job_template = self._validate_and_get_job_template(job_template_id)
+
+        # Check user access
+        if not self.request.user.can_access(models.JobTemplate, 'start', job_template):
+            raise PermissionDenied(_('You do not have access to job template with id: %(id)s.') % {'id': job_template.id})
+
+        # Generate workload identity token
+        jwt_token = self._get_workload_identity_token(job_template, backend_kwargs.get('url'))
+        backend_kwargs['workload_identity_token'] = jwt_token
+
+        return {'details': {'sent_jwt_payload': self._decode_jwt_payload_for_display(jwt_token)}}
+
+    def _call_backend_with_error_handling(self, plugin, backend_kwargs, response_body):
+        """Call credential backend and handle errors."""
+        try:
+            with set_environ(**settings.AWX_TASK_ENV):
+                plugin.backend(**backend_kwargs)
+                return Response(response_body, status=status.HTTP_202_ACCEPTED)
+        except requests.exceptions.HTTPError as exc:
+            message = self._extract_http_error_message(exc)
+            self._add_error_to_response(response_body, message)
+            return Response(response_body, status=status.HTTP_400_BAD_REQUEST)
+        except Exception as exc:
+            message = self._extract_generic_error_message(exc)
+            self._add_error_to_response(response_body, message)
+            return Response(response_body, status=status.HTTP_400_BAD_REQUEST)
+
+    @staticmethod
+    def _extract_http_error_message(exc):
+        """Extract error message from HTTPError, checking response JSON and text."""
+        message = str(exc)
+        if not hasattr(exc, 'response') or exc.response is None:
+            return message
+
+        try:
+            error_data = exc.response.json()
+            if 'errors' in error_data and error_data['errors']:
+                return ', '.join(error_data['errors'])
+            if 'error' in error_data:
+                return error_data['error']
+        except (ValueError, KeyError):
+            if exc.response.text:
+                return exc.response.text
+        return message
+
+    @staticmethod
+    def _extract_generic_error_message(exc):
+        """Extract error message from exception, handling ConnectTimeoutError specially."""
+        message = str(exc) if str(exc) else exc.__class__.__name__
+        for arg in getattr(exc, 'args', []):
+            if isinstance(getattr(arg, 'reason', None), ConnectTimeoutError):
+                return str(arg.reason)
+        return message
+
+    @staticmethod
+    def _add_error_to_response(response_body, message):
+        """Add error message to both 'detail' and 'details.error_message' fields."""
+        response_body['detail'] = message
+        if 'details' in response_body:
+            response_body['details']['error_message'] = message
+
+
+class CredentialExternalTest(OIDCCredentialTestMixin, SubDetailAPIView):
     """
     Test updates to the input values and metadata of an external credential
     before saving them.
@@ -1603,9 +1765,15 @@ class CredentialExternalTest(SubDetailAPIView):
     obj_permission_type = 'use'
     resource_purpose = 'test external credential'
 
-    @extend_schema_if_available(extensions={"x-ai-description": "Test update the input values and metadata of an external credential"})
+    @extend_schema_if_available(extensions={"x-ai-description": """Test update the input values and metadata of an external credential.
+        This endpoint supports testing credentials that connect to external secret management systems
+        such as CyberArk AIM, CyberArk Conjur, HashiCorp Vault, AWS Secrets Manager, Azure Key Vault,
+        Centrify Vault, Thycotic DevOps Secrets Vault, and GitHub App Installation Access Token Lookup.
+        It does not support standard credential types such as Machine, SCM, and Cloud."""})
     def post(self, request, *args, **kwargs):
         obj = self.get_object()
+        if obj.credential_type.kind != 'external':
+            raise ParseError(_('Credential is not testable.'))
         backend_kwargs = {}
         for field_name, value in obj.inputs.items():
             backend_kwargs[field_name] = obj.get_input(field_name)
@@ -1613,20 +1781,22 @@ class CredentialExternalTest(SubDetailAPIView):
             if value != '$encrypted$':
                 backend_kwargs[field_name] = value
         backend_kwargs.update(request.data.get('metadata', {}))
-        try:
-            with set_environ(**settings.AWX_TASK_ENV):
-                obj.credential_type.plugin.backend(**backend_kwargs)
-                return Response({}, status=status.HTTP_202_ACCEPTED)
-        except requests.exceptions.HTTPError as exc:
-            message = 'HTTP {}'.format(exc.response.status_code)
-            return Response({'inputs': message}, status=status.HTTP_400_BAD_REQUEST)
-        except Exception as exc:
-            message = exc.__class__.__name__
-            args = getattr(exc, 'args', [])
-            for a in args:
-                if isinstance(getattr(a, 'reason', None), ConnectTimeoutError):
-                    message = str(a.reason)
-            return Response({'inputs': message}, status=status.HTTP_400_BAD_REQUEST)
+
+        # Handle OIDC workload identity token generation if enabled
+        response_body = {}
+        if flag_enabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED') and self._has_workload_identity_token(obj.credential_type.inputs):
+            try:
+                oidc_response_body = self._handle_oidc_credential_test(backend_kwargs)
+                response_body.update(oidc_response_body)
+            except PermissionDenied:
+                raise
+            except Exception as exc:
+                error_message = str(exc.detail) if hasattr(exc, 'detail') else str(exc)
+                response_body['detail'] = error_message
+                response_body['details'] = {'error_message': error_message}
+                return Response(response_body, status=status.HTTP_400_BAD_REQUEST)
+
+        return self._call_backend_with_error_handling(obj.credential_type.plugin, backend_kwargs, response_body)
 
 
 class CredentialInputSourceDetail(RetrieveUpdateDestroyAPIView):
@@ -1656,7 +1826,7 @@ class CredentialInputSourceSubList(SubListCreateAPIView):
     parent_key = 'target_credential'
 
 
-class CredentialTypeExternalTest(SubDetailAPIView):
+class CredentialTypeExternalTest(OIDCCredentialTestMixin, SubDetailAPIView):
     """
     Test a complete set of input values for an external credential before
     saving it.
@@ -1671,21 +1841,26 @@ class CredentialTypeExternalTest(SubDetailAPIView):
     @extend_schema_if_available(extensions={"x-ai-description": "Test a complete set of input values for an external credential"})
     def post(self, request, *args, **kwargs):
         obj = self.get_object()
+        if obj.kind != 'external':
+            raise ParseError(_('Credential type is not testable.'))
         backend_kwargs = request.data.get('inputs', {})
         backend_kwargs.update(request.data.get('metadata', {}))
-        try:
-            obj.plugin.backend(**backend_kwargs)
-            return Response({}, status=status.HTTP_202_ACCEPTED)
-        except requests.exceptions.HTTPError as exc:
-            message = 'HTTP {}'.format(exc.response.status_code)
-            return Response({'inputs': message}, status=status.HTTP_400_BAD_REQUEST)
-        except Exception as exc:
-            message = exc.__class__.__name__
-            args = getattr(exc, 'args', [])
-            for a in args:
-                if isinstance(getattr(a, 'reason', None), ConnectTimeoutError):
-                    message = str(a.reason)
-            return Response({'inputs': message}, status=status.HTTP_400_BAD_REQUEST)
+
+        # Handle OIDC workload identity token generation if enabled
+        response_body = {}
+        if flag_enabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED') and self._has_workload_identity_token(obj.inputs):
+            try:
+                oidc_response_body = self._handle_oidc_credential_test(backend_kwargs)
+                response_body.update(oidc_response_body)
+            except PermissionDenied:
+                raise
+            except Exception as exc:
+                error_message = str(exc.detail) if hasattr(exc, 'detail') else str(exc)
+                response_body['detail'] = error_message
+                response_body['details'] = {'error_message': error_message}
+                return Response(response_body, status=status.HTTP_400_BAD_REQUEST)
+
+        return self._call_backend_with_error_handling(obj.plugin, backend_kwargs, response_body)
 
 
 class HostRelatedSearchMixin(object):
@@ -1751,7 +1926,7 @@ class HostList(HostRelatedSearchMixin, ListCreateAPIView):
         if filter_string:
             filter_qs = SmartFilter.query_from_string(filter_string)
             qs &= filter_qs
-        return qs.distinct()
+        return qs.distinct().with_latest_summary_id()
 
     def list(self, *args, **kwargs):
         try:
@@ -1766,6 +1941,9 @@ class HostDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIView):
     serializer_class = serializers.HostSerializer
     resource_purpose = 'host detail'
 
+    def get_queryset(self):
+        return super().get_queryset().with_latest_summary_id()
+
     @extend_schema_if_available(extensions={"x-ai-description": "Delete a host"})
     def delete(self, request, *args, **kwargs):
         if self.get_object().inventory.pending_deletion:
@@ -1799,6 +1977,9 @@ class InventoryHostsList(HostRelatedSearchMixin, SubListCreateAttachDetachAPIVie
     filter_read_permission = False
     resource_purpose = 'hosts of an inventory'
 
+    def get_queryset(self):
+        return super().get_queryset().with_latest_summary_id()
+
 
 class HostGroupsList(SubListCreateAttachDetachAPIView):
     '''the list of groups a host is directly a member of'''
@@ -1982,6 +2163,9 @@ class GroupHostsList(HostRelatedSearchMixin, SubListCreateAttachDetachAPIView):
     relationship = 'hosts'
     resource_purpose = 'hosts of a group'
 
+    def get_queryset(self):
+        return super().get_queryset().with_latest_summary_id()
+
     def update_raw_data(self, data):
         data.pop('inventory', None)
         return super(GroupHostsList, self).update_raw_data(data)
@@ -2013,7 +2197,7 @@ class GroupAllHostsList(HostRelatedSearchMixin, SubListAPIView):
         self.check_parent_access(parent)
         qs = self.request.user.get_queryset(self.model).distinct()  # need distinct for '&' operator
         sublist_qs = parent.all_hosts.distinct()
-        return qs & sublist_qs
+        return (qs & sublist_qs).with_latest_summary_id()
 
 
 class GroupInventorySourcesList(SubListAPIView):
@@ -2306,6 +2490,9 @@ class InventorySourceHostsList(HostRelatedSearchMixin, SubListDestroyAPIView):
     check_sub_obj_permission = False
     resource_purpose = 'hosts of an inventory source'
 
+    def get_queryset(self):
+        return super().get_queryset().with_latest_summary_id()
+
     def perform_list_destroy(self, instance_list):
         inv_source = self.get_parent_object()
         with ignore_inventory_computed_fields():
@@ -2469,6 +2656,11 @@ class JobTemplateDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIV
     resource_purpose = 'job template detail'
 
 
+@extend_schema_view(
+    retrieve=extend_schema(
+        extensions={'x-ai-description': 'List job template launch criteria'},
+    )
+)
 class JobTemplateLaunch(RetrieveAPIView):
     model = models.JobTemplate
     obj_permission_type = 'start'
@@ -2477,6 +2669,9 @@ class JobTemplateLaunch(RetrieveAPIView):
     resource_purpose = 'launch a job from a job template'
 
     def update_raw_data(self, data):
+        """
+        Use the ID of a job template to retrieve its launch details.
+        """
         try:
             obj = self.get_object()
         except PermissionDenied:
@@ -3310,6 +3505,11 @@ class WorkflowJobTemplateLabelList(JobTemplateLabelList):
     resource_purpose = 'labels of a workflow job template'
 
 
+@extend_schema_view(
+    retrieve=extend_schema(
+        extensions={'x-ai-description': 'List workflow job template launch criteria.'},
+    )
+)
 class WorkflowJobTemplateLaunch(RetrieveAPIView):
     model = models.WorkflowJobTemplate
     obj_permission_type = 'start'
@@ -3318,6 +3518,9 @@ class WorkflowJobTemplateLaunch(RetrieveAPIView):
     resource_purpose = 'launch a workflow job from a workflow job template'
 
     def update_raw_data(self, data):
+        """
+        Use the ID of a workflow job template to retrieve its launch details.
+        """
         try:
             obj = self.get_object()
         except PermissionDenied:
@@ -3710,6 +3913,11 @@ class JobCancel(GenericCancelView):
         return super().post(request, *args, **kwargs)
 
 
+@extend_schema_view(
+    retrieve=extend_schema(
+        extensions={'x-ai-description': 'List job relaunch criteria'},
+    )
+)
 class JobRelaunch(RetrieveAPIView):
     model = models.Job
     obj_permission_type = 'start'
@@ -3717,6 +3925,7 @@ class JobRelaunch(RetrieveAPIView):
     resource_purpose = 'relaunch a job'
 
     def update_raw_data(self, data):
+        """Use the ID of a job to retrieve data on retry attempts and necessary passwords."""
         data = super(JobRelaunch, self).update_raw_data(data)
         try:
             obj = self.get_object()
@@ -4661,19 +4870,12 @@ class RoleUsersList(SubListAttachDetachAPIView):
         if not sub_id:
             return super(RoleUsersList, self).post(request)
 
-        user = get_object_or_400(models.User, pk=sub_id)
-        role = self.get_parent_object()
-
-        content_types = ContentType.objects.get_for_models(models.Organization, models.Team, models.Credential)  # dict of {model: content_type}
-        credential_content_type = content_types[models.Credential]
-        if role.content_type == credential_content_type:
-            if 'disassociate' not in request.data and role.content_object.organization and user not in role.content_object.organization.member_role:
-                data = dict(msg=_("You cannot grant credential access to a user not in the credentials' organization"))
-                return Response(data, status=status.HTTP_400_BAD_REQUEST)
-
-            if not role.content_object.organization and not request.user.is_superuser:
-                data = dict(msg=_("You cannot grant private credential access to another user"))
-                return Response(data, status=status.HTTP_400_BAD_REQUEST)
+        if not request.data.get('disassociate'):
+            user = get_object_or_400(models.User, pk=sub_id)
+            role = self.get_parent_object()
+            content_object = role.content_object
+            if hasattr(content_object, 'validate_role_assignment'):
+                content_object.validate_role_assignment(user, role_definition=None, requesting_user=request.user)
 
         return super(RoleUsersList, self).post(request, *args, **kwargs)
 
@@ -4706,24 +4908,6 @@ class RoleTeamsList(SubListAttachDetachAPIView):
             data = dict(msg=_("You cannot assign an Organization participation role as a child role for a Team."))
             return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
-        credential_content_type = ContentType.objects.get_for_model(models.Credential)
-        if role.content_type == credential_content_type:
-            # Private credentials (no organization) are never allowed for teams
-            if not role.content_object.organization:
-                data = dict(
-                    msg=_("You cannot grant access to a credential that is not assigned to an organization (private credentials cannot be assigned to teams)")
-                )
-                return Response(data, status=status.HTTP_400_BAD_REQUEST)
-            # Cross-organization credentials are only allowed for superusers
-            elif role.content_object.organization.id != team.organization.id:
-                if not request.user.is_superuser:
-                    data = dict(
-                        msg=_(
-                            "You cannot grant a team access to a credential in a different organization. Only superusers can grant cross-organization credential access to teams"
-                        )
-                    )
-                    return Response(data, status=status.HTTP_400_BAD_REQUEST)
-
         action = 'attach'
         if request.data.get('disassociate', None):
             action = 'unattach'
@@ -4732,6 +4916,11 @@ class RoleTeamsList(SubListAttachDetachAPIView):
             data = dict(msg=_("You cannot grant system-level permissions to a team."))
             return Response(data, status=status.HTTP_400_BAD_REQUEST)
 
+        if action == 'attach':
+            content_object = role.content_object
+            if hasattr(content_object, 'validate_role_assignment'):
+                content_object.validate_role_assignment(team, role_definition=None, requesting_user=request.user)
+
         if not request.user.can_access(self.parent_model, action, role, team, self.relationship, request.data, skip_sub_obj_read_check=False):
             raise PermissionDenied()
         if request.data.get('disassociate', None):

awx/api/views/analytics.py

@@ -49,7 +49,6 @@ class GetNotAllowedMixin(object):
 class AnalyticsRootView(APIView):
     permission_classes = (AnalyticsPermission,)
     name = _('Automation Analytics')
-    swagger_topic = 'Automation Analytics'
     resource_purpose = 'automation analytics endpoints'
 
     @extend_schema_if_available(extensions={"x-ai-description": "A list of additional API endpoints related to analytics"})
@@ -306,7 +305,6 @@ class AnalyticsAuthorizedView(AnalyticsGenericListView):
 
 class AnalyticsReportsList(GetNotAllowedMixin, AnalyticsGenericListView):
     name = _("Reports")
-    swagger_topic = "Automation Analytics"
     resource_purpose = 'automation analytics reports'
 
 

awx/api/views/mixin.py

@@ -4,7 +4,8 @@
 import dateutil
 import logging
 
-from django.db.models import Count
+from django.db.models import Count, IntegerField, OuterRef, Subquery
+from django.db.models.functions import Coalesce
 from django.db import transaction
 from django.shortcuts import get_object_or_404
 from django.utils.timezone import now
@@ -15,7 +16,7 @@ from rest_framework.response import Response
 from rest_framework import status
 
 from awx.main.constants import ACTIVE_STATES
-from awx.main.models import Organization
+from awx.main.models import Organization, Role
 from awx.main.utils import get_object_or_400
 from awx.main.models.ha import Instance, InstanceGroup, schedule_policy_task
 from awx.main.models.organization import Team
@@ -178,9 +179,22 @@ class OrganizationCountsMixin(object):
         db_results['projects'] = project_qs.values('organization').annotate(Count('organization')).order_by('organization')
 
         # Other members and admins of organization are always viewable
-        db_results['users'] = org_qs.annotate(users=Count('member_role__members', distinct=True), admins=Count('admin_role__members', distinct=True)).values(
-            'id', 'users', 'admins'
+        #
+        # Use independent subqueries instead of double-JOIN Count to avoid
+        # cartesian product.
+        RoleMember = Role.members.through
+        member_count = Subquery(
+            RoleMember.objects.filter(role_id=OuterRef('member_role_id')).values('role_id').annotate(cnt=Count('user_id', distinct=True)).values('cnt'),
+            output_field=IntegerField(),
         )
+        admin_count = Subquery(
+            RoleMember.objects.filter(role_id=OuterRef('admin_role_id')).values('role_id').annotate(cnt=Count('user_id', distinct=True)).values('cnt'),
+            output_field=IntegerField(),
+        )
+        db_results['users'] = org_qs.annotate(
+            users=Coalesce(member_count, 0),
+            admins=Coalesce(admin_count, 0),
+        ).values('id', 'users', 'admins')
 
         count_context = {}
         for org in org_id_list:

awx/api/views/organization.py

@@ -5,7 +5,8 @@
 import logging
 
 # Django
-from django.db.models import Count
+from django.db.models import Count, IntegerField, OuterRef, Subquery
+from django.db.models.functions import Coalesce
 from django.contrib.contenttypes.models import ContentType
 from django.utils.translation import gettext_lazy as _
 
@@ -77,10 +78,19 @@ class OrganizationDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPI
 
         org_counts = {}
         access_kwargs = {'accessor': self.request.user, 'role_field': 'read_role'}
+        # Use independent subqueries instead of double-JOIN Count to avoid
+        # cartesian product.
+        RoleMember = Role.members.through
+        member_count = Subquery(
+            RoleMember.objects.filter(role_id=OuterRef('member_role_id')).values('role_id').annotate(cnt=Count('user_id', distinct=True)).values('cnt'),
+            output_field=IntegerField(),
+        )
+        admin_count = Subquery(
+            RoleMember.objects.filter(role_id=OuterRef('admin_role_id')).values('role_id').annotate(cnt=Count('user_id', distinct=True)).values('cnt'),
+            output_field=IntegerField(),
+        )
         direct_counts = (
-            Organization.objects.filter(id=org_id)
-            .annotate(users=Count('member_role__members', distinct=True), admins=Count('admin_role__members', distinct=True))
-            .values('users', 'admins')
+            Organization.objects.filter(id=org_id).annotate(users=Coalesce(member_count, 0), admins=Coalesce(admin_count, 0)).values('users', 'admins')
         )
 
         if not direct_counts:

awx/api/views/root.py

@@ -25,7 +25,6 @@ import requests
 
 from ansible_base.lib.utils.schema import extend_schema_if_available
 
-from awx import MODE
 from awx.api.generics import APIView
 from awx.conf.registry import settings_registry
 from awx.main.analytics import all_collectors
@@ -33,7 +32,7 @@ from awx.main.ha import is_ha_environment
 from awx.main.tasks.system import clear_setting_cache
 from awx.main.utils import get_awx_version, get_custom_venv_choices
 from awx.main.utils.licensing import validate_entitlement_manifest
-from awx.api.versioning import URLPathVersioning, reverse, drf_reverse
+from awx.api.versioning import URLPathVersioning, reverse
 from awx.main.constants import PRIVILEGE_ESCALATION_METHODS
 from awx.main.models import Project, Organization, Instance, InstanceGroup, JobTemplate
 from awx.main.utils import set_environ
@@ -62,8 +61,6 @@ class ApiRootView(APIView):
         data['custom_logo'] = settings.CUSTOM_LOGO
         data['custom_login_info'] = settings.CUSTOM_LOGIN_INFO
         data['login_redirect_override'] = settings.LOGIN_REDIRECT_OVERRIDE
-        if MODE == 'development':
-            data['docs'] = drf_reverse('api:schema-swagger-ui')
         return Response(data)
 
 
@@ -347,13 +344,22 @@ class ApiV2ConfigView(APIView):
             become_methods=PRIVILEGE_ESCALATION_METHODS,
         )
 
-        if (
-            request.user.is_superuser
-            or request.user.is_system_auditor
-            or Organization.accessible_objects(request.user, 'admin_role').exists()
-            or Organization.accessible_objects(request.user, 'auditor_role').exists()
-            or Organization.accessible_objects(request.user, 'project_admin_role').exists()
-        ):
+        # Check superuser/auditor first
+        if request.user.is_superuser or request.user.is_system_auditor:
+            has_org_access = True
+        else:
+            # Single query checking all three organization role types at once
+            has_org_access = (
+                (
+                    Organization.access_qs(request.user, 'change')
+                    | Organization.access_qs(request.user, 'audit')
+                    | Organization.access_qs(request.user, 'add_project')
+                )
+                .distinct()
+                .exists()
+            )
+
+        if has_org_access:
             data.update(
                 dict(
                     project_base_dir=settings.PROJECTS_ROOT,
@@ -361,8 +367,10 @@ class ApiV2ConfigView(APIView):
                     custom_virtualenvs=get_custom_venv_choices(),
                 )
             )
-        elif JobTemplate.accessible_objects(request.user, 'admin_role').exists():
-            data['custom_virtualenvs'] = get_custom_venv_choices()
+        else:
+            # Only check JobTemplate access if org check failed
+            if JobTemplate.accessible_objects(request.user, 'admin_role').exists():
+                data['custom_virtualenvs'] = get_custom_venv_choices()
 
         return Response(data)
 

awx/api/views/webhooks.py

@@ -17,7 +17,7 @@ from awx.api import serializers
 from awx.api.generics import APIView, GenericAPIView
 from awx.api.permissions import WebhookKeyPermission
 from awx.main.models import Job, JobTemplate, WorkflowJob, WorkflowJobTemplate
-from awx.main.constants import JOB_VARIABLE_PREFIXES
+from awx.main.utils.common import get_job_variable_prefixes
 
 logger = logging.getLogger('awx.api.views.webhooks')
 
@@ -133,7 +133,7 @@ class WebhookReceiverBase(APIView):
 
     @csrf_exempt
     @extend_schema_if_available(extensions={"x-ai-description": "Receive a webhook event and trigger a job"})
-    def post(self, request, *args, **kwargs):
+    def post(self, request, *args, **kwargs_in):
         # Ensure that the full contents of the request are captured for multiple uses.
         request.body
 
@@ -166,7 +166,7 @@ class WebhookReceiverBase(APIView):
             'extra_vars': {},
         }
 
-        for name in JOB_VARIABLE_PREFIXES:
+        for name in get_job_variable_prefixes():
             kwargs['extra_vars']['{}_webhook_event_type'.format(name)] = event_type
             kwargs['extra_vars']['{}_webhook_event_guid'.format(name)] = event_guid
             kwargs['extra_vars']['{}_webhook_event_ref'.format(name)] = event_ref

awx/main/access.py

@@ -897,8 +897,6 @@ class HostAccess(BaseAccess):
         'created_by',
         'modified_by',
         'inventory',
-        'last_job__job_template',
-        'last_job_host_summary__job',
     )
     prefetch_related = ('groups', 'inventory_sources')
 

awx/main/analytics/core.py

@@ -8,6 +8,7 @@ import pathlib
 import shutil
 import tarfile
 import tempfile
+from urllib.parse import urlparse, urlunparse
 
 from django.conf import settings
 from django.core.serializers.json import DjangoJSONEncoder
@@ -23,6 +24,8 @@ from awx.main.models import Job
 from awx.main.access import access_registry
 from awx.main.utils import get_awx_http_client_headers, set_environ, datetime_hook
 from awx.main.utils.analytics_proxy import OIDCClient
+from awx.main.utils.candlepin import get_or_generate_candlepin_certificate
+from awx.main.utils.candlepin.client import _temp_cert_files
 
 __all__ = ['register', 'gather', 'ship']
 
@@ -41,6 +44,76 @@ def _valid_license():
     return True
 
 
+def _get_cert_upload_url(url):
+    """
+    Convert analytics URL to use 'cert.' subdomain for mTLS uploads.
+
+    Some analytics services use different hostnames for different auth methods:
+    - cert.example.com - for mTLS (certificate-based) uploads
+    - example.com - for OIDC (token-based) uploads
+
+    Args:
+        url: Original analytics URL
+
+    Returns:
+        URL with 'cert.' prepended to hostname if not already present
+    """
+    try:
+        parsed = urlparse(url)
+        hostname = parsed.hostname
+
+        # Only modify if hostname doesn't already start with 'cert.'
+        if hostname and not hostname.startswith('cert.'):
+            new_hostname = f'cert.{hostname}'
+            # Reconstruct URL with new hostname
+            netloc = new_hostname
+            if parsed.port:
+                netloc = f'{new_hostname}:{parsed.port}'
+
+            new_parsed = parsed._replace(netloc=netloc)
+            return urlunparse(new_parsed)
+
+        return url
+    except Exception as e:
+        logger.warning(f'Could not modify URL for cert upload: {e}, using original URL')
+        return url
+
+
+def _get_analytics_credentials():
+    """
+    Get Red Hat Insights credentials from settings.
+
+    Attempts to retrieve credentials in the following priority order:
+    1. REDHAT_USERNAME / REDHAT_PASSWORD
+    2. SUBSCRIPTIONS_USERNAME / SUBSCRIPTIONS_PASSWORD
+    3. SUBSCRIPTIONS_CLIENT_ID / SUBSCRIPTIONS_CLIENT_SECRET
+
+    Returns:
+        tuple: (username, password) if credentials are found, (None, None) otherwise
+    """
+    rh_id = getattr(settings, 'REDHAT_USERNAME', None)
+    rh_secret = getattr(settings, 'REDHAT_PASSWORD', None)
+
+    if rh_id and rh_secret:
+        return rh_id, rh_secret
+
+    # Try SUBSCRIPTIONS_USERNAME / SUBSCRIPTIONS_PASSWORD
+    rh_id = getattr(settings, 'SUBSCRIPTIONS_USERNAME', None)
+    rh_secret = getattr(settings, 'SUBSCRIPTIONS_PASSWORD', None)
+
+    if rh_id and rh_secret:
+        return rh_id, rh_secret
+
+    # Try SUBSCRIPTIONS_CLIENT_ID / SUBSCRIPTIONS_CLIENT_SECRET
+    rh_id = getattr(settings, 'SUBSCRIPTIONS_CLIENT_ID', None)
+    rh_secret = getattr(settings, 'SUBSCRIPTIONS_CLIENT_SECRET', None)
+
+    if rh_id and rh_secret:
+        return rh_id, rh_secret
+
+    return None, None
+
+
 def all_collectors():
     from awx.main.analytics import collectors
 
@@ -184,10 +257,8 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti
             logger.log(log_level, "Automation Analytics not enabled. Use --dry-run to gather locally without sending.")
             return None
 
-        if not (
-            settings.AUTOMATION_ANALYTICS_URL
-            and ((settings.REDHAT_USERNAME and settings.REDHAT_PASSWORD) or (settings.SUBSCRIPTIONS_CLIENT_ID and settings.SUBSCRIPTIONS_CLIENT_SECRET))
-        ):
+        rh_id, rh_secret = _get_analytics_credentials()
+        if not (settings.AUTOMATION_ANALYTICS_URL and rh_id and rh_secret):
             logger.log(log_level, "Not gathering analytics, configuration is invalid. Use --dry-run to gather locally without sending.")
             return None
 
@@ -368,19 +439,14 @@ def ship(path):
         logger.error('AUTOMATION_ANALYTICS_URL is not set')
         return False
 
-    rh_id = getattr(settings, 'REDHAT_USERNAME', None)
-    rh_secret = getattr(settings, 'REDHAT_PASSWORD', None)
-
-    if not (rh_id and rh_secret):
-        rh_id = getattr(settings, 'SUBSCRIPTIONS_CLIENT_ID', None)
-        rh_secret = getattr(settings, 'SUBSCRIPTIONS_CLIENT_SECRET', None)
+    rh_id, rh_secret = _get_analytics_credentials()
 
     if not rh_id:
-        logger.error('Neither REDHAT_USERNAME nor SUBSCRIPTIONS_CLIENT_ID are set')
+        logger.error('No valid username found. Tried: REDHAT_USERNAME, SUBSCRIPTIONS_USERNAME, SUBSCRIPTIONS_CLIENT_ID')
         return False
 
     if not rh_secret:
-        logger.error('Neither REDHAT_PASSWORD nor SUBSCRIPTIONS_CLIENT_SECRET are set')
+        logger.error('No valid password found. Tried: REDHAT_PASSWORD, SUBSCRIPTIONS_PASSWORD, SUBSCRIPTIONS_CLIENT_SECRET')
         return False
 
     with open(path, 'rb') as f:
@@ -388,17 +454,40 @@ def ship(path):
         s = requests.Session()
         s.headers = get_awx_http_client_headers()
         s.headers.pop('Content-Type')
+
         with set_environ(**settings.AWX_TASK_ENV):
+            # Try Certificate-based mTLS authentication (zero-touch)
+            cert_pem, key_pem = get_or_generate_candlepin_certificate()
+            if cert_pem and key_pem:
+                # Use cert. subdomain for mTLS uploads
+                cert_url = _get_cert_upload_url(url)
+                logger.debug("Attempting certificate-based authentication for analytics upload")
+                try:
+                    with _temp_cert_files(cert_pem, key_pem) as (cert_path, key_path):
+                        response = s.post(
+                            cert_url, files=files, cert=(cert_path, key_path), verify=settings.INSIGHTS_CERT_PATH, headers=s.headers, timeout=(31, 31)
+                        )
+                        if response.status_code < 300:
+                            return True
+                        else:
+                            logger.warning(
+                                f'Certificate-based authentication failed with status {response.status_code}, {response.text}. Falling back to OIDC auth'
+                            )
+                except Exception as e:
+                    logger.warning(f"Certificate-based authentication failed: {e}, falling back to OIDC auth")
+
+            # Try OIDC authentication
+            logger.debug("Attempting OIDC authentication for analytics upload")
+            f.seek(0)  # requests POST may read from the handler, so seek to beginning of file for the next POST attempt
             try:
                 client = OIDCClient(rh_id, rh_secret)
                 response = client.make_request("POST", url, headers=s.headers, files=files, verify=settings.INSIGHTS_CERT_PATH, timeout=(31, 31))
-            except requests.RequestException:
-                logger.error("Automation Analytics API request failed, trying base auth method")
-                response = s.post(url, files=files, verify=settings.INSIGHTS_CERT_PATH, auth=(rh_id, rh_secret), headers=s.headers, timeout=(31, 31))
 
-        # Accept 2XX status_codes
-        if response.status_code >= 300:
-            logger.error('Upload failed with status {}, {}'.format(response.status_code, response.text))
-            return False
-
-        return True
+                if response.status_code < 300:
+                    return True
+                else:
+                    logger.error(f'OIDC authentication failed with status {response.status_code}, {response.text}')
+                    return False
+            except requests.RequestException as e:
+                logger.error(f"OIDC authentication failed: {e}")
+                return False

awx/main/analytics/dispatcherd_metrics.py

@@ -0,0 +1,41 @@
+import http.client
+import socket
+import urllib.error
+import urllib.request
+import logging
+
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
+
+
+def get_dispatcherd_metrics(request):
+    metrics_cfg = settings.METRICS_SUBSYSTEM_CONFIG.get('server', {}).get(settings.METRICS_SERVICE_DISPATCHER, {})
+    host = metrics_cfg.get('host', 'localhost')
+    port = metrics_cfg.get('port', 8015)
+    metrics_filter = []
+    if request is not None and hasattr(request, "query_params"):
+        try:
+            nodes_filter = request.query_params.getlist("node")
+        except Exception:
+            nodes_filter = []
+        if nodes_filter and settings.CLUSTER_HOST_ID not in nodes_filter:
+            return ''
+        try:
+            metrics_filter = request.query_params.getlist("metric")
+        except Exception:
+            metrics_filter = []
+    if metrics_filter:
+        # Right now we have no way of filtering the dispatcherd metrics
+        # so just avoid getting in the way if another metric is filtered for
+        return ''
+    url = f"http://{host}:{port}/metrics"
+    try:
+        with urllib.request.urlopen(url, timeout=1.0) as response:
+            payload = response.read()
+            if not payload:
+                return ''
+            return payload.decode('utf-8')
+    except (urllib.error.URLError, UnicodeError, socket.timeout, TimeoutError, http.client.HTTPException) as exc:
+        logger.debug(f"Failed to collect dispatcherd metrics from {url}: {exc}")
+        return ''

awx/main/analytics/subsystem_metrics.py

@@ -15,6 +15,7 @@ from rest_framework.request import Request
 from awx.main.consumers import emit_channel_notification
 from awx.main.utils import is_testing
 from awx.main.utils.redis import get_redis_client
+from .dispatcherd_metrics import get_dispatcherd_metrics
 
 root_key = settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX
 logger = logging.getLogger('awx.main.analytics')
@@ -398,11 +399,6 @@ class DispatcherMetrics(Metrics):
         SetFloatM('workflow_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
         SetFloatM('workflow_manager_spawn_workflow_graph_jobs_seconds', 'Time spent spawning workflow tasks'),
         SetFloatM('workflow_manager_get_tasks_seconds', 'Time spent loading workflow tasks from db'),
-        # dispatcher subsystem metrics
-        SetIntM('dispatcher_pool_scale_up_events', 'Number of times local dispatcher scaled up a worker since startup'),
-        SetIntM('dispatcher_pool_active_task_count', 'Number of active tasks in the worker pool when last task was submitted'),
-        SetIntM('dispatcher_pool_max_worker_count', 'Highest number of workers in worker pool in last collection interval, about 20s'),
-        SetFloatM('dispatcher_availability', 'Fraction of time (in last collection interval) dispatcher was able to receive messages'),
     ]
 
     def __init__(self, *args, **kwargs):
@@ -430,8 +426,12 @@ class CallbackReceiverMetrics(Metrics):
 
 def metrics(request):
     output_text = ''
-    for m in [DispatcherMetrics(), CallbackReceiverMetrics()]:
-        output_text += m.generate_metrics(request)
+    output_text += DispatcherMetrics().generate_metrics(request)
+    output_text += CallbackReceiverMetrics().generate_metrics(request)
+
+    dispatcherd_metrics = get_dispatcherd_metrics(request)
+    if dispatcherd_metrics:
+        output_text += dispatcherd_metrics
     return output_text
 
 
@@ -481,13 +481,6 @@ class CallbackReceiverMetricsServer(MetricsServer):
         super().__init__(settings.METRICS_SERVICE_CALLBACK_RECEIVER, registry)
 
 
-class DispatcherMetricsServer(MetricsServer):
-    def __init__(self):
-        registry = CollectorRegistry(auto_describe=True)
-        registry.register(CustomToPrometheusMetricsCollector(DispatcherMetrics(metrics_have_changed=False)))
-        super().__init__(settings.METRICS_SERVICE_DISPATCHER, registry)
-
-
 class WebsocketsMetricsServer(MetricsServer):
     def __init__(self):
         registry = CollectorRegistry(auto_describe=True)

awx/main/apps.py

@@ -1,22 +1,25 @@
-import os
-
 from dispatcherd.config import setup as dispatcher_setup
 
 from django.apps import AppConfig
 from django.db import connection
 from django.utils.translation import gettext_lazy as _
-from awx.main.utils.common import bypass_in_test, load_all_entry_points_for
-from awx.main.utils.migration import is_database_synchronized
-from awx.main.utils.named_url_graph import _customize_graph, generate_graph
-from awx.conf import register, fields
+from django.core.management.base import CommandError
+from django.db.models.signals import pre_migrate
 
-from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name
+from awx.main.utils.named_url_graph import _customize_graph, generate_graph
+from awx.main.utils.db import db_requirement_violations
+from awx.conf import register, fields
 
 
 class MainConfig(AppConfig):
     name = 'awx.main'
     verbose_name = _('Main')
 
+    def check_db_requirement(self, *args, **kwargs):
+        violations = db_requirement_violations()
+        if violations:
+            raise CommandError(violations)
+
     def load_named_url_feature(self):
         models = [m for m in self.get_models() if hasattr(m, 'get_absolute_url')]
         generate_graph(models)
@@ -43,46 +46,10 @@ class MainConfig(AppConfig):
             category_slug='named-url',
         )
 
-    def _load_credential_types_feature(self):
-        """
-        Create CredentialType records for any discovered credentials.
-
-        Note that Django docs advise _against_ interacting with the database using
-        the ORM models in the ready() path. Specifically, during testing.
-        However, we explicitly use the @bypass_in_test decorator to avoid calling this
-        method during testing.
-
-        Django also advises against running pattern because it runs everywhere i.e.
-        every management command. We use an advisory lock to ensure correctness and
-        we will deal performance if it becomes an issue.
-        """
-        from awx.main.models.credential import CredentialType
-
-        if is_database_synchronized():
-            CredentialType.setup_tower_managed_defaults(app_config=self)
-
-    @bypass_in_test
-    def load_credential_types_feature(self):
-        from awx.main.models.credential import load_credentials
-
-        load_credentials()
-        return self._load_credential_types_feature()
-
-    def load_inventory_plugins(self):
-        from awx.main.models.inventory import InventorySourceOptions
-
-        is_awx = detect_server_product_name() == 'AWX'
-        extra_entry_point_groups = () if is_awx else ('inventory.supported',)
-        entry_points = load_all_entry_points_for(['inventory', *extra_entry_point_groups])
-
-        for entry_point_name, entry_point in entry_points.items():
-            cls = entry_point.load()
-            InventorySourceOptions.injectors[entry_point_name] = cls
-
     def configure_dispatcherd(self):
         """This implements the default configuration for dispatcherd
 
-        If running the tasking service like awx-manage run_dispatcher,
+        If running the tasking service like awx-manage dispatcherd,
         some additional config will be applied on top of this.
         This configuration provides the minimum such that code can submit
         tasks to pg_notify to run those tasks.
@@ -100,13 +67,5 @@ class MainConfig(AppConfig):
         super().ready()
 
         self.configure_dispatcherd()
-
-        """
-        Credential loading triggers database operations. There are cases we want to call
-        awx-manage collectstatic without a database. All management commands invoke the ready() code
-        path. Using settings.AWX_SKIP_CREDENTIAL_TYPES_DISCOVER _could_ invoke a database operation.
-        """
-        if not os.environ.get('AWX_SKIP_CREDENTIAL_TYPES_DISCOVER', None):
-            self.load_credential_types_feature()
         self.load_named_url_feature()
-        self.load_inventory_plugins()
+        pre_migrate.connect(self.check_db_requirement, sender=self)

awx/main/cache.py

@@ -1,87 +0,0 @@
-import functools
-
-from django.conf import settings
-from django.core.cache.backends.base import DEFAULT_TIMEOUT
-from django.core.cache.backends.redis import RedisCache
-
-from redis.exceptions import ConnectionError, ResponseError, TimeoutError
-import socket
-
-# This list comes from what django-redis ignores and the behavior we are trying
-# to retain while dropping the dependency on django-redis.
-IGNORED_EXCEPTIONS = (TimeoutError, ResponseError, ConnectionError, socket.timeout)
-
-CONNECTION_INTERRUPTED_SENTINEL = object()
-
-
-def optionally_ignore_exceptions(func=None, return_value=None):
-    if func is None:
-        return functools.partial(optionally_ignore_exceptions, return_value=return_value)
-
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except IGNORED_EXCEPTIONS as e:
-            if settings.DJANGO_REDIS_IGNORE_EXCEPTIONS:
-                return return_value
-            raise e.__cause__ or e
-
-    return wrapper
-
-
-class AWXRedisCache(RedisCache):
-    """
-    We just want to wrap the upstream RedisCache class so that we can ignore
-    the exceptions that it raises when the cache is unavailable.
-    """
-
-    @optionally_ignore_exceptions
-    def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
-        return super().add(key, value, timeout, version)
-
-    @optionally_ignore_exceptions(return_value=CONNECTION_INTERRUPTED_SENTINEL)
-    def _get(self, key, default=None, version=None):
-        return super().get(key, default, version)
-
-    def get(self, key, default=None, version=None):
-        value = self._get(key, default, version)
-        if value is CONNECTION_INTERRUPTED_SENTINEL:
-            return default
-        return value
-
-    @optionally_ignore_exceptions
-    def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
-        return super().set(key, value, timeout, version)
-
-    @optionally_ignore_exceptions
-    def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None):
-        return super().touch(key, timeout, version)
-
-    @optionally_ignore_exceptions
-    def delete(self, key, version=None):
-        return super().delete(key, version)
-
-    @optionally_ignore_exceptions
-    def get_many(self, keys, version=None):
-        return super().get_many(keys, version)
-
-    @optionally_ignore_exceptions
-    def has_key(self, key, version=None):
-        return super().has_key(key, version)
-
-    @optionally_ignore_exceptions
-    def incr(self, key, delta=1, version=None):
-        return super().incr(key, delta, version)
-
-    @optionally_ignore_exceptions
-    def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None):
-        return super().set_many(data, timeout, version)
-
-    @optionally_ignore_exceptions
-    def delete_many(self, keys, version=None):
-        return super().delete_many(keys, version)
-
-    @optionally_ignore_exceptions
-    def clear(self):
-        return super().clear()

awx/main/conf.py

@@ -213,6 +213,40 @@ register(
     category_slug='system',
 )
 
+register(
+    'AWX_ANALYTICS_CANDLEPIN_CA',
+    field_class=fields.CharField,
+    default='/etc/rhsm/ca/redhat-uep.pem',
+    allow_blank=True,
+    label=_('Candlepin CA Certificate Path'),
+    help_text=_('Path to the CA certificate file for verifying TLS connections to Candlepin. Leave blank to use system certificates.'),
+    category=_('System'),
+    category_slug='system',
+)
+
+register(
+    'AWX_ANALYTICS_CANDLEPIN_RENEWAL_THRESHOLD_DAYS',
+    field_class=fields.IntegerField,
+    default=90,
+    min_value=1,
+    label=_('Candlepin Certificate Renewal Threshold'),
+    help_text=_('Number of days before certificate expiry to trigger automatic renewal of Candlepin identity certificates.'),
+    category=_('System'),
+    category_slug='system',
+    unit=_('days'),
+)
+
+register(
+    'AWX_ANALYTICS_CANDLEPIN_PROXY_URL',
+    field_class=fields.CharField,
+    default='',
+    allow_blank=True,
+    label=_('Candlepin Proxy URL'),
+    help_text=_('HTTP/HTTPS proxy URL for Candlepin API requests (e.g., http://proxy.example.com:8080). Leave blank for no proxy.'),
+    category=_('System'),
+    category_slug='system',
+)
+
 register(
     'INSTALL_UUID',
     field_class=fields.CharField,
@@ -291,6 +325,22 @@ register(
     category_slug='jobs',
 )
 
+register(
+    'INCLUDE_DEPRECATED_AWX_VAR_PREFIX',
+    field_class=fields.BooleanField,
+    default=True,
+    label=_('Include Deprecated AWX Variable Prefix'),
+    help_text=_(
+        'When enabled (default), auto-generated job variables are emitted '
+        'with both the tower_ prefix and the deprecated awx_ prefix for '
+        'backward compatibility. Disable to emit only tower_ prefixed '
+        'variables and eliminate duplicates. The awx_ prefix is deprecated '
+        'and this setting will default to False in a future release.'
+    ),
+    category=_('Jobs'),
+    category_slug='jobs',
+)
+
 register(
     'AWX_ISOLATION_BASE_PATH',
     field_class=fields.CharField,
@@ -824,6 +874,58 @@ register(
     unit=_('seconds'),
 )
 
+register(
+    'CANDLEPIN_CONSUMER_UUID',
+    field_class=fields.CharField,
+    default='',
+    allow_blank=True,
+    encrypted=False,
+    label=_('Candlepin Consumer UUID'),
+    help_text=_('UUID of the registered Candlepin consumer for this AAP instance.'),
+    category=_('System'),
+    category_slug='system',
+    hidden=True,
+)
+
+register(
+    'CANDLEPIN_CERT_PEM',
+    field_class=fields.CharField,
+    default='',
+    allow_blank=True,
+    encrypted=True,
+    label=_('Candlepin Identity Certificate'),
+    help_text=_('PEM-encoded Candlepin identity certificate for mTLS authentication.'),
+    category=_('System'),
+    category_slug='system',
+    hidden=True,
+)
+
+register(
+    'CANDLEPIN_KEY_PEM',
+    field_class=fields.CharField,
+    default='',
+    allow_blank=True,
+    encrypted=True,
+    label=_('Candlepin Identity Key'),
+    help_text=_('PEM-encoded private key for Candlepin identity certificate.'),
+    category=_('System'),
+    category_slug='system',
+    hidden=True,
+)
+
+register(
+    'CANDLEPIN_SERIAL_NUMBER',
+    field_class=fields.CharField,
+    default='',
+    allow_blank=True,
+    encrypted=False,
+    label=_('Candlepin Certificate Serial Number'),
+    help_text=_('Serial number of the Candlepin identity certificate for tracking.'),
+    category=_('System'),
+    category_slug='system',
+    hidden=True,
+)
+
 register(
     'IS_K8S',
     field_class=fields.BooleanField,

awx/main/constants.py

@@ -11,6 +11,7 @@ __all__ = [
     'CAN_CANCEL',
     'ACTIVE_STATES',
     'STANDARD_INVENTORY_UPDATE_ENV',
+    'OIDC_CREDENTIAL_TYPE_NAMESPACES',
 ]
 
 PRIVILEGE_ESCALATION_METHODS = [
@@ -99,10 +100,6 @@ MAX_ISOLATED_PATH_COLON_DELIMITER = 2
 
 SURVEY_TYPE_MAPPING = {'text': str, 'textarea': str, 'password': str, 'multiplechoice': str, 'multiselect': str, 'integer': int, 'float': (float, int)}
 
-JOB_VARIABLE_PREFIXES = [
-    'awx',
-    'tower',
-]
 
 # Note, the \u001b[... are ansi color codes. We don't currenly import any of the python modules which define the codes.
 # Importing a library just for this message seemed like overkill
@@ -140,3 +137,6 @@ org_role_to_permission = {
     'execution_environment_admin_role': 'add_executionenvironment',
     'auditor_role': 'view_project',  # TODO: also doesnt really work
 }
+
+# OIDC credential type namespaces for feature flag filtering
+OIDC_CREDENTIAL_TYPE_NAMESPACES = ['hashivault-kv-oidc', 'hashivault-ssh-oidc']

awx/main/dispatch/config.py

@@ -25,12 +25,17 @@ def get_dispatcherd_config(for_service: bool = False, mock_publish: bool = False
         "version": 2,
         "service": {
             "pool_kwargs": {
-                "min_workers": settings.JOB_EVENT_WORKERS,
+                "min_workers": settings.DISPATCHER_MIN_WORKERS,
                 "max_workers": max_workers,
+                # This must be less than max_workers to make sense, which is usually 4
+                # With reserve of 1, after a burst of tasks, load needs to down to 4-1=3
+                # before we return to min_workers
+                "scaledown_reserve": 1,
+                "worker_max_lifetime_seconds": settings.WORKER_MAX_LIFETIME_SECONDS,
             },
             "main_kwargs": {"node_id": settings.CLUSTER_HOST_ID},
             "process_manager_cls": "ForkServerManager",
-            "process_manager_kwargs": {"preload_modules": ['awx.main.dispatch.hazmat']},
+            "process_manager_kwargs": {"preload_modules": ['awx.main.dispatch.prefork']},
         },
         "brokers": {},
         "publish": {},
@@ -38,8 +43,8 @@ def get_dispatcherd_config(for_service: bool = False, mock_publish: bool = False
     }
 
     if mock_publish:
-        config["brokers"]["noop"] = {}
-        config["publish"]["default_broker"] = "noop"
+        config["brokers"]["dispatcherd.testing.brokers.noop"] = {}
+        config["publish"]["default_broker"] = "dispatcherd.testing.brokers.noop"
     else:
         config["brokers"]["pg_notify"] = {
             "config": get_pg_notify_params(),
@@ -56,5 +61,11 @@ def get_dispatcherd_config(for_service: bool = False, mock_publish: bool = False
         }
 
         config["brokers"]["pg_notify"]["channels"] = ['tower_broadcast_all', 'tower_settings_change', get_task_queuename()]
+        metrics_cfg = settings.METRICS_SUBSYSTEM_CONFIG.get('server', {}).get(settings.METRICS_SERVICE_DISPATCHER)
+        if metrics_cfg:
+            config["service"]["metrics_kwargs"] = {
+                "host": metrics_cfg.get("host", "localhost"),
+                "port": metrics_cfg.get("port", 8015),
+            }
 
     return config

awx/main/dispatch/pool.py

@@ -1,6 +1,4 @@
 import logging
-import os
-import time
 
 from multiprocessing import Process
 
@@ -15,13 +13,12 @@ class PoolWorker(object):
     """
     A simple wrapper around a multiprocessing.Process that tracks a worker child process.
 
-    The worker process runs the provided target function and tracks its creation time.
+    The worker process runs the provided target function.
     """
 
-    def __init__(self, target, args, **kwargs):
+    def __init__(self, target, args):
         self.process = Process(target=target, args=args)
         self.process.daemon = True
-        self.creation_time = time.monotonic()
 
     def start(self):
         self.process.start()
@@ -38,44 +35,20 @@ class WorkerPool(object):
         pool = WorkerPool(workers_num=4)  # spawn four worker processes
     """
 
-    pool_cls = PoolWorker
-    debug_meta = ''
-
     def __init__(self, workers_num=None):
-        self.name = settings.CLUSTER_HOST_ID
-        self.pid = os.getpid()
         self.workers_num = workers_num or settings.JOB_EVENT_WORKERS
-        self.workers = []
 
-    def __len__(self):
-        return len(self.workers)
-
-    def init_workers(self, target, *target_args):
-        self.target = target
-        self.target_args = target_args
+    def init_workers(self, target):
         for idx in range(self.workers_num):
-            self.up()
-
-    def up(self):
-        idx = len(self.workers)
-        # It's important to close these because we're _about_ to fork, and we
-        # don't want the forked processes to inherit the open sockets
-        # for the DB and cache connections (that way lies race conditions)
-        django_connection.close()
-        django_cache.close()
-        worker = self.pool_cls(self.target, (idx,) + self.target_args)
-        self.workers.append(worker)
-        try:
-            worker.start()
-        except Exception:
-            logger.exception('could not fork')
-        else:
-            logger.debug('scaling up worker pid:{}'.format(worker.process.pid))
-        return idx, worker
-
-    def stop(self, signum):
-        try:
-            for worker in self.workers:
-                os.kill(worker.pid, signum)
-        except Exception:
-            logger.exception('could not kill {}'.format(worker.pid))
+            # It's important to close these because we're _about_ to fork, and we
+            # don't want the forked processes to inherit the open sockets
+            # for the DB and cache connections (that way lies race conditions)
+            django_connection.close()
+            django_cache.close()
+            worker = PoolWorker(target, (idx,))
+            try:
+                worker.start()
+            except Exception:
+                logger.exception('could not fork')
+            else:
+                logger.debug('scaling up worker pid:{}'.format(worker.process.pid))

awx/main/dispatch/prefork.py

@@ -18,7 +18,7 @@ django.setup()  # noqa
 from django.conf import settings
 
 # Preload all periodic tasks so their imports will be in shared memory
-for name, options in settings.CELERYBEAT_SCHEDULE.items():
+for name, options in settings.DISPATCHER_SCHEDULE.items():
     resolve_callable(options['task'])
 
 

awx/main/dispatch/reaper.py

@@ -1,9 +1,6 @@
-from datetime import timedelta
 import logging
 
 from django.db.models import Q
-from django.conf import settings
-from django.utils.timezone import now as tz_now
 from django.contrib.contenttypes.models import ContentType
 
 from awx.main.models import Instance, UnifiedJob, WorkflowJob
@@ -50,26 +47,6 @@ def reap_job(j, status, job_explanation=None):
     logger.error(f'{j.log_format} is no longer {status_before}; reaping')
 
 
-def reap_waiting(instance=None, status='failed', job_explanation=None, grace_period=None, excluded_uuids=None, ref_time=None):
-    """
-    Reap all jobs in waiting for this instance.
-    """
-    if grace_period is None:
-        grace_period = settings.JOB_WAITING_GRACE_PERIOD + settings.TASK_MANAGER_TIMEOUT
-
-    if instance is None:
-        hostname = Instance.objects.my_hostname()
-    else:
-        hostname = instance.hostname
-    if ref_time is None:
-        ref_time = tz_now()
-    jobs = UnifiedJob.objects.filter(status='waiting', modified__lte=ref_time - timedelta(seconds=grace_period), controller_node=hostname)
-    if excluded_uuids:
-        jobs = jobs.exclude(celery_task_id__in=excluded_uuids)
-    for j in jobs:
-        reap_job(j, status, job_explanation=job_explanation)
-
-
 def reap(instance=None, status='failed', job_explanation=None, excluded_uuids=None, ref_time=None):
     """
     Reap all jobs in running for this instance.

awx/main/dispatch/worker/base.py

@@ -19,49 +19,24 @@ def signame(sig):
     return dict((k, v) for v, k in signal.__dict__.items() if v.startswith('SIG') and not v.startswith('SIG_'))[sig]
 
 
-class WorkerSignalHandler:
-    def __init__(self):
-        self.kill_now = False
-        signal.signal(signal.SIGTERM, signal.SIG_DFL)
-        signal.signal(signal.SIGINT, self.exit_gracefully)
-
-    def exit_gracefully(self, *args, **kwargs):
-        self.kill_now = True
-
-
-class AWXConsumerBase(object):
-    last_stats = time.time()
-
-    def __init__(self, name, worker, queues=[], pool=None):
-        self.should_stop = False
+class AWXConsumerRedis(object):
 
+    def __init__(self, name, worker):
         self.name = name
-        self.total_messages = 0
-        self.queues = queues
-        self.worker = worker
-        self.pool = pool
-        if pool is None:
-            self.pool = WorkerPool()
-        self.pool.init_workers(self.worker.work_loop)
+        self.pool = WorkerPool()
+        self.pool.init_workers(worker.work_loop)
         self.redis = get_redis_client()
 
-    def run(self, *args, **kwargs):
+    def run(self):
         signal.signal(signal.SIGINT, self.stop)
         signal.signal(signal.SIGTERM, self.stop)
 
-        # Child should implement other things here
-
-    def stop(self, signum, frame):
-        self.should_stop = True
-        logger.warning('received {}, stopping'.format(signame(signum)))
-        raise SystemExit()
-
-
-class AWXConsumerRedis(AWXConsumerBase):
-    def run(self, *args, **kwargs):
-        super(AWXConsumerRedis, self).run(*args, **kwargs)
         logger.info(f'Callback receiver started with pid={os.getpid()}')
         db.connection.close()  # logs use database, so close connection
 
         while True:
             time.sleep(60)
+
+    def stop(self, signum, frame):
+        logger.warning('received {}, stopping'.format(signame(signum)))
+        raise SystemExit()

awx/main/dispatch/worker/callback.py

@@ -26,7 +26,6 @@ from awx.main.models.events import emit_event_detail
 from awx.main.utils.profiling import AWXProfiler
 from awx.main.tasks.system import events_processed_hook
 import awx.main.analytics.subsystem_metrics as s_metrics
-from .base import WorkerSignalHandler
 
 logger = logging.getLogger('awx.main.commands.run_callback_receiver')
 
@@ -57,6 +56,16 @@ def job_stats_wrapup(job_identifier, event=None):
         logger.exception('Worker failed to save stats or emit notifications: Job {}'.format(job_identifier))
 
 
+class WorkerSignalHandler:
+    def __init__(self):
+        self.kill_now = False
+        signal.signal(signal.SIGTERM, signal.SIG_DFL)
+        signal.signal(signal.SIGINT, self.exit_gracefully)
+
+    def exit_gracefully(self, *args, **kwargs):
+        self.kill_now = True
+
+
 class CallbackBrokerWorker:
     """
     A worker implementation that deserializes callback event data and persists
@@ -68,13 +77,13 @@ class CallbackBrokerWorker:
 
     MAX_RETRIES = 2
     INDIVIDUAL_EVENT_RETRIES = 3
-    last_stats = time.time()
-    last_flush = time.time()
     total = 0
     last_event = ''
     prof = None
 
     def __init__(self):
+        self.last_stats = time.time()
+        self.last_flush = time.time()
         self.buff = {}
         self.redis = get_redis_client()
         self.subsystem_metrics = s_metrics.CallbackReceiverMetrics(auto_pipe_execute=False)

awx/main/dispatch/worker/task.py

@@ -1,4 +1,3 @@
-import inspect
 import logging
 import importlib
 import time
@@ -37,18 +36,13 @@ def run_callable(body):
     if 'guid' in body:
         set_guid(body.pop('guid'))
     _call = resolve_callable(task)
-    if inspect.isclass(_call):
-        # the callable is a class, e.g., RunJob; instantiate and
-        # return its `run()` method
-        _call = _call().run
     log_extra = ''
     logger_method = logger.debug
-    if ('time_ack' in body) and ('time_pub' in body):
-        time_publish = body['time_ack'] - body['time_pub']
-        time_waiting = time.time() - body['time_ack']
-        if time_waiting > 5.0 or time_publish > 5.0:
+    if 'time_pub' in body:
+        time_publish = time.time() - body['time_pub']
+        if time_publish > 5.0:
             # If task too a very long time to process, add this information to the log
-            log_extra = f' took {time_publish:.4f} to ack, {time_waiting:.4f} in local dispatcher'
+            log_extra = f' took {time_publish:.4f} to send message'
             logger_method = logger.info
     # don't print kwargs, they often contain launch-time secrets
     logger_method(f'task {uuid} starting {task}(*{args}){log_extra}')

awx/main/fields.py

@@ -428,6 +428,9 @@ class CredentialInputField(JSONSchemaField):
         # determine the defined fields for the associated credential type
         properties = {}
         for field in model_instance.credential_type.inputs.get('fields', []):
+            # Prevent users from providing values for internally resolved fields
+            if 'internal' in field:
+                continue
             field = field.copy()
             properties[field['id']] = field
             if field.get('choices', []):
@@ -566,6 +569,7 @@ class CredentialTypeInputField(JSONSchemaField):
                             },
                             'label': {'type': 'string'},
                             'help_text': {'type': 'string'},
+                            'internal': {'type': 'boolean'},
                             'multiline': {'type': 'boolean'},
                             'secret': {'type': 'boolean'},
                             'ask_at_runtime': {'type': 'boolean'},

awx/main/management/commands/candlepin_cert.py

@@ -0,0 +1,330 @@
+import sys
+
+from argparse import RawDescriptionHelpFormatter
+
+from django.core.management.base import BaseCommand
+
+from awx.main.utils.candlepin.client import CandlepinClient
+from awx.main.utils.candlepin.lifecycle import (
+    get_candlepin_ca,
+    get_candlepin_url,
+    get_proxy_url,
+    get_renewal_days,
+    needs_renewal,
+    parse_cert,
+)
+from awx.main.utils.candlepin import (
+    _fetch_candlepin_cert_from_db,
+    _save_candlepin_cert_to_db,
+    _save_candlepin_registration_to_db,
+    resolve_registration_credentials,
+)
+
+
+class Command(BaseCommand):
+    """
+    Manage Candlepin consumer registration and certificate lifecycle.
+
+    Subcommands:
+      register  Register this AAP instance as a Candlepin consumer and obtain an
+                identity certificate for mTLS analytics uploads.
+      renew     Perform a manual check-in and, if needed, renew the stored identity
+                certificate.
+    """
+
+    help = 'Manage Candlepin consumer registration and certificate lifecycle'
+
+    def create_parser(self, prog_name, subcommand, **kwargs):
+        return super().create_parser(
+            prog_name,
+            subcommand,
+            formatter_class=RawDescriptionHelpFormatter,
+            epilog='\n'.join(
+                [
+                    'SUBCOMMANDS',
+                    '',
+                    '  register  Register this instance as a Candlepin consumer.',
+                    '            Credentials are read from AWX database by default',
+                    '            (REDHAT_USERNAME, REDHAT_PASSWORD). The organization is',
+                    '            discovered automatically from the Candlepin account.',
+                    '            Pass --username / --password-stdin / --org to override.',
+                    '            Example: echo "password" | awx-manage candlepin_cert register --username user --password-stdin',
+                    '',
+                    '  renew     Perform a manual check-in and proactive cert renewal.',
+                    '            Reads the stored cert/key/UUID from database.',
+                    '            Use --force to renew even if the cert is not near expiry.',
+                    '',
+                    'CONFIGURATION',
+                    '',
+                    '  Settings can be configured via Django settings (awx/settings/defaults.py):',
+                    '',
+                    '  AWX_ANALYTICS_CANDLEPIN_URL              Candlepin base URL',
+                    '                                           (default: https://subscription.example.com/candlepin)',
+                    '  AWX_ANALYTICS_CANDLEPIN_CA               Path to Candlepin CA cert for TLS verification',
+                    '  AWX_ANALYTICS_CANDLEPIN_RENEWAL_THRESHOLD_DAYS  Days before expiry to trigger renewal (default: 90)',
+                    '  AWX_ANALYTICS_CANDLEPIN_PROXY_URL        HTTP/HTTPS proxy for Candlepin API calls',
+                ]
+            ),
+            **kwargs,
+        )
+
+    def add_arguments(self, parser):
+        subparsers = parser.add_subparsers(dest='subcommand', metavar='subcommand')
+        subparsers.required = True
+
+        # --- register ---
+        reg = subparsers.add_parser(
+            'register',
+            help='Register this instance as a Candlepin consumer',
+            formatter_class=RawDescriptionHelpFormatter,
+        )
+        reg.add_argument('--username', help='Red Hat subscription username (overrides REDHAT_USERNAME from database)')
+        reg.add_argument(
+            '--password-stdin', dest='password_stdin', action='store_true', help='Read password from stdin (overrides REDHAT_PASSWORD from database)'
+        )
+        reg.add_argument('--org', help='Candlepin owner/org key (overrides auto-discovered organization)')
+        reg.add_argument('--candlepin-url', dest='candlepin_url', help='Candlepin base URL (overrides AWX_ANALYTICS_CANDLEPIN_URL setting)')
+        reg.add_argument(
+            '--candlepin-ca', dest='candlepin_ca', help='Path to Candlepin CA cert for TLS verification (overrides AWX_ANALYTICS_CANDLEPIN_CA setting)'
+        )
+        reg.add_argument('--proxy', help='HTTP/HTTPS proxy URL (overrides AWX_ANALYTICS_CANDLEPIN_PROXY_URL setting)')
+        reg.add_argument('--no-verify-tls', dest='no_verify_tls', action='store_true', help='Disable TLS certificate verification for Candlepin API calls')
+        reg.add_argument('--force', action='store_true', help='Re-register even if a certificate already exists in database')
+        reg.add_argument('--dry-run', dest='dry_run', action='store_true', help='Perform registration but do not save the result to database')
+
+        # --- renew ---
+        ren = subparsers.add_parser(
+            'renew',
+            help='Check in and renew the Candlepin identity certificate',
+            formatter_class=RawDescriptionHelpFormatter,
+        )
+        ren.add_argument('--candlepin-url', dest='candlepin_url', help='Candlepin base URL (overrides AWX_ANALYTICS_CANDLEPIN_URL setting)')
+        ren.add_argument(
+            '--candlepin-ca', dest='candlepin_ca', help='Path to Candlepin CA cert for TLS verification (overrides AWX_ANALYTICS_CANDLEPIN_CA setting)'
+        )
+        ren.add_argument('--proxy', help='HTTP/HTTPS proxy URL (overrides AWX_ANALYTICS_CANDLEPIN_PROXY_URL setting)')
+        ren.add_argument('--no-verify-tls', dest='no_verify_tls', action='store_true', help='Disable TLS certificate verification for Candlepin API calls')
+        ren.add_argument('--force', action='store_true', help='Renew the certificate even if it is not near expiry')
+        ren.add_argument('--dry-run', dest='dry_run', action='store_true', help='Perform check-in and renewal but do not save the result to database')
+
+    def handle(self, *args, **options):
+        subcommand = options['subcommand']
+        if subcommand == 'register':
+            ok = self._handle_register(options)
+        elif subcommand == 'renew':
+            ok = self._handle_renew(options)
+        else:
+            self.stderr.write(f'Unknown subcommand: {subcommand}')
+            sys.exit(1)
+
+        if not ok:
+            sys.exit(1)
+
+    # ------------------------------------------------------------------
+    # register
+    # ------------------------------------------------------------------
+
+    def _resolve_and_validate_credentials(self, options):
+        """Merge CLI options with DB values and validate all required fields are present.
+
+        Returns ``(username, password, org, db_install_uuid)`` on success, or ``None``
+        if any required field is missing (errors are written to ``self.stderr``).
+        """
+        username_override = options.get('username')
+        org_override = options.get('org')
+        verify_tls = not options.get('no_verify_tls', False)
+
+        # Read password from stdin if --password-stdin is set
+        if options.get('password_stdin'):
+            password_override = sys.stdin.read().strip()
+            if not password_override:
+                self.stderr.write('--password-stdin specified but no password provided on stdin')
+                return None
+        else:
+            password_override = None
+
+        # Use shared resolution and validation function
+        username, password, org, install_uuid, errors = resolve_registration_credentials(
+            username_override=username_override, password_override=password_override, org_override=org_override, verify_tls=verify_tls
+        )
+
+        if errors:
+            for error in errors:
+                self.stderr.write(f'Missing required value: {error}')
+            return None
+
+        return username, password, org, install_uuid
+
+    def _handle_register(self, options):
+        dry_run = options['dry_run']
+        force = options['force']
+
+        # Check whether a cert is already stored unless --force.
+        existing_cert, existing_key, _ = _fetch_candlepin_cert_from_db()
+        if existing_cert and existing_key and not force:
+            self.stdout.write('A Candlepin identity certificate is already stored in database. Use --force to re-register and replace it.')
+            return True
+
+        # Resolve credentials: CLI flags take precedence over database.
+        resolved = self._resolve_and_validate_credentials(options)
+        if resolved is None:
+            return False
+        username, password, org, db_install_uuid = resolved
+
+        candlepin_url = options.get('candlepin_url') or get_candlepin_url()
+        candlepin_ca = options.get('candlepin_ca') or get_candlepin_ca()
+        proxy = options.get('proxy') or get_proxy_url()
+        verify_tls = not options.get('no_verify_tls', False)
+
+        # If dry-run, display what would happen and exit early before any Candlepin operations
+        if dry_run:
+            self.stdout.write('[dry-run] Would register with Candlepin:')
+            self.stdout.write(f'  URL           : {candlepin_url}')
+            self.stdout.write(f'  Organization  : {org}')
+            self.stdout.write(f'  Username      : {username}')
+            self.stdout.write(f'  Install UUID  : {db_install_uuid}')
+            if candlepin_ca:
+                self.stdout.write(f'  CA cert       : {candlepin_ca}')
+            if proxy:
+                self.stdout.write(f'  Proxy         : {proxy}')
+            self.stdout.write(f'  Verify TLS    : {verify_tls}')
+            self.stdout.write('[dry-run] No Candlepin operations performed.')
+            return True
+
+        client = CandlepinClient(base_url=candlepin_url, candlepin_ca=candlepin_ca, proxy=proxy, verify_tls=verify_tls)
+
+        self.stdout.write(f'Registering with Candlepin at {candlepin_url} (org={org}) ...')
+        try:
+            cert_pem, key_pem, consumer_uuid = client.register_consumer(username, password, org, install_uuid=db_install_uuid)
+        except Exception as e:
+            self.stderr.write(f'Registration failed: {e}')
+            return False
+
+        self.stdout.write('Registered successfully.')
+        self.stdout.write(f'  Consumer UUID : {consumer_uuid}')
+
+        # Save to database
+        if _save_candlepin_registration_to_db(cert_pem, key_pem, consumer_uuid):
+            self.stdout.write('Certificate, key, and consumer UUID saved to database.')
+        else:
+            self.stderr.write('Failed to save registration to database.')
+            return False
+
+        # Best-effort certificate metadata display
+        try:
+            info = parse_cert(cert_pem)
+            self.stdout.write(f'  Cert serial   : {info["serial"]}')
+            self.stdout.write(f'  Cert CN       : {info["cn"]}')
+            self.stdout.write(f'  Valid until   : {info["not_after"]} ({info["days_remaining"]} days remaining)')
+        except ValueError as e:
+            self.stdout.write(f'Certificate metadata unavailable: {e}')
+
+        return True
+
+    # ------------------------------------------------------------------
+    # renew
+    # ------------------------------------------------------------------
+
+    def _handle_renew(self, options):
+        dry_run = options['dry_run']
+        force = options['force']
+
+        cert_pem, key_pem, consumer_uuid = _fetch_candlepin_cert_from_db()
+
+        if not cert_pem or not key_pem:
+            self.stderr.write('No Candlepin identity certificate found in database. Run the register subcommand first.')
+            return False
+
+        if not consumer_uuid:
+            self.stderr.write('CANDLEPIN_CONSUMER_UUID is not set. Run the register subcommand first.')
+            return False
+
+        try:
+            info = parse_cert(cert_pem)
+            self.stdout.write('Current certificate:')
+            self.stdout.write(f'  Serial        : {info["serial"]}')
+            self.stdout.write(f'  CN            : {info["cn"]}')
+            self.stdout.write(f'  Valid until   : {info["not_after"]} ({info["days_remaining"]} days remaining)')
+        except ValueError as e:
+            self.stdout.write('Current certificate:')
+            self.stdout.write(f'  Certificate metadata unavailable: {e}')
+            info = None
+
+        candlepin_url = options.get('candlepin_url') or get_candlepin_url()
+        candlepin_ca = options.get('candlepin_ca') or get_candlepin_ca()
+        proxy = options.get('proxy') or get_proxy_url()
+        verify_tls = not options.get('no_verify_tls', False)
+        renewal_days = get_renewal_days()
+
+        # Check if renewal is needed (without force, just check cert expiry locally)
+        renewal_needed = force or needs_renewal(cert_pem, renewal_days)
+
+        # If dry-run, display what would happen and exit early before any Candlepin operations
+        if dry_run:
+            self.stdout.write('[dry-run] Would perform the following operations:')
+            self.stdout.write(f'  URL           : {candlepin_url}')
+            self.stdout.write(f'  Consumer UUID : {consumer_uuid}')
+            if candlepin_ca:
+                self.stdout.write(f'  CA cert       : {candlepin_ca}')
+            if proxy:
+                self.stdout.write(f'  Proxy         : {proxy}')
+            self.stdout.write(f'  Verify TLS    : {verify_tls}')
+            self.stdout.write('  1. Check in with Candlepin')
+            if renewal_needed:
+                reason = 'forced via --force' if force else f'expiry within {renewal_days} days'
+                self.stdout.write(f'  2. Renew certificate ({reason})')
+            else:
+                if info:
+                    self.stdout.write(f'  2. No renewal needed ({info["days_remaining"]} days remaining, threshold: {renewal_days} days)')
+                else:
+                    self.stdout.write(f'  2. No renewal needed (threshold: {renewal_days} days)')
+            self.stdout.write('[dry-run] No Candlepin operations performed.')
+            return True
+
+        client = CandlepinClient(base_url=candlepin_url, candlepin_ca=candlepin_ca, proxy=proxy, verify_tls=verify_tls)
+
+        self.stdout.write(f'Checking in with Candlepin at {candlepin_url} (consumer={consumer_uuid}) ...')
+        checkin_success = client.checkin(consumer_uuid, cert_pem, key_pem)
+
+        if not checkin_success:
+            self.stderr.write('Check-in with Candlepin failed. Unable to verify certificate status.')
+            self.stderr.write('Certificate renewal may still be needed. Use --force to renew anyway, or check logs for details.')
+            return False
+
+        self.stdout.write('Check-in successful.')
+
+        if not renewal_needed:
+            if info:
+                self.stdout.write(f'Certificate has {info["days_remaining"]} days remaining (renewal threshold: {renewal_days} days). No renewal needed.')
+            else:
+                self.stdout.write(f'Certificate renewal threshold is {renewal_days} days. No renewal needed.')
+            return True
+
+        reason = 'forced via --force' if force else f'expiry within {renewal_days} days'
+        self.stdout.write(f'Renewing certificate ({reason}) ...')
+        try:
+            new_cert_pem, new_key_pem = client.regenerate_cert(consumer_uuid, cert_pem, key_pem)
+        except Exception as e:
+            self.stderr.write(f'Certificate renewal failed: {e}')
+            return False
+
+        self.stdout.write('Certificate renewed successfully.')
+
+        # Save to database
+        if _save_candlepin_cert_to_db(new_cert_pem, new_key_pem):
+            self.stdout.write('Renewed certificate and key saved to database.')
+        else:
+            self.stderr.write('Failed to save renewed certificate to database.')
+            return False
+
+        # Best-effort certificate metadata display
+        try:
+            new_info = parse_cert(new_cert_pem)
+            if info:
+                self.stdout.write(f'  Old serial    : {info["serial"]}')
+            self.stdout.write(f'  New serial    : {new_info["serial"]}')
+            self.stdout.write(f'  Valid until   : {new_info["not_after"]} ({new_info["days_remaining"]} days remaining)')
+        except ValueError as e:
+            self.stdout.write(f'Certificate metadata unavailable: {e}')
+
+        return True

awx/main/management/commands/check_db.py

@@ -1,9 +1,11 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved
 
-from django.core.management.base import BaseCommand
+from django.core.management.base import BaseCommand, CommandError
 from django.db import connection
 
+from awx.main.utils.db import db_requirement_violations
+
 
 class Command(BaseCommand):
     """Checks connection to the database, and prints out connection info if not connected"""
@@ -13,4 +15,8 @@ class Command(BaseCommand):
             cursor.execute("SELECT version()")
             version = str(cursor.fetchone()[0])
 
+        violations = db_requirement_violations()
+        if violations:
+            raise CommandError(violations)
+
         return "Database Version: {}".format(version)

awx/main/management/commands/create_preload_data.py

@@ -52,7 +52,11 @@ class Command(BaseCommand):
 
                     ssh_type = CredentialType.objects.filter(namespace='ssh').first()
                     c, _ = Credential.objects.get_or_create(
-                        credential_type=ssh_type, name='Demo Credential', inputs={'username': getattr(superuser, 'username', 'null')}, created_by=superuser
+                        credential_type=ssh_type,
+                        name='Demo Credential',
+                        inputs={'username': getattr(superuser, 'username', 'null')},
+                        created_by=superuser,
+                        organization=o,
                     )
 
                     if superuser:

awx/main/management/commands/dispatcherctl.py

@@ -0,0 +1,88 @@
+import argparse
+import inspect
+import logging
+import os
+import sys
+
+import yaml
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import connection
+
+from dispatcherd.cli import (
+    CONTROL_ARG_SCHEMAS,
+    DEFAULT_CONFIG_FILE,
+    _base_cli_parent,
+    _control_common_parent,
+    _register_control_arguments,
+    _build_command_data_from_args,
+)
+from dispatcherd.config import setup as dispatcher_setup
+from dispatcherd.factories import get_control_from_settings
+from dispatcherd.service import control_tasks
+
+from awx.main.dispatch.config import get_dispatcherd_config
+from awx.main.management.commands.dispatcherd import ensure_no_dispatcherd_env_config
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = 'Dispatcher control operations'
+
+    def add_arguments(self, parser):
+        parser.description = 'Run dispatcherd control commands using awx-manage.'
+        base_parent = _base_cli_parent()
+        control_parent = _control_common_parent()
+        parser._add_container_actions(base_parent)
+        parser._add_container_actions(control_parent)
+
+        subparsers = parser.add_subparsers(dest='command', metavar='command')
+        subparsers.required = True
+        shared_parents = [base_parent, control_parent]
+        for command in control_tasks.__all__:
+            func = getattr(control_tasks, command, None)
+            doc = inspect.getdoc(func) or ''
+            summary = doc.splitlines()[0] if doc else None
+            command_parser = subparsers.add_parser(
+                command,
+                help=summary,
+                description=doc,
+                parents=shared_parents,
+            )
+            _register_control_arguments(command_parser, CONTROL_ARG_SCHEMAS.get(command))
+
+    def handle(self, *args, **options):
+        command = options.pop('command', None)
+        if not command:
+            raise CommandError('No dispatcher control command specified')
+
+        for django_opt in ('verbosity', 'traceback', 'no_color', 'force_color', 'skip_checks'):
+            options.pop(django_opt, None)
+
+        log_level = options.pop('log_level', 'DEBUG')
+        config_path = os.path.abspath(options.pop('config', DEFAULT_CONFIG_FILE))
+        expected_replies = options.pop('expected_replies', 1)
+
+        logging.basicConfig(level=getattr(logging, log_level), stream=sys.stdout)
+        logger.debug(f"Configured standard out logging at {log_level} level")
+
+        default_config = os.path.abspath(DEFAULT_CONFIG_FILE)
+        ensure_no_dispatcherd_env_config()
+        if config_path != default_config:
+            raise CommandError('The config path CLI option is not allowed for the awx-manage command')
+        if connection.vendor == 'sqlite':
+            raise CommandError('dispatcherctl is not supported with sqlite3; use a PostgreSQL database')
+        else:
+            logger.info('Using config generated from awx.main.dispatch.config.get_dispatcherd_config')
+            dispatcher_setup(get_dispatcherd_config())
+
+        schema_namespace = argparse.Namespace(**options)
+        data = _build_command_data_from_args(schema_namespace, command)
+
+        ctl = get_control_from_settings()
+        returned = ctl.control_with_reply(command, data=data, expected_replies=expected_replies)
+        self.stdout.write(yaml.dump(returned, default_flow_style=False))
+        if len(returned) < expected_replies:
+            logger.error(f'Obtained only {len(returned)} of {expected_replies}, exiting with non-zero code')
+            raise CommandError('dispatcherctl returned fewer replies than expected')

awx/main/management/commands/dispatcherd.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+import copy
+import hashlib
+import json
+import logging
+import logging.config
+import os
+
+from django.conf import settings
+from django.core.cache import cache as django_cache
+from django.core.management.base import BaseCommand, CommandError
+from django.db import connection
+
+from dispatcherd.config import setup as dispatcher_setup
+
+from awx.main.dispatch.config import get_dispatcherd_config
+
+logger = logging.getLogger('awx.main.dispatch')
+
+
+from dispatcherd import run_service
+
+
+def _json_default(value):
+    if isinstance(value, set):
+        return sorted(value)
+    if isinstance(value, tuple):
+        return list(value)
+    return str(value)
+
+
+def _hash_config(config):
+    serialized = json.dumps(config, sort_keys=True, separators=(',', ':'), default=_json_default)
+    return hashlib.sha256(serialized.encode('utf-8')).hexdigest()
+
+
+def ensure_no_dispatcherd_env_config():
+    if os.getenv('DISPATCHERD_CONFIG_FILE'):
+        raise CommandError('DISPATCHERD_CONFIG_FILE is set but awx-manage dispatcherd uses dynamic config from code')
+
+
+class Command(BaseCommand):
+    help = (
+        'Run the background task service, this is the supported entrypoint since the introduction of dispatcherd as a library. '
+        'This replaces the prior awx-manage run_dispatcher service, and control actions are at awx-manage dispatcherctl.'
+    )
+
+    def add_arguments(self, parser):
+        return
+
+    def handle(self, *arg, **options):
+        ensure_no_dispatcherd_env_config()
+
+        self.configure_dispatcher_logging()
+        config = get_dispatcherd_config(for_service=True)
+        config_hash = _hash_config(config)
+        logger.info(
+            'Using dispatcherd config generated from awx.main.dispatch.config.get_dispatcherd_config (sha256=%s)',
+            config_hash,
+        )
+
+        # Close the connection, because the pg_notify broker will create new async connection
+        connection.close()
+        django_cache.close()
+        dispatcher_setup(config)
+
+        run_service()
+
+    def configure_dispatcher_logging(self):
+        # Apply special log rule for the parent process
+        special_logging = copy.deepcopy(settings.LOGGING)
+        changed_handlers = []
+        for handler_name, handler_config in special_logging.get('handlers', {}).items():
+            filters = handler_config.get('filters', [])
+            if 'dynamic_level_filter' in filters:
+                handler_config['filters'] = [flt for flt in filters if flt != 'dynamic_level_filter']
+                changed_handlers.append(handler_name)
+        logger.info(f'Dispatcherd main process replaced log level filter for handlers: {changed_handlers}')
+
+        # Apply the custom logging level here, before the asyncio code starts
+        special_logging.setdefault('loggers', {}).setdefault('dispatcherd', {})
+        special_logging['loggers']['dispatcherd']['level'] = settings.LOG_AGGREGATOR_LEVEL
+
+        logging.config.dictConfig(special_logging)

awx/main/management/commands/inventory_import.py

@@ -409,10 +409,12 @@ class Command(BaseCommand):
             del_child_group_pks = list(set(db_children_name_pk_map.values()))
             for offset in range(0, len(del_child_group_pks), self._batch_size):
                 child_group_pks = del_child_group_pks[offset : (offset + self._batch_size)]
-                for db_child in db_children.filter(pk__in=child_group_pks):
-                    group_group_count += 1
-                    db_group.children.remove(db_child)
-                    logger.debug('Group "%s" removed from group "%s"', db_child.name, db_group.name)
+                children_to_remove = list(db_children.filter(pk__in=child_group_pks))
+                if children_to_remove:
+                    group_group_count += len(children_to_remove)
+                    db_group.children.remove(*children_to_remove)
+                    for db_child in children_to_remove:
+                        logger.debug('Group "%s" removed from group "%s"', db_child.name, db_group.name)
             # FIXME: Inventory source group relationships
             # Delete group/host relationships not present in imported data.
             db_hosts = db_group.hosts
@@ -441,12 +443,12 @@ class Command(BaseCommand):
             del_host_pks = list(del_host_pks)
             for offset in range(0, len(del_host_pks), self._batch_size):
                 del_pks = del_host_pks[offset : (offset + self._batch_size)]
-                for db_host in db_hosts.filter(pk__in=del_pks):
-                    group_host_count += 1
-                    if db_host not in db_group.hosts.all():
-                        continue
-                    db_group.hosts.remove(db_host)
-                    logger.debug('Host "%s" removed from group "%s"', db_host.name, db_group.name)
+                hosts_to_remove = list(db_hosts.filter(pk__in=del_pks))
+                if hosts_to_remove:
+                    group_host_count += len(hosts_to_remove)
+                    db_group.hosts.remove(*hosts_to_remove)
+                    for db_host in hosts_to_remove:
+                        logger.debug('Host "%s" removed from group "%s"', db_host.name, db_group.name)
         if settings.SQL_DEBUG:
             logger.warning(
                 'group-group and group-host deletions took %d queries for %d relationships',

awx/main/management/commands/run_callback_receiver.py

@@ -3,7 +3,6 @@
 
 import redis
 
-from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 import redis.exceptions
 
@@ -36,11 +35,7 @@ class Command(BaseCommand):
             raise CommandError(f'Callback receiver could not connect to redis, error: {exc}')
 
         try:
-            consumer = AWXConsumerRedis(
-                'callback_receiver',
-                CallbackBrokerWorker(),
-                queues=[getattr(settings, 'CALLBACK_QUEUE', '')],
-            )
+            consumer = AWXConsumerRedis('callback_receiver', CallbackBrokerWorker())
             consumer.run()
         except KeyboardInterrupt:
             print('Terminating Callback Receiver')

awx/main/management/commands/run_dispatcher.py

@@ -1,26 +1,20 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved.
 import logging
-import logging.config
-import yaml
-import copy
 
-from django.conf import settings
-from django.core.management.base import BaseCommand, CommandError
-from django.core.cache import cache as django_cache
-from django.db import connection
+import yaml
+
+from django.core.management.base import CommandError
 
 from dispatcherd.factories import get_control_from_settings
-from dispatcherd import run_service
-from dispatcherd.config import setup as dispatcher_setup
 
-from awx.main.dispatch.config import get_dispatcherd_config
+from awx.main.management.commands.dispatcherd import Command as DispatcherdCommand
 
 logger = logging.getLogger('awx.main.dispatch')
 
 
-class Command(BaseCommand):
-    help = 'Launch the task dispatcher'
+class Command(DispatcherdCommand):
+    help = 'Launch the task dispatcher (deprecated; use awx-manage dispatcherd)'
 
     def add_arguments(self, parser):
         parser.add_argument('--status', dest='status', action='store_true', help='print the internal state of any running dispatchers')
@@ -34,8 +28,10 @@ class Command(BaseCommand):
                 'Only running tasks can be canceled, queued tasks must be started before they can be canceled.'
             ),
         )
+        super().add_arguments(parser)
 
-    def handle(self, *arg, **options):
+    def handle(self, *args, **options):
+        logger.warning('awx-manage run_dispatcher is deprecated; use awx-manage dispatcherd')
         if options.get('status'):
             ctl = get_control_from_settings()
             running_data = ctl.control_with_reply('status')
@@ -65,28 +61,4 @@ class Command(BaseCommand):
                 results.append(result)
             print(yaml.dump(results, default_flow_style=False))
             return
-
-        self.configure_dispatcher_logging()
-        # Close the connection, because the pg_notify broker will create new async connection
-        connection.close()
-        django_cache.close()
-        dispatcher_setup(get_dispatcherd_config(for_service=True))
-        run_service()
-
-        dispatcher_setup(get_dispatcherd_config(for_service=True))
-        run_service()
-
-    def configure_dispatcher_logging(self):
-        # Apply special log rule for the parent process
-        special_logging = copy.deepcopy(settings.LOGGING)
-        for handler_name, handler_config in special_logging.get('handlers', {}).items():
-            filters = handler_config.get('filters', [])
-            if 'dynamic_level_filter' in filters:
-                handler_config['filters'] = [flt for flt in filters if flt != 'dynamic_level_filter']
-                logger.info(f'Dispatcherd main process replaced log level filter for {handler_name} handler')
-
-        # Apply the custom logging level here, before the asyncio code starts
-        special_logging.setdefault('loggers', {}).setdefault('dispatcherd', {})
-        special_logging['loggers']['dispatcherd']['level'] = settings.LOG_AGGREGATOR_LEVEL
-
-        logging.config.dictConfig(special_logging)
+        return super().handle(*args, **options)

awx/main/managers.py

@@ -5,6 +5,7 @@ import logging
 import uuid
 from django.db import models
 from django.conf import settings
+from django.db.models import OuterRef, Subquery
 from django.db.models.functions import Lower
 
 from ansible_base.lib.utils.db import advisory_lock
@@ -23,7 +24,65 @@ class DeferJobCreatedManager(models.Manager):
         return super(DeferJobCreatedManager, self).get_queryset().defer('job_created')
 
 
-class HostManager(models.Manager):
+class HostLatestSummaryQuerySet(models.QuerySet):
+    """Queryset that annotates and bulk-attaches the latest JobHostSummary
+    at queryset evaluation time, similar to prefetch_related().
+
+    Why not use Django's Prefetch?
+    Django's Prefetch with [:1] slicing fetches 1 record globally, not per-host
+    (Django ticket #26780). Window-function workarounds require Django 4.2+ and
+    are more complex. Prefetching all summaries then filtering in Python wastes
+    memory for hosts with many job runs. The approach here — annotate the latest
+    ID via Subquery, then in_bulk() only those IDs — is the same 2-query pattern
+    prefetch_related uses internally, customized for "latest per group."
+
+    Not streaming-safe: relies on _result_cache existing after _fetch_all().
+    """
+
+    _awx_latest_summary_attached = False
+
+    def _clone(self):
+        clone = super()._clone()
+        clone._awx_latest_summary_attached = self._awx_latest_summary_attached
+        return clone
+
+    def with_latest_summary_id(self):
+        from awx.main.models.jobs import JobHostSummary
+
+        latest_summary = JobHostSummary.objects.filter(host_id=OuterRef('pk')).order_by('-id')
+        return self.annotate(
+            _latest_summary_id=Subquery(latest_summary.values('id')[:1]),
+        )
+
+    def _fetch_all(self):
+        super()._fetch_all()
+
+        if self._awx_latest_summary_attached or not self._result_cache:
+            return
+
+        # Only bulk-attach if the queryset was annotated via with_latest_summary_id().
+        # Without this guard, we'd set _latest_summary_cache=None on every host,
+        # masking the per-object fallback query in Host.latest_summary.
+        if not hasattr(self._result_cache[0], '_latest_summary_id'):
+            return
+
+        from awx.main.models.jobs import JobHostSummary
+
+        latest_summary_ids = [host._latest_summary_id for host in self._result_cache if host._latest_summary_id is not None]
+
+        if latest_summary_ids:
+            summaries_by_id = JobHostSummary.objects.select_related('job', 'job__job_template').in_bulk(latest_summary_ids)
+        else:
+            summaries_by_id = {}
+
+        for host in self._result_cache:
+            latest_summary_id = getattr(host, '_latest_summary_id', None)
+            host._latest_summary_cache = summaries_by_id.get(latest_summary_id)
+
+        self._awx_latest_summary_attached = True
+
+
+class HostManager(models.Manager.from_queryset(HostLatestSummaryQuerySet)):
     """Custom manager class for Hosts model."""
 
     def active_count(self):
@@ -31,38 +90,46 @@ class HostManager(models.Manager):
         Construction of query involves:
          - remove any ordering specified in model's Meta
          - Exclude hosts sourced from another Tower
+         - Exclude hosts in constructed inventories (these are shadow rows of source-inventory hosts)
          - Restrict the query to only return the name column
          - Only consider results that are unique
          - Return the count of this query
         """
-        return self.order_by().exclude(inventory_sources__source='controller').values(name_lower=Lower('name')).distinct().count()
+        return (
+            self.order_by()
+            .exclude(inventory_sources__source='controller')
+            .exclude(inventory__kind='constructed')
+            .values(name_lower=Lower('name'))
+            .distinct()
+            .count()
+        )
 
     def org_active_count(self, org_id):
         """Return count of active, unique hosts used by an organization.
         Construction of query involves:
          - remove any ordering specified in model's Meta
          - Exclude hosts sourced from another Tower
+         - Exclude hosts in constructed inventories (these are shadow rows of source-inventory hosts)
          - Consider only hosts where the canonical inventory is owned by the organization
          - Restrict the query to only return the name column
          - Only consider results that are unique
          - Return the count of this query
         """
-        return self.order_by().exclude(inventory_sources__source='controller').filter(inventory__organization=org_id).values('name').distinct().count()
+        return (
+            self.order_by()
+            .exclude(inventory_sources__source='controller')
+            .exclude(inventory__kind='constructed')
+            .filter(inventory__organization=org_id)
+            .values('name')
+            .distinct()
+            .count()
+        )
 
     def get_queryset(self):
         """When the parent instance of the host query set has a `kind=smart` and a `host_filter`
         set. Use the `host_filter` to generate the queryset for the hosts.
         """
-        qs = (
-            super(HostManager, self)
-            .get_queryset()
-            .defer(
-                'last_job__extra_vars',
-                'last_job_host_summary__job__extra_vars',
-                'last_job__artifacts',
-                'last_job_host_summary__job__artifacts',
-            )
-        )
+        qs = super().get_queryset().defer('ansible_facts')
 
         if hasattr(self, 'instance') and hasattr(self.instance, 'host_filter') and hasattr(self.instance, 'kind'):
             if self.instance.kind == 'smart' and self.instance.host_filter is not None:

awx/main/migrations/0201_create_managed_creds.py

@@ -21,6 +21,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(setup_tower_managed_defaults),
-        migrations.RunPython(setup_rbac_role_system_administrator),
+        migrations.RunPython(setup_tower_managed_defaults, migrations.RunPython.noop),
+        migrations.RunPython(setup_rbac_role_system_administrator, migrations.RunPython.noop),
     ]

awx/main/migrations/0202_convert_controller_role_definitions.py

@@ -98,5 +98,5 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(convert_controller_role_definitions),
+        migrations.RunPython(convert_controller_role_definitions, migrations.RunPython.noop),
     ]

awx/main/migrations/0204_squashed_deletions.py

@@ -3,19 +3,15 @@ from django.db import migrations, models
 from awx.main.migrations._create_system_jobs import delete_clear_tokens_sjt
 
 
-# --- START of function merged from 0203_rename_github_app_kind.py ---
 def update_github_app_kind(apps, schema_editor):
     """
-    Updates the 'kind' field for CredentialType records
+    Updates the 'namespace' field for CredentialType records
     from 'github_app' to 'github_app_lookup'.
     This addresses a change in the entry point key for the GitHub App plugin.
     """
     CredentialType = apps.get_model('main', 'CredentialType')
     db_alias = schema_editor.connection.alias
-    CredentialType.objects.using(db_alias).filter(kind='github_app').update(kind='github_app_lookup')
-
-
-# --- END of function merged from 0203_rename_github_app_kind.py ---
+    CredentialType.objects.using(db_alias).filter(namespace='github_app').update(namespace='github_app_lookup')
 
 
 class Migration(migrations.Migration):
@@ -118,7 +114,5 @@ class Migration(migrations.Migration):
                 max_length=32,
             ),
         ),
-        # --- START of operations merged from 0203_rename_github_app_kind.py ---
         migrations.RunPython(update_github_app_kind, migrations.RunPython.noop),
-        # --- END of operations merged from 0203_rename_github_app_kind.py ---
     ]

awx/main/migrations/0205_add_ordering_to_instancegroup_and_workflow_nodes.py

@@ -0,0 +1,29 @@
+# Generated by Django 5.2.8 on 2026-02-20 03:39
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0204_squashed_deletions'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='instancegroup',
+            options={
+                'default_permissions': ('change', 'delete', 'view'),
+                'ordering': ('pk',),
+                'permissions': [('use_instancegroup', 'Can use instance group in a preference list of a resource')],
+            },
+        ),
+        migrations.AlterModelOptions(
+            name='workflowjobnode',
+            options={'ordering': ('pk',)},
+        ),
+        migrations.AlterModelOptions(
+            name='workflowjobtemplatenode',
+            options={'ordering': ('pk',)},
+        ),
+    ]

awx/main/migrations/_inventory_source_vars.py

@@ -386,7 +386,6 @@ class gce(PluginFileInjector):
         # auth related items
         ret['auth_kind'] = "serviceaccount"
 
-        filters = []
         # TODO: implement gce group_by options
         # gce never processed the group_by field, if it had, we would selectively
         # apply those options here, but it did not, so all groups are added here
@@ -420,8 +419,6 @@ class gce(PluginFileInjector):
 
         if keyed_groups:
             ret['keyed_groups'] = keyed_groups
-        if filters:
-            ret['filters'] = filters
         if compose_dict:
             ret['compose'] = compose_dict
         if inventory_source.source_regions and 'all' not in inventory_source.source_regions:

awx/main/models/ad_hoc_commands.py

@@ -211,7 +211,7 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
         return AdHocCommand.objects.create(**data)
 
     def save(self, *args, **kwargs):
-        update_fields = kwargs.get('update_fields', [])
+        update_fields = kwargs.get('update_fields') or []
 
         def add_to_update_fields(name):
             if name not in update_fields:

awx/main/models/base.py

@@ -177,7 +177,7 @@ class CreatedModifiedModel(BaseModel):
     )
 
     def save(self, *args, **kwargs):
-        update_fields = list(kwargs.get('update_fields', []))
+        update_fields = list(kwargs.get('update_fields') or [])
         # Manually perform auto_now_add and auto_now logic.
         if not self.pk and not self.created:
             self.created = now()
@@ -207,7 +207,7 @@ class PasswordFieldsModel(BaseModel):
         new_instance = not bool(self.pk)
         # If update_fields has been specified, add our field names to it,
         # if it hasn't been specified, then we're just doing a normal save.
-        update_fields = kwargs.get('update_fields', [])
+        update_fields = kwargs.get('update_fields') or []
         # When first saving to the database, don't store any password field
         # values, but instead save them until after the instance is created.
         # Otherwise, store encrypted values to the database.
@@ -315,15 +315,14 @@ class PrimordialModel(HasEditsMixin, CreatedModifiedModel):
     )
 
     def __init__(self, *args, **kwargs):
-        r = super(PrimordialModel, self).__init__(*args, **kwargs)
+        super(PrimordialModel, self).__init__(*args, **kwargs)
         if self.pk:
             self._prior_values_store = self._get_fields_snapshot()
         else:
             self._prior_values_store = {}
-        return r
 
     def save(self, *args, **kwargs):
-        update_fields = kwargs.get('update_fields', [])
+        update_fields = kwargs.get('update_fields') or []
         user = get_current_user()
         if user and not user.id:
             user = None

awx/main/models/credential.py

@@ -28,6 +28,7 @@ from rest_framework.serializers import ValidationError as DRFValidationError
 from ansible_base.lib.utils.db import advisory_lock
 
 # AWX
+from awx.main.constants import OIDC_CREDENTIAL_TYPE_NAMESPACES
 from awx.api.versioning import reverse
 from awx.main.fields import (
     ImplicitRoleField,
@@ -46,12 +47,9 @@ from awx.main.models.rbac import (
 )
 from awx.main.models import Team, Organization
 from awx.main.utils import encrypt_field
+from awx.main.utils.lazy_registry import LazyLoadDict
 from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name
 
-# DAB
-from ansible_base.resource_registry.tasks.sync import get_resource_server_client
-from ansible_base.resource_registry.utils.settings import resource_server_defined
-
 __all__ = ['Credential', 'CredentialType', 'CredentialInputSource', 'build_safe_env']
 
 logger = logging.getLogger('awx.main.models.credential')
@@ -79,46 +77,6 @@ def build_safe_env(env):
     return safe_env
 
 
-def check_resource_server_for_user_in_organization(user, organization, requesting_user):
-    if not resource_server_defined():
-        return False
-
-    if not requesting_user:
-        return False
-
-    client = get_resource_server_client(settings.RESOURCE_SERVICE_PATH, jwt_user_id=str(requesting_user.resource.ansible_id), raise_if_bad_request=False)
-    # need to get the organization object_id in resource server, by querying with ansible_id
-    response = client._make_request(path=f'resources/?ansible_id={str(organization.resource.ansible_id)}', method='GET')
-    response_json = response.json()
-    if response.status_code != 200:
-        logger.error(f'Failed to get organization object_id in resource server: {response_json.get("detail", "")}')
-        return False
-
-    if response_json.get('count', 0) == 0:
-        return False
-    org_id_in_resource_server = response_json['results'][0]['object_id']
-
-    client.base_url = client.base_url.replace('/api/gateway/v1/service-index/', '/api/gateway/v1/')
-    # find role assignments with:
-    # - roles Organization Member or Organization Admin
-    # - user ansible id
-    # - organization object id
-
-    response = client._make_request(
-        path=f'role_user_assignments/?role_definition__name__in=Organization Member,Organization Admin&user__resource__ansible_id={str(user.resource.ansible_id)}&object_id={org_id_in_resource_server}',
-        method='GET',
-    )
-    response_json = response.json()
-    if response.status_code != 200:
-        logger.error(f'Failed to get role user assignments in resource server: {response_json.get("detail", "")}')
-        return False
-
-    if response_json.get('count', 0) > 0:
-        return True
-
-    return False
-
-
 class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
     """
     A credential contains information about how to talk to a remote resource
@@ -242,6 +200,29 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
                 needed.append('vault_password')
         return needed
 
+    @functools.cached_property
+    def context(self):
+        """
+        Property for storing runtime context during credential resolution.
+
+        The context is a dict keyed by CredentialInputSource PK, where each value
+        is a dict of runtime fields for that input source. Example::
+
+            {
+                <input_source_pk>: {
+                    "workload_identity_token": "<jwt_token>"
+                },
+                <another_input_source_pk>: {
+                    "workload_identity_token": "<different_jwt_token>"
+                },
+            }
+
+        This structure allows each input source to have its own set of runtime
+        values, avoiding conflicts when a credential has multiple input sources
+        with different configurations (e.g., different JWT audiences).
+        """
+        return {}
+
     @cached_property
     def dynamic_input_fields(self):
         # if the credential is not yet saved we can't access the input_sources
@@ -367,21 +348,20 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
     def _get_dynamic_input(self, field_name):
         for input_source in self.input_sources.all():
             if input_source.input_field_name == field_name:
-                return input_source.get_input_value()
+                return input_source.get_input_value(context=self.context)
         else:
             raise ValueError('{} is not a dynamic input field'.format(field_name))
 
     def validate_role_assignment(self, actor, role_definition, **kwargs):
+        requesting_user = kwargs.get('requesting_user', None)
+        if requesting_user and requesting_user.is_superuser:
+            return
         if self.organization:
             if isinstance(actor, User):
                 if actor.is_superuser:
                     return
                 if Organization.access_qs(actor, 'member').filter(id=self.organization.id).exists():
                     return
-
-                requesting_user = kwargs.get('requesting_user', None)
-                if check_resource_server_for_user_in_organization(actor, self.organization, requesting_user):
-                    return
             if isinstance(actor, Team):
                 if actor.organization == self.organization:
                     return
@@ -435,13 +415,15 @@ class CredentialType(CommonModelNameNotUnique):
     def from_db(cls, db, field_names, values):
         instance = super(CredentialType, cls).from_db(db, field_names, values)
         if instance.managed and instance.namespace and instance.kind != "external":
-            native = ManagedCredentialType.registry[instance.namespace]
-            instance.inputs = native.inputs
-            instance.injectors = native.injectors
-            instance.custom_injectors = getattr(native, 'custom_injectors', None)
+            native = ManagedCredentialType.registry.get(instance.namespace)
+            if native:
+                instance.inputs = native.inputs
+                instance.injectors = native.injectors
+                instance.custom_injectors = getattr(native, 'custom_injectors', None)
         elif instance.namespace and instance.kind == "external":
-            native = ManagedCredentialType.registry[instance.namespace]
-            instance.inputs = native.inputs
+            native = ManagedCredentialType.registry.get(instance.namespace)
+            if native:
+                instance.inputs = native.inputs
 
         return instance
 
@@ -505,6 +487,7 @@ class CredentialType(CommonModelNameNotUnique):
             existing = ct_class.objects.filter(name=default.name, kind=default.kind).first()
             if existing is not None:
                 existing.namespace = default.namespace
+                existing.description = getattr(default, 'description', '')
                 existing.inputs = {}
                 existing.injectors = {}
                 existing.save()
@@ -544,7 +527,14 @@ class CredentialType(CommonModelNameNotUnique):
     @classmethod
     def load_plugin(cls, ns, plugin):
         # TODO: User "side-loaded" credential custom_injectors isn't supported
-        ManagedCredentialType.registry[ns] = SimpleNamespace(namespace=ns, name=plugin.name, kind='external', inputs=plugin.inputs, backend=plugin.backend)
+        ManagedCredentialType.registry[ns] = SimpleNamespace(
+            namespace=ns,
+            name=plugin.name,
+            kind='external',
+            inputs=plugin.inputs,
+            backend=plugin.backend,
+            description=getattr(plugin, 'plugin_description', ''),
+        )
 
     def inject_credential(self, credential, env, safe_env, args, private_data_dir, container_root=None):
         from awx_plugins.interfaces._temporary_private_inject_api import inject_credential
@@ -556,7 +546,13 @@ class CredentialTypeHelper:
     @classmethod
     def get_creation_params(cls, cred_type):
         if cred_type.kind == 'external':
-            return dict(namespace=cred_type.namespace, kind=cred_type.kind, name=cred_type.name, managed=True)
+            return {
+                'namespace': cred_type.namespace,
+                'kind': cred_type.kind,
+                'name': cred_type.name,
+                'managed': True,
+                'description': getattr(cred_type, 'description', ''),
+            }
         return dict(
             namespace=cred_type.namespace,
             kind=cred_type.kind,
@@ -574,7 +570,7 @@ class CredentialTypeHelper:
 
 
 class ManagedCredentialType(SimpleNamespace):
-    registry = {}
+    registry = None  # initialized as LazyLoadDict after load_credentials is defined
 
 
 class CredentialInputSource(PrimordialModel):
@@ -622,7 +618,15 @@ class CredentialInputSource(PrimordialModel):
             raise ValidationError(_('Input field must be defined on target credential (options are {}).'.format(', '.join(sorted(defined_fields)))))
         return self.input_field_name
 
-    def get_input_value(self):
+    def get_input_value(self, context: dict | None = None):
+        """
+        Retrieve the value from the external credential backend.
+
+        Args:
+            context: Optional runtime context dict passed from the target credential.
+        """
+        if context is None:
+            context = {}
         backend = self.source_credential.credential_type.plugin.backend
         backend_kwargs = {}
         for field_name, value in self.source_credential.inputs.items():
@@ -633,6 +637,17 @@ class CredentialInputSource(PrimordialModel):
 
         backend_kwargs.update(self.metadata)
 
+        # Resolve internal fields from the per-input-source context.
+        # The context dict is keyed by input source PK, e.g.:
+        #   {42: {"workload_identity_token": "eyJ..."}, 43: {"workload_identity_token": "eyX..."}}
+        # This allows each input source to carry its own runtime values.
+        input_source_context = context.get(self.pk, {})
+        for field in self.source_credential.credential_type.inputs.get('fields', []):
+            if field.get('internal'):
+                value = input_source_context.get(field['id'])
+                if value is not None:
+                    backend_kwargs[field['id']] = value
+
         with set_environ(**settings.AWX_TASK_ENV):
             return backend(**backend_kwargs)
 
@@ -641,13 +656,22 @@ class CredentialInputSource(PrimordialModel):
         return reverse(view_name, kwargs={'pk': self.pk}, request=request)
 
 
+def _is_oidc_namespace_disabled(ns):
+    """Check if a credential namespace should be skipped based on the OIDC feature flag."""
+    return ns in OIDC_CREDENTIAL_TYPE_NAMESPACES and not getattr(settings, 'FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED', False)
+
+
 def load_credentials():
+    ManagedCredentialType.registry.clear()
 
     awx_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials')}
     supported_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials.supported')}
     plugin_entry_points = awx_entry_points if detect_server_product_name() == 'AWX' else {**awx_entry_points, **supported_entry_points}
 
     for ns, ep in plugin_entry_points.items():
+        if _is_oidc_namespace_disabled(ns):
+            continue
+
         cred_plugin = ep.load()
         if not hasattr(cred_plugin, 'inputs'):
             setattr(cred_plugin, 'inputs', {})
@@ -666,5 +690,13 @@ def load_credentials():
         credential_plugins = {}
 
     for ns, ep in credential_plugins.items():
+        if _is_oidc_namespace_disabled(ns):
+            continue
+
         plugin = ep.load()
         CredentialType.load_plugin(ns, plugin)
+
+
+# load_credentials writes directly into this dict via registry[ns] = ...,
+# LazyLoadDict just ensures it runs once before the first read access
+ManagedCredentialType.registry = LazyLoadDict(load_credentials)

awx/main/models/events.py

@@ -24,7 +24,6 @@ from awx.main.managers import DeferJobCreatedManager
 from awx.main.constants import MINIMAL_EVENTS
 from awx.main.models.base import CreatedModifiedModel
 from awx.main.utils import ignore_inventory_computed_fields, camelcase_to_underscore
-from awx.main.utils.db import bulk_update_sorted_by_id
 
 analytics_logger = logging.getLogger('awx.analytics.job_events')
 
@@ -590,20 +589,8 @@ class JobEvent(BasePlaybookEvent):
 
             JobHostSummary.objects.bulk_create(summaries.values())
 
-            # update the last_job_id and last_job_host_summary_id
-            # in single queries
-            host_mapping = dict((summary['host_id'], summary['id']) for summary in JobHostSummary.objects.filter(job_id=job.id).values('id', 'host_id'))
-            updated_hosts = set()
-            for h in all_hosts:
-                # if the hostname *shows up* in the playbook_on_stats event
-                if h.name in hostnames:
-                    h.last_job_id = job.id
-                    updated_hosts.add(h)
-                if h.id in host_mapping:
-                    h.last_job_host_summary_id = host_mapping[h.id]
-                    updated_hosts.add(h)
-
-            bulk_update_sorted_by_id(Host, updated_hosts, ['last_job_id', 'last_job_host_summary_id'])
+            # last_job and last_job_host_summary are now derived via
+            # JobHostSummary.latest_for_host / latest_job_for_host
 
             # Create/update Host Metrics
             self._update_host_metrics(updated_hosts_list)

awx/main/models/execution_environments.py

@@ -58,8 +58,6 @@ class ExecutionEnvironment(CommonModel):
         return reverse('api:execution_environment_detail', kwargs={'pk': self.pk}, request=request)
 
     def validate_role_assignment(self, actor, role_definition, **kwargs):
-        from awx.main.models.credential import check_resource_server_for_user_in_organization
-
         if self.managed:
             raise ValidationError({'object_id': _('Can not assign object roles to managed Execution Environments')})
         if self.organization_id is None:
@@ -69,8 +67,4 @@ class ExecutionEnvironment(CommonModel):
             if actor.has_obj_perm(self.organization, 'view'):
                 return
 
-            requesting_user = kwargs.get('requesting_user', None)
-            if check_resource_server_for_user_in_organization(actor, self.organization, requesting_user):
-                return
-
             raise ValidationError({'user': _('User must have view permission to Execution Environment organization')})

awx/main/models/ha.py

@@ -50,9 +50,8 @@ class HasPolicyEditsMixin(HasEditsMixin):
         abstract = True
 
     def __init__(self, *args, **kwargs):
-        r = super(BaseModel, self).__init__(*args, **kwargs)
+        super(BaseModel, self).__init__(*args, **kwargs)
         self._prior_values_store = self._get_fields_snapshot()
-        return r
 
     def save(self, *args, **kwargs):
         super(BaseModel, self).save(*args, **kwargs)
@@ -486,6 +485,7 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin, ResourceMi
 
     class Meta:
         app_label = 'main'
+        ordering = ('pk',)
         permissions = [('use_instancegroup', 'Can use instance group in a preference list of a resource')]
         # Since this has no direct organization field only superuser can add, so remove add permission
         default_permissions = ('change', 'delete', 'view')

awx/main/models/inventory.py

@@ -18,7 +18,7 @@ from django.db import transaction
 from django.core.exceptions import ValidationError
 from django.urls import resolve
 from django.utils.timezone import now
-from django.db.models import Q
+from django.db.models import Q, Subquery, OuterRef
 
 # REST Framework
 from rest_framework.exceptions import ParseError
@@ -27,7 +27,10 @@ from ansible_base.lib.utils.models import prevent_search
 
 # AWX
 from awx.api.versioning import reverse
+from awx.main.utils.common import load_all_entry_points_for
+from awx.main.utils.lazy_registry import LazyLoadDict
 from awx.main.utils.plugins import discover_available_cloud_provider_plugin_names, compute_cloud_inventory_sources
+from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name
 from awx.main.consumers import emit_channel_notification
 from awx.main.fields import (
     ImplicitRoleField,
@@ -386,7 +389,10 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin, OpaQu
         logger.debug("Going to update inventory computed fields, pk={0}".format(self.pk))
         start_time = time.time()
         active_hosts = self.hosts
-        failed_hosts = active_hosts.filter(last_job_host_summary__failed=True)
+        from awx.main.models.jobs import JobHostSummary  # circular import: inventory.py loads before jobs.py
+
+        latest_summary_failed = Subquery(JobHostSummary.objects.filter(host_id=OuterRef('pk')).order_by('-id').values('failed')[:1])
+        failed_hosts = active_hosts.annotate(_latest_failed=latest_summary_failed).filter(_latest_failed=True)
         active_groups = self.groups
         if self.kind == 'smart':
             active_groups = active_groups.none()
@@ -582,6 +588,23 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
 
     objects = HostManager()
 
+    @property
+    def latest_summary(self):
+        if hasattr(self, '_latest_summary_cache'):
+            return self._latest_summary_cache
+        from awx.main.models.jobs import JobHostSummary
+
+        summary = JobHostSummary.objects.filter(host_id=self.pk).order_by('-id').select_related('job', 'job__job_template').first()
+        self._latest_summary_cache = summary
+        return summary
+
+    @property
+    def latest_job(self):
+        summary = self.latest_summary
+        if summary is None:
+            return None
+        return summary.job
+
     def get_absolute_url(self, request=None):
         return reverse('api:host_detail', kwargs={'pk': self.pk}, request=request)
 
@@ -906,12 +929,22 @@ class HostMetricSummaryMonthly(models.Model):
     indirectly_managed_hosts = models.IntegerField(default=0, help_text=("Manually entered number indirectly managed hosts for a certain month"))
 
 
+def _load_inventory_plugins():
+    is_awx = detect_server_product_name() == 'AWX'
+    extra_entry_point_groups = () if is_awx else ('inventory.supported',)
+    all_entry_points = load_all_entry_points_for(['inventory', *extra_entry_point_groups])
+
+    for entry_point_name, entry_point in all_entry_points.items():
+        cls = entry_point.load()
+        InventorySourceOptions.injectors[entry_point_name] = cls
+
+
 class InventorySourceOptions(BaseModel):
     """
     Common fields for InventorySource and InventoryUpdate.
     """
 
-    injectors = dict()
+    injectors = LazyLoadDict(_load_inventory_plugins)
 
     # From the options of the Django management base command
     INVENTORY_UPDATE_VERBOSITY_CHOICES = [
@@ -1129,7 +1162,7 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE
 
         # If update_fields has been specified, add our field names to it,
         # if it hasn't been specified, then we're just doing a normal save.
-        update_fields = kwargs.get('update_fields', [])
+        update_fields = kwargs.get('update_fields') or []
         is_new_instance = not bool(self.pk)
 
         # Set name automatically. Include PK (or placeholder) to make sure the names are always unique.

awx/main/models/jobs.py

@@ -52,7 +52,7 @@ from awx.main.models.mixins import (
     WebhookTemplateMixin,
     OpaQueryPathMixin,
 )
-from awx.main.constants import JOB_VARIABLE_PREFIXES
+from awx.main.utils.common import get_job_variable_prefixes
 
 logger = logging.getLogger('awx.main.models.jobs')
 
@@ -347,7 +347,7 @@ class JobTemplate(
             return actual_slice_count
 
     def save(self, *args, **kwargs):
-        update_fields = kwargs.get('update_fields', [])
+        update_fields = kwargs.get('update_fields') or []
         # if project is deleted for some reason, then keep the old organization
         # to retain ownership for organization admins
         if self.project and self.project.organization_id != self.organization_id:
@@ -817,19 +817,20 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
 
     def awx_meta_vars(self):
         r = super(Job, self).awx_meta_vars()
+        prefixes = get_job_variable_prefixes()
         if self.project:
-            for name in JOB_VARIABLE_PREFIXES:
+            for name in prefixes:
                 r['{}_project_revision'.format(name)] = self.project.scm_revision
                 r['{}_project_scm_branch'.format(name)] = self.project.scm_branch
         if self.scm_branch:
-            for name in JOB_VARIABLE_PREFIXES:
+            for name in prefixes:
                 r['{}_job_scm_branch'.format(name)] = self.scm_branch
         if self.job_template:
-            for name in JOB_VARIABLE_PREFIXES:
+            for name in prefixes:
                 r['{}_job_template_id'.format(name)] = self.job_template.pk
                 r['{}_job_template_name'.format(name)] = self.job_template.name
         if self.execution_node:
-            for name in JOB_VARIABLE_PREFIXES:
+            for name in prefixes:
                 r['{}_execution_node'.format(name)] = self.execution_node
         return r
 
@@ -845,6 +846,21 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
     def get_notification_friendly_name(self):
         return "Job"
 
+    def get_source_hosts_for_constructed_inventory(self):
+        """Return a QuerySet of the source (input inventory) hosts for a constructed inventory.
+
+        Constructed inventory hosts have an instance_id pointing to the real
+        host in the input inventory.  This resolves those references and returns
+        a proper QuerySet (never a list), suitable for use with finish_fact_cache.
+        """
+        Host = JobHostSummary._meta.get_field('host').related_model
+        if not self.inventory_id:
+            return Host.objects.none()
+        id_field = Host._meta.get_field('id')
+        return Host.objects.filter(id__in=self.inventory.hosts.exclude(instance_id='').values_list(Cast('instance_id', output_field=id_field))).only(
+            *HOST_FACTS_FIELDS
+        )
+
     def get_hosts_for_fact_cache(self):
         """
         Builds the queryset to use for writing or finalizing the fact cache
@@ -852,17 +868,15 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
         For constructed inventories, that means the original (input inventory) hosts
         when slicing, that means only returning hosts in that slice
         """
-        Host = JobHostSummary._meta.get_field('host').related_model
         if not self.inventory_id:
+            Host = JobHostSummary._meta.get_field('host').related_model
             return Host.objects.none()
 
         if self.inventory.kind == 'constructed':
-            id_field = Host._meta.get_field('id')
-            host_qs = Host.objects.filter(id__in=self.inventory.hosts.exclude(instance_id='').values_list(Cast('instance_id', output_field=id_field)))
+            host_qs = self.get_source_hosts_for_constructed_inventory()
         else:
-            host_qs = self.inventory.hosts
+            host_qs = self.inventory.hosts.only(*HOST_FACTS_FIELDS)
 
-        host_qs = host_qs.only(*HOST_FACTS_FIELDS)
         host_qs = self.inventory.get_sliced_hosts(host_qs, self.job_slice_number, self.job_slice_count)
         return host_qs
 
@@ -1127,6 +1141,22 @@ class JobHostSummary(CreatedModifiedModel):
             self.skipped,
         )
 
+    @classmethod
+    def latest_for_host(cls, host_id):
+        """Return the most recent JobHostSummary for a given host, or None."""
+        return cls.objects.filter(host_id=host_id).order_by('-id').first()
+
+    @classmethod
+    def latest_job_for_host(cls, host_id):
+        """Return the Job from the most recent JobHostSummary for a host, or None."""
+        summary = cls.latest_for_host(host_id)
+        if summary:
+            try:
+                return summary.job
+            except cls.job.field.related_model.DoesNotExist:
+                return None
+        return None
+
     def get_absolute_url(self, request=None):
         return reverse('api:job_host_summary_detail', kwargs={'pk': self.pk}, request=request)
 
@@ -1135,7 +1165,7 @@ class JobHostSummary(CreatedModifiedModel):
         # if it hasn't been specified, then we're just doing a normal save.
         if self.host is not None:
             self.host_name = self.host.name
-        update_fields = kwargs.get('update_fields', [])
+        update_fields = kwargs.get('update_fields') or []
         self.failed = bool(self.dark or self.failures)
         update_fields.append('failed')
         super(JobHostSummary, self).save(*args, **kwargs)

awx/main/models/mixins.py

@@ -188,6 +188,16 @@ class SurveyJobTemplateMixin(models.Model):
                             runtime_extra_vars.pop(variable_key)
 
                 if default is not None:
+                    # do not add variables that contain an empty string, are not required and are not present in extra_vars
+                    # password fields must be skipped, because default values have special behaviour
+                    if (
+                        default == ''
+                        and not survey_element.get('required')
+                        and survey_element.get('type') != 'password'
+                        and variable_key not in runtime_extra_vars
+                    ):
+                        continue
+
                     decrypted_default = default
                     if survey_element['type'] == "password" and isinstance(decrypted_default, str) and decrypted_default.startswith('$encrypted$'):
                         decrypted_default = decrypt_value(get_encryption_key('value', pk=None), decrypted_default)

awx/main/models/notifications.py

@@ -99,7 +99,7 @@ class NotificationTemplate(CommonModelNameNotUnique):
 
     def save(self, *args, **kwargs):
         new_instance = not bool(self.pk)
-        update_fields = kwargs.get('update_fields', [])
+        update_fields = kwargs.get('update_fields') or []
 
         # preserve existing notification messages if not overwritten by new messages
         if not new_instance:

awx/main/models/projects.py

@@ -367,7 +367,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
         pre_save_vals = getattr(self, '_prior_values_store', {})
         # If update_fields has been specified, add our field names to it,
         # if it hasn't been specified, then we're just doing a normal save.
-        update_fields = kwargs.get('update_fields', [])
+        update_fields = kwargs.get('update_fields') or []
         self._skip_update = bool(kwargs.pop('skip_update', False))
         # Create auto-generated local path if project uses SCM.
         if self.pk and self.scm_type and not self.local_path.startswith('_'):

awx/main/models/rbac.py

@@ -613,7 +613,7 @@ def get_role_from_object_role(object_role):
         model_name, role_name = rd.name.split()
         role_name = role_name.lower()
         role_name += '_role'
-    return getattr(object_role.content_object, role_name)
+    return getattr(object_role.content_object, role_name, None)
 
 
 def give_or_remove_permission(role, actor, giving=True, rd=None):
@@ -649,6 +649,8 @@ def give_creator_permissions(user, obj):
     if assignment:
         with disable_rbac_sync():
             old_role = get_role_from_object_role(assignment.object_role)
+            if old_role is None:
+                return
             old_role.members.add(user)
 
 

awx/main/models/schedules.py

@@ -72,10 +72,10 @@ def _fast_forward_rrule(rrule, ref_dt=None):
     if ref_dt is None:
         ref_dt = now()
 
-    ref_dt = ref_dt.astimezone(datetime.timezone.utc)
+    dtstart_tz = rrule._dtstart.tzinfo
+    ref_dt = ref_dt.astimezone(dtstart_tz)
 
-    rrule_dtstart_utc = rrule._dtstart.astimezone(datetime.timezone.utc)
-    if rrule_dtstart_utc > ref_dt:
+    if rrule._dtstart > ref_dt:
         return rrule
 
     interval = rrule._interval if rrule._interval else 1
@@ -84,20 +84,14 @@ def _fast_forward_rrule(rrule, ref_dt=None):
     elif rrule._freq == dateutil.rrule.MINUTELY:
         interval *= 60
 
-    # if after converting to seconds the interval is still a fraction,
-    # just return original rrule
     if isinstance(interval, float) and not interval.is_integer():
         return rrule
 
-    seconds_since_dtstart = (ref_dt - rrule_dtstart_utc).total_seconds()
+    seconds_since_dtstart = (ref_dt - rrule._dtstart).total_seconds()
 
-    # it is important to fast forward by a number that is divisible by
-    # interval. For example, if interval is 7 hours, we fast forward by 7, 14, 21, etc. hours.
-    # Otherwise, the occurrences after the fast forward might not match the ones before.
-    # x // y is integer division, lopping off any remainder, so that we get the outcome we want.
     interval_aligned_offset = datetime.timedelta(seconds=(seconds_since_dtstart // interval) * interval)
-    new_start = rrule_dtstart_utc + interval_aligned_offset
-    new_rrule = rrule.replace(dtstart=new_start.astimezone(rrule._dtstart.tzinfo))
+    new_start = rrule._dtstart + interval_aligned_offset
+    new_rrule = rrule.replace(dtstart=new_start)
     return new_rrule
 
 

awx/main/models/unified_jobs.py

@@ -10,7 +10,6 @@ import json
 import logging
 import os
 import re
-import socket
 import subprocess
 import tempfile
 from collections import OrderedDict
@@ -59,7 +58,8 @@ from awx.main.utils.common import (
 )
 from awx.main.utils.encryption import encrypt_dict, decrypt_field
 from awx.main.utils import polymorphic
-from awx.main.constants import ACTIVE_STATES, CAN_CANCEL, JOB_VARIABLE_PREFIXES
+from awx.main.constants import ACTIVE_STATES, CAN_CANCEL
+from awx.main.utils.common import get_job_variable_prefixes
 from awx.main.redact import UriCleaner, REPLACE_STR
 from awx.main.consumers import emit_channel_notification
 from awx.main.fields import AskForField, OrderedManyToManyField
@@ -305,7 +305,7 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
     def save(self, *args, **kwargs):
         # If update_fields has been specified, add our field names to it,
         # if it hasn't been specified, then we're just doing a normal save.
-        update_fields = kwargs.get('update_fields', [])
+        update_fields = kwargs.get('update_fields') or []
         # Update status and last_updated fields.
         if not getattr(_inventory_updates, 'is_updating', False):
             updated_fields = self._set_status_and_last_job_run(save=False)
@@ -877,7 +877,7 @@ class UnifiedJob(
         """
         # If update_fields has been specified, add our field names to it,
         # if it hasn't been specified, then we're just doing a normal save.
-        update_fields = kwargs.get('update_fields', [])
+        update_fields = kwargs.get('update_fields') or []
 
         # Get status before save...
         status_before = self.status or 'new'
@@ -919,7 +919,7 @@ class UnifiedJob(
 
         # If we have a start and finished time, and haven't already calculated
         # out the time that elapsed, do so.
-        if self.started and self.finished and self.elapsed == 0.0:
+        if self.started and self.finished and self.elapsed == decimal.Decimal(0):
             td = self.finished - self.started
             elapsed = decimal.Decimal(td.total_seconds())
             self.elapsed = elapsed.quantize(dq)
@@ -1355,8 +1355,6 @@ class UnifiedJob(
                     status_data['instance_group_name'] = None
             elif status in ['successful', 'failed', 'canceled'] and self.finished:
                 status_data['finished'] = datetime.datetime.strftime(self.finished, "%Y-%m-%dT%H:%M:%S.%fZ")
-            elif status == 'running':
-                status_data['started'] = datetime.datetime.strftime(self.finished, "%Y-%m-%dT%H:%M:%S.%fZ")
             status_data.update(self.websocket_emit_data())
             status_data['group_name'] = 'jobs'
             if getattr(self, 'unified_job_template_id', None):
@@ -1488,40 +1486,17 @@ class UnifiedJob(
             return 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (self.model_to_str(), self.name, self.id)
         return None
 
-    def fallback_cancel(self):
-        if not self.celery_task_id:
-            self.refresh_from_db(fields=['celery_task_id'])
-        self.cancel_dispatcher_process()
-
     def cancel_dispatcher_process(self):
         """Returns True if dispatcher running this job acknowledged request and sent SIGTERM"""
         if not self.celery_task_id:
             return False
 
-        # Special case for task manager (used during workflow job cancellation)
-        if not connection.get_autocommit():
-            try:
-
-                ctl = get_control_from_settings()
-                ctl.control('cancel', data={'uuid': self.celery_task_id})
-            except Exception:
-                logger.exception("Error sending cancel command to dispatcher")
-            return True  # task manager itself needs to act under assumption that cancel was received
-
-        # Standard case with reply
         try:
-            timeout = 5
-
-            ctl = get_control_from_settings()
-            results = ctl.control_with_reply('cancel', data={'uuid': self.celery_task_id}, expected_replies=1, timeout=timeout)
-            # Check if cancel was successful by checking if we got any results
-            return bool(results and len(results) > 0)
-        except socket.timeout:
-            logger.error(f'could not reach dispatcher on {self.controller_node} within {timeout}s')
+            logger.info(f'Sending cancel message to pg_notify channel {self.controller_node} for task {self.celery_task_id}')
+            ctl = get_control_from_settings(default_publish_channel=self.controller_node)
+            ctl.control('cancel', data={'uuid': self.celery_task_id})
         except Exception:
-            logger.exception("error encountered when checking task status")
-
-        return False  # whether confirmation was obtained
+            logger.exception("Error sending cancel command to dispatcher")
 
     def cancel(self, job_explanation=None, is_chain=False):
         if self.can_cancel:
@@ -1544,19 +1519,13 @@ class UnifiedJob(
                 # the job control process will use the cancel_flag to distinguish a shutdown from a cancel
                 self.save(update_fields=cancel_fields)
 
-            controller_notified = False
-            if self.celery_task_id:
-                controller_notified = self.cancel_dispatcher_process()
+            # Be extra sure we have the task id, in case job is transitioning into running right now
+            if not self.celery_task_id:
+                self.refresh_from_db(fields=['celery_task_id', 'controller_node'])
 
-            # If a SIGTERM signal was sent to the control process, and acked by the dispatcher
-            # then we want to let its own cleanup change status, otherwise change status now
-            if not controller_notified:
-                if self.status != 'canceled':
-                    self.status = 'canceled'
-                    self.save(update_fields=['status'])
-                # Avoid race condition where we have stale model from pending state but job has already started,
-                # its checking signal but not cancel_flag, so re-send signal after updating cancel fields
-                self.fallback_cancel()
+            # send pg_notify message to cancel, will not send until transaction completes
+            if self.celery_task_id:
+                self.cancel_dispatcher_process()
 
         return self.cancel_flag
 
@@ -1600,7 +1569,8 @@ class UnifiedJob(
         by AWX, for purposes of client playbook hooks
         """
         r = {}
-        for name in JOB_VARIABLE_PREFIXES:
+        prefixes = get_job_variable_prefixes()
+        for name in prefixes:
             r['{}_job_id'.format(name)] = self.pk
             r['{}_job_launch_type'.format(name)] = self.launch_type
 
@@ -1609,7 +1579,7 @@ class UnifiedJob(
         wj = self.get_workflow_job()
         if wj:
             schedule = getattr_dne(wj, 'schedule')
-            for name in JOB_VARIABLE_PREFIXES:
+            for name in prefixes:
                 r['{}_workflow_job_id'.format(name)] = wj.pk
                 r['{}_workflow_job_name'.format(name)] = wj.name
                 r['{}_workflow_job_launch_type'.format(name)] = wj.launch_type
@@ -1620,12 +1590,12 @@ class UnifiedJob(
         if not created_by:
             schedule = getattr_dne(self, 'schedule')
             if schedule:
-                for name in JOB_VARIABLE_PREFIXES:
+                for name in prefixes:
                     r['{}_schedule_id'.format(name)] = schedule.pk
                     r['{}_schedule_name'.format(name)] = schedule.name
 
         if created_by:
-            for name in JOB_VARIABLE_PREFIXES:
+            for name in prefixes:
                 r['{}_user_id'.format(name)] = created_by.pk
                 r['{}_user_name'.format(name)] = created_by.username
                 r['{}_user_email'.format(name)] = created_by.email
@@ -1634,7 +1604,7 @@ class UnifiedJob(
 
         inventory = getattr_dne(self, 'inventory')
         if inventory:
-            for name in JOB_VARIABLE_PREFIXES:
+            for name in prefixes:
                 r['{}_inventory_id'.format(name)] = inventory.pk
                 r['{}_inventory_name'.format(name)] = inventory.name
 

awx/main/models/workflow.py

@@ -200,6 +200,7 @@ class WorkflowJobTemplateNode(WorkflowNodeBase):
         indexes = [
             models.Index(fields=['identifier']),
         ]
+        ordering = ('pk',)
 
     def get_absolute_url(self, request=None):
         return reverse('api:workflow_job_template_node_detail', kwargs={'pk': self.pk}, request=request)
@@ -286,6 +287,7 @@ class WorkflowJobNode(WorkflowNodeBase):
             models.Index(fields=["identifier", "workflow_job"]),
             models.Index(fields=['identifier']),
         ]
+        ordering = ('pk',)
 
     @property
     def event_processing_finished(self):
@@ -343,7 +345,11 @@ class WorkflowJobNode(WorkflowNodeBase):
                 )
             data.update(accepted_fields)  # missing fields are handled in the scheduler
         # build ancestor artifacts, save them to node model for later
-        aa_dict = {}
+        # initialize from pre-seeded ancestor_artifacts (set on root nodes of
+        # child workflows via seed_root_ancestor_artifacts to carry artifacts
+        # from the parent workflow); exclude job_slice which is internal
+        # metadata handled separately below
+        aa_dict = {k: v for k, v in self.ancestor_artifacts.items() if k != 'job_slice'} if self.ancestor_artifacts else {}
         is_root_node = True
         for parent_node in self.get_parent_nodes():
             is_root_node = False
@@ -364,11 +370,13 @@ class WorkflowJobNode(WorkflowNodeBase):
             data['survey_passwords'] = password_dict
         # process extra_vars
         extra_vars = data.get('extra_vars', {})
-        if ujt_obj and isinstance(ujt_obj, (JobTemplate, WorkflowJobTemplate)):
+        if ujt_obj and isinstance(ujt_obj, JobTemplate):
             if aa_dict:
                 functional_aa_dict = copy(aa_dict)
                 functional_aa_dict.pop('_ansible_no_log', None)
                 extra_vars.update(functional_aa_dict)
+        elif ujt_obj and isinstance(ujt_obj, WorkflowJobTemplate):
+            pass  # artifacts are applied via seed_root_ancestor_artifacts in the task manager
 
         # Workflow Job extra_vars higher precedence than ancestor artifacts
         extra_vars.update(wj_special_vars)
@@ -732,6 +740,18 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificatio
             wj = wj.get_workflow_job()
         return ancestors
 
+    def seed_root_ancestor_artifacts(self, artifacts):
+        """Apply parent workflow artifacts to root nodes so they propagate
+        through the normal ancestor_artifacts channel instead of being
+        baked into this workflow's extra_vars."""
+        self.workflow_job_nodes.exclude(
+            workflowjobnodes_success__isnull=False,
+        ).exclude(
+            workflowjobnodes_failure__isnull=False,
+        ).exclude(
+            workflowjobnodes_always__isnull=False,
+        ).update(ancestor_artifacts=artifacts)
+
     def get_effective_artifacts(self, **kwargs):
         """
         For downstream jobs of a workflow nested inside of a workflow,
@@ -785,7 +805,7 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificatio
     def cancel_dispatcher_process(self):
         # WorkflowJobs don't _actually_ run anything in the dispatcher, so
         # there's no point in asking the dispatcher if it knows about this task
-        return True
+        return
 
 
 class WorkflowApprovalTemplate(UnifiedJobTemplate, RelatedJobsMixin):
@@ -880,7 +900,7 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin):
         return 'workflow_approval_template'
 
     def save(self, *args, **kwargs):
-        update_fields = list(kwargs.get('update_fields', []))
+        update_fields = list(kwargs.get('update_fields') or [])
         if self.timeout != 0 and ((not self.pk) or (not update_fields) or ('timeout' in update_fields)):
             if not self.created:  # on creation, created will be set by parent class, so we fudge it here
                 created = now()
@@ -916,6 +936,17 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin):
         ScheduleWorkflowManager().schedule()
         return reverse('api:workflow_approval_deny', kwargs={'pk': self.pk}, request=request)
 
+    def cancel(self, job_explanation=None, is_chain=False):
+        # WorkflowApprovals have no dispatcher process (they wait for human
+        # input) and are excluded from TaskManager processing, so the base
+        # cancel() would only set cancel_flag without ever transitioning the
+        # status.  We call super() for the flag, then transition directly.
+        has_already_canceled = bool(self.status == 'canceled')
+        super().cancel(job_explanation=job_explanation, is_chain=is_chain)
+        if self.status != 'canceled' and not has_already_canceled:
+            self.status = 'canceled'
+            self.save(update_fields=['status'])
+
     def signal_start(self, **kwargs):
         can_start = super(WorkflowApproval, self).signal_start(**kwargs)
         self.started = self.created

awx/main/notifications/grafana_backend.py

@@ -76,10 +76,12 @@ class GrafanaBackend(AWXBaseEmailBackend, CustomNotificationBase):
             grafana_headers = {}
             if 'started' in m.body:
                 try:
-                    epoch = datetime.datetime.utcfromtimestamp(0)
-                    grafana_data['time'] = grafana_data['timeEnd'] = int((dp.parse(m.body['started']).replace(tzinfo=None) - epoch).total_seconds() * 1000)
+                    epoch = datetime.datetime.fromtimestamp(0, tz=datetime.timezone.utc)
+                    grafana_data['time'] = grafana_data['timeEnd'] = int(
+                        (dp.parse(m.body['started']).replace(tzinfo=datetime.timezone.utc) - epoch).total_seconds() * 1000
+                    )
                     if m.body.get('finished'):
-                        grafana_data['timeEnd'] = int((dp.parse(m.body['finished']).replace(tzinfo=None) - epoch).total_seconds() * 1000)
+                        grafana_data['timeEnd'] = int((dp.parse(m.body['finished']).replace(tzinfo=datetime.timezone.utc) - epoch).total_seconds() * 1000)
                 except ValueError:
                     logger.error(smart_str(_("Error converting time {} or timeEnd {} to int.").format(m.body['started'], m.body['finished'])))
                     if not self.fail_silently:

awx/main/notifications/webhook_backend.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2016 Ansible, Inc.
 # All Rights Reserved.
 
+import base64
 import json
 import logging
 import requests
@@ -84,20 +85,25 @@ class WebhookBackend(AWXBaseEmailBackend, CustomNotificationBase):
                 if resp.status_code not in [301, 307]:
                     break
 
+                # convert the url to a base64 encoded string for safe logging
+                url_log_safe = base64.b64encode(url.encode('UTF-8'))
+
+                # get the next URL to try
+                url_next = resp.headers.get("Location", None)
+                url_next_log_safe = base64.b64encode(url_next.encode('UTF-8')) if url_next else b'None'
+
                 # we've hit a redirect. extract the redirect URL out of the first response header and try again
-                logger.warning(
-                    f"Received a {resp.status_code} from {url}, trying to reach redirect url {resp.headers.get('Location', None)}; attempt #{retries+1}"
-                )
+                logger.warning(f"Received a {resp.status_code} from {url_log_safe}, trying to reach redirect url {url_next_log_safe}; attempt #{retries+1}")
 
                 # take the first redirect URL in the response header and try that
-                url = resp.headers.get("Location", None)
+                url = url_next
 
                 if url is None:
-                    err = f"Webhook notification received redirect to a blank URL from {url}. Response headers={resp.headers}"
+                    err = f"Webhook notification received redirect to a blank URL from {url_log_safe}. Response headers={resp.headers}"
                     break
             else:
                 # no break condition in the loop encountered; therefore we have hit the maximum number of retries
-                err = f"Webhook notification max number of retries [{self.MAX_RETRIES}] exceeded. Failed to send webhook notification to {url}"
+                err = f"Webhook notification max number of retries [{self.MAX_RETRIES}] exceeded. Failed to send webhook notification to {url_log_safe}"
 
             if resp.status_code >= 400:
                 err = f"Error sending webhook notification: {resp.status_code}"

awx/main/registrar.py

@@ -19,13 +19,8 @@ class ActivityStreamRegistrar(object):
             pre_delete.connect(activity_stream_delete, sender=model, dispatch_uid=str(self.__class__) + str(model) + "_delete")
 
             for m2mfield in model._meta.many_to_many:
-                try:
-                    m2m_attr = getattr(model, m2mfield.name)
-                    m2m_changed.connect(
-                        activity_stream_associate, sender=m2m_attr.through, dispatch_uid=str(self.__class__) + str(m2m_attr.through) + "_associate"
-                    )
-                except AttributeError:
-                    pass
+                m2m_attr = getattr(model, m2mfield.name)
+                m2m_changed.connect(activity_stream_associate, sender=m2m_attr.through, dispatch_uid=str(self.__class__) + str(m2m_attr.through) + "_associate")
 
     def disconnect(self, model):
         if model in self.models:

awx/main/scheduler/dag_simple.py

@@ -48,11 +48,6 @@ class SimpleDAG(object):
         '''
         self.node_to_edges_by_label = dict()
 
-    def __contains__(self, obj):
-        if self.node['node_object'] in self.node_obj_to_node_index:
-            return True
-        return False
-
     def __len__(self):
         return len(self.nodes)
 

awx/main/scheduler/dag_workflow.py

@@ -122,8 +122,11 @@ class WorkflowDAG(SimpleDAG):
             if not job:
                 continue
             elif job.can_cancel:
-                cancel_finished = False
                 job.cancel()
+                # If the job is not yet in a terminal state after .cancel(),
+                # the TaskManager still needs to process it.
+                if job.status not in ('successful', 'failed', 'canceled', 'error'):
+                    cancel_finished = False
         return cancel_finished
 
     def is_workflow_done(self):

awx/main/scheduler/task_manager.py

@@ -196,6 +196,10 @@ class WorkflowManager(TaskBase):
                     workflow_job.start_args = ''  # blank field to remove encrypted passwords
                     workflow_job.save(update_fields=['status', 'start_args'])
                     status_changed = True
+                else:
+                    # Speed-up: schedule the task manager so it can process the
+                    # canceled pending jobs without waiting for the next cycle.
+                    ScheduleTaskManager().schedule()
             else:
                 dnr_nodes = dag.mark_dnr_nodes()
                 WorkflowJobNode.objects.bulk_update(dnr_nodes, ['do_not_run'])
@@ -237,6 +241,8 @@ class WorkflowManager(TaskBase):
                     job = spawn_node.unified_job_template.create_unified_job(**kv)
                     spawn_node.job = job
                     spawn_node.save()
+                    if spawn_node.ancestor_artifacts and isinstance(spawn_node.unified_job_template, WorkflowJobTemplate):
+                        job.seed_root_ancestor_artifacts(spawn_node.ancestor_artifacts)
                     logger.debug('Spawned %s in %s for node %s', job.log_format, workflow_job.log_format, spawn_node.pk)
                     can_start = True
                     if isinstance(spawn_node.unified_job_template, WorkflowJobTemplate):
@@ -443,17 +449,29 @@ class TaskManager(TaskBase):
         self.controlplane_ig = self.tm_models.instance_groups.controlplane_ig
 
     def process_job_dep_failures(self, task):
-        """If job depends on a job that has failed, mark as failed and handle misc stuff."""
+        """If job depends on a job that has failed or been canceled, mark as failed.
+
+        Returns True if a dep failure was found, False otherwise.
+        """
         for dep in task.dependent_jobs.all():
-            # if we detect a failed or error dependency, go ahead and fail this task.
-            if dep.status in ("error", "failed"):
+            # if we detect a failed, error, or canceled dependency, go ahead and fail this task.
+            if dep.status in ("error", "failed", "canceled"):
                 task.status = 'failed'
-                logger.warning(f'Previous task failed task: {task.id} dep: {dep.id} task manager')
-                task.job_explanation = 'Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (
-                    get_type_for_model(type(dep)),
-                    dep.name,
-                    dep.id,
-                )
+                if dep.status == 'canceled':
+                    logger.warning(f'Previous task canceled, failing task: {task.id} dep: {dep.id} task manager')
+                    task.job_explanation = 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (
+                        get_type_for_model(type(dep)),
+                        dep.name,
+                        dep.id,
+                    )
+                    ScheduleWorkflowManager().schedule()  # speedup for dependency chains in workflow, on workflow cancel
+                else:
+                    logger.warning(f'Previous task failed, failing task: {task.id} dep: {dep.id} task manager')
+                    task.job_explanation = 'Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (
+                        get_type_for_model(type(dep)),
+                        dep.name,
+                        dep.id,
+                    )
                 task.save(update_fields=['status', 'job_explanation'])
                 task.websocket_emit_status('failed')
                 self.pre_start_failed.append(task.id)
@@ -545,8 +563,17 @@ class TaskManager(TaskBase):
                 logger.warning("Task manager has reached time out while processing pending jobs, exiting loop early")
                 break
 
-            has_failed = self.process_job_dep_failures(task)
-            if has_failed:
+            if task.cancel_flag:
+                logger.debug(f"Canceling pending task {task.log_format} because cancel_flag is set")
+                task.status = 'canceled'
+                task.job_explanation = gettext_noop("This job was canceled before it started.")
+                task.save(update_fields=['status', 'job_explanation'])
+                task.websocket_emit_status('canceled')
+                self.pre_start_failed.append(task.id)
+                ScheduleWorkflowManager().schedule()
+                continue
+
+            if self.process_job_dep_failures(task):
                 continue
 
             blocked_by = self.job_blocked_by(task)
@@ -661,6 +688,17 @@ class TaskManager(TaskBase):
                 logger.error(f'{j.execution_node} is not a registered instance; reaping {j.log_format}')
                 reap_job(j, 'failed')
 
+        # Reset waiting jobs whose controller_node was deprovisioned (e.g. K8s pod replaced).
+        # These jobs will never be picked up because no live node is listening for them.
+        registered_control_nodes = Instance.objects.filter(node_type__in=('control', 'hybrid')).values_list('hostname', flat=True)
+        orphaned_waiting = UnifiedJob.objects.filter(status='waiting').exclude(controller_node__in=registered_control_nodes)
+        for j in orphaned_waiting:
+            logger.warning(f'{j.controller_node} is not a registered instance; resetting {j.log_format} to pending')
+            j.status = 'pending'
+            j.controller_node = ''
+            j.execution_node = ''
+            j.save(update_fields=['status', 'controller_node', 'execution_node'])
+
     def process_tasks(self):
         # maintain a list of jobs that went to an early failure state,
         # meaning the dispatcher never got these jobs,

awx/main/signals.py

@@ -36,7 +36,6 @@ from awx.main.models import (
     Inventory,
     InventorySource,
     Job,
-    JobHostSummary,
     Organization,
     Project,
     Role,
@@ -251,45 +250,9 @@ def migrate_children_from_deleted_group_to_parent_groups(sender, **kwargs):
                         pass
 
 
-# Update host pointers to last_job and last_job_host_summary when a job is deleted
-
-
-def _update_host_last_jhs(host):
-    jhs_qs = JobHostSummary.objects.filter(host__pk=host.pk)
-    try:
-        jhs = jhs_qs.order_by('-job__pk')[0]
-    except IndexError:
-        jhs = None
-    update_fields = []
-    try:
-        last_job = jhs.job if jhs else None
-    except Job.DoesNotExist:
-        # The job (and its summaries) have already been/are currently being
-        # deleted, so there's no need to update the host w/ a reference to it
-        return
-    if host.last_job != last_job:
-        host.last_job = last_job
-        update_fields.append('last_job')
-    if host.last_job_host_summary != jhs:
-        host.last_job_host_summary = jhs
-        update_fields.append('last_job_host_summary')
-    if update_fields:
-        host.save(update_fields=update_fields)
-
-
-@receiver(pre_delete, sender=Job)
-def save_host_pks_before_job_delete(sender, **kwargs):
-    instance = kwargs['instance']
-    hosts_qs = Host.objects.filter(last_job__pk=instance.pk)
-    instance._saved_hosts_pks = set(hosts_qs.values_list('pk', flat=True))
-
-
-@receiver(post_delete, sender=Job)
-def update_host_last_job_after_job_deleted(sender, **kwargs):
-    instance = kwargs['instance']
-    hosts_pks = getattr(instance, '_saved_hosts_pks', [])
-    for host in Host.objects.filter(pk__in=hosts_pks):
-        _update_host_last_jhs(host)
+# Host.last_job and Host.last_job_host_summary are now derived from
+# JobHostSummary.latest_for_host / latest_job_for_host.
+# No signal handlers needed to maintain these denormalized FKs.
 
 
 # Set via ActivityStreamRegistrar to record activity stream events

awx/main/tasks/callback.py

@@ -54,9 +54,6 @@ def try_load_query_file(artifact_dir) -> Tuple[bool, Optional[dict]]:
     returns the contents of ansible_data.json if present
     """
 
-    if not flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
-        return False, None
-
     queries_path = os.path.join(artifact_dir, COLLECTION_FILENAME)
     if not os.path.isfile(queries_path):
         logger.info(f"no query file found: {queries_path}")
@@ -277,20 +274,6 @@ class RunnerCallback:
     def artifacts_handler(self, artifact_dir):
         success, query_file_contents = try_load_query_file(artifact_dir)
         if success:
-            self.delay_update(event_queries_processed=False)
-            collections_info = collect_queries(query_file_contents)
-            for collection, data in collections_info.items():
-                version = data['version']
-                event_query = data['host_query']
-                instance = EventQuery(fqcn=collection, collection_version=version, event_query=event_query)
-                try:
-                    instance.validate_unique()
-                    instance.save()
-
-                    logger.info(f"eventy query for collection {collection}, version {version} created")
-                except ValidationError as e:
-                    logger.info(e)
-
             if 'installed_collections' in query_file_contents:
                 self.delay_update(installed_collections=query_file_contents['installed_collections'])
             else:
@@ -301,6 +284,21 @@ class RunnerCallback:
             else:
                 logger.warning(f'The file {COLLECTION_FILENAME} unexpectedly did not contain ansible_version')
 
+            if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
+                self.delay_update(event_queries_processed=False)
+                collections_info = collect_queries(query_file_contents)
+                for collection, data in collections_info.items():
+                    version = data['version']
+                    event_query = data['host_query']
+                    instance = EventQuery(fqcn=collection, collection_version=version, event_query=event_query)
+                    try:
+                        instance.validate_unique()
+                        instance.save()
+
+                        logger.info(f"event query for collection {collection}, version {version} created")
+                    except ValidationError as e:
+                        logger.info(e)
+
         self.artifacts_processed = True
 
 

awx/main/tasks/facts.py

@@ -25,7 +25,8 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
     log_data = log_data or {}
     log_data['inventory_id'] = inventory_id
     log_data['written_ct'] = 0
-    hosts_cached = []
+    # Dict mapping host name -> bool (True if a fact file was written)
+    hosts_cached = {}
 
     # Create the fact_cache directory inside artifacts_dir
     fact_cache_dir = os.path.join(artifacts_dir, 'fact_cache')
@@ -37,13 +38,14 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
     last_write_time = None
 
     for host in hosts:
-        hosts_cached.append(host.name)
         if not host.ansible_facts_modified or (timeout and host.ansible_facts_modified < now() - datetime.timedelta(seconds=timeout)):
+            hosts_cached[host.name] = False
             continue  # facts are expired - do not write them
 
         filepath = os.path.join(fact_cache_dir, host.name)
         if not os.path.realpath(filepath).startswith(fact_cache_dir):
             logger.error(f'facts for host {smart_str(host.name)} could not be cached')
+            hosts_cached[host.name] = False
             continue
 
         try:
@@ -51,9 +53,18 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
                 os.chmod(f.name, 0o600)
                 json.dump(host.ansible_facts, f)
                 log_data['written_ct'] += 1
-                last_write_time = os.path.getmtime(filepath)
+            # Backdate the file by 2 seconds so finish_fact_cache can reliably
+            # distinguish these reference files from files updated by ansible.
+            # This guarantees fact file mtime < summary file mtime even with
+            # zipfile's 2-second timestamp rounding during artifact transfer.
+            mtime = os.path.getmtime(filepath)
+            backdated = mtime - 2
+            os.utime(filepath, (backdated, backdated))
+            last_write_time = backdated
+            hosts_cached[host.name] = True
         except IOError:
             logger.error(f'facts for host {smart_str(host.name)} could not be cached')
+            hosts_cached[host.name] = False
             continue
 
     # Write summary file directly to the artifacts_dir
@@ -62,7 +73,6 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
         summary_data = {
             'last_write_time': last_write_time,
             'hosts_cached': hosts_cached,
-            'written_ct': log_data['written_ct'],
         }
         with open(summary_file, 'w', encoding='utf-8') as f:
             json.dump(summary_data, f, indent=2)
@@ -74,7 +84,7 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
     msg='Inventory {inventory_id} host facts: updated {updated_ct}, cleared {cleared_ct}, unchanged {unmodified_ct}, took {delta:.3f} s',
     add_log_data=True,
 )
-def finish_fact_cache(artifacts_dir, job_id=None, inventory_id=None, log_data=None):
+def finish_fact_cache(host_qs, artifacts_dir, job_id=None, inventory_id=None, job_created=None, log_data=None):
     log_data = log_data or {}
     log_data['inventory_id'] = inventory_id
     log_data['updated_ct'] = 0
@@ -89,63 +99,118 @@ def finish_fact_cache(artifacts_dir, job_id=None, inventory_id=None, log_data=No
     try:
         with open(summary_path, 'r', encoding='utf-8') as f:
             summary = json.load(f)
-        facts_write_time = os.path.getmtime(summary_path)  # After successful read
+        facts_write_time = os.path.getmtime(summary_path)
     except (json.JSONDecodeError, OSError) as e:
         logger.error(f'Error reading summary file at {summary_path}: {e}')
         return
 
-    host_names = summary.get('hosts_cached', [])
-    hosts_cached = Host.objects.filter(name__in=host_names).order_by('id').iterator()
-    # Path where individual fact files were written
+    hosts_cached_map = summary.get('hosts_cached', {})
     fact_cache_dir = os.path.join(artifacts_dir, 'fact_cache')
-    hosts_to_update = []
 
-    for host in hosts_cached:
-        filepath = os.path.join(fact_cache_dir, host.name)
-        if not os.path.realpath(filepath).startswith(fact_cache_dir):
-            logger.error(f'Invalid path for facts file: {filepath}')
-            continue
+    # Phase 1: Scan files on disk to discover which hosts have updated or missing facts
+    hosts_with_updates = set()  # hostnames whose fact file was modified by Ansible
+    hosts_to_clear = []  # hostnames where Ansible removed the fact file
+    seen_in_dir = set()  # hostnames we found as files on disk
 
-        if os.path.exists(filepath):
-            # If the file changed since we wrote the last facts file, pre-playbook run...
-            modified = os.path.getmtime(filepath)
-            if not facts_write_time or modified >= facts_write_time:
-                try:
-                    with codecs.open(filepath, 'r', encoding='utf-8') as f:
-                        ansible_facts = json.load(f)
-                except ValueError:
-                    continue
+    if os.path.isdir(fact_cache_dir):
+        for filename in os.listdir(fact_cache_dir):
+            if filename not in hosts_cached_map:
+                continue  # not an expected host for this job
 
-                if ansible_facts != host.ansible_facts:
-                    host.ansible_facts = ansible_facts
-                    host.ansible_facts_modified = now()
-                    hosts_to_update.append(host)
-                    logger.info(
-                        f'New fact for inventory {smart_str(host.inventory.name)} host {smart_str(host.name)}',
-                        extra=dict(
-                            inventory_id=host.inventory.id,
-                            host_name=host.name,
-                            ansible_facts=host.ansible_facts,
-                            ansible_facts_modified=host.ansible_facts_modified.isoformat(),
-                            job_id=job_id,
-                        ),
-                    )
-                    log_data['updated_ct'] += 1
-                else:
-                    log_data['unmodified_ct'] += 1
+            filepath = os.path.join(fact_cache_dir, filename)
+            if os.path.islink(filepath):
+                logger.error(f'Invalid path for facts file: {filepath}')
+                continue
+            if not os.path.isfile(filepath):
+                continue
+
+            seen_in_dir.add(filename)
+            try:
+                modified = os.path.getmtime(filepath)
+            except OSError as e:
+                logger.warning(f'Could not stat facts file {filepath}: {e}')
+                continue
+            if modified >= facts_write_time:
+                hosts_with_updates.add(filename)
             else:
                 log_data['unmodified_ct'] += 1
+
+    # Check for files we wrote pre-job that are now missing (Ansible cleared facts)
+    for hostname, was_written in hosts_cached_map.items():
+        if hostname in seen_in_dir:
+            continue  # already handled above
+        if was_written:
+            hosts_to_clear.append(hostname)
         else:
-            # if the file goes missing, ansible removed it (likely via clear_facts)
-            # if the file goes missing, but the host has not started facts, then we should not clear the facts
-            host.ansible_facts = {}
-            host.ansible_facts_modified = now()
-            hosts_to_update.append(host)
-            logger.info(f'Facts cleared for inventory {smart_str(host.inventory.name)} host {smart_str(host.name)}')
-            log_data['cleared_ct'] += 1
+            log_data['unmodified_ct'] += 1
 
-        if len(hosts_to_update) >= 100:
-            bulk_update_sorted_by_id(Host, hosts_to_update, fields=['ansible_facts', 'ansible_facts_modified'])
-            hosts_to_update = []
+    # Phase 2: Stream updated facts to database in batches
+    if hosts_with_updates:
+        hosts_to_save = []
+        total_rows_updated = 0
+        for host in host_qs.filter(name__in=list(hosts_with_updates)).select_related('inventory').iterator():
+            filepath = os.path.join(fact_cache_dir, host.name)
+            try:
+                with codecs.open(filepath, 'r', encoding='utf-8') as f:
+                    new_facts = json.load(f)
+            except (ValueError, OSError):
+                continue
 
-    bulk_update_sorted_by_id(Host, hosts_to_update, fields=['ansible_facts', 'ansible_facts_modified'])
+            if new_facts != host.ansible_facts:
+                host.ansible_facts = new_facts
+                host.ansible_facts_modified = now()
+                hosts_to_save.append(host)
+                logger.info(
+                    f'New fact for inventory {smart_str(host.inventory.name)} host {smart_str(host.name)}',
+                    extra=dict(
+                        inventory_id=host.inventory.id,
+                        host_name=host.name,
+                        ansible_facts=host.ansible_facts,
+                        ansible_facts_modified=host.ansible_facts_modified.isoformat(),
+                        job_id=job_id,
+                    ),
+                )
+                log_data['updated_ct'] += 1
+            else:
+                log_data['unmodified_ct'] += 1
+
+            if len(hosts_to_save) >= 100:
+                total_rows_updated += bulk_update_sorted_by_id(Host, hosts_to_save, fields=['ansible_facts', 'ansible_facts_modified'])
+                hosts_to_save = []
+
+        if hosts_to_save:
+            total_rows_updated += bulk_update_sorted_by_id(Host, hosts_to_save, fields=['ansible_facts', 'ansible_facts_modified'])
+
+        # Mismatch means a concurrent process changed or deleted hosts between our read and bulk update
+        if total_rows_updated != log_data['updated_ct']:
+            logger.warning(
+                f'Fact update for inventory {inventory_id} job {job_id}: expected to update {log_data["updated_ct"]} hosts but {total_rows_updated} rows were changed'
+            )
+
+    # Phase 3: Clear facts for hosts whose files were removed by Ansible
+    if hosts_to_clear:
+        hosts = list(host_qs.filter(name__in=hosts_to_clear).select_related('inventory'))
+        clear_hosts = []
+        for host in hosts:
+            if job_created and host.ansible_facts_modified and host.ansible_facts_modified > job_created:
+                logger.warning(
+                    f'Skipping fact clear for host {smart_str(host.name)} in job {job_id} '
+                    f'inventory {inventory_id}: host ansible_facts_modified '
+                    f'({host.ansible_facts_modified.isoformat()}) is after this job\'s '
+                    f'created time ({job_created.isoformat()}). '
+                    f'A concurrent job likely updated this host\'s facts while this job was running.'
+                )
+                log_data['unmodified_ct'] += 1
+            else:
+                host.ansible_facts = {}
+                host.ansible_facts_modified = now()
+                clear_hosts.append(host)
+                logger.info(f'Facts cleared for inventory {smart_str(host.inventory.name)} host {smart_str(host.name)}')
+                log_data['cleared_ct'] += 1
+
+        if clear_hosts:
+            rows = bulk_update_sorted_by_id(Host, clear_hosts, fields=['ansible_facts', 'ansible_facts_modified'])
+            if rows != len(clear_hosts):
+                logger.warning(f'Fact clear for inventory {inventory_id} job {job_id}: expected to clear {len(clear_hosts)} hosts but {rows} rows were changed')
+
+    logger.debug(f'Updated {log_data["updated_ct"]} host facts for inventory {inventory_id} in job {job_id}')

awx/main/tasks/jobs.py

@@ -17,7 +17,6 @@ import urllib.parse as urlparse
 
 # Django
 from django.conf import settings
-from django.db import transaction
 
 # Shared code for the AWX platform
 from awx_plugins.interfaces._temporary_private_container_api import CONTAINER_ROOT, get_incontainer_path
@@ -84,6 +83,7 @@ from awx.main.utils.common import (
     create_partition,
     ScheduleWorkflowManager,
     ScheduleTaskManager,
+    getattr_dne,
 )
 from awx.conf.license import get_license
 from awx.main.utils.handlers import SpecialInventoryHandler
@@ -92,9 +92,90 @@ from awx.main.utils.update_model import update_model
 # Django flags
 from flags.state import flag_enabled
 
+# Workload Identity
+from ansible_base.lib.workload_identity.controller import AutomationControllerJobScope
+from awx.main.utils.workload_identity import retrieve_workload_identity_jwt_with_claims
+
 logger = logging.getLogger('awx.main.tasks.jobs')
 
 
+def populate_claims_for_workload(unified_job) -> dict:
+    """
+    Extract JWT claims from a Controller workload for the aap_controller_automation_job scope.
+    """
+
+    claims = {
+        AutomationControllerJobScope.CLAIM_JOB_ID: unified_job.id,
+        AutomationControllerJobScope.CLAIM_JOB_NAME: unified_job.name,
+        AutomationControllerJobScope.CLAIM_LAUNCH_TYPE: unified_job.launch_type,
+    }
+
+    # Related objects in the UnifiedJob model, applies to all job types
+    # null cases are omitted because of OIDC
+    if organization := getattr_dne(unified_job, 'organization'):
+        claims[AutomationControllerJobScope.CLAIM_ORGANIZATION_NAME] = organization.name
+        claims[AutomationControllerJobScope.CLAIM_ORGANIZATION_ID] = organization.id
+
+    if ujt := getattr_dne(unified_job, 'unified_job_template'):
+        claims[AutomationControllerJobScope.CLAIM_UNIFIED_JOB_TEMPLATE_NAME] = ujt.name
+        claims[AutomationControllerJobScope.CLAIM_UNIFIED_JOB_TEMPLATE_ID] = ujt.id
+
+    if instance_group := getattr_dne(unified_job, 'instance_group'):
+        claims[AutomationControllerJobScope.CLAIM_INSTANCE_GROUP_NAME] = instance_group.name
+        claims[AutomationControllerJobScope.CLAIM_INSTANCE_GROUP_ID] = instance_group.id
+
+    # Related objects on concrete models, may not be valid for type of unified_job
+    if inventory := getattr_dne(unified_job, 'inventory', None):
+        claims[AutomationControllerJobScope.CLAIM_INVENTORY_NAME] = inventory.name
+        claims[AutomationControllerJobScope.CLAIM_INVENTORY_ID] = inventory.id
+
+    if execution_environment := getattr_dne(unified_job, 'execution_environment', None):
+        claims[AutomationControllerJobScope.CLAIM_EXECUTION_ENVIRONMENT_NAME] = execution_environment.name
+        claims[AutomationControllerJobScope.CLAIM_EXECUTION_ENVIRONMENT_ID] = execution_environment.id
+
+    if project := getattr_dne(unified_job, 'project', None):
+        claims[AutomationControllerJobScope.CLAIM_PROJECT_NAME] = project.name
+        claims[AutomationControllerJobScope.CLAIM_PROJECT_ID] = project.id
+
+    if jt := getattr_dne(unified_job, 'job_template', None):
+        claims[AutomationControllerJobScope.CLAIM_JOB_TEMPLATE_NAME] = jt.name
+        claims[AutomationControllerJobScope.CLAIM_JOB_TEMPLATE_ID] = jt.id
+
+    # Only valid for job templates
+    if hasattr(unified_job, 'playbook'):
+        claims[AutomationControllerJobScope.CLAIM_PLAYBOOK_NAME] = unified_job.playbook
+
+    # Not valid for inventory updates and system jobs
+    if hasattr(unified_job, 'job_type'):
+        claims[AutomationControllerJobScope.CLAIM_JOB_TYPE] = unified_job.job_type
+
+    launched_by: dict = unified_job.launched_by
+    if 'name' in launched_by:
+        claims[AutomationControllerJobScope.CLAIM_LAUNCHED_BY_NAME] = launched_by['name']
+    if 'id' in launched_by:
+        claims[AutomationControllerJobScope.CLAIM_LAUNCHED_BY_ID] = launched_by['id']
+
+    return claims
+
+
+def retrieve_workload_identity_jwt(
+    unified_job: UnifiedJob,
+    audience: str,
+    scope: str,
+    workload_ttl_seconds: int | None = None,
+) -> str:
+    """Retrieve JWT token from workload claims.
+    Raises:
+        RuntimeError: if the workload identity client is not configured.
+    """
+    return retrieve_workload_identity_jwt_with_claims(
+        populate_claims_for_workload(unified_job),
+        audience,
+        scope,
+        workload_ttl_seconds,
+    )
+
+
 def with_path_cleanup(f):
     @functools.wraps(f)
     def _wrapped(self, *args, **kwargs):
@@ -121,6 +202,7 @@ def dispatch_waiting_jobs(binder):
         if not kwargs:
             kwargs = {}
         binder.control('run', data={'task': serialize_task(uj._get_task_class()), 'args': [uj.id], 'kwargs': kwargs, 'uuid': uj.celery_task_id})
+        UnifiedJob.objects.filter(pk=uj.pk, status='waiting').update(status='running', start_args='')
 
 
 class BaseTask(object):
@@ -135,6 +217,63 @@ class BaseTask(object):
         self.update_attempts = int(getattr(settings, 'DISPATCHER_DB_DOWNTOWN_TOLLERANCE', settings.DISPATCHER_DB_DOWNTIME_TOLERANCE) / 5)
         self.runner_callback = self.callback_class(model=self.model)
 
+    @functools.cached_property
+    def _credentials(self):
+        """
+        Credentials for the task execution.
+        Fetches credentials once using build_credentials_list() and stores
+        them for the duration of the task to avoid redundant database queries.
+        """
+        credentials_list = self.build_credentials_list(self.instance)
+        # Convert to list to prevent re-evaluation of QuerySet
+        return list(credentials_list)
+
+    def populate_workload_identity_tokens(self, additional_credentials=None):
+        """
+        Populate credentials with workload identity tokens.
+
+        Sets the context on Credential objects that have input sources
+        using compatible external credential types.
+        """
+        credentials = list(self._credentials)
+        if additional_credentials:
+            credentials.extend(additional_credentials)
+        credential_input_sources = (
+            (credential.context, src)
+            for credential in credentials
+            for src in credential.input_sources.all()
+            if any(
+                field.get('id') == 'workload_identity_token' and field.get('internal')
+                for field in src.source_credential.credential_type.inputs.get('fields', [])
+            )
+        )
+        for credential_ctx, input_src in credential_input_sources:
+            if flag_enabled("FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED"):
+                effective_timeout = self.get_instance_timeout(self.instance)
+                workload_ttl = effective_timeout if effective_timeout else None
+                try:
+                    jwt = retrieve_workload_identity_jwt(
+                        self.instance,
+                        audience=input_src.source_credential.get_input('url'),
+                        scope=AutomationControllerJobScope.name,
+                        workload_ttl_seconds=workload_ttl,
+                    )
+                    # Store token keyed by input source PK, since a credential can have
+                    # multiple input sources (one per field), each potentially with a different audience
+                    credential_ctx[input_src.pk] = {"workload_identity_token": jwt}
+                except Exception as e:
+                    self.instance.job_explanation = (
+                        f'Could not generate workload identity token for credential {input_src.source_credential.name} used in this job. Error:\n{e}'
+                    )
+                    self.instance.status = 'error'
+                    self.instance.save()
+            else:
+                self.instance.job_explanation = (
+                    f'Flag FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED is not enabled, required for credential {input_src.source_credential.name} used in this job.'
+                )
+                self.instance.status = 'error'
+                self.instance.save()
+
     def update_model(self, pk, _attempt=0, **updates):
         return update_model(self.model, pk, _attempt=0, _max_attempts=self.update_attempts, **updates)
 
@@ -286,6 +425,19 @@ class BaseTask(object):
                     private_data_files['credentials'][credential] = self.write_private_data_file(private_data_dir, None, data, sub_dir='env')
             for credential, data in private_data.get('certificates', {}).items():
                 self.write_private_data_file(private_data_dir, 'ssh_key_data-cert.pub', data, sub_dir=os.path.join('artifacts', str(self.instance.id)))
+
+        # Copy vendor collections to private_data_dir for indirect node counting
+        # This makes external query files available to the callback plugin in EEs
+        if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
+            vendor_src = '/var/lib/awx/vendor_collections'
+            vendor_dest = os.path.join(private_data_dir, 'vendor_collections')
+            if os.path.exists(vendor_src):
+                try:
+                    shutil.copytree(vendor_src, vendor_dest)
+                    logger.debug(f"Copied vendor collections from {vendor_src} to {vendor_dest}")
+                except Exception as e:
+                    logger.warning(f"Failed to copy vendor collections: {e}")
+
         return private_data_files, ssh_key_data
 
     def build_passwords(self, instance, runtime_passwords):
@@ -359,6 +511,7 @@ class BaseTask(object):
         return []
 
     def get_instance_timeout(self, instance):
+        """Return the effective job timeout in seconds."""
         global_timeout_setting_name = instance._global_timeout_setting()
         if global_timeout_setting_name:
             global_timeout = getattr(settings, global_timeout_setting_name, 0)
@@ -467,48 +620,32 @@ class BaseTask(object):
     def should_use_fact_cache(self):
         return False
 
-    def transition_status(self, pk: int) -> bool:
-        """Atomically transition status to running, if False returned, another process got it"""
-        with transaction.atomic():
-            # Explanation of parts for the fetch:
-            # .values - avoid loading a full object, this is known to lead to deadlocks due to signals
-            #   the signals load other related rows which another process may be locking, and happens in practice
-            # of=('self',) - keeps FK tables out of the lock list, another way deadlocks can happen
-            # .get - just load the single job
-            instance_data = UnifiedJob.objects.select_for_update(of=('self',)).values('status', 'cancel_flag').get(pk=pk)
-
-            # If status is not waiting (obtained under lock) then this process does not have clearence to run
-            if instance_data['status'] == 'waiting':
-                if instance_data['cancel_flag']:
-                    updated_status = 'canceled'
-                else:
-                    updated_status = 'running'
-                # Explanation of the update:
-                # .filter - again, do not load the full object
-                # .update - a bulk update on just that one row, avoid loading unintended data
-                UnifiedJob.objects.filter(pk=pk).update(status=updated_status, start_args='')
-            elif instance_data['status'] == 'running':
-                logger.info(f'Job {pk} is being ran by another process, exiting')
-                return False
-        return True
-
     @with_path_cleanup
     @with_signal_handling
     def run(self, pk, **kwargs):
         """
         Run the job/task and capture its output.
         """
-        if not self.instance:  # Used to skip fetch for local runs
-            if not self.transition_status(pk):
-                logger.info(f'Job {pk} is being ran by another process, exiting')
-                return
 
-        # Load the instance
-        self.instance = self.update_model(pk)
+        if not self.instance:  # Used to skip fetch for local runs
+            # Load the instance
+            self.instance = self.update_model(pk)
+
+        # status should be "running" from dispatch_waiting_jobs,
+        # but may still be "waiting" if the worker picked this up before the status update landed.
+        if self.instance.status == 'waiting':
+            UnifiedJob.objects.filter(pk=pk).update(status="running", start_args='')
+            self.instance.refresh_from_db()
+
         if self.instance.status != 'running':
             logger.error(f'Not starting {self.instance.status} task pk={pk} because its status "{self.instance.status}" is not expected')
             return
 
+        if self.instance.cancel_flag:
+            self.instance = self.update_model(pk, status='canceled')
+            self.instance.websocket_emit_status('canceled')
+            return
+
         self.instance.websocket_emit_status("running")
         status, rc = 'error', None
         self.runner_callback.event_ct = 0
@@ -547,6 +684,12 @@ class BaseTask(object):
             if not os.path.exists(settings.AWX_ISOLATION_BASE_PATH):
                 raise RuntimeError('AWX_ISOLATION_BASE_PATH=%s does not exist' % settings.AWX_ISOLATION_BASE_PATH)
 
+            if flag_enabled("FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED"):
+                logger.info(f'Generating workload identity tokens for {self.instance.log_format}')
+                self.populate_workload_identity_tokens()
+                if self.instance.status == 'error':
+                    raise RuntimeError('not starting %s task' % self.instance.status)
+
             # May have to serialize the value
             private_data_files, ssh_key_data = self.build_private_data_files(self.instance, private_data_dir)
             passwords = self.build_passwords(self.instance, kwargs)
@@ -564,7 +707,7 @@ class BaseTask(object):
 
             self.runner_callback.job_created = str(self.instance.created)
 
-            credentials = self.build_credentials_list(self.instance)
+            credentials = self._credentials
 
             container_root = None
             if settings.IS_K8S and isinstance(self.instance, ProjectUpdate):
@@ -859,6 +1002,29 @@ class RunJob(SourceControlMixin, BaseTask):
     model = Job
     event_model = JobEvent
 
+    def _extract_credentials_of_kind(self, kind: str):
+        return (cred for cred in self._credentials if cred.credential_type.kind == kind)
+
+    @property
+    def _machine_credential(self) -> object:
+        """Get machine credential."""
+        return next(self._extract_credentials_of_kind('ssh'), None)
+
+    @property
+    def _vault_credentials(self) -> list[object]:
+        """Get vault credentials."""
+        return list(self._extract_credentials_of_kind('vault'))
+
+    @property
+    def _network_credentials(self) -> list[object]:
+        """Get network credentials."""
+        return list(self._extract_credentials_of_kind('net'))
+
+    @property
+    def _cloud_credentials(self) -> list[object]:
+        """Get cloud credentials."""
+        return list(self._extract_credentials_of_kind('cloud'))
+
     def build_private_data(self, job, private_data_dir):
         """
         Returns a dict of the form
@@ -876,7 +1042,7 @@ class RunJob(SourceControlMixin, BaseTask):
         }
         """
         private_data = {'credentials': {}}
-        for credential in job.credentials.prefetch_related('input_sources__source_credential').all():
+        for credential in self._credentials:
             # If we were sent SSH credentials, decrypt them and send them
             # back (they will be written to a temporary file).
             if credential.has_input('ssh_key_data'):
@@ -892,14 +1058,14 @@ class RunJob(SourceControlMixin, BaseTask):
         and ansible-vault.
         """
         passwords = super(RunJob, self).build_passwords(job, runtime_passwords)
-        cred = job.machine_credential
+        cred = self._machine_credential
         if cred:
             for field in ('ssh_key_unlock', 'ssh_password', 'become_password', 'vault_password'):
                 value = runtime_passwords.get(field, cred.get_input('password' if field == 'ssh_password' else field, default=''))
                 if value not in ('', 'ASK'):
                     passwords[field] = value
 
-        for cred in job.vault_credentials:
+        for cred in self._vault_credentials:
             field = 'vault_password'
             vault_id = cred.get_input('vault_id', default=None)
             if vault_id:
@@ -915,7 +1081,7 @@ class RunJob(SourceControlMixin, BaseTask):
         key unlock over network key unlock.
         '''
         if 'ssh_key_unlock' not in passwords:
-            for cred in job.network_credentials:
+            for cred in self._network_credentials:
                 if cred.inputs.get('ssh_key_unlock'):
                     passwords['ssh_key_unlock'] = runtime_passwords.get('ssh_key_unlock', cred.get_input('ssh_key_unlock', default=''))
                     break
@@ -950,11 +1116,11 @@ class RunJob(SourceControlMixin, BaseTask):
 
         # Set environment variables for cloud credentials.
         cred_files = private_data_files.get('credentials', {})
-        for cloud_cred in job.cloud_credentials:
+        for cloud_cred in self._cloud_credentials:
             if cloud_cred and cloud_cred.credential_type.namespace == 'openstack' and cred_files.get(cloud_cred, ''):
                 env['OS_CLIENT_CONFIG_FILE'] = get_incontainer_path(cred_files.get(cloud_cred, ''), private_data_dir)
 
-        for network_cred in job.network_credentials:
+        for network_cred in self._network_credentials:
             env['ANSIBLE_NET_USERNAME'] = network_cred.get_input('username', default='')
             env['ANSIBLE_NET_PASSWORD'] = network_cred.get_input('password', default='')
 
@@ -972,12 +1138,11 @@ class RunJob(SourceControlMixin, BaseTask):
             ('ANSIBLE_COLLECTIONS_PATH', 'collections_path', 'requirements_collections', '~/.ansible/collections:/usr/share/ansible/collections'),
         ]
 
-        if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
-            path_vars.append(
-                ('ANSIBLE_CALLBACK_PLUGINS', 'callback_plugins', 'plugins_path', '~/.ansible/plugins:/plugins/callback:/usr/share/ansible/plugins/callback'),
-            )
+        path_vars.append(
+            ('ANSIBLE_CALLBACK_PLUGINS', 'callback_plugins', 'plugins_path', '~/.ansible/plugins:/plugins/callback:/usr/share/ansible/plugins/callback'),
+        )
 
-        config_values = read_ansible_config(os.path.join(private_data_dir, 'project'), list(map(lambda x: x[1], path_vars)))
+        config_values = read_ansible_config(os.path.join(private_data_dir, 'project'), list(map(lambda x: x[1], path_vars)) + ['callbacks_enabled'])
 
         for env_key, config_setting, folder, default in path_vars:
             paths = default.split(':')
@@ -992,10 +1157,16 @@ class RunJob(SourceControlMixin, BaseTask):
             paths = [os.path.join(CONTAINER_ROOT, folder)] + paths
             env[env_key] = os.pathsep.join(paths)
 
+        env['ANSIBLE_CALLBACKS_ENABLED'] = 'indirect_instance_count'
+        if 'callbacks_enabled' in config_values:
+            env['ANSIBLE_CALLBACKS_ENABLED'] += ',' + config_values['callbacks_enabled']
+
         if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
-            env['ANSIBLE_CALLBACKS_ENABLED'] = 'indirect_instance_count'
-            if 'callbacks_enabled' in config_values:
-                env['ANSIBLE_CALLBACKS_ENABLED'] += ':' + config_values['callbacks_enabled']
+            env['AWX_COLLECT_HOST_QUERIES'] = '1'
+            # Add vendor collections path for external query file discovery
+            vendor_collections_path = os.path.join(CONTAINER_ROOT, 'vendor_collections')
+            env['ANSIBLE_COLLECTIONS_PATH'] = f"{vendor_collections_path}:{env['ANSIBLE_COLLECTIONS_PATH']}"
+            logger.debug(f"ANSIBLE_COLLECTIONS_PATH updated for vendor collections: {env['ANSIBLE_COLLECTIONS_PATH']}")
 
         return env
 
@@ -1004,7 +1175,7 @@ class RunJob(SourceControlMixin, BaseTask):
         Build command line argument list for running ansible-playbook,
         optionally using ssh-agent for public/private key authentication.
         """
-        creds = job.machine_credential
+        creds = self._machine_credential
 
         ssh_username, become_username, become_method = '', '', ''
         if creds:
@@ -1156,10 +1327,17 @@ class RunJob(SourceControlMixin, BaseTask):
             return
         if self.should_use_fact_cache() and self.runner_callback.artifacts_processed:
             job.log_lifecycle("finish_job_fact_cache")
+            if job.inventory.kind == 'constructed':
+                hosts_qs = job.get_source_hosts_for_constructed_inventory()
+            else:
+                hosts_qs = job.inventory.hosts
+            hosts_qs = hosts_qs.only(*HOST_FACTS_FIELDS)
             finish_fact_cache(
+                hosts_qs,
                 artifacts_dir=os.path.join(private_data_dir, 'artifacts', str(job.id)),
                 job_id=job.id,
                 inventory_id=job.inventory_id,
+                job_created=job.created,
             )
 
     def final_run_hook(self, job, status, private_data_dir):
@@ -1328,7 +1506,6 @@ class RunProjectUpdate(BaseTask):
                 'local_path': os.path.basename(project_update.project.local_path),
                 'project_path': project_update.get_project_path(check_if_exists=False),  # deprecated
                 'insights_url': settings.INSIGHTS_URL_BASE,
-                'oidc_endpoint': settings.INSIGHTS_OIDC_ENDPOINT,
                 'awx_license_type': get_license().get('license_type', 'UNLICENSED'),
                 'awx_version': get_awx_version(),
                 'scm_url': scm_url,
@@ -1435,16 +1612,14 @@ class RunProjectUpdate(BaseTask):
                 shutil.copytree(cache_subpath, dest_subpath, symlinks=True)
                 logger.debug('{0} {1} prepared {2} from cache'.format(type(project).__name__, project.pk, dest_subpath))
 
-        if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
-            # copy the special callback (not stdout type) plugin to get list of collections
-            pdd_plugins_path = os.path.join(job_private_data_dir, 'plugins_path')
-            if not os.path.exists(pdd_plugins_path):
-                os.mkdir(pdd_plugins_path)
-            from awx.playbooks import library
+        pdd_plugins_path = os.path.join(job_private_data_dir, 'plugins_path')
+        if not os.path.exists(pdd_plugins_path):
+            os.mkdir(pdd_plugins_path)
+        from awx.playbooks import library
 
-            plugin_file_source = os.path.join(library.__path__._path[0], 'indirect_instance_count.py')
-            plugin_file_dest = os.path.join(pdd_plugins_path, 'indirect_instance_count.py')
-            shutil.copyfile(plugin_file_source, plugin_file_dest)
+        plugin_file_source = os.path.join(library.__path__[0], 'indirect_instance_count.py')
+        plugin_file_dest = os.path.join(pdd_plugins_path, 'indirect_instance_count.py')
+        shutil.copyfile(plugin_file_source, plugin_file_dest)
 
     def post_run_hook(self, instance, status):
         super(RunProjectUpdate, self).post_run_hook(instance, status)
@@ -1507,7 +1682,7 @@ class RunProjectUpdate(BaseTask):
         return params
 
     def build_credentials_list(self, project_update):
-        if project_update.scm_type == 'insights' and project_update.credential:
+        if project_update.credential:
             return [project_update.credential]
         return []
 
@@ -1690,6 +1865,24 @@ class RunInventoryUpdate(SourceControlMixin, BaseTask):
         # All credentials not used by inventory source injector
         return inventory_update.get_extra_credentials()
 
+    def populate_workload_identity_tokens(self, additional_credentials=None):
+        """Also generate OIDC tokens for the cloud credential.
+
+        The cloud credential is not in _credentials (it is handled by the
+        inventory source injector), but it may still need a workload identity
+        token generated for it.
+        """
+        cloud_cred = self.instance.get_cloud_credential()
+        creds = list(additional_credentials or [])
+        if cloud_cred:
+            creds.append(cloud_cred)
+        super().populate_workload_identity_tokens(additional_credentials=creds or None)
+        # Override get_cloud_credential on this instance so the injector
+        # uses the credential with OIDC context instead of doing a fresh
+        # DB fetch that would lose it.
+        if cloud_cred and cloud_cred.context:
+            self.instance.get_cloud_credential = lambda: cloud_cred
+
     def build_project_dir(self, inventory_update, private_data_dir):
         source_project = None
         if inventory_update.inventory_source:

awx/main/tasks/policy.py

@@ -393,9 +393,9 @@ def evaluate_policy(instance):
             raise PolicyEvaluationError(_('Following certificate settings are missing for OPA_AUTH_TYPE=Certificate: {}').format(cert_settings_missing))
 
     query_paths = [
-        ('Organization', instance.organization.opa_query_path),
-        ('Inventory', instance.inventory.opa_query_path),
-        ('Job template', instance.job_template.opa_query_path),
+        ('Organization', instance.organization.opa_query_path if instance.organization else None),
+        ('Inventory', instance.inventory.opa_query_path if instance.inventory else None),
+        ('Job template', instance.job_template.opa_query_path if instance.job_template else None),
     ]
     violations = dict()
     errors = dict()

awx/main/tasks/signals.py

@@ -69,7 +69,7 @@ def signal_callback():
 
 def with_signal_handling(f):
     """
-    Change signal handling to make signal_callback return True in event of SIGTERM or SIGINT.
+    Change signal handling to make signal_callback return True in event of SIGTERM, SIGINT, or SIGUSR1.
     """
 
     @functools.wraps(f)

awx/main/tasks/system.py

@@ -19,6 +19,7 @@ from dispatcherd.publish import task
 # Runner
 import ansible_runner.cleanup
 import psycopg
+from ansible_base.lib.cache.tasks import clear_cache as dab_clear_cache
 from ansible_base.lib.utils.db import advisory_lock
 
 # django-ansible-base
@@ -68,10 +69,12 @@ from awx.main.models import (
     UnifiedJob,
     convert_jsonfields,
 )
+from awx.main.models.credential import CredentialType
 from awx.main.tasks.helpers import is_run_threshold_reached
 from awx.main.tasks.host_indirect import save_indirect_host_entries
 from awx.main.tasks.receptor import administrative_workunit_reaper, get_receptor_ctl, worker_cleanup, worker_info, write_receptor_config
 from awx.main.utils.common import ignore_inventory_computed_fields, ignore_inventory_group_removal
+from awx.main.utils.migration import is_database_synchronized
 from awx.main.utils.reload import stop_local_services
 
 logger = logging.getLogger('awx.main.tasks.system')
@@ -83,6 +86,16 @@ Try upgrading OpenSSH or providing your private key in an different format. \
 '''
 
 
+def _sync_credential_types_to_db():
+    """Ensure CredentialType DB rows match the installed plugins.
+
+    The in-memory registry is populated lazily on first access via LazyLoadDict.
+    This function only handles the DB sync step.
+    """
+    if is_database_synchronized():
+        CredentialType.setup_tower_managed_defaults()
+
+
 def _run_dispatch_startup_common():
     """
     Execute the common startup initialization steps.
@@ -93,7 +106,15 @@ def _run_dispatch_startup_common():
 
     # TODO: Enable this on VM installs
     if settings.IS_K8S:
-        write_receptor_config()
+        try:
+            write_receptor_config()
+        except Exception:
+            logger.exception("Failed to write receptor config, skipping.")
+
+    try:
+        _sync_credential_types_to_db()
+    except Exception:
+        logger.exception("Failed to sync credential types to DB, skipping.")
 
     try:
         convert_jsonfields()
@@ -237,12 +258,17 @@ def apply_cluster_membership_policies():
         # Process policy instance list first, these will represent manually managed memberships
         instance_hostnames_map = {inst.hostname: inst for inst in all_instances}
         for ig in all_groups:
+            # we don't want to allow execution nodes in the control plane
+            exclude_type = 'execution' if ig.name == settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME else 'control'
             group_actual = Group(obj=ig, instances=[], prior_instances=[instance.pk for instance in ig.instances.all()])  # obtained in prefetch
             for hostname in ig.policy_instance_list:
                 if hostname not in instance_hostnames_map:
                     logger.info("Unknown instance {} in {} policy list".format(hostname, ig.name))
                     continue
                 inst = instance_hostnames_map[hostname]
+                if inst.node_type == exclude_type:
+                    logger.info("Instance {} is excluded in {} policy list".format(hostname, ig.name))
+                    continue
                 group_actual.instances.append(inst.id)
                 # NOTE: arguable behavior: policy-list-group is not added to
                 # instance's group count for consideration in minimum-policy rules
@@ -323,24 +349,22 @@ def apply_cluster_membership_policies():
         logger.debug('Cluster policy computation finished in {} seconds'.format(time.time() - started_compute))
 
 
-@task(queue='tower_settings_change', timeout=600)
-def clear_setting_cache(setting_keys):
-    # log that cache is being cleared
-    logger.info(f"clear_setting_cache of keys {setting_keys}")
-    orig_len = len(setting_keys)
-    for i in range(orig_len):
-        for dependent_key in settings_registry.get_dependent_settings(setting_keys[i]):
-            setting_keys.append(dependent_key)
-    cache_keys = set(setting_keys)
-    logger.debug('cache delete_many(%r)', cache_keys)
-    cache.delete_many(cache_keys)
+def _resolve_setting_dependents(key):
+    return settings_registry.get_dependent_settings(key)
 
-    if 'LOG_AGGREGATOR_LEVEL' in setting_keys:
+
+def _post_setting_invalidation(invalidated_keys):
+    if 'LOG_AGGREGATOR_LEVEL' in invalidated_keys:
         ctl = get_control_from_settings()
         ctl.queuename = get_task_queuename()
         ctl.control('set_log_level', data={'level': settings.LOG_AGGREGATOR_LEVEL})
 
 
+@task(queue='tower_settings_change', timeout=600)
+def clear_setting_cache(setting_keys):
+    dab_clear_cache(setting_keys, _resolve_setting_dependents, _post_setting_invalidation)
+
+
 @task(queue='tower_broadcast_all', timeout=600)
 def delete_project_files(project_path):
     # TODO: possibly implement some retry logic
@@ -757,14 +781,16 @@ def _heartbeat_check_versions(this_inst, instance_list):
 
 
 def _heartbeat_handle_lost_instances(lost_instances, this_inst):
-    """Handle lost instances by reaping their jobs and marking them offline."""
+    """Handle lost instances by reaping their running jobs and marking them offline."""
     for other_inst in lost_instances:
         try:
+            # Any jobs marked as running will be marked as error
             explanation = "Job reaped due to instance shutdown"
             reaper.reap(other_inst, job_explanation=explanation)
-            reaper.reap_waiting(other_inst, grace_period=0, job_explanation=explanation)
+            # Any jobs that were waiting to be processed by this node will be handed back to task manager
+            UnifiedJob.objects.filter(status='waiting', controller_node=other_inst.hostname).update(status='pending', controller_node='', execution_node='')
         except Exception:
-            logger.exception('failed to reap jobs for {}'.format(other_inst.hostname))
+            logger.exception('failed to re-process jobs for lost instance {}'.format(other_inst.hostname))
         try:
             if settings.AWX_AUTO_DEPROVISION_INSTANCES and other_inst.node_type == "control":
                 deprovision_hostname = other_inst.hostname

awx/main/tests/data/collections/host_query_external_v1_0_0/galaxy.yml

@@ -0,0 +1,19 @@
+---
+authors:
+  - AWX Project Contributors <awx-project@googlegroups.com>
+dependencies: {}
+description: External query testing collection. No embedded query file. Not for use in production.
+documentation: https://github.com/ansible/awx
+homepage: https://github.com/ansible/awx
+issues: https://github.com/ansible/awx
+license:
+  - GPL-3.0-or-later
+name: external
+namespace: demo
+readme: README.md
+repository: https://github.com/ansible/awx
+tags:
+  - demo
+  - testing
+  - external_query
+version: 1.0.0

awx/main/tests/data/collections/host_query_external_v1_0_0/plugins/modules/example.py

@@ -0,0 +1,78 @@
+#!/usr/bin/python
+
+# Same licensing as AWX
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+DOCUMENTATION = r'''
+---
+module: example
+
+short_description: Module for specific live tests
+
+version_added: "2.0.0"
+
+description: This module is part of a test collection in local source. Used for external query testing.
+
+options:
+    host_name:
+        description: Name to return as the host name.
+        required: false
+        type: str
+
+author:
+    - AWX Live Tests
+'''
+
+EXAMPLES = r'''
+- name: Test with defaults
+  demo.external.example:
+
+- name: Test with custom host name
+  demo.external.example:
+    host_name: foo_host
+'''
+
+RETURN = r'''
+direct_host_name:
+    description: The name of the host, this will be collected with the feature.
+    type: str
+    returned: always
+    sample: 'foo_host'
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def run_module():
+    module_args = dict(
+        host_name=dict(type='str', required=False, default='foo_host_default'),
+    )
+
+    result = dict(
+        changed=False,
+        other_data='sample_string',
+    )
+
+    module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
+
+    if module.check_mode:
+        module.exit_json(**result)
+
+    result['direct_host_name'] = module.params['host_name']
+    result['nested_host_name'] = {'host_name': module.params['host_name']}
+    result['name'] = 'vm-foo'
+
+    # non-cononical facts
+    result['device_type'] = 'Fake Host'
+
+    module.exit_json(**result)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()

awx/main/tests/data/collections/host_query_external_v1_5_0/galaxy.yml

@@ -0,0 +1,19 @@
+---
+authors:
+  - AWX Project Contributors <awx-project@googlegroups.com>
+dependencies: {}
+description: External query testing collection v1.5.0. No embedded query file. Not for use in production.
+documentation: https://github.com/ansible/awx
+homepage: https://github.com/ansible/awx
+issues: https://github.com/ansible/awx
+license:
+  - GPL-3.0-or-later
+name: external
+namespace: demo
+readme: README.md
+repository: https://github.com/ansible/awx
+tags:
+  - demo
+  - testing
+  - external_query
+version: 1.5.0

awx/main/tests/data/collections/host_query_external_v1_5_0/plugins/modules/example.py

@@ -0,0 +1,78 @@
+#!/usr/bin/python
+
+# Same licensing as AWX
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+DOCUMENTATION = r'''
+---
+module: example
+
+short_description: Module for specific live tests
+
+version_added: "2.0.0"
+
+description: This module is part of a test collection in local source. Used for external query testing.
+
+options:
+    host_name:
+        description: Name to return as the host name.
+        required: false
+        type: str
+
+author:
+    - AWX Live Tests
+'''
+
+EXAMPLES = r'''
+- name: Test with defaults
+  demo.external.example:
+
+- name: Test with custom host name
+  demo.external.example:
+    host_name: foo_host
+'''
+
+RETURN = r'''
+direct_host_name:
+    description: The name of the host, this will be collected with the feature.
+    type: str
+    returned: always
+    sample: 'foo_host'
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def run_module():
+    module_args = dict(
+        host_name=dict(type='str', required=False, default='foo_host_default'),
+    )
+
+    result = dict(
+        changed=False,
+        other_data='sample_string',
+    )
+
+    module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
+
+    if module.check_mode:
+        module.exit_json(**result)
+
+    result['direct_host_name'] = module.params['host_name']
+    result['nested_host_name'] = {'host_name': module.params['host_name']}
+    result['name'] = 'vm-foo'
+
+    # non-cononical facts
+    result['device_type'] = 'Fake Host'
+
+    module.exit_json(**result)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()

awx/main/tests/data/collections/host_query_external_v3_0_0/galaxy.yml

@@ -0,0 +1,19 @@
+---
+authors:
+  - AWX Project Contributors <awx-project@googlegroups.com>
+dependencies: {}
+description: External query testing collection v3.0.0. No embedded query file. Not for use in production.
+documentation: https://github.com/ansible/awx
+homepage: https://github.com/ansible/awx
+issues: https://github.com/ansible/awx
+license:
+  - GPL-3.0-or-later
+name: external
+namespace: demo
+readme: README.md
+repository: https://github.com/ansible/awx
+tags:
+  - demo
+  - testing
+  - external_query
+version: 3.0.0

awx/main/tests/data/collections/host_query_external_v3_0_0/plugins/modules/example.py

@@ -0,0 +1,78 @@
+#!/usr/bin/python
+
+# Same licensing as AWX
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+DOCUMENTATION = r'''
+---
+module: example
+
+short_description: Module for specific live tests
+
+version_added: "2.0.0"
+
+description: This module is part of a test collection in local source. Used for external query testing.
+
+options:
+    host_name:
+        description: Name to return as the host name.
+        required: false
+        type: str
+
+author:
+    - AWX Live Tests
+'''
+
+EXAMPLES = r'''
+- name: Test with defaults
+  demo.external.example:
+
+- name: Test with custom host name
+  demo.external.example:
+    host_name: foo_host
+'''
+
+RETURN = r'''
+direct_host_name:
+    description: The name of the host, this will be collected with the feature.
+    type: str
+    returned: always
+    sample: 'foo_host'
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def run_module():
+    module_args = dict(
+        host_name=dict(type='str', required=False, default='foo_host_default'),
+    )
+
+    result = dict(
+        changed=False,
+        other_data='sample_string',
+    )
+
+    module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
+
+    if module.check_mode:
+        module.exit_json(**result)
+
+    result['direct_host_name'] = module.params['host_name']
+    result['nested_host_name'] = {'host_name': module.params['host_name']}
+    result['name'] = 'vm-foo'
+
+    # non-cononical facts
+    result['device_type'] = 'Fake Host'
+
+    module.exit_json(**result)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()

awx/main/tests/data/projects/debug/set_stats.yml

@@ -0,0 +1,11 @@
+---
+- hosts: all
+  gather_facts: false
+  connection: local
+  tasks:
+    - name: Set artifacts via set_stats
+      ansible.builtin.set_stats:
+        data: "{{ stats_data }}"
+        per_host: false
+        aggregate: false
+      when: stats_data is defined

awx/main/tests/data/projects/facts/gather_slow.yml

@@ -0,0 +1,21 @@
+---
+# Generated by Claude Opus 4.6 (claude-opus-4-6).
+
+- hosts: all
+  vars:
+    extra_value: ""
+  gather_facts: false
+  connection: local
+  tasks:
+    - name: set a custom fact
+      set_fact:
+        foo: "bar{{ extra_value }}"
+        bar:
+          a:
+            b:
+              - "c"
+              - "d"
+        cacheable: true
+    - name: sleep to create overlap window for concurrent job testing
+      wait_for:
+        timeout: 2

awx/main/tests/data/projects/test_host_query_external_v1_0_0/collections/requirements.yml

@@ -0,0 +1,5 @@
+---
+collections:
+  - name: 'file:///tmp/live_tests/host_query_external_v1_0_0'
+    type: git
+    version: devel

awx/main/tests/data/projects/test_host_query_external_v1_0_0/run_task.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  gather_facts: false
+  connection: local
+  tasks:
+    - demo.external.example:
+      register: result
+    - debug: var=result

awx/main/tests/data/projects/test_host_query_external_v1_5_0/collections/requirements.yml

@@ -0,0 +1,5 @@
+---
+collections:
+  - name: 'file:///tmp/live_tests/host_query_external_v1_5_0'
+    type: git
+    version: devel

awx/main/tests/data/projects/test_host_query_external_v1_5_0/run_task.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  gather_facts: false
+  connection: local
+  tasks:
+    - demo.external.example:
+      register: result
+    - debug: var=result

awx/main/tests/data/projects/test_host_query_external_v3_0_0/collections/requirements.yml

@@ -0,0 +1,5 @@
+---
+collections:
+  - name: 'file:///tmp/live_tests/host_query_external_v3_0_0'
+    type: git
+    version: devel

awx/main/tests/data/projects/test_host_query_external_v3_0_0/run_task.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  gather_facts: false
+  connection: local
+  tasks:
+    - demo.external.example:
+      register: result
+    - debug: var=result

awx/main/tests/functional/analytics/test_core.py

@@ -74,9 +74,9 @@ def temp_analytic_tar():
 
 @pytest.fixture
 def mock_analytic_post():
-    # Patch the Session.post method to return a mock response with status_code 200
-    with mock.patch('awx.main.analytics.core.requests.Session.post', return_value=mock.Mock(status_code=200)) as mock_post:
-        yield mock_post
+    # Patch get_or_generate_candlepin_certificate to skip mTLS path
+    with mock.patch('awx.main.analytics.core.get_or_generate_candlepin_certificate', return_value=(None, None)):
+        yield
 
 
 @pytest.mark.parametrize(
@@ -141,15 +141,22 @@ def mock_analytic_post():
 )
 @pytest.mark.django_db
 def test_ship_credential(setting_map, expected_result, expected_auth, temp_analytic_tar, mock_analytic_post):
-    with override_settings(**setting_map):
-        result = ship(temp_analytic_tar)
+    with override_settings(**setting_map, AUTOMATION_ANALYTICS_URL='https://example.com/api'):
+        with mock.patch('awx.main.analytics.core.OIDCClient') as mock_oidc:
+            mock_oidc_instance = mock.Mock()
+            mock_oidc_instance.make_request.return_value = mock.Mock(status_code=200)
+            mock_oidc.return_value = mock_oidc_instance
 
-        assert result == expected_result
-        if expected_auth:
-            mock_analytic_post.assert_called_once()
-            assert mock_analytic_post.call_args[1]['auth'] == expected_auth
-        else:
-            mock_analytic_post.assert_not_called()
+            result = ship(temp_analytic_tar)
+
+            assert result == expected_result
+            if expected_auth:
+                # Verify OIDC client was instantiated with correct credentials
+                mock_oidc.assert_called_once_with(expected_auth[0], expected_auth[1])
+                mock_oidc_instance.make_request.assert_called_once()
+            else:
+                # When credentials are missing, OIDCClient should not be called
+                mock_oidc.assert_not_called()
 
 
 @pytest.mark.django_db

awx/main/tests/functional/analytics/test_metrics.py

@@ -1,8 +1,11 @@
 import pytest
 
+from django.test import RequestFactory
 from prometheus_client.parser import text_string_to_metric_families
+from rest_framework.request import Request
 from awx.main import models
 from awx.main.analytics.metrics import metrics
+from awx.main.analytics.dispatcherd_metrics import get_dispatcherd_metrics
 from awx.api.versioning import reverse
 
 EXPECTED_VALUES = {
@@ -77,3 +80,55 @@ def test_metrics_http_methods(get, post, patch, put, options, admin):
     assert patch(get_metrics_view_db_only(), user=admin).status_code == 405
     assert post(get_metrics_view_db_only(), user=admin).status_code == 405
     assert options(get_metrics_view_db_only(), user=admin).status_code == 200
+
+
+class DummyMetricsResponse:
+    def __init__(self, payload):
+        self._payload = payload
+
+    def read(self):
+        return self._payload
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        return False
+
+
+def test_dispatcherd_metrics_node_filter_match(mocker, settings):
+    settings.CLUSTER_HOST_ID = "awx-1"
+    payload = b'# HELP test_metric A test metric\n# TYPE test_metric gauge\ntest_metric 1\n'
+
+    def fake_urlopen(url, timeout=1.0):
+        return DummyMetricsResponse(payload)
+
+    mocker.patch('urllib.request.urlopen', fake_urlopen)
+
+    request = Request(RequestFactory().get('/api/v2/metrics/', {'node': 'awx-1'}))
+
+    assert get_dispatcherd_metrics(request) == payload.decode('utf-8')
+
+
+def test_dispatcherd_metrics_node_filter_excludes_local(mocker, settings):
+    settings.CLUSTER_HOST_ID = "awx-1"
+
+    def fake_urlopen(*args, **kwargs):
+        raise AssertionError("urlopen should not be called when node filter excludes local node")
+
+    mocker.patch('urllib.request.urlopen', fake_urlopen)
+
+    request = Request(RequestFactory().get('/api/v2/metrics/', {'node': 'awx-2'}))
+
+    assert get_dispatcherd_metrics(request) == ''
+
+
+def test_dispatcherd_metrics_metric_filter_excludes_unrelated(mocker):
+    def fake_urlopen(*args, **kwargs):
+        raise AssertionError("urlopen should not be called when metric filter excludes dispatcherd metrics")
+
+    mocker.patch('urllib.request.urlopen', fake_urlopen)
+
+    request = Request(RequestFactory().get('/api/v2/metrics/', {'metric': 'awx_system_info'}))
+
+    assert get_dispatcherd_metrics(request) == ''

awx/main/tests/functional/api/test_config_endpoint.py

@@ -0,0 +1,84 @@
+import pytest
+from awx.api.versioning import reverse
+from rest_framework import status
+
+from awx.main.models.jobs import JobTemplate
+
+
+@pytest.mark.django_db
+class TestConfigEndpointFields:
+    def test_base_fields_all_users(self, get, rando):
+        url = reverse('api:api_v2_config_view')
+        response = get(url, rando, expect=200)
+
+        assert 'time_zone' in response.data
+        assert 'license_info' in response.data
+        assert 'version' in response.data
+        assert 'eula' in response.data
+        assert 'analytics_status' in response.data
+        assert 'analytics_collectors' in response.data
+        assert 'become_methods' in response.data
+
+    @pytest.mark.parametrize(
+        "role_type",
+        [
+            "superuser",
+            "system_auditor",
+            "org_admin",
+            "org_auditor",
+            "org_project_admin",
+        ],
+    )
+    def test_privileged_users_conditional_fields(self, get, user, organization, admin, role_type):
+        url = reverse('api:api_v2_config_view')
+
+        if role_type == "superuser":
+            test_user = admin
+        elif role_type == "system_auditor":
+            test_user = user('system-auditor', is_superuser=False)
+            test_user.is_system_auditor = True
+            test_user.save()
+        elif role_type == "org_admin":
+            test_user = user('org-admin', is_superuser=False)
+            organization.admin_role.members.add(test_user)
+        elif role_type == "org_auditor":
+            test_user = user('org-auditor', is_superuser=False)
+            organization.auditor_role.members.add(test_user)
+        elif role_type == "org_project_admin":
+            test_user = user('org-project-admin', is_superuser=False)
+            organization.project_admin_role.members.add(test_user)
+
+        response = get(url, test_user, expect=200)
+
+        assert 'project_base_dir' in response.data
+        assert 'project_local_paths' in response.data
+        assert 'custom_virtualenvs' in response.data
+
+    def test_job_template_admin_gets_venvs_only(self, get, user, organization, project, inventory):
+        """Test that JobTemplate admin without org access gets only custom_virtualenvs"""
+        jt_admin = user('jt-admin', is_superuser=False)
+
+        jt = JobTemplate.objects.create(name='test-jt', organization=organization, project=project, inventory=inventory)
+        jt.admin_role.members.add(jt_admin)
+
+        url = reverse('api:api_v2_config_view')
+        response = get(url, jt_admin, expect=200)
+
+        assert 'custom_virtualenvs' in response.data
+        assert 'project_base_dir' not in response.data
+        assert 'project_local_paths' not in response.data
+
+    def test_normal_user_no_conditional_fields(self, get, rando):
+        url = reverse('api:api_v2_config_view')
+        response = get(url, rando, expect=200)
+
+        assert 'project_base_dir' not in response.data
+        assert 'project_local_paths' not in response.data
+        assert 'custom_virtualenvs' not in response.data
+
+    def test_unauthenticated_denied(self, get):
+        """Test that unauthenticated requests are denied"""
+        url = reverse('api:api_v2_config_view')
+        response = get(url, None, expect=401)
+
+        assert response.status_code == status.HTTP_401_UNAUTHORIZED

awx/main/tests/functional/api/test_credential.py

@@ -200,6 +200,7 @@ def test_grant_org_credential_to_org_user_through_user_roles(post, credential, o
 
 @pytest.mark.django_db
 def test_grant_org_credential_to_non_org_user_through_role_users(post, credential, organization, org_admin, alice):
+    # NOTE: this endpoint is going away soon
     credential.organization = organization
     credential.save()
     response = post(reverse('api:role_users_list', kwargs={'pk': credential.use_role.id}), {'id': alice.id}, org_admin)
@@ -208,6 +209,7 @@ def test_grant_org_credential_to_non_org_user_through_role_users(post, credentia
 
 @pytest.mark.django_db
 def test_grant_org_credential_to_non_org_user_through_user_roles(post, credential, organization, org_admin, alice):
+    # NOTE: this endpoint is going away soon
     credential.organization = organization
     credential.save()
     response = post(reverse('api:user_roles_list', kwargs={'pk': alice.id}), {'id': credential.use_role.id}, org_admin)
@@ -216,18 +218,18 @@ def test_grant_org_credential_to_non_org_user_through_user_roles(post, credentia
 
 @pytest.mark.django_db
 def test_grant_private_credential_to_user_through_role_users(post, credential, alice, bob):
-    # normal users can't do this
+    # NOTE: this endpoint is going away soon
     credential.admin_role.members.add(alice)
     response = post(reverse('api:role_users_list', kwargs={'pk': credential.use_role.id}), {'id': bob.id}, alice)
-    assert response.status_code == 400
+    assert response.status_code == 403
 
 
 @pytest.mark.django_db
 def test_grant_private_credential_to_org_user_through_role_users(post, credential, org_admin, org_member):
-    # org admins can't either
+    # NOTE: this endpoint is going away soon
     credential.admin_role.members.add(org_admin)
     response = post(reverse('api:role_users_list', kwargs={'pk': credential.use_role.id}), {'id': org_member.id}, org_admin)
-    assert response.status_code == 400
+    assert response.status_code == 204
 
 
 @pytest.mark.django_db
@@ -239,18 +241,18 @@ def test_sa_grant_private_credential_to_user_through_role_users(post, credential
 
 @pytest.mark.django_db
 def test_grant_private_credential_to_user_through_user_roles(post, credential, alice, bob):
-    # normal users can't do this
+    # NOTE: this endpoint is going away soon
     credential.admin_role.members.add(alice)
     response = post(reverse('api:user_roles_list', kwargs={'pk': bob.id}), {'id': credential.use_role.id}, alice)
-    assert response.status_code == 400
+    assert response.status_code == 403
 
 
 @pytest.mark.django_db
 def test_grant_private_credential_to_org_user_through_user_roles(post, credential, org_admin, org_member):
-    # org admins can't either
+    # NOTE: this endpoint is going away soon
     credential.admin_role.members.add(org_admin)
     response = post(reverse('api:user_roles_list', kwargs={'pk': org_member.id}), {'id': credential.use_role.id}, org_admin)
-    assert response.status_code == 400
+    assert response.status_code == 204
 
 
 @pytest.mark.django_db
@@ -282,14 +284,14 @@ def test_grant_org_credential_to_team_through_team_roles(post, credential, organ
 
 @pytest.mark.django_db
 def test_sa_grant_private_credential_to_team_through_role_teams(post, credential, admin, team):
-    # not even a system admin can grant a private cred to a team though
+    # NOTE: this endpoint is going away soon
     response = post(reverse('api:role_teams_list', kwargs={'pk': credential.use_role.id}), {'id': team.id}, admin)
-    assert response.status_code == 400
+    assert response.status_code == 204
 
 
 @pytest.mark.django_db
 def test_grant_credential_to_team_different_organization_through_role_teams(post, get, credential, organizations, admin, org_admin, team, team_member):
-    # # Test that credential from different org can be assigned to team by a superuser through role_teams_list endpoint
+    # NOTE: this endpoint is going away soon
     orgs = organizations(2)
     credential.organization = orgs[0]
     credential.save()
@@ -299,10 +301,7 @@ def test_grant_credential_to_team_different_organization_through_role_teams(post
     # Non-superuser (org_admin) trying cross-org assignment should be denied
     response = post(reverse('api:role_teams_list', kwargs={'pk': credential.use_role.id}), {'id': team.id}, org_admin)
     assert response.status_code == 400
-    assert (
-        "You cannot grant a team access to a credential in a different organization. Only superusers can grant cross-organization credential access to teams"
-        in response.data['msg']
-    )
+    assert "You cannot grant credential access to a Team not in the credentials' organization" in str(response.data['detail'])
 
     # Superuser (admin) can do cross-org assignment
     response = post(reverse('api:role_teams_list', kwargs={'pk': credential.use_role.id}), {'id': team.id}, admin)
@@ -316,20 +315,17 @@ def test_grant_credential_to_team_different_organization_through_role_teams(post
 
 @pytest.mark.django_db
 def test_grant_credential_to_team_different_organization(post, get, credential, organizations, admin, org_admin, team, team_member):
-    # Test that credential from different org can be assigned to team by a superuser
+    # NOTE: this endpoint is going away soon
     orgs = organizations(2)
     credential.organization = orgs[0]
     credential.save()
     team.organization = orgs[1]
     team.save()
 
-    # Non-superuser (org_admin, ...) trying cross-org assignment should be denied
+    # Non-superuser (org_admin) trying cross-org assignment should be denied
     response = post(reverse('api:team_roles_list', kwargs={'pk': team.id}), {'id': credential.use_role.id}, org_admin)
     assert response.status_code == 400
-    assert (
-        "You cannot grant a team access to a credential in a different organization. Only superusers can grant cross-organization credential access to teams"
-        in response.data['msg']
-    )
+    assert "You cannot grant credential access to a Team not in the credentials' organization" in str(response.data['detail'])
 
     # Superuser (system admin) can do cross-org assignment
     response = post(reverse('api:team_roles_list', kwargs={'pk': team.id}), {'id': credential.use_role.id}, admin)