Disable checkbox while job is running in project and inventory source lists (#11841)

Add a CI check for the development environment

.github/ISSUE_TEMPLATE.md

@@ -16,7 +16,7 @@ https://www.ansible.com/security
 <!-- Pick the area of AWX for this issue, you can have multiple, delete the rest: -->
  - API
  - UI
- - Installer
+ - Collection
 
 ##### SUMMARY
 <!-- Briefly describe the problem. -->

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,26 +1,24 @@
 ---
 name: Bug Report
 description: Create a report to help us improve
-labels:
-  - bug
 body:
   - type: markdown
     attributes:
       value: |
         Issues are for **concrete, actionable bugs and feature requests** only. For debugging help or technical support, please use:
         - The #ansible-awx channel on irc.libera.chat
-        - https://groups.google.com/forum/#!forum/awx-project
+        - The awx project mailing list, https://groups.google.com/forum/#!forum/awx-project
 
   - type: checkboxes
     id: terms
     attributes:
       label: Please confirm the following
       options:
-        - label: I agree to follow this project's [code of conduct](http://docs.ansible.com/ansible/latest/community/code_of_conduct.html).
+        - label: I agree to follow this project's [code of conduct](https://docs.ansible.com/ansible/latest/community/code_of_conduct.html).
           required: true
         - label: I have checked the [current issues](https://github.com/ansible/awx/issues) for duplicates.
           required: true
-        - label: I understand that AWX is open source software provided for free and that I am not entitled to status updates or other assurances.
+        - label: I understand that AWX is open source software provided for free and that I might not receive a timely response.
           required: true
 
   - type: textarea
@@ -39,6 +37,15 @@ body:
     validations:
       required: true
 
+  - type: checkboxes
+    id: components
+    attributes:
+      label: Select the relevant components
+      options:
+        - label: UI
+        - label: API
+        - label: Docs
+
   - type: dropdown
     id: awx-install-method
     attributes:

.github/PULL_REQUEST_TEMPLATE.md

@@ -25,6 +25,7 @@ the change does.
 <!--- Name of the module/plugin/module/task -->
  - API
  - UI
+ - Collection
 
 ##### AWX VERSION
 <!--- Paste verbatim output from `make VERSION` between quotes below -->

.github/issue_labeler.yml

@@ -0,0 +1,12 @@
+needs_triage:
+  - '.*'
+"type:bug":
+  - "Please confirm the following"
+"type:enhancement":
+  - "Feature Idea"
+"component:ui":
+  - "\\[X\\] UI"
+"component:api":
+  - "\\[X\\] API"
+"component:docs":
+  - "\\[X\\] Docs"

.github/pr_labeler.yml

@@ -0,0 +1,14 @@
+"component:api":
+  - any: ['awx/**/*', '!awx/ui/*']
+
+"component:ui":
+  - any: ['awx/ui/**/*']
+
+"component:docs":
+  - any: ['docs/**/*']
+
+"component:cli":
+  - any: ['awxkit/**/*']
+
+"component:collection":
+  - any: ['awx_collection/**/*']

.github/workflows/ci.yml

@@ -5,14 +5,51 @@ env:
 on:
   pull_request:
 jobs:
-  api-test:
+  common-tests:
+    name: ${{ matrix.tests.name }}
     runs-on: ubuntu-latest
     permissions:
       packages: write
       contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        tests:
+          - name: api-test
+            command: /start_tests.sh
+            label: Run API Tests
+          - name: api-lint
+            command: /var/lib/awx/venv/awx/bin/tox -e linters
+            label: Run API Linters
+          - name: api-swagger
+            command: /start_tests.sh swagger
+            label: Generate API Reference
+          - name: awx-collection
+            command: /start_tests.sh test_collection_all
+            label: Run Collection Tests
+          - name: api-schema
+            label: Check API Schema
+            command: /start_tests.sh detect-schema-change SCHEMA_DIFF_BASE_BRANCH=${{ github.event.pull_request.base.ref }}
+          - name: ui-lint
+            label: Run UI Linters
+            command: make ui-lint
+          - name: ui-test-screens
+            label: Run UI Screens Tests
+            command: make ui-test-screens
+          - name: ui-test-general
+            label: Run UI General Tests
+            command: make ui-test-general
     steps:
       - uses: actions/checkout@v2
 
+      - name: Get python version from Makefile
+        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
+
+      - name: Install python ${{ env.py_version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.py_version }}
+
       - name: Log in to registry
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
@@ -25,18 +62,23 @@ jobs:
         run: |
           DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build
 
-      - name: Run API Tests
+      - name: ${{ matrix.texts.label }}
         run: |
           docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /start_tests.sh
-  api-lint:
+            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} ${{ matrix.tests.command }}
+  dev-env:
     runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
     steps:
       - uses: actions/checkout@v2
 
+      - name: Get python version from Makefile
+        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
+
+      - name: Install python ${{ env.py_version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.py_version }}
+
       - name: Log in to registry
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
@@ -49,130 +91,12 @@ jobs:
         run: |
           DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build
 
-      - name: Run API Linters
+      - name: Run smoke test
         run: |
-          docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /var/lib/awx/venv/awx/bin/tox -e linters
-  api-swagger:
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-    steps:
-      - uses: actions/checkout@v2
+          export DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }}
+          export COMPOSE_TAG=${{ env.BRANCH }}
+          ansible-playbook tools/docker-compose/ansible/smoke-test.yml -e repo_dir=$(pwd) -v
 
-      - name: Log in to registry
-        run: |
-          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Pre-pull image to warm build cache
-        run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || :
-
-      - name: Build image
-        run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }}  make docker-compose-build
-
-      - name: Generate API Reference
-        run: |
-          docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /start_tests.sh swagger
-  awx-collection:
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Log in to registry
-        run: |
-          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Pre-pull image to warm build cache
-        run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || :
-
-      - name: Build image
-        run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }}  make docker-compose-build
-
-      - name: Run Collection Tests
-        run: |
-          docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /start_tests.sh test_collection_all
-  api-schema:
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Log in to registry
-        run: |
-          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Pre-pull image to warm build cache
-        run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || :
-
-      - name: Build image
-        run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }}  make docker-compose-build
-
-      - name: Check API Schema
-        run: |
-          docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /start_tests.sh detect-schema-change
-  ui-lint:
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Log in to registry
-        run: |
-          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Pre-pull image to warm build cache
-        run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || :
-
-      - name: Build image
-        run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build
-
-      - name: Run UI Linters
-        run: |
-          docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} make ui-lint
-  ui-test:
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Log in to registry
-        run: |
-          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Pre-pull image to warm build cache
-        run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || :
-
-      - name: Build image
-        run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build
-
-      - name: Run UI Tests
-        run: |
-          docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} make ui-test
   awx-operator:
     runs-on: ubuntu-latest
     steps:
@@ -207,7 +131,7 @@ jobs:
           ansible-galaxy collection install -r molecule/requirements.yml
           sudo rm -f $(which kustomize)
           make kustomize
-          KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind
+          KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule -v test -s kind
         env:
           AWX_TEST_IMAGE: awx
           AWX_TEST_VERSION: ci

.github/workflows/devel_image.yml

@@ -13,6 +13,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: Get python version from Makefile
+        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
+
+      - name: Install python ${{ env.py_version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.py_version }}
+
       - name: Log in to registry
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin

.github/workflows/e2e_test.yml

@@ -18,6 +18,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: Get python version from Makefile
+        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
+
+      - name: Install python ${{ env.py_version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.py_version }}
+
       - name: Install system deps
         run: sudo apt-get install -y gettext
 

.github/workflows/label_issue.yml

@@ -0,0 +1,22 @@
+name: Label Issue
+
+on:
+  issues:
+    types:
+      - opened
+      - reopened
+      - edited
+
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    name: Label Issue
+
+    steps:
+      - name: Label Issue
+        uses: github/issue-labeler@v2.4.1
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"
+          not-before: 2021-12-07T07:00:00Z
+          configuration-path: .github/issue_labeler.yml
+          enable-versioned-regex: 0

.github/workflows/label_pr.yml

@@ -0,0 +1,20 @@
+name: Label PR
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+      - synchronize
+
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    name: Label PR
+
+    steps:
+      - name: Label PR
+        uses: actions/labeler@v3
+        with:
+          repo-token: "${{ secrets.GITHUB_TOKEN }}"
+          configuration-path: .github/pr_labeler.yml

.github/workflows/promote.yml

@@ -8,6 +8,53 @@ jobs:
   promote:
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout awx
+        uses: actions/checkout@v2
+
+      - name: Get python version from Makefile
+        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
+
+      - name: Install python ${{ env.py_version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.py_version }}
+
+      - name: Install dependencies
+        run: |
+          python${{ env.py_version }} -m pip install wheel twine
+
+      - name: Set official collection namespace
+        run: echo collection_namespace=awx >> $GITHUB_ENV
+        if: ${{ github.repository_owner == 'ansible' }}
+
+      - name: Set unofficial collection namespace
+        run: echo collection_namespace=${{ github.repository_owner }} >> $GITHUB_ENV
+        if: ${{ github.repository_owner != 'ansible' }}
+
+      - name: Build collection and publish to galaxy
+        run: |
+          COLLECTION_NAMESPACE=${{ env.collection_namespace }} make build_collection
+          ansible-galaxy collection publish \
+            --token=${{ secrets.GALAXY_TOKEN }} \
+            awx_collection_build/${{ env.collection_namespace }}-awx-${{ github.event.release.tag_name }}.tar.gz
+
+      - name: Set official pypi info
+        run: echo pypi_repo=pypi >> $GITHUB_ENV
+        if: ${{ github.repository_owner == 'ansible' }}
+
+      - name: Set unofficial pypi info
+        run: echo pypi_repo=testpypi >> $GITHUB_ENV
+        if: ${{ github.repository_owner != 'ansible' }}
+
+      - name: Build awxkit and upload to pypi
+        run: |
+          cd awxkit && python3 setup.py bdist_wheel
+          twine upload \
+            -r ${{ env.pypi_repo }} \
+            -u ${{ secrets.PYPI_USERNAME }} \
+            -p ${{ secrets.PYPI_PASSWORD }} \
+            dist/*
+
       - name: Log in to GHCR
         run: |
           echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin

.github/workflows/stage.yml

@@ -43,6 +43,14 @@ jobs:
         with:
           path: awx
 
+      - name: Get python version from Makefile
+        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
+
+      - name: Install python ${{ env.py_version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.py_version }}
+
       - name: Checkout awx-logos
         uses: actions/checkout@v2
         with:

.github/workflows/upload_schema.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - devel
+      - release_4.1
 jobs:
   push:
     runs-on: ubuntu-latest
@@ -13,6 +14,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: Get python version from Makefile
+        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
+
+      - name: Install python ${{ env.py_version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.py_version }}       
+
       - name: Log in to registry
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
@@ -38,6 +47,6 @@ jobs:
         run: |
           ansible localhost -c local, -m command -a "{{ ansible_python_interpreter + ' -m pip install boto3'}}"
           ansible localhost -c local -m aws_s3 \
-            -a 'src=${{ github.workspace }}/schema.json bucket=awx-public-ci-files object=schema.json mode=put permission=public-read'
+            -a "src=${{ github.workspace }}/schema.json bucket=awx-public-ci-files object=${GITHUB_REF##*/}/schema.json mode=put permission=public-read"
 
 

.gitignore

@@ -42,6 +42,7 @@ tools/docker-compose/_build
 tools/docker-compose/_sources
 tools/docker-compose/overrides/
 tools/docker-compose-minikube/_sources
+tools/docker-compose/keycloak.awx.realm.json
 
 # Tower setup playbook testing
 setup/test/roles/postgresql

Makefile

@@ -1,5 +1,4 @@
-PYTHON ?= python3.8
-PYTHON_VERSION = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_version; print(get_python_version())")
+PYTHON ?= python3.9
 OFFICIAL ?= no
 NODE ?= node
 NPM_BIN ?= npm
@@ -11,14 +10,17 @@ COLLECTION_VERSION := $(shell $(PYTHON) setup.py --version | cut -d . -f 1-3)
 
 # NOTE: This defaults the container image version to the branch that's active
 COMPOSE_TAG ?= $(GIT_BRANCH)
-COMPOSE_HOST ?= $(shell hostname)
 MAIN_NODE_TYPE ?= hybrid
+# If set to true docker-compose will also start a keycloak instance
+KEYCLOAK ?= false
 
 VENV_BASE ?= /var/lib/awx/venv
 
 DEV_DOCKER_TAG_BASE ?= quay.io/awx
 DEVEL_IMAGE_NAME ?= $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG)
 
+RECEPTOR_IMAGE ?= quay.io/ansible/receptor:devel
+
 # Python packages to install only from source (not from binary wheels)
 # Comma separated list
 SRC_ONLY_PKGS ?= cffi,pycparser,psycopg2,twilio
@@ -41,7 +43,7 @@ I18N_FLAG_FILE = .i18n_built
 	receiver test test_unit test_coverage coverage_html \
 	dev_build release_build sdist \
 	ui-release ui-devel \
-	VERSION docker-compose-sources \
+	VERSION PYTHON_VERSION docker-compose-sources \
 	.git/hooks/pre-commit
 
 clean-tmp:
@@ -143,24 +145,6 @@ version_file:
 	fi; \
 	$(PYTHON) -c "import awx; print(awx.__version__)" > /var/lib/awx/.awx_version; \
 
-# Do any one-time init tasks.
-comma := ,
-init:
-	if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/awx/bin/activate; \
-	fi; \
-	$(MANAGEMENT_COMMAND) provision_instance --hostname=$(COMPOSE_HOST) --node_type=$(MAIN_NODE_TYPE); \
-	$(MANAGEMENT_COMMAND) register_queue --queuename=controlplane --instance_percent=100;\
-	$(MANAGEMENT_COMMAND) register_queue --queuename=default --instance_percent=100;
-	if [ ! -f /etc/receptor/certs/awx.key ]; then \
-		rm -f /etc/receptor/certs/*; \
-		receptor --cert-init commonname="AWX Test CA" bits=2048 outcert=/etc/receptor/certs/ca.crt outkey=/etc/receptor/certs/ca.key; \
-		for node in $(RECEPTOR_MUTUAL_TLS); do \
-			receptor --cert-makereq bits=2048 commonname="$$node test cert" dnsname=$$node nodeid=$$node outreq=/etc/receptor/certs/$$node.csr outkey=/etc/receptor/certs/$$node.key; \
-			receptor --cert-signreq req=/etc/receptor/certs/$$node.csr cacert=/etc/receptor/certs/ca.crt cakey=/etc/receptor/certs/ca.key outcert=/etc/receptor/certs/$$node.crt verify=yes; \
-		done; \
-	fi
-
 # Refresh development environment after pulling new code.
 refresh: clean requirements_dev version_file develop migrate
 
@@ -281,16 +265,16 @@ api-lint:
 
 awx-link:
 	[ -d "/awx_devel/awx.egg-info" ] || $(PYTHON) /awx_devel/setup.py egg_info_dev
-	cp -f /tmp/awx.egg-link /var/lib/awx/venv/awx/lib/python$(PYTHON_VERSION)/site-packages/awx.egg-link
+	cp -f /tmp/awx.egg-link /var/lib/awx/venv/awx/lib/$(PYTHON)/site-packages/awx.egg-link
 
 TEST_DIRS ?= awx/main/tests/unit awx/main/tests/functional awx/conf/tests awx/sso/tests
-
+PYTEST_ARGS ?= -n auto
 # Run all API unit tests.
 test:
 	if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	PYTHONDONTWRITEBYTECODE=1 py.test -p no:cacheprovider -n auto $(TEST_DIRS)
+	PYTHONDONTWRITEBYTECODE=1 py.test -p no:cacheprovider $(PYTEST_ARGS) $(TEST_DIRS)
 	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py3
 	awx-manage check_migrations --dry-run --check  -n 'missing_migration_file'
 
@@ -321,7 +305,7 @@ symlink_collection:
 	mkdir -p ~/.ansible/collections/ansible_collections/$(COLLECTION_NAMESPACE)  # in case it does not exist
 	ln -s $(shell pwd)/awx_collection $(COLLECTION_INSTALL)
 
-build_collection:
+awx_collection_build: $(shell find awx_collection -type f)
 	ansible-playbook -i localhost, awx_collection/tools/template_galaxy.yml \
 	  -e collection_package=$(COLLECTION_PACKAGE) \
 	  -e collection_namespace=$(COLLECTION_NAMESPACE) \
@@ -329,6 +313,8 @@ build_collection:
 	  -e '{"awx_template_version":false}'
 	ansible-galaxy collection build awx_collection_build --force --output-path=awx_collection_build
 
+build_collection: awx_collection_build
+
 install_collection: build_collection
 	rm -rf $(COLLECTION_INSTALL)
 	ansible-galaxy collection install awx_collection_build/$(COLLECTION_NAMESPACE)-$(COLLECTION_PACKAGE)-$(COLLECTION_VERSION).tar.gz
@@ -382,7 +368,7 @@ clean-ui:
 	rm -rf $(UI_BUILD_FLAG_FILE)
 
 awx/ui/node_modules:
-	NODE_OPTIONS=--max-old-space-size=4096 $(NPM_BIN) --prefix awx/ui --loglevel warn ci
+	NODE_OPTIONS=--max-old-space-size=6144 $(NPM_BIN) --prefix awx/ui --loglevel warn ci
 
 $(UI_BUILD_FLAG_FILE): awx/ui/node_modules
 	$(PYTHON) tools/scripts/compilemessages.py
@@ -416,9 +402,18 @@ ui-lint:
 
 ui-test:
 	$(NPM_BIN) --prefix awx/ui install
-	$(NPM_BIN) run --prefix awx/ui test -- --coverage --maxWorkers=4 --watchAll=false
+	$(NPM_BIN) run --prefix awx/ui test
 
+ui-test-screens:
+	$(NPM_BIN) --prefix awx/ui install
+	$(NPM_BIN) run --prefix awx/ui pretest
+	$(NPM_BIN) run --prefix awx/ui test-screens --runInBand
 
+ui-test-general:
+	$(NPM_BIN) --prefix awx/ui install
+	$(NPM_BIN) run --prefix awx/ui pretest
+	$(NPM_BIN) run --prefix awx/ui/ test-general --runInBand
+	
 # Build a pip-installable package into dist/ with a timestamped version number.
 dev_build:
 	$(PYTHON) setup.py dev_build
@@ -463,9 +458,11 @@ docker-compose-sources: .git/hooks/pre-commit
 	ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/sources.yml \
 	    -e awx_image=$(DEV_DOCKER_TAG_BASE)/awx_devel \
 	    -e awx_image_tag=$(COMPOSE_TAG) \
+	    -e receptor_image=$(RECEPTOR_IMAGE) \
 	    -e control_plane_node_count=$(CONTROL_PLANE_NODE_COUNT) \
 	    -e execution_node_count=$(EXECUTION_NODE_COUNT) \
-	    -e minikube_container_group=$(MINIKUBE_CONTAINER_GROUP)
+	    -e minikube_container_group=$(MINIKUBE_CONTAINER_GROUP) \
+	    -e enable_keycloak=$(KEYCLOAK)
 
 
 docker-compose: awx/projects docker-compose-sources
@@ -484,8 +481,9 @@ docker-compose-runtest: awx/projects docker-compose-sources
 docker-compose-build-swagger: awx/projects docker-compose-sources
 	docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports --no-deps awx_1 /start_tests.sh swagger
 
+SCHEMA_DIFF_BASE_BRANCH ?= devel
 detect-schema-change: genschema
-	curl https://s3.amazonaws.com/awx-public-ci-files/schema.json -o reference-schema.json
+	curl https://s3.amazonaws.com/awx-public-ci-files/$(SCHEMA_DIFF_BASE_BRANCH)/schema.json -o reference-schema.json
 	# Ignore differences in whitespace with -b
 	diff -u -b reference-schema.json schema.json
 
@@ -500,7 +498,7 @@ docker-compose-container-group-clean:
 
 # Base development image build
 docker-compose-build:
-	ansible-playbook tools/ansible/dockerfile.yml -e build_dev=True
+	ansible-playbook tools/ansible/dockerfile.yml -e build_dev=True -e receptor_image=$(RECEPTOR_IMAGE)
 	DOCKER_BUILDKIT=1 docker build -t $(DEVEL_IMAGE_NAME) \
 	    --build-arg BUILDKIT_INLINE_CACHE=1 \
 	    --cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) .
@@ -543,14 +541,18 @@ psql-container:
 VERSION:
 	@echo "awx: $(VERSION)"
 
+PYTHON_VERSION:
+	@echo "$(PYTHON)" | sed 's:python::'
+
 Dockerfile: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
-	ansible-playbook tools/ansible/dockerfile.yml
+	ansible-playbook tools/ansible/dockerfile.yml -e receptor_image=$(RECEPTOR_IMAGE)
 
 Dockerfile.kube-dev: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
 	ansible-playbook tools/ansible/dockerfile.yml \
 	    -e dockerfile_name=Dockerfile.kube-dev \
 	    -e kube_dev=True \
-	    -e template_dest=_build_kube_dev
+	    -e template_dest=_build_kube_dev \
+	    -e receptor_image=$(RECEPTOR_IMAGE)
 
 awx-kube-dev-build: Dockerfile.kube-dev
 	docker build -f Dockerfile.kube-dev \
@@ -576,3 +578,6 @@ messages:
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
 	$(PYTHON) manage.py makemessages -l $(LANG) --keep-pot
+
+print-%:
+	@echo $($*)

awx/api/generics.py

@@ -44,6 +44,7 @@ from awx.main.views import ApiErrorView
 from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer, UserSerializer
 from awx.api.versioning import URLPathVersioning
 from awx.api.metadata import SublistAttachDetatchMetadata, Metadata
+from awx.conf import settings_registry
 
 __all__ = [
     'APIView',
@@ -98,6 +99,7 @@ class LoggedLoginView(auth_views.LoginView):
             current_user = smart_text(JSONRenderer().render(current_user.data))
             current_user = urllib.parse.quote('%s' % current_user, '')
             ret.set_cookie('current_user', current_user, secure=settings.SESSION_COOKIE_SECURE or None)
+            ret.setdefault('X-API-Session-Cookie-Name', getattr(settings, 'SESSION_COOKIE_NAME', 'awx_sessionid'))
 
             return ret
         else:
@@ -208,12 +210,27 @@ class APIView(views.APIView):
             return response
 
         if response.status_code >= 400:
-            status_msg = "status %s received by user %s attempting to access %s from %s" % (
-                response.status_code,
-                request.user,
-                request.path,
-                request.META.get('REMOTE_ADDR', None),
-            )
+            msg_data = {
+                'status_code': response.status_code,
+                'user_name': request.user,
+                'url_path': request.path,
+                'remote_addr': request.META.get('REMOTE_ADDR', None),
+            }
+
+            if type(response.data) is dict:
+                msg_data['error'] = response.data.get('error', response.status_text)
+            elif type(response.data) is list:
+                msg_data['error'] = ", ".join(list(map(lambda x: x.get('error', response.status_text), response.data)))
+            else:
+                msg_data['error'] = response.status_text
+
+            try:
+                status_msg = getattr(settings, 'API_400_ERROR_LOG_FORMAT').format(**msg_data)
+            except Exception as e:
+                if getattr(settings, 'API_400_ERROR_LOG_FORMAT', None):
+                    logger.error("Unable to format API_400_ERROR_LOG_FORMAT setting, defaulting log message: {}".format(e))
+                status_msg = settings_registry.get_setting_field('API_400_ERROR_LOG_FORMAT').get_default().format(**msg_data)
+
             if hasattr(self, '__init_request_error__'):
                 response = self.handle_exception(self.__init_request_error__)
             if response.status_code == 401:
@@ -221,6 +238,7 @@ class APIView(views.APIView):
                 logger.info(status_msg)
             else:
                 logger.warning(status_msg)
+
         response = super(APIView, self).finalize_response(request, response, *args, **kwargs)
         time_started = getattr(self, 'time_started', None)
         response['X-API-Product-Version'] = get_awx_version()
@@ -817,7 +835,7 @@ class ResourceAccessList(ParentMixin, ListAPIView):
 
 
 def trigger_delayed_deep_copy(*args, **kwargs):
-    from awx.main.tasks import deep_copy_model_obj
+    from awx.main.tasks.system import deep_copy_model_obj
 
     connection.on_commit(lambda: deep_copy_model_obj.delay(*args, **kwargs))
 

awx/api/permissions.py

@@ -243,7 +243,7 @@ class IsSystemAdminOrAuditor(permissions.BasePermission):
     """
 
     def has_permission(self, request, view):
-        if not request.user:
+        if not (request.user and request.user.is_authenticated):
             return False
         if request.method == 'GET':
             return request.user.is_superuser or request.user.is_system_auditor

awx/api/serializers.py

@@ -57,6 +57,7 @@ from awx.main.models import (
     Host,
     Instance,
     InstanceGroup,
+    InstanceLink,
     Inventory,
     InventorySource,
     InventoryUpdate,
@@ -378,19 +379,22 @@ class BaseSerializer(serializers.ModelSerializer, metaclass=BaseSerializerMetacl
     def _get_related(self, obj):
         return {} if obj is None else self.get_related(obj)
 
-    def _generate_named_url(self, url_path, obj, node):
-        url_units = url_path.split('/')
+    def _generate_friendly_id(self, obj, node):
         reset_counters()
-        named_url = node.generate_named_url(obj)
-        url_units[4] = named_url
-        return '/'.join(url_units)
+        return node.generate_named_url(obj)
 
     def get_related(self, obj):
         res = OrderedDict()
         view = self.context.get('view', None)
         if view and (hasattr(view, 'retrieve') or view.request.method == 'POST') and type(obj) in settings.NAMED_URL_GRAPH:
-            original_url = self.get_url(obj)
-            res['named_url'] = self._generate_named_url(original_url, obj, settings.NAMED_URL_GRAPH[type(obj)])
+            original_path = self.get_url(obj)
+            path_components = original_path.lstrip('/').rstrip('/').split('/')
+
+            friendly_id = self._generate_friendly_id(obj, settings.NAMED_URL_GRAPH[type(obj)])
+            path_components[-1] = friendly_id
+
+            new_path = '/' + '/'.join(path_components) + '/'
+            res['named_url'] = new_path
         if getattr(obj, 'created_by', None):
             res['created_by'] = self.reverse('api:user_detail', kwargs={'pk': obj.created_by.pk})
         if getattr(obj, 'modified_by', None):
@@ -861,7 +865,7 @@ class UnifiedJobSerializer(BaseSerializer):
         if 'elapsed' in ret:
             if obj and obj.pk and obj.started and not obj.finished:
                 td = now() - obj.started
-                ret['elapsed'] = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / (10 ** 6 * 1.0)
+                ret['elapsed'] = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / (10**6 * 1.0)
             ret['elapsed'] = float(ret['elapsed'])
         # Because this string is saved in the db in the source language,
         # it must be marked for translation after it is pulled from the db, not when set
@@ -1639,7 +1643,25 @@ class BaseSerializerWithVariables(BaseSerializer):
         return vars_validate_or_raise(value)
 
 
-class InventorySerializer(BaseSerializerWithVariables):
+class LabelsListMixin(object):
+    def _summary_field_labels(self, obj):
+        label_list = [{'id': x.id, 'name': x.name} for x in obj.labels.all()[:10]]
+        if has_model_field_prefetched(obj, 'labels'):
+            label_ct = len(obj.labels.all())
+        else:
+            if len(label_list) < 10:
+                label_ct = len(label_list)
+            else:
+                label_ct = obj.labels.count()
+        return {'count': label_ct, 'results': label_list}
+
+    def get_summary_fields(self, obj):
+        res = super(LabelsListMixin, self).get_summary_fields(obj)
+        res['labels'] = self._summary_field_labels(obj)
+        return res
+
+
+class InventorySerializer(LabelsListMixin, BaseSerializerWithVariables):
     show_capabilities = ['edit', 'delete', 'adhoc', 'copy']
     capabilities_prefetch = ['admin', 'adhoc', {'copy': 'organization.inventory_admin'}]
 
@@ -1680,6 +1702,7 @@ class InventorySerializer(BaseSerializerWithVariables):
                 object_roles=self.reverse('api:inventory_object_roles_list', kwargs={'pk': obj.pk}),
                 instance_groups=self.reverse('api:inventory_instance_groups_list', kwargs={'pk': obj.pk}),
                 copy=self.reverse('api:inventory_copy', kwargs={'pk': obj.pk}),
+                labels=self.reverse('api:inventory_label_list', kwargs={'pk': obj.pk}),
             )
         )
         if obj.organization:
@@ -2749,24 +2772,6 @@ class OrganizationCredentialSerializerCreate(CredentialSerializerCreate):
         fields = ('*', '-user', '-team')
 
 
-class LabelsListMixin(object):
-    def _summary_field_labels(self, obj):
-        label_list = [{'id': x.id, 'name': x.name} for x in obj.labels.all()[:10]]
-        if has_model_field_prefetched(obj, 'labels'):
-            label_ct = len(obj.labels.all())
-        else:
-            if len(label_list) < 10:
-                label_ct = len(label_list)
-            else:
-                label_ct = obj.labels.count()
-        return {'count': label_ct, 'results': label_list}
-
-    def get_summary_fields(self, obj):
-        res = super(LabelsListMixin, self).get_summary_fields(obj)
-        res['labels'] = self._summary_field_labels(obj)
-        return res
-
-
 class JobOptionsSerializer(LabelsListMixin, BaseSerializer):
     class Meta:
         fields = (
@@ -4767,6 +4772,28 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria
         return super(ScheduleSerializer, self).validate(attrs)
 
 
+class InstanceLinkSerializer(BaseSerializer):
+    class Meta:
+        model = InstanceLink
+        fields = ('source', 'target')
+
+    source = serializers.SlugRelatedField(slug_field="hostname", read_only=True)
+    target = serializers.SlugRelatedField(slug_field="hostname", read_only=True)
+
+
+class InstanceNodeSerializer(BaseSerializer):
+    class Meta:
+        model = Instance
+        fields = ('id', 'hostname', 'node_type', 'node_state')
+
+    node_state = serializers.SerializerMethodField()
+
+    def get_node_state(self, obj):
+        if not obj.enabled:
+            return "disabled"
+        return "error" if obj.errors else "healthy"
+
+
 class InstanceSerializer(BaseSerializer):
 
     consumed_capacity = serializers.SerializerMethodField()
@@ -4810,7 +4837,8 @@ class InstanceSerializer(BaseSerializer):
         res['jobs'] = self.reverse('api:instance_unified_jobs_list', kwargs={'pk': obj.pk})
         res['instance_groups'] = self.reverse('api:instance_instance_groups_list', kwargs={'pk': obj.pk})
         if self.context['request'].user.is_superuser or self.context['request'].user.is_system_auditor:
-            res['health_check'] = self.reverse('api:instance_health_check', kwargs={'pk': obj.pk})
+            if obj.node_type != 'hop':
+                res['health_check'] = self.reverse('api:instance_health_check', kwargs={'pk': obj.pk})
         return res
 
     def get_consumed_capacity(self, obj):

awx/api/templates/api/mesh_visualizer.md

@@ -0,0 +1 @@
+Make a GET request to this resource to obtain a list all Receptor Nodes and their links.

awx/api/urls/inventory.py

@@ -20,6 +20,7 @@ from awx.api.views import (
     InventoryAccessList,
     InventoryObjectRolesList,
     InventoryInstanceGroupsList,
+    InventoryLabelList,
     InventoryCopy,
 )
 
@@ -41,6 +42,7 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/access_list/$', InventoryAccessList.as_view(), name='inventory_access_list'),
     url(r'^(?P<pk>[0-9]+)/object_roles/$', InventoryObjectRolesList.as_view(), name='inventory_object_roles_list'),
     url(r'^(?P<pk>[0-9]+)/instance_groups/$', InventoryInstanceGroupsList.as_view(), name='inventory_instance_groups_list'),
+    url(r'^(?P<pk>[0-9]+)/labels/$', InventoryLabelList.as_view(), name='inventory_label_list'),
     url(r'^(?P<pk>[0-9]+)/copy/$', InventoryCopy.as_view(), name='inventory_copy'),
 ]
 

awx/api/urls/urls.py

@@ -28,6 +28,7 @@ from awx.api.views import (
     OAuth2TokenList,
     ApplicationOAuth2TokenList,
     OAuth2ApplicationDetail,
+    MeshVisualizer,
 )
 
 from awx.api.views.metrics import MetricsView
@@ -95,6 +96,7 @@ v2_urls = [
     url(r'^me/$', UserMeList.as_view(), name='user_me_list'),
     url(r'^dashboard/$', DashboardView.as_view(), name='dashboard_view'),
     url(r'^dashboard/graphs/jobs/$', DashboardJobsGraphView.as_view(), name='dashboard_jobs_graph_view'),
+    url(r'^mesh_visualizer/', MeshVisualizer.as_view(), name='mesh_visualizer_view'),
     url(r'^settings/', include('awx.conf.urls')),
     url(r'^instances/', include(instance_urls)),
     url(r'^instance_groups/', include(instance_group_urls)),

awx/api/views/__init__.py

@@ -62,7 +62,7 @@ import pytz
 from wsgiref.util import FileWrapper
 
 # AWX
-from awx.main.tasks import send_notifications, update_inventory_computed_fields
+from awx.main.tasks.system import send_notifications, update_inventory_computed_fields
 from awx.main.access import get_user_queryset, HostAccess
 from awx.api.generics import (
     APIView,
@@ -113,7 +113,7 @@ from awx.api.permissions import (
 from awx.api import renderers
 from awx.api import serializers
 from awx.api.metadata import RoleMetadata
-from awx.main.constants import ACTIVE_STATES
+from awx.main.constants import ACTIVE_STATES, SURVEY_TYPE_MAPPING
 from awx.main.scheduler.dag_workflow import WorkflowDAG
 from awx.api.views.mixin import (
     ControlledByScmMixin,
@@ -157,8 +157,10 @@ from awx.api.views.inventory import (  # noqa
     InventoryAccessList,
     InventoryObjectRolesList,
     InventoryJobTemplateList,
+    InventoryLabelList,
     InventoryCopy,
 )
+from awx.api.views.mesh_visualizer import MeshVisualizer  # noqa
 from awx.api.views.root import (  # noqa
     ApiRootView,
     ApiOAuthAuthorizationRootView,
@@ -406,6 +408,8 @@ class InstanceInstanceGroupsList(InstanceGroupMembershipMixin, SubListCreateAtta
     def is_valid_relation(self, parent, sub, created=False):
         if parent.node_type == 'control':
             return {'msg': _(f"Cannot change instance group membership of control-only node: {parent.hostname}.")}
+        if parent.node_type == 'hop':
+            return {'msg': _(f"Cannot change instance group membership of hop node: {parent.hostname}.")}
         return None
 
 
@@ -416,6 +420,10 @@ class InstanceHealthCheck(GenericAPIView):
     serializer_class = serializers.InstanceHealthCheckSerializer
     permission_classes = (IsSystemAdminOrAuditor,)
 
+    def get_queryset(self):
+        # FIXME: For now, we don't have a good way of checking the health of a hop node.
+        return super().get_queryset().exclude(node_type='hop')
+
     def get(self, request, *args, **kwargs):
         obj = self.get_object()
         data = self.get_serializer(data=request.data).to_representation(obj)
@@ -425,7 +433,7 @@ class InstanceHealthCheck(GenericAPIView):
         obj = self.get_object()
 
         if obj.node_type == 'execution':
-            from awx.main.tasks import execution_node_health_check
+            from awx.main.tasks.system import execution_node_health_check
 
             runner_data = execution_node_health_check(obj.hostname)
             obj.refresh_from_db()
@@ -435,7 +443,7 @@ class InstanceHealthCheck(GenericAPIView):
                 if extra_field in runner_data:
                     data[extra_field] = runner_data[extra_field]
         else:
-            from awx.main.tasks import cluster_node_health_check
+            from awx.main.tasks.system import cluster_node_health_check
 
             if settings.CLUSTER_HOST_ID == obj.hostname:
                 cluster_node_health_check(obj.hostname)
@@ -503,6 +511,8 @@ class InstanceGroupInstanceList(InstanceGroupMembershipMixin, SubListAttachDetac
     def is_valid_relation(self, parent, sub, created=False):
         if sub.node_type == 'control':
             return {'msg': _(f"Cannot change instance group membership of control-only node: {sub.hostname}.")}
+        if sub.node_type == 'hop':
+            return {'msg': _(f"Cannot change instance group membership of hop node: {sub.hostname}.")}
         return None
 
 
@@ -2458,8 +2468,6 @@ class JobTemplateSurveySpec(GenericAPIView):
     obj_permission_type = 'admin'
     serializer_class = serializers.EmptySerializer
 
-    ALLOWED_TYPES = {'text': str, 'textarea': str, 'password': str, 'multiplechoice': str, 'multiselect': str, 'integer': int, 'float': float}
-
     def get(self, request, *args, **kwargs):
         obj = self.get_object()
         return Response(obj.display_survey_spec())
@@ -2530,17 +2538,17 @@ class JobTemplateSurveySpec(GenericAPIView):
             # Type-specific validation
             # validate question type <-> default type
             qtype = survey_item["type"]
-            if qtype not in JobTemplateSurveySpec.ALLOWED_TYPES:
+            if qtype not in SURVEY_TYPE_MAPPING:
                 return Response(
                     dict(
                         error=_("'{survey_item[type]}' in survey question {idx} is not one of '{allowed_types}' allowed question types.").format(
-                            allowed_types=', '.join(JobTemplateSurveySpec.ALLOWED_TYPES.keys()), **context
+                            allowed_types=', '.join(SURVEY_TYPE_MAPPING.keys()), **context
                         )
                     ),
                     status=status.HTTP_400_BAD_REQUEST,
                 )
             if 'default' in survey_item and survey_item['default'] != '':
-                if not isinstance(survey_item['default'], JobTemplateSurveySpec.ALLOWED_TYPES[qtype]):
+                if not isinstance(survey_item['default'], SURVEY_TYPE_MAPPING[qtype]):
                     type_label = 'string'
                     if qtype in ['integer', 'float']:
                         type_label = qtype

awx/api/views/inventory.py

@@ -16,17 +16,21 @@ from rest_framework.response import Response
 from rest_framework import status
 
 # AWX
-from awx.main.models import (
-    ActivityStream,
-    Inventory,
-    JobTemplate,
-    Role,
-    User,
-    InstanceGroup,
-    InventoryUpdateEvent,
-    InventoryUpdate,
+from awx.main.models import ActivityStream, Inventory, JobTemplate, Role, User, InstanceGroup, InventoryUpdateEvent, InventoryUpdate
+
+from awx.main.models.label import Label
+
+from awx.api.generics import (
+    ListCreateAPIView,
+    RetrieveUpdateDestroyAPIView,
+    SubListAPIView,
+    SubListAttachDetachAPIView,
+    ResourceAccessList,
+    CopyAPIView,
+    DeleteLastUnattachLabelMixin,
+    SubListCreateAttachDetachAPIView,
 )
-from awx.api.generics import ListCreateAPIView, RetrieveUpdateDestroyAPIView, SubListAPIView, SubListAttachDetachAPIView, ResourceAccessList, CopyAPIView
+
 
 from awx.api.serializers import (
     InventorySerializer,
@@ -35,6 +39,7 @@ from awx.api.serializers import (
     InstanceGroupSerializer,
     InventoryUpdateEventSerializer,
     JobTemplateSerializer,
+    LabelSerializer,
 )
 from awx.api.views.mixin import RelatedJobsPreventDeleteMixin, ControlledByScmMixin
 
@@ -152,6 +157,30 @@ class InventoryJobTemplateList(SubListAPIView):
         return qs.filter(inventory=parent)
 
 
+class InventoryLabelList(DeleteLastUnattachLabelMixin, SubListCreateAttachDetachAPIView, SubListAPIView):
+
+    model = Label
+    serializer_class = LabelSerializer
+    parent_model = Inventory
+    relationship = 'labels'
+
+    def post(self, request, *args, **kwargs):
+        # If a label already exists in the database, attach it instead of erroring out
+        # that it already exists
+        if 'id' not in request.data and 'name' in request.data and 'organization' in request.data:
+            existing = Label.objects.filter(name=request.data['name'], organization_id=request.data['organization'])
+            if existing.exists():
+                existing = existing[0]
+                request.data['id'] = existing.id
+                del request.data['name']
+                del request.data['organization']
+        if Label.objects.filter(inventory_labels=self.kwargs['pk']).count() > 100:
+            return Response(
+                dict(msg=_('Maximum number of labels for {} reached.'.format(self.parent_model._meta.verbose_name_raw))), status=status.HTTP_400_BAD_REQUEST
+            )
+        return super(InventoryLabelList, self).post(request, *args, **kwargs)
+
+
 class InventoryCopy(CopyAPIView):
 
     model = Inventory

awx/api/views/mesh_visualizer.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2018 Red Hat, Inc.
+# All Rights Reserved.
+
+from django.utils.translation import ugettext_lazy as _
+
+from awx.api.generics import APIView, Response
+from awx.api.permissions import IsSystemAdminOrAuditor
+from awx.api.serializers import InstanceLinkSerializer, InstanceNodeSerializer
+from awx.main.models import InstanceLink, Instance
+
+
+class MeshVisualizer(APIView):
+
+    name = _("Mesh Visualizer")
+    permission_classes = (IsSystemAdminOrAuditor,)
+    swagger_topic = "System Configuration"
+
+    def get(self, request, format=None):
+
+        data = {
+            'nodes': InstanceNodeSerializer(Instance.objects.all(), many=True).data,
+            'links': InstanceLinkSerializer(InstanceLink.objects.select_related('target', 'source'), many=True).data,
+        }
+
+        return Response(data)

awx/api/views/root.py

@@ -123,6 +123,7 @@ class ApiVersionRootView(APIView):
         data['workflow_approvals'] = reverse('api:workflow_approval_list', request=request)
         data['workflow_job_template_nodes'] = reverse('api:workflow_job_template_node_list', request=request)
         data['workflow_job_nodes'] = reverse('api:workflow_job_node_list', request=request)
+        data['mesh_visualizer'] = reverse('api:mesh_visualizer_view', request=request)
         return Response(data)
 
 
@@ -149,13 +150,13 @@ class ApiV2PingView(APIView):
         response = {'ha': is_ha_environment(), 'version': get_awx_version(), 'active_node': settings.CLUSTER_HOST_ID, 'install_uuid': settings.INSTALL_UUID}
 
         response['instances'] = []
-        for instance in Instance.objects.all():
+        for instance in Instance.objects.exclude(node_type='hop'):
             response['instances'].append(
                 dict(
                     node=instance.hostname,
                     node_type=instance.node_type,
                     uuid=instance.uuid,
-                    heartbeat=instance.modified,
+                    heartbeat=instance.last_seen,
                     capacity=instance.capacity,
                     version=instance.version,
                 )

awx/conf/fields.py

@@ -13,6 +13,9 @@ from django.utils.translation import ugettext_lazy as _
 from rest_framework.fields import BooleanField, CharField, ChoiceField, DictField, DateTimeField, EmailField, IntegerField, ListField, NullBooleanField  # noqa
 from rest_framework.serializers import PrimaryKeyRelatedField  # noqa
 
+# AWX
+from awx.main.constants import CONTAINER_VOLUMES_MOUNT_TYPES, MAX_ISOLATED_PATH_COLON_DELIMITER
+
 logger = logging.getLogger('awx.conf.fields')
 
 # Use DRF fields to convert/validate settings:
@@ -109,6 +112,49 @@ class StringListPathField(StringListField):
             self.fail('type_error', input_type=type(paths))
 
 
+class StringListIsolatedPathField(StringListField):
+    # Valid formats
+    # '/etc/pki/ca-trust'
+    # '/etc/pki/ca-trust:/etc/pki/ca-trust'
+    # '/etc/pki/ca-trust:/etc/pki/ca-trust:O'
+
+    default_error_messages = {
+        'type_error': _('Expected list of strings but got {input_type} instead.'),
+        'path_error': _('{path} is not a valid path choice. You must provide an absolute path.'),
+        'mount_error': _('{scontext} is not a valid mount option. Allowed types are {mount_types}'),
+        'syntax_error': _('Invalid syntax. A string HOST-DIR[:CONTAINER-DIR[:OPTIONS]] is expected but got {path}.'),
+    }
+
+    def to_internal_value(self, paths):
+
+        if isinstance(paths, (list, tuple)):
+            for p in paths:
+                if not isinstance(p, str):
+                    self.fail('type_error', input_type=type(p))
+                if not p.startswith('/'):
+                    self.fail('path_error', path=p)
+
+                if p.count(':'):
+                    if p.count(':') > MAX_ISOLATED_PATH_COLON_DELIMITER:
+                        self.fail('syntax_error', path=p)
+                    try:
+                        src, dest, scontext = p.split(':')
+                    except ValueError:
+                        scontext = 'z'
+                        src, dest = p.split(':')
+                    finally:
+                        for sp in [src, dest]:
+                            if not len(sp):
+                                self.fail('syntax_error', path=sp)
+                            if not sp.startswith('/'):
+                                self.fail('path_error', path=sp)
+                        if scontext not in CONTAINER_VOLUMES_MOUNT_TYPES:
+                            self.fail('mount_error', scontext=scontext, mount_types=CONTAINER_VOLUMES_MOUNT_TYPES)
+            return super(StringListIsolatedPathField, self).to_internal_value(sorted(paths))
+        else:
+            self.fail('type_error', input_type=type(paths))
+
+
 class URLField(CharField):
     # these lines set up a custom regex that allow numbers in the
     # top-level domain

awx/conf/views.py

@@ -26,7 +26,7 @@ from awx.api.generics import APIView, GenericAPIView, ListAPIView, RetrieveUpdat
 from awx.api.permissions import IsSystemAdminOrAuditor
 from awx.api.versioning import reverse
 from awx.main.utils import camelcase_to_underscore
-from awx.main.tasks import handle_setting_changes
+from awx.main.tasks.system import handle_setting_changes
 from awx.conf.models import Setting
 from awx.conf.serializers import SettingCategorySerializer, SettingSingletonSerializer
 from awx.conf import settings_registry

awx/locale/fr/LC_MESSAGES/django.po

@@ -1,6 +1,3 @@
-# SOME DESCRIPTIVE TITLE.
-# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
-# This file is distributed under the same license as the PACKAGE package.
 # FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
 #
 msgid ""
@@ -2255,7 +2252,6 @@ msgid ""
 "Maximum number of messages to update the UI live job output with per second. "
 "Value of 0 means no limit."
 msgstr "Nombre maximal de messages pour mettre à jour la sortie du Job dans l'interface live utilisateur, par seconde. La valeur de 0 signifie qu'il n'y a pas de limite."
-
 #: awx/main/conf.py:380
 msgid "Maximum Scheduled Jobs"
 msgstr "Nombre max. de tâches planifiées"
@@ -2812,7 +2808,7 @@ msgstr "URL du coffre HashiCorp"
 #: awx/main/models/credential/__init__.py:920
 #: awx/main/models/credential/__init__.py:939
 msgid "Token"
-msgstr "Token"
+msgstr "Jeton"
 
 #: awx/main/credential_plugins/hashivault.py:25
 msgid "The access token used to authenticate to the Vault server"
@@ -3724,7 +3720,7 @@ msgstr "interne : à la non-importation pour l'hôte"
 
 #: awx/main/models/events.py:193
 msgid "Play Started"
-msgstr "Scène démarrée"
+msgstr "Play Démarrage"
 
 #: awx/main/models/events.py:194
 msgid "Playbook Complete"
@@ -3981,7 +3977,7 @@ msgid ""
 "The host would be marked enabled. If power_state where any value other than "
 "powered_on then the host would be disabled when imported. If the key is not "
 "found then the host will be enabled"
-msgstr "Utilisé uniquement lorsque enabled_var est défini. Valeur lorsque l'hôte est considéré comme activé. Par exemple, si enabled_var=\"status.power_state \" et enabled_value=\"powered_on\" avec les variables de l'hôte:{ \"status\" : { \"power_state\" : \"powered_on\", \"created\" : \"2020-08-04T18:13:04+00:00\", \"healthy\" : true }, \"name\" : \"foobar\", \"ip_address\" : \"192.168.2.1\"}, l'hôte serait marqué comme étant activé. Si power_state contient une valeur autre que power_on, alors l'hôte sera désactivé lors de l'importation. Si la clé n'est pas trouvée, alors l'hôte sera activé"
+msgstr "Utilisé uniquement lorsque enabled_var est défini. Valeur lorsque l'hôte est considéré comme activé. Par exemple, si enabled_var=\"status.power_state \" et enabled_value=\"powered_on\" avec les variables de l'hôte:{   \"status\": {     \"power_state\": \"powered_on\",     \"created\": \"2020-08-04T18:13:04+00:00\",     \"healthy\": true    },\"name\" : \"foobar\", \"ip_address\" : \"192.168.2.1\"}, l'hôte serait marqué comme étant activé. Si power_state contient une valeur autre que power_on, alors l'hôte sera désactivé lors de l'importation. Si la clé n'est pas trouvée, alors l'hôte sera activé"
 
 #: awx/main/models/inventory.py:878
 msgid "Regex where only matching hosts will be imported."
@@ -4301,7 +4297,7 @@ msgstr "Utilisé pour une vérification plus rigoureuse de l'accès à une appli
 #: awx/main/models/oauth.py:74
 msgid ""
 "Set to Public or Confidential depending on how secure the client device is."
-msgstr "Défini sur sur Public ou Confidentiel selon le degré de sécurité du périphérique client."
+msgstr "Définir sur sur Public ou Confidentiel selon le degré de sécurité du périphérique client."
 
 #: awx/main/models/oauth.py:76
 msgid ""
@@ -4555,7 +4551,7 @@ msgstr "Utilisation"
 
 #: awx/main/models/rbac.py:52
 msgid "Approve"
-msgstr "Approbation"
+msgstr "Approuver"
 
 #: awx/main/models/rbac.py:56
 msgid "Can manage all aspects of the system"
@@ -5147,7 +5143,7 @@ msgstr "Au moins %(min_certs)d certificats sont requis, seulement %(cert_count)d
 #: awx/main/validators.py:152
 #, python-format
 msgid "Only one certificate is allowed, %(cert_count)d provided."
-msgstr "Un seul certificat est autorisé, %(cert_count)d ont été fournis."
+msgstr "Un seul certificat est autorisé, %(cert_count) ont été fournis."
 
 #: awx/main/validators.py:154
 #, python-format
@@ -5633,7 +5629,7 @@ msgstr "Nom de l'organisation GitHub"
 msgid ""
 "The name of your GitHub organization, as used in your organization's URL: "
 "https://github.com/<yourorg>/."
-msgstr "Nom de votre organisation GitHub, tel qu'utilisé dans l'URL de votre organisation : https://github.com/<votreorg>/."
+msgstr "Nom de votre organisation GitHub, tel qu'utilisé dans l'URL de votre organisation : https://github.com/<yourorg>/."
 
 #: awx/sso/conf.py:762
 msgid "GitHub Organization OAuth2 Organization Map"
@@ -5653,7 +5649,7 @@ msgid ""
 "<yourorg>/settings/applications and obtain an OAuth2 key (Client ID) and "
 "secret (Client Secret). Provide this URL as the callback URL for your "
 "application."
-msgstr "Créez une application appartenant à une organisation sur https://github.com/organizations/<votreorg>/settings/applications et obtenez une clé OAuth2 (ID client) et un secret (secret client). Entrez cette URL comme URL de rappel de votre application."
+msgstr "Créez une application appartenant à une organisation sur https://github.com/organizations/<yourorg>/settings/applications et obtenez une clé OAuth2 (ID client) et un secret (secret client). Entrez cette URL comme URL de rappel de votre application."
 
 #: awx/sso/conf.py:797 awx/sso/conf.py:809 awx/sso/conf.py:820
 #: awx/sso/conf.py:832 awx/sso/conf.py:843 awx/sso/conf.py:855
@@ -5789,7 +5785,7 @@ msgstr "Nom de l'organisation GitHub Enterprise"
 msgid ""
 "The name of your GitHub Enterprise organization, as used in your "
 "organization's URL: https://github.com/<yourorg>/."
-msgstr "Nom de votre organisation GitHub Enterprise, tel qu'utilisé dans l'URL de votre organisation : https://github.com/<votreorg>/."
+msgstr "Nom de votre organisation GitHub Enterprise, tel qu'utilisé dans l'URL de votre organisation : https://github.com/<yourorg>/."
 
 #: awx/sso/conf.py:1030
 msgid "GitHub Enterprise Organization OAuth2 Organization Map"

awx/locale/ja/LC_MESSAGES/django.po

@@ -1,6 +1,3 @@
-# SOME DESCRIPTIVE TITLE.
-# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
-# This file is distributed under the same license as the PACKAGE package.
 # FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
 #
 msgid ""
@@ -596,7 +593,7 @@ msgstr "指定された変数 {} には置き換えるデータベースの値
 
 #: awx/api/serializers.py:3739
 msgid "\"$encrypted$ is a reserved keyword, may not be used for {}.\""
-msgstr "\"$encrypted$ は予約されたキーワードで、{} には使用できません。\""
+msgstr "\"$encrypted は予約されたキーワードで {} には使用できません。\""
 
 #: awx/api/serializers.py:4212
 msgid "A project is required to run a job."
@@ -824,7 +821,7 @@ msgstr "ポリシーインスタンスの割合"
 msgid ""
 "Minimum percentage of all instances that will be automatically assigned to "
 "this group when new instances come online."
-msgstr "新規インスタンスがオンラインになると、このグループに自動的に最小限割り当てられるインスタンスの割合を選択します。"
+msgstr "新規インスタンスがオンラインになると、このグループに自動的に最小限割り当てられるインスタンスの割合"
 
 #: awx/api/serializers.py:4853
 msgid "Policy Instance Minimum"
@@ -1253,7 +1250,7 @@ msgstr "デフォルトで指定されている選択項目は、一覧から回
 msgid ""
 "$encrypted$ is a reserved keyword for password question defaults, survey "
 "question {idx} is type {survey_item[type]}."
-msgstr "$encrypted$ は、デフォルト設定されているパスワードの質問に予約されたキーワードで、Survey の質問 {idx} は {survey_item[type]} タイプです。"
+msgstr "$encrypted$ はパスワードの質問のデフォルトの予約されたキーワードで、Survey の質問 {idx} はタイプ {survey_item[type]} です。"
 
 #: awx/api/views/__init__.py:2567
 #, python-brace-format
@@ -3638,7 +3635,7 @@ msgstr "ホスト OK"
 
 #: awx/main/models/events.py:169
 msgid "Host Failure"
-msgstr "ホストの失敗"
+msgstr "ホストの障害"
 
 #: awx/main/models/events.py:170 awx/main/models/events.py:767
 msgid "Host Skipped"
@@ -4694,7 +4691,7 @@ msgstr "取り消されました"
 
 #: awx/main/models/unified_jobs.py:84
 msgid "Never Updated"
-msgstr "更新されていません"
+msgstr "未更新"
 
 #: awx/main/models/unified_jobs.py:88
 msgid "OK"
@@ -6261,4 +6258,3 @@ msgstr "%s が現在アップグレード中です。"
 #: awx/ui/urls.py:24
 msgid "This page will refresh when complete."
 msgstr "このページは完了すると更新されます。"
-

awx/locale/zh/LC_MESSAGES/django.po

@@ -1,6 +1,3 @@
-# SOME DESCRIPTIVE TITLE.
-# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
-# This file is distributed under the same license as the PACKAGE package.
 # FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
 #
 msgid ""
@@ -596,7 +593,7 @@ msgstr "提供的变量 {} 没有要替换的数据库值。"
 
 #: awx/api/serializers.py:3739
 msgid "\"$encrypted$ is a reserved keyword, may not be used for {}.\""
-msgstr "\"$encrypted$ 是一个保留的关键字，可能不能用于 {}\""
+msgstr "\"$encrypted$ 是一个保留关键字，可能无法用于 {}。\""
 
 #: awx/api/serializers.py:4212
 msgid "A project is required to run a job."
@@ -1031,7 +1028,7 @@ msgstr "对于受管执行环境，只能编辑 'pull' 字段。"
 
 #: awx/api/views/__init__.py:805
 msgid "Project Schedules"
-msgstr "项目调度"
+msgstr "项目计划"
 
 #: awx/api/views/__init__.py:816
 msgid "Project SCM Inventory Sources"
@@ -1253,14 +1250,14 @@ msgstr "默认的选择必须从列出的选择中回答。"
 msgid ""
 "$encrypted$ is a reserved keyword for password question defaults, survey "
 "question {idx} is type {survey_item[type]}."
-msgstr "$encrypted$ 是密码问题默认值的保留关键字，问卷调查问题 {idx} 的类型是 {survey_item[type]}。"
+msgstr "$encrypted$ 是密码问题默认值的保留关键字，问卷调查问题 {idx} 是类型 {survey_item[type]}。"
 
 #: awx/api/views/__init__.py:2567
 #, python-brace-format
 msgid ""
 "$encrypted$ is a reserved keyword, may not be used for new default in "
 "position {idx}."
-msgstr "$encrypted$ 是一个保留关键字，可能无法用于位置 {idx} 中的新默认值。"
+msgstr "$encrypted$ 是一个保留关键字，无法用于位置 {idx} 中的新默认值。"
 
 #: awx/api/views/__init__.py:2639
 #, python-brace-format
@@ -2865,7 +2862,7 @@ msgstr "Secret 的路径"
 
 #: awx/main/credential_plugins/hashivault.py:78
 msgid "Path to Auth"
-msgstr "Auth 的路径"
+msgstr "到 Auth 的路径"
 
 #: awx/main/credential_plugins/hashivault.py:81
 msgid "The path where the Authentication method is mounted e.g, approle"
@@ -3979,7 +3976,7 @@ msgid ""
 "The host would be marked enabled. If power_state where any value other than "
 "powered_on then the host would be disabled when imported. If the key is not "
 "found then the host will be enabled"
-msgstr "仅在设置 enabled_var 时使用。 主机被视为启用时的值。 例如： if enabled_var=\"status.power_state\"and enabled_value=\"powered_on\" with host variables:{ \"status\": { \"power_state\": \"powered_on\", \"created\": \"2020-08-04T18:13:04+00:00\", \"healthy\": true }, \"name\": \"foobar\", \"ip_address\": \"192.168.2.1\"}The host would be marked enabled. 如果 power_state 在除 powered_on 以外的任何值，则会在导入时禁用主机。如果没有找到密钥，则会启用主机"
+msgstr "仅在设置 enabled_var 时使用。 主机被视为启用时的值。 例如：是否 enabled_var=\"status.power_state\"and enabled_value=\"powered_on\" with host variables:{   \"status\": {     \"power_state\": \"powered_on\",     \"created\": \"2020-08-04T18:13:04+00:00\",     \"healthy\": true    },    \"name\": \"foobar\",    \"ip_address\": \"192.168.2.1\"}。如果 power_state 在除 powered_on 以外的任何值，则会在导入时禁用主机。如果没有找到密钥，则会启用主机"
 
 #: awx/main/models/inventory.py:878
 msgid "Regex where only matching hosts will be imported."
@@ -6133,7 +6130,7 @@ msgstr "您的帐户不活跃"
 #: awx/sso/validators.py:24 awx/sso/validators.py:51
 #, python-format
 msgid "DN must include \"%%(user)s\" placeholder for username: %s"
-msgstr "DN 必须包含 \"%%\" 占位符用于用户名：%s"
+msgstr "DN 必须包含 \"%%(user)s\" 占位符用于用户名：%s"
 
 #: awx/sso/validators.py:31
 #, python-format
@@ -6263,4 +6260,3 @@ msgstr "%s 当前正在升级。"
 #: awx/ui/urls.py:24
 msgid "This page will refresh when complete."
 msgstr "完成后，此页面会刷新。"
-

awx/main/access.py

@@ -853,7 +853,12 @@ class InventoryAccess(BaseAccess):
     """
 
     model = Inventory
-    prefetch_related = ('created_by', 'modified_by', 'organization')
+    prefetch_related = (
+        'created_by',
+        'modified_by',
+        'organization',
+        Prefetch('labels', queryset=Label.objects.all().order_by('name')),
+    )
 
     def filtered_queryset(self, allowed=None, ad_hoc=None):
         return self.model.accessible_objects(self.user, 'read_role')

awx/main/analytics/collectors.py

@@ -211,7 +211,7 @@ def projects_by_scm_type(since, **kwargs):
     return counts
 
 
-@register('instance_info', '1.1', description=_('Cluster topology and capacity'))
+@register('instance_info', '1.2', description=_('Cluster topology and capacity'))
 def instance_info(since, include_hostnames=False, **kwargs):
     info = {}
     instances = models.Instance.objects.values_list('hostname').values(
@@ -337,6 +337,7 @@ def _events_table(since, full_path, until, tbl, where_column, project_job_create
                           {tbl}.parent_uuid,
                           {tbl}.event,
                           task_action,
+                          resolved_action,
                           -- '-' operator listed here:
                           -- https://www.postgresql.org/docs/12/functions-json.html
                           -- note that operator is only supported by jsonb objects
@@ -356,7 +357,7 @@ def _events_table(since, full_path, until, tbl, where_column, project_job_create
                           x.duration AS duration,
                           x.res->'warnings' AS warnings,
                           x.res->'deprecations' AS deprecations
-                          FROM {tbl}, json_to_record({event_data}) AS x("res" json, "duration" text, "task_action" text, "start" text, "end" text)
+                          FROM {tbl}, jsonb_to_record({event_data}) AS x("res" json, "duration" text, "task_action" text, "resolved_action" text, "start" text, "end" text)
                           WHERE ({tbl}.{where_column} > '{since.isoformat()}' AND {tbl}.{where_column} <= '{until.isoformat()}')) TO STDOUT WITH CSV HEADER'''
         return query
 
@@ -366,23 +367,24 @@ def _events_table(since, full_path, until, tbl, where_column, project_job_create
         return _copy_table(table='events', query=query(f"replace({tbl}.event_data::text, '\\u0000', '')::jsonb"), path=full_path)
 
 
-@register('events_table', '1.3', format='csv', description=_('Automation task records'), expensive=four_hour_slicing)
+@register('events_table', '1.4', format='csv', description=_('Automation task records'), expensive=four_hour_slicing)
 def events_table_unpartitioned(since, full_path, until, **kwargs):
     return _events_table(since, full_path, until, '_unpartitioned_main_jobevent', 'created', **kwargs)
 
 
-@register('events_table', '1.3', format='csv', description=_('Automation task records'), expensive=four_hour_slicing)
+@register('events_table', '1.4', format='csv', description=_('Automation task records'), expensive=four_hour_slicing)
 def events_table_partitioned_modified(since, full_path, until, **kwargs):
     return _events_table(since, full_path, until, 'main_jobevent', 'modified', project_job_created=True, **kwargs)
 
 
-@register('unified_jobs_table', '1.2', format='csv', description=_('Data on jobs run'), expensive=four_hour_slicing)
+@register('unified_jobs_table', '1.3', format='csv', description=_('Data on jobs run'), expensive=four_hour_slicing)
 def unified_jobs_table(since, full_path, until, **kwargs):
     unified_job_query = '''COPY (SELECT main_unifiedjob.id,
                                  main_unifiedjob.polymorphic_ctype_id,
                                  django_content_type.model,
                                  main_unifiedjob.organization_id,
                                  main_organization.name as organization_name,
+                                 main_executionenvironment.image as execution_environment_image,
                                  main_job.inventory_id,
                                  main_inventory.name as inventory_name,
                                  main_unifiedjob.created,
@@ -407,6 +409,7 @@ def unified_jobs_table(since, full_path, until, **kwargs):
                                  LEFT JOIN main_job ON main_unifiedjob.id = main_job.unifiedjob_ptr_id
                                  LEFT JOIN main_inventory ON main_job.inventory_id = main_inventory.id
                                  LEFT JOIN main_organization ON main_organization.id = main_unifiedjob.organization_id
+                                 LEFT JOIN main_executionenvironment ON main_executionenvironment.id = main_unifiedjob.execution_environment_id
                                  WHERE ((main_unifiedjob.created > '{0}' AND main_unifiedjob.created <= '{1}')
                                        OR (main_unifiedjob.finished > '{0}' AND main_unifiedjob.finished <= '{1}'))
                                        AND main_unifiedjob.launch_type != 'sync'
@@ -417,11 +420,12 @@ def unified_jobs_table(since, full_path, until, **kwargs):
     return _copy_table(table='unified_jobs', query=unified_job_query, path=full_path)
 
 
-@register('unified_job_template_table', '1.0', format='csv', description=_('Data on job templates'))
+@register('unified_job_template_table', '1.1', format='csv', description=_('Data on job templates'))
 def unified_job_template_table(since, full_path, **kwargs):
     unified_job_template_query = '''COPY (SELECT main_unifiedjobtemplate.id,
                                  main_unifiedjobtemplate.polymorphic_ctype_id,
                                  django_content_type.model,
+                                 main_executionenvironment.image as execution_environment_image,
                                  main_unifiedjobtemplate.created,
                                  main_unifiedjobtemplate.modified,
                                  main_unifiedjobtemplate.created_by_id,
@@ -434,7 +438,8 @@ def unified_job_template_table(since, full_path, **kwargs):
                                  main_unifiedjobtemplate.next_job_run,
                                  main_unifiedjobtemplate.next_schedule_id,
                                  main_unifiedjobtemplate.status
-                                 FROM main_unifiedjobtemplate, django_content_type
+                                 FROM main_unifiedjobtemplate
+                                 LEFT JOIN main_executionenvironment ON main_executionenvironment.id = main_unifiedjobtemplate.execution_environment_id, django_content_type
                                  WHERE main_unifiedjobtemplate.polymorphic_ctype_id = django_content_type.id
                                  ORDER BY main_unifiedjobtemplate.id ASC) TO STDOUT WITH CSV HEADER'''
     return _copy_table(table='unified_job_template', query=unified_job_template_query, path=full_path)

awx/main/analytics/core.py

@@ -90,7 +90,7 @@ def package(target, data, timestamp):
                     if isinstance(item, str):
                         f.add(item, arcname=f'./{name}')
                     else:
-                        buf = json.dumps(item).encode('utf-8')
+                        buf = json.dumps(item, cls=DjangoJSONEncoder).encode('utf-8')
                         info = tarfile.TarInfo(f'./{name}')
                         info.size = len(buf)
                         info.mtime = timestamp.timestamp()
@@ -230,7 +230,7 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti
             try:
                 last_entry = max(last_entries.get(key) or last_gather, until - timedelta(weeks=4))
                 results = (func(since or last_entry, collection_type=collection_type, until=until), func.__awx_analytics_version__)
-                json.dumps(results)  # throwaway check to see if the data is json-serializable
+                json.dumps(results, cls=DjangoJSONEncoder)  # throwaway check to see if the data is json-serializable
                 data[filename] = results
             except Exception:
                 logger.exception("Could not generate metric {}".format(filename))

awx/main/analytics/subsystem_metrics.py

@@ -160,6 +160,7 @@ class Metrics:
             IntM('callback_receiver_batch_events_errors', 'Number of times batch insertion failed'),
             FloatM('callback_receiver_events_insert_db_seconds', 'Time spent saving events to database'),
             IntM('callback_receiver_events_insert_db', 'Number of events batch inserted into database'),
+            IntM('callback_receiver_events_broadcast', 'Number of events broadcast to other control plane nodes'),
             HistogramM(
                 'callback_receiver_batch_events_insert_db', 'Number of events batch inserted into database', settings.SUBSYSTEM_METRICS_BATCH_INSERT_BUCKETS
             ),

awx/main/conf.py

@@ -72,8 +72,8 @@ register(
         'HTTP headers and meta keys to search to determine remote host '
         'name or IP. Add additional items to this list, such as '
         '"HTTP_X_FORWARDED_FOR", if behind a reverse proxy. '
-        'See the "Proxy Support" section of the Adminstrator guide for '
-        'more details.'
+        'See the "Proxy Support" section of the AAP Installation guide '
+        'for more details.'
     ),
     category=_('System'),
     category_slug='system',
@@ -259,10 +259,14 @@ register(
 
 register(
     'AWX_ISOLATION_SHOW_PATHS',
-    field_class=fields.StringListField,
+    field_class=fields.StringListIsolatedPathField,
     required=False,
     label=_('Paths to expose to isolated jobs'),
-    help_text=_('List of paths that would otherwise be hidden to expose to isolated jobs. Enter one path per line.'),
+    help_text=_(
+        'List of paths that would otherwise be hidden to expose to isolated jobs. Enter one path per line. '
+        'Volumes will be mounted from the execution node to the container. '
+        'The supported format is HOST-DIR[:CONTAINER-DIR[:OPTIONS]]. '
+    ),
     category=_('Jobs'),
     category_slug='jobs',
 )
@@ -330,6 +334,19 @@ register(
     category_slug='jobs',
 )
 
+register(
+    'AWX_MOUNT_ISOLATED_PATHS_ON_K8S',
+    field_class=fields.BooleanField,
+    default=False,
+    label=_('Expose host paths for Container Groups'),
+    help_text=_(
+        'Expose paths via hostPath for the Pods created by a Container Group. '
+        'HostPath volumes present many security risks, and it is a best practice to avoid the use of HostPaths when possible. '
+    ),
+    category=_('Jobs'),
+    category_slug='jobs',
+)
+
 register(
     'GALAXY_IGNORE_CERTS',
     field_class=fields.BooleanField,
@@ -674,6 +691,24 @@ register(
     category=_('Logging'),
     category_slug='logging',
 )
+register(
+    'API_400_ERROR_LOG_FORMAT',
+    field_class=fields.CharField,
+    default='status {status_code} received by user {user_name} attempting to access {url_path} from {remote_addr}',
+    label=_('Log Format For API 4XX Errors'),
+    help_text=_(
+        'The format of logged messages when an API 4XX error occurs, '
+        'the following variables will be substituted: \n'
+        'status_code - The HTTP status code of the error\n'
+        'user_name - The user name attempting to use the API\n'
+        'url_path - The URL path to the API endpoint called\n'
+        'remote_addr - The remote address seen for the user\n'
+        'error - The error set by the api endpoint\n'
+        'Variables need to be in the format {<variable name>}.'
+    ),
+    category=_('Logging'),
+    category_slug='logging',
+)
 
 
 register(
@@ -687,7 +722,7 @@ register(
 register(
     'AUTOMATION_ANALYTICS_LAST_ENTRIES',
     field_class=fields.CharField,
-    label=_('Last gathered entries for expensive collectors for Insights for Ansible Automation Platform.'),
+    label=_('Last gathered entries from the data collection service of Insights for Ansible Automation Platform'),
     default='',
     allow_blank=True,
     category=_('System'),

awx/main/constants.py

@@ -85,3 +85,13 @@ RECEPTOR_PENDING = 'ansible-runner-???'
 # Naming pattern for AWX jobs in /tmp folder, like /tmp/awx_42_xiwm
 # also update awxkit.api.pages.unified_jobs if changed
 JOB_FOLDER_PREFIX = 'awx_%s_'
+
+# :z option tells Podman that two containers share the volume content with r/w
+# :O option tells Podman to mount the directory from the host as a temporary storage using the overlay file system.
+# :ro or :rw option to mount a volume in read-only or read-write mode, respectively. By default, the volumes are mounted read-write.
+# see podman-run manpage for further details
+# /HOST-DIR:/CONTAINER-DIR:OPTIONS
+CONTAINER_VOLUMES_MOUNT_TYPES = ['z', 'O', 'ro', 'rw']
+MAX_ISOLATED_PATH_COLON_DELIMITER = 2
+
+SURVEY_TYPE_MAPPING = {'text': str, 'textarea': str, 'password': str, 'multiplechoice': str, 'multiselect': str, 'integer': int, 'float': (float, int)}

awx/main/dispatch/pool.py

@@ -22,6 +22,7 @@ import psutil
 
 from awx.main.models import UnifiedJob
 from awx.main.dispatch import reaper
+from awx.main.utils.common import convert_mem_str_to_bytes, get_mem_effective_capacity
 
 if 'run_callback_receiver' in sys.argv:
     logger = logging.getLogger('awx.main.commands.run_callback_receiver')
@@ -248,7 +249,7 @@ class WorkerPool(object):
         except Exception:
             logger.exception('could not fork')
         else:
-            logger.warn('scaling up worker pid:{}'.format(worker.pid))
+            logger.debug('scaling up worker pid:{}'.format(worker.pid))
         return idx, worker
 
     def debug(self, *args, **kwargs):
@@ -319,11 +320,13 @@ class AutoscalePool(WorkerPool):
         if self.max_workers is None:
             settings_absmem = getattr(settings, 'SYSTEM_TASK_ABS_MEM', None)
             if settings_absmem is not None:
-                total_memory_gb = int(settings_absmem)
+                # There are 1073741824 bytes in a gigabyte. Convert bytes to gigabytes by dividing by 2**30
+                total_memory_gb = convert_mem_str_to_bytes(settings_absmem) // 2**30
             else:
                 total_memory_gb = (psutil.virtual_memory().total >> 30) + 1  # noqa: round up
-            # 5 workers per GB of total memory
-            self.max_workers = total_memory_gb * 5
+
+            # Get same number as max forks based on memory, this function takes memory as bytes
+            self.max_workers = get_mem_effective_capacity(total_memory_gb * 2**30)
 
         # max workers can't be less than min_workers
         self.max_workers = max(self.min_workers, self.max_workers)
@@ -387,7 +390,7 @@ class AutoscalePool(WorkerPool):
                 # more processes in the pool than we need (> min)
                 # send this process a message so it will exit gracefully
                 # at the next opportunity
-                logger.warn('scaling down worker pid:{}'.format(w.pid))
+                logger.debug('scaling down worker pid:{}'.format(w.pid))
                 w.quit()
                 self.workers.remove(w)
             if w.alive:

awx/main/dispatch/worker/base.py

@@ -60,7 +60,7 @@ class AWXConsumerBase(object):
         return f'listening on {self.queues}'
 
     def control(self, body):
-        logger.warn(body)
+        logger.warn(f'Received control signal:\n{body}')
         control = body.get('control')
         if control in ('status', 'running'):
             reply_queue = body['reply_to']
@@ -137,7 +137,7 @@ class AWXConsumerPG(AWXConsumerBase):
     def run(self, *args, **kwargs):
         super(AWXConsumerPG, self).run(*args, **kwargs)
 
-        logger.warn(f"Running worker {self.name} listening to queues {self.queues}")
+        logger.info(f"Running worker {self.name} listening to queues {self.queues}")
         init = False
 
         while True:
@@ -188,7 +188,7 @@ class BaseWorker(object):
                 if 'uuid' in body:
                     uuid = body['uuid']
                     finished.put(uuid)
-        logger.warn('worker exiting gracefully pid:{}'.format(os.getpid()))
+        logger.debug('worker exiting gracefully pid:{}'.format(os.getpid()))
 
     def perform_work(self, body):
         raise NotImplementedError()

awx/main/dispatch/worker/callback.py

@@ -17,7 +17,7 @@ import redis
 
 from awx.main.consumers import emit_channel_notification
 from awx.main.models import JobEvent, AdHocCommandEvent, ProjectUpdateEvent, InventoryUpdateEvent, SystemJobEvent, UnifiedJob, Job
-from awx.main.tasks import handle_success_and_failure_notifications
+from awx.main.tasks.system import handle_success_and_failure_notifications
 from awx.main.models.events import emit_event_detail
 from awx.main.utils.profiling import AWXProfiler
 import awx.main.analytics.subsystem_metrics as s_metrics
@@ -116,19 +116,20 @@ class CallbackBrokerWorker(BaseWorker):
     def flush(self, force=False):
         now = tz_now()
         if force or (time.time() - self.last_flush) > settings.JOB_EVENT_BUFFER_SECONDS or any([len(events) >= 1000 for events in self.buff.values()]):
-            bulk_events_saved = 0
-            singular_events_saved = 0
+            metrics_bulk_events_saved = 0
+            metrics_singular_events_saved = 0
             metrics_events_batch_save_errors = 0
+            metrics_events_broadcast = 0
             for cls, events in self.buff.items():
                 logger.debug(f'{cls.__name__}.objects.bulk_create({len(events)})')
                 for e in events:
                     if not e.created:
                         e.created = now
                     e.modified = now
-                duration_to_save = time.perf_counter()
+                metrics_duration_to_save = time.perf_counter()
                 try:
                     cls.objects.bulk_create(events)
-                    bulk_events_saved += len(events)
+                    metrics_bulk_events_saved += len(events)
                 except Exception:
                     # if an exception occurs, we should re-attempt to save the
                     # events one-by-one, because something in the list is
@@ -137,22 +138,24 @@ class CallbackBrokerWorker(BaseWorker):
                     for e in events:
                         try:
                             e.save()
-                            singular_events_saved += 1
+                            metrics_singular_events_saved += 1
                         except Exception:
                             logger.exception('Database Error Saving Job Event')
-                duration_to_save = time.perf_counter() - duration_to_save
+                metrics_duration_to_save = time.perf_counter() - metrics_duration_to_save
                 for e in events:
                     if not getattr(e, '_skip_websocket_message', False):
+                        metrics_events_broadcast += 1
                         emit_event_detail(e)
             self.buff = {}
             self.last_flush = time.time()
             # only update metrics if we saved events
-            if (bulk_events_saved + singular_events_saved) > 0:
+            if (metrics_bulk_events_saved + metrics_singular_events_saved) > 0:
                 self.subsystem_metrics.inc('callback_receiver_batch_events_errors', metrics_events_batch_save_errors)
-                self.subsystem_metrics.inc('callback_receiver_events_insert_db_seconds', duration_to_save)
-                self.subsystem_metrics.inc('callback_receiver_events_insert_db', bulk_events_saved + singular_events_saved)
-                self.subsystem_metrics.observe('callback_receiver_batch_events_insert_db', bulk_events_saved)
-                self.subsystem_metrics.inc('callback_receiver_events_in_memory', -(bulk_events_saved + singular_events_saved))
+                self.subsystem_metrics.inc('callback_receiver_events_insert_db_seconds', metrics_duration_to_save)
+                self.subsystem_metrics.inc('callback_receiver_events_insert_db', metrics_bulk_events_saved + metrics_singular_events_saved)
+                self.subsystem_metrics.observe('callback_receiver_batch_events_insert_db', metrics_bulk_events_saved)
+                self.subsystem_metrics.inc('callback_receiver_events_in_memory', -(metrics_bulk_events_saved + metrics_singular_events_saved))
+                self.subsystem_metrics.inc('callback_receiver_events_broadcast', metrics_events_broadcast)
             if self.subsystem_metrics.should_pipe_execute() is True:
                 self.subsystem_metrics.pipe_execute()
 

awx/main/dispatch/worker/task.py

@@ -9,7 +9,7 @@ from kubernetes.config import kube_config
 from django.conf import settings
 from django_guid.middleware import GuidMiddleware
 
-from awx.main.tasks import dispatch_startup, inform_cluster_of_shutdown
+from awx.main.tasks.system import dispatch_startup, inform_cluster_of_shutdown
 
 from .base import BaseWorker
 
@@ -30,8 +30,8 @@ class TaskWorker(BaseWorker):
         """
         Transform a dotted notation task into an imported, callable function, e.g.,
 
-        awx.main.tasks.delete_inventory
-        awx.main.tasks.RunProjectUpdate
+        awx.main.tasks.system.delete_inventory
+        awx.main.tasks.jobs.RunProjectUpdate
         """
         if not task.startswith('awx.'):
             raise ValueError('{} is not a valid awx task'.format(task))
@@ -73,15 +73,15 @@ class TaskWorker(BaseWorker):
             'callbacks': [{
                 'args': [],
                 'kwargs': {}
-                'task': u'awx.main.tasks.handle_work_success'
+                'task': u'awx.main.tasks.system.handle_work_success'
             }],
             'errbacks': [{
                 'args': [],
                 'kwargs': {},
-                'task': 'awx.main.tasks.handle_work_error'
+                'task': 'awx.main.tasks.system.handle_work_error'
             }],
             'kwargs': {},
-            'task': u'awx.main.tasks.RunProjectUpdate'
+            'task': u'awx.main.tasks.jobs.RunProjectUpdate'
         }
         """
         settings.__clean_on_fork__()

awx/main/management/commands/create_preload_data.py

@@ -23,44 +23,54 @@ class Command(BaseCommand):
         with impersonate(superuser):
             with disable_computed_fields():
                 if not Organization.objects.exists():
-                    o = Organization.objects.create(name='Default')
+                    o, _ = Organization.objects.get_or_create(name='Default')
 
-                    p = Project(
-                        name='Demo Project',
-                        scm_type='git',
-                        scm_url='https://github.com/ansible/ansible-tower-samples',
-                        scm_update_on_launch=True,
-                        scm_update_cache_timeout=0,
-                        organization=o,
-                    )
+                    # Avoid calling directly the get_or_create() to bypass project update
+                    p = Project.objects.filter(name='Demo Project', scm_type='git').first()
+                    if not p:
+                        p = Project(
+                            name='Demo Project',
+                            scm_type='git',
+                            scm_url='https://github.com/ansible/ansible-tower-samples',
+                            scm_update_on_launch=True,
+                            scm_update_cache_timeout=0,
+                        )
+
+                    p.organization = o
                     p.save(skip_update=True)
 
                     ssh_type = CredentialType.objects.filter(namespace='ssh').first()
-                    c = Credential.objects.create(
+                    c, _ = Credential.objects.get_or_create(
                         credential_type=ssh_type, name='Demo Credential', inputs={'username': superuser.username}, created_by=superuser
                     )
 
                     c.admin_role.members.add(superuser)
 
-                    public_galaxy_credential = Credential(
+                    public_galaxy_credential, _ = Credential.objects.get_or_create(
                         name='Ansible Galaxy',
                         managed=True,
                         credential_type=CredentialType.objects.get(kind='galaxy'),
                         inputs={'url': 'https://galaxy.ansible.com/'},
                     )
-                    public_galaxy_credential.save()
                     o.galaxy_credentials.add(public_galaxy_credential)
 
-                    i = Inventory.objects.create(name='Demo Inventory', organization=o, created_by=superuser)
+                    i, _ = Inventory.objects.get_or_create(name='Demo Inventory', organization=o, created_by=superuser)
 
-                    Host.objects.create(
+                    Host.objects.get_or_create(
                         name='localhost',
                         inventory=i,
                         variables="ansible_connection: local\nansible_python_interpreter: '{{ ansible_playbook_python }}'",
                         created_by=superuser,
                     )
 
-                    jt = JobTemplate.objects.create(name='Demo Job Template', playbook='hello_world.yml', project=p, inventory=i)
+                    jt = JobTemplate.objects.filter(name='Demo Job Template').first()
+                    if jt:
+                        jt.project = p
+                        jt.inventory = i
+                        jt.playbook = 'hello_world.yml'
+                        jt.save()
+                    else:
+                        jt, _ = JobTemplate.objects.get_or_create(name='Demo Job Template', playbook='hello_world.yml', project=p, inventory=i)
                     jt.credentials.add(c)
 
                     print('Default organization added.')

awx/main/management/commands/inventory_import.py

@@ -78,6 +78,20 @@ class AnsibleInventoryLoader(object):
             bargs.extend(['-e', '{0}={1}'.format(key, value)])
         ee = get_default_execution_environment()
 
+        if settings.IS_K8S:
+            logger.warn('This command is not able to run on kubernetes-based deployment. This action should be done using the API.')
+            sys.exit(1)
+
+        if ee.credential:
+            process = subprocess.run(['podman', 'image', 'exists', ee.image], capture_output=True)
+            if process.returncode != 0:
+                logger.warn(
+                    f'The default execution environment (id={ee.id}, name={ee.name}, image={ee.image}) is not available on this node. '
+                    'The image needs to be available locally before using this command, due to registry authentication. '
+                    'To pull this image, either run a job on this node or manually pull the image.'
+                )
+                sys.exit(1)
+
         bargs.extend([ee.image])
 
         bargs.extend(['ansible-inventory', '-i', self.source])

awx/main/management/commands/list_instances.py

@@ -11,13 +11,16 @@ class Ungrouped(object):
     policy_instance_percentage = None
     policy_instance_minimum = None
 
+    def __init__(self):
+        self.qs = Instance.objects.filter(rampart_groups__isnull=True)
+
     @property
     def instances(self):
-        return Instance.objects.filter(rampart_groups__isnull=True)
+        return self.qs
 
     @property
     def capacity(self):
-        return sum(x.capacity for x in self.instances)
+        return sum(x.capacity for x in self.instances.all())
 
 
 class Command(BaseCommand):
@@ -29,26 +32,29 @@ class Command(BaseCommand):
 
         groups = list(InstanceGroup.objects.all())
         ungrouped = Ungrouped()
-        if len(ungrouped.instances):
+        if len(ungrouped.instances.all()):
             groups.append(ungrouped)
 
-        for instance_group in groups:
-            fmt = '[{0.name} capacity={0.capacity}'
-            if instance_group.policy_instance_percentage:
-                fmt += ' policy={0.policy_instance_percentage}%'
-            if instance_group.policy_instance_minimum:
-                fmt += ' policy>={0.policy_instance_minimum}'
-            print((fmt + ']').format(instance_group))
-            for x in instance_group.instances.all():
+        for ig in groups:
+            policy = ''
+            if ig.policy_instance_percentage:
+                policy = f' policy={ig.policy_instance_percentage}%'
+            if ig.policy_instance_minimum:
+                policy = f' policy>={ig.policy_instance_minimum}'
+            print(f'[{ig.name} capacity={ig.capacity}{policy}]')
+
+            for x in ig.instances.all():
                 color = '\033[92m'
-                if x.capacity == 0:
+                if x.capacity == 0 and x.node_type != 'hop':
                     color = '\033[91m'
-                if x.enabled is False:
+                if not x.enabled:
                     color = '\033[90m[DISABLED] '
                 if no_color:
                     color = ''
-                fmt = '\t' + color + '{0.hostname} capacity={0.capacity} node_type={0.node_type} version={1}'
-                if x.capacity:
-                    fmt += ' heartbeat="{0.modified:%Y-%m-%d %H:%M:%S}"'
-                print((fmt + '\033[0m').format(x, x.version or '?'))
-            print('')
+
+                capacity = f' capacity={x.capacity}' if x.node_type != 'hop' else ''
+                version = f" version={x.version or '?'}" if x.node_type != 'hop' else ''
+                heartbeat = f' heartbeat="{x.modified:%Y-%m-%d %H:%M:%S}"' if x.capacity or x.node_type == 'hop' else ''
+                print(f'\t{color}{x.hostname}{capacity} node_type={x.node_type}{version}{heartbeat}\033[0m')
+
+            print()

awx/main/management/commands/profile_sql.py

@@ -1,6 +1,6 @@
 from django.core.management.base import BaseCommand
 
-from awx.main.tasks import profile_sql
+from awx.main.tasks.system import profile_sql
 
 
 class Command(BaseCommand):

awx/main/management/commands/provision_instance.py

@@ -13,19 +13,19 @@ class Command(BaseCommand):
     Register this instance with the database for HA tracking.
     """
 
-    help = 'Add instance to the database. ' 'Specify `--hostname` to use this command.'
+    help = "Add instance to the database. Specify `--hostname` to use this command."
 
     def add_arguments(self, parser):
-        parser.add_argument('--hostname', dest='hostname', type=str, help='Hostname used during provisioning')
-        parser.add_argument('--node_type', type=str, default="hybrid", choices=["control", "execution", "hybrid"], help='Instance Node type')
-        parser.add_argument('--uuid', type=str, help='Instance UUID')
+        parser.add_argument('--hostname', dest='hostname', type=str, help="Hostname used during provisioning")
+        parser.add_argument('--node_type', type=str, default='hybrid', choices=['control', 'execution', 'hop', 'hybrid'], help="Instance Node type")
+        parser.add_argument('--uuid', type=str, help="Instance UUID")
 
     def _register_hostname(self, hostname, node_type, uuid):
         if not hostname:
             return
         (changed, instance) = Instance.objects.register(hostname=hostname, node_type=node_type, uuid=uuid)
         if changed:
-            print('Successfully registered instance {}'.format(hostname))
+            print("Successfully registered instance {}".format(hostname))
         else:
             print("Instance already registered {}".format(instance.hostname))
         self.changed = changed
@@ -37,4 +37,4 @@ class Command(BaseCommand):
         self.changed = False
         self._register_hostname(options.get('hostname'), options.get('node_type'), options.get('uuid'))
         if self.changed:
-            print('(changed: True)')
+            print("(changed: True)")

awx/main/management/commands/regenerate_secret_key.py

@@ -16,13 +16,26 @@ from awx.main.utils.encryption import encrypt_field, decrypt_field, encrypt_valu
 
 class Command(BaseCommand):
     """
-    Regenerate a new SECRET_KEY value and re-encrypt every secret in the database.
+    Re-encrypt every secret in the database, using regenerated new SECRET_KEY or user provided key.
     """
 
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--use-custom-key',
+            dest='use_custom_key',
+            action='store_true',
+            default=False,
+            help='Use existing key provided as TOWER_SECRET_KEY environment variable',
+        )
+
     @transaction.atomic
     def handle(self, **options):
         self.old_key = settings.SECRET_KEY
-        self.new_key = base64.encodebytes(os.urandom(33)).decode().rstrip()
+        custom_key = os.environ.get("TOWER_SECRET_KEY")
+        if options.get("use_custom_key") and custom_key:
+            self.new_key = custom_key
+        else:
+            self.new_key = base64.encodebytes(os.urandom(33)).decode().rstrip()
         self._notification_templates()
         self._credentials()
         self._unified_jobs()

awx/main/management/commands/register_peers.py

@@ -0,0 +1,87 @@
+import warnings
+
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+
+from awx.main.models import Instance, InstanceLink
+
+
+class Command(BaseCommand):
+    """
+    Internal tower command.
+    Register the peers of a receptor node.
+    """
+
+    help = "Register or remove links between Receptor nodes."
+
+    def add_arguments(self, parser):
+        parser.add_argument('source', type=str, help="Receptor node opening the connections.")
+        parser.add_argument('--peers', type=str, nargs='+', required=False, help="Nodes that the source node connects out to.")
+        parser.add_argument('--disconnect', type=str, nargs='+', required=False, help="Nodes that should no longer be connected to by the source node.")
+        parser.add_argument(
+            '--exact',
+            type=str,
+            nargs='*',
+            required=False,
+            help="The exact set of nodes the source node should connect out to. Any existing links registered in the database that do not match will be removed. May be empty.",
+        )
+
+    def handle(self, **options):
+        nodes = Instance.objects.in_bulk(field_name='hostname')
+        if options['source'] not in nodes:
+            raise CommandError(f"Host {options['source']} is not a registered instance.")
+        if not (options['peers'] or options['disconnect'] or options['exact'] is not None):
+            raise CommandError("One of the options --peers, --disconnect, or --exact is required.")
+        if options['exact'] is not None and options['peers']:
+            raise CommandError("The option --peers may not be used with --exact.")
+        if options['exact'] is not None and options['disconnect']:
+            raise CommandError("The option --disconnect may not be used with --exact.")
+
+        # No 1-cycles
+        for collection in ('peers', 'disconnect', 'exact'):
+            if options[collection] is not None and options['source'] in options[collection]:
+                raise CommandError(f"Source node {options['source']} may not also be in --{collection}.")
+
+        # No 2-cycles
+        if options['peers'] or options['exact'] is not None:
+            peers = set(options['peers'] or options['exact'])
+            incoming = set(InstanceLink.objects.filter(target=nodes[options['source']]).values_list('source__hostname', flat=True))
+            if peers & incoming:
+                warnings.warn(f"Source node {options['source']} should not link to nodes already peering to it: {peers & incoming}.")
+
+        if options['peers']:
+            missing_peers = set(options['peers']) - set(nodes)
+            if missing_peers:
+                missing = ' '.join(missing_peers)
+                raise CommandError(f"Peers not currently registered as instances: {missing}")
+
+            results = 0
+            for target in options['peers']:
+                _, created = InstanceLink.objects.get_or_create(source=nodes[options['source']], target=nodes[target])
+                if created:
+                    results += 1
+
+            print(f"{results} new peer links added to the database.")
+
+        if options['disconnect']:
+            results = 0
+            for target in options['disconnect']:
+                if target not in nodes:  # Be permissive, the node might have already been de-registered.
+                    continue
+                n, _ = InstanceLink.objects.filter(source=nodes[options['source']], target=nodes[target]).delete()
+                results += n
+
+            print(f"{results} peer links removed from the database.")
+
+        if options['exact'] is not None:
+            additions = 0
+            with transaction.atomic():
+                peers = set(options['exact'])
+                links = set(InstanceLink.objects.filter(source=nodes[options['source']]).values_list('target__hostname', flat=True))
+                removals, _ = InstanceLink.objects.filter(source=nodes[options['source']], target__hostname__in=links - peers).delete()
+                for target in peers - links:
+                    _, created = InstanceLink.objects.get_or_create(source=nodes[options['source']], target=nodes[target])
+                    if created:
+                        additions += 1
+
+            print(f"{additions} peer links added and {removals} deleted from the database.")

awx/main/management/commands/register_queue.py

@@ -17,13 +17,14 @@ class InstanceNotFound(Exception):
 
 
 class RegisterQueue:
-    def __init__(self, queuename, instance_percent, inst_min, hostname_list, is_container_group=None):
+    def __init__(self, queuename, instance_percent, inst_min, hostname_list, is_container_group=None, pod_spec_override=None):
         self.instance_not_found_err = None
         self.queuename = queuename
         self.instance_percent = instance_percent
         self.instance_min = inst_min
         self.hostname_list = hostname_list
         self.is_container_group = is_container_group
+        self.pod_spec_override = pod_spec_override
 
     def get_create_update_instance_group(self):
         created = False
@@ -40,6 +41,10 @@ class RegisterQueue:
             ig.is_container_group = self.is_container_group
             changed = True
 
+        if self.pod_spec_override and (ig.pod_spec_override != self.pod_spec_override):
+            ig.pod_spec_override = self.pod_spec_override
+            changed = True
+
         if changed:
             ig.save()
 
@@ -48,14 +53,14 @@ class RegisterQueue:
     def add_instances_to_group(self, ig):
         changed = False
 
-        instance_list_unique = set([x.strip() for x in self.hostname_list if x])
+        instance_list_unique = {x for x in (x.strip() for x in self.hostname_list) if x}
         instances = []
         for inst_name in instance_list_unique:
-            instance = Instance.objects.filter(hostname=inst_name)
+            instance = Instance.objects.filter(hostname=inst_name).exclude(node_type='hop')
             if instance.exists():
                 instances.append(instance[0])
             else:
-                raise InstanceNotFound("Instance does not exist: {}".format(inst_name), changed)
+                raise InstanceNotFound("Instance does not exist or cannot run jobs: {}".format(inst_name), changed)
 
         ig.instances.add(*instances)
 

awx/main/managers.py

@@ -179,15 +179,13 @@ class InstanceManager(models.Manager):
             else:
                 registered = self.register(ip_address=pod_ip, uuid=settings.SYSTEM_UUID)
             RegisterQueue(settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME, 100, 0, [], is_container_group=False).register()
-            RegisterQueue(settings.DEFAULT_EXECUTION_QUEUE_NAME, 100, 0, [], is_container_group=True).register()
+            RegisterQueue(
+                settings.DEFAULT_EXECUTION_QUEUE_NAME, 100, 0, [], is_container_group=True, pod_spec_override=settings.DEFAULT_EXECUTION_QUEUE_POD_SPEC_OVERRIDE
+            ).register()
             return registered
         else:
             return (False, self.me())
 
-    def active_count(self):
-        """Return count of active Tower nodes for licensing."""
-        return self.all().count()
-
 
 class InstanceGroupManager(models.Manager):
     """A custom manager class for the Instance model.
@@ -245,7 +243,13 @@ class InstanceGroupManager(models.Manager):
         for t in tasks:
             # TODO: dock capacity for isolated job management tasks running in queue
             impact = t.task_impact
-            if t.status == 'waiting' or not t.execution_node:
+            control_groups = []
+            if t.controller_node:
+                control_groups = instance_ig_mapping.get(t.controller_node, [])
+                if not control_groups:
+                    logger.warn(f"No instance group found for {t.controller_node}, capacity consumed may be innaccurate.")
+
+            if t.status == 'waiting' or (not t.execution_node and not t.is_container_group_task):
                 # Subtract capacity from any peer groups that share instances
                 if not t.instance_group:
                     impacted_groups = []
@@ -262,6 +266,12 @@ class InstanceGroupManager(models.Manager):
                     graph[group_name][f'consumed_{capacity_type}_capacity'] += impact
                     if breakdown:
                         graph[group_name]['committed_capacity'] += impact
+                for group_name in control_groups:
+                    if group_name not in graph:
+                        self.zero_out_group(graph, group_name, breakdown)
+                    graph[group_name][f'consumed_control_capacity'] += settings.AWX_CONTROL_NODE_TASK_IMPACT
+                    if breakdown:
+                        graph[group_name]['committed_capacity'] += settings.AWX_CONTROL_NODE_TASK_IMPACT
             elif t.status == 'running':
                 # Subtract capacity from all groups that contain the instance
                 if t.execution_node not in instance_ig_mapping:
@@ -273,6 +283,7 @@ class InstanceGroupManager(models.Manager):
                         impacted_groups = []
                 else:
                     impacted_groups = instance_ig_mapping[t.execution_node]
+
                 for group_name in impacted_groups:
                     if group_name not in graph:
                         self.zero_out_group(graph, group_name, breakdown)
@@ -281,6 +292,12 @@ class InstanceGroupManager(models.Manager):
                     graph[group_name][f'consumed_{capacity_type}_capacity'] += impact
                     if breakdown:
                         graph[group_name]['running_capacity'] += impact
+                for group_name in control_groups:
+                    if group_name not in graph:
+                        self.zero_out_group(graph, group_name, breakdown)
+                    graph[group_name][f'consumed_control_capacity'] += settings.AWX_CONTROL_NODE_TASK_IMPACT
+                    if breakdown:
+                        graph[group_name]['running_capacity'] += settings.AWX_CONTROL_NODE_TASK_IMPACT
             else:
                 logger.error('Programming error, %s not in ["running", "waiting"]', t.log_format)
         return graph

awx/main/middleware.py

@@ -180,11 +180,7 @@ class URLModificationMiddleware(MiddlewareMixin):
         return '/'.join(url_units)
 
     def process_request(self, request):
-        if hasattr(request, 'environ') and 'REQUEST_URI' in request.environ:
-            old_path = urllib.parse.urlsplit(request.environ['REQUEST_URI']).path
-            old_path = old_path[request.path.find(request.path_info) :]
-        else:
-            old_path = request.path_info
+        old_path = request.path_info
         new_path = self._convert_named_url(old_path)
         if request.path_info != new_path:
             request.environ['awx.named_url_rewritten'] = request.path

awx/main/migrations/0156_capture_mesh_topology.py

@@ -0,0 +1,44 @@
+# Generated by Django 2.2.20 on 2021-12-17 19:26
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0155_improved_health_check'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='instance',
+            name='node_type',
+            field=models.CharField(
+                choices=[
+                    ('control', 'Control plane node'),
+                    ('execution', 'Execution plane node'),
+                    ('hybrid', 'Controller and execution'),
+                    ('hop', 'Message-passing node, no execution capability'),
+                ],
+                default='hybrid',
+                max_length=16,
+            ),
+        ),
+        migrations.CreateModel(
+            name='InstanceLink',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='main.Instance')),
+                ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='reverse_peers', to='main.Instance')),
+            ],
+            options={
+                'unique_together': {('source', 'target')},
+            },
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='peers',
+            field=models.ManyToManyField(through='main.InstanceLink', to='main.Instance'),
+        ),
+    ]

awx/main/migrations/0157_inventory_labels.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.20 on 2022-01-18 16:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0156_capture_mesh_topology'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='inventory',
+            name='labels',
+            field=models.ManyToManyField(blank=True, help_text='Labels associated with this inventory.', related_name='inventory_labels', to='main.Label'),
+        ),
+    ]

awx/main/migrations/0158_make_instance_cpu_decimal.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.24 on 2022-02-14 17:37
+
+from decimal import Decimal
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0157_inventory_labels'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='instance',
+            name='cpu',
+            field=models.DecimalField(decimal_places=1, default=Decimal('0'), editable=False, max_digits=4),
+        ),
+    ]

awx/main/models/__init__.py

@@ -47,6 +47,7 @@ from awx.main.models.execution_environments import ExecutionEnvironment  # noqa
 from awx.main.models.activity_stream import ActivityStream  # noqa
 from awx.main.models.ha import (  # noqa
     Instance,
+    InstanceLink,
     InstanceGroup,
     TowerScheduleState,
 )

awx/main/models/ad_hoc_commands.py

@@ -144,7 +144,7 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
 
     @classmethod
     def _get_task_class(cls):
-        from awx.main.tasks import RunAdHocCommand
+        from awx.main.tasks.jobs import RunAdHocCommand
 
         return RunAdHocCommand
 
@@ -160,9 +160,7 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
 
     @property
     def notification_templates(self):
-        all_orgs = set()
-        for h in self.hosts.all():
-            all_orgs.add(h.inventory.organization)
+        all_orgs = {h.inventory.organization for h in self.hosts.all()}
         active_templates = dict(error=set(), success=set(), started=set())
         base_notification_templates = NotificationTemplate.objects
         for org in all_orgs:

awx/main/models/events.py

@@ -388,7 +388,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
                     job.get_event_queryset().filter(uuid__in=failed).update(failed=True)
 
                     # send success/failure notifications when we've finished handling the playbook_on_stats event
-                    from awx.main.tasks import handle_success_and_failure_notifications  # circular import
+                    from awx.main.tasks.system import handle_success_and_failure_notifications  # circular import
 
                     def _send_notifications():
                         handle_success_and_failure_notifications.apply_async([job.id])
@@ -541,8 +541,7 @@ class JobEvent(BasePlaybookEvent):
                 return
             job = self.job
 
-            from awx.main.models import Host, JobHostSummary  # circular import
-            from awx.main.models import Host, JobHostSummary, HostMetric
+            from awx.main.models import Host, JobHostSummary, HostMetric  # circular import
 
             all_hosts = Host.objects.filter(pk__in=self.host_map.values()).only('id', 'name')
             existing_host_ids = set(h.id for h in all_hosts)

awx/main/models/ha.py

@@ -29,7 +29,7 @@ from awx.main.models.mixins import RelatedJobsMixin
 # ansible-runner
 from ansible_runner.utils.capacity import get_cpu_count, get_mem_in_bytes
 
-__all__ = ('Instance', 'InstanceGroup', 'TowerScheduleState')
+__all__ = ('Instance', 'InstanceGroup', 'InstanceLink', 'TowerScheduleState')
 
 logger = logging.getLogger('awx.main.models.ha')
 
@@ -54,6 +54,14 @@ class HasPolicyEditsMixin(HasEditsMixin):
         return self._values_have_edits(new_values)
 
 
+class InstanceLink(BaseModel):
+    source = models.ForeignKey('Instance', on_delete=models.CASCADE, related_name='+')
+    target = models.ForeignKey('Instance', on_delete=models.CASCADE, related_name='reverse_peers')
+
+    class Meta:
+        unique_together = ('source', 'target')
+
+
 class Instance(HasPolicyEditsMixin, BaseModel):
     """A model representing an AWX instance running against this database."""
 
@@ -74,8 +82,10 @@ class Instance(HasPolicyEditsMixin, BaseModel):
     modified = models.DateTimeField(auto_now=True)
     # Fields defined in health check or heartbeat
     version = models.CharField(max_length=120, blank=True)
-    cpu = models.IntegerField(
-        default=0,
+    cpu = models.DecimalField(
+        default=Decimal(0.0),
+        max_digits=4,
+        decimal_places=1,
         editable=False,
     )
     memory = models.BigIntegerField(
@@ -116,9 +126,16 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         default=0,
         editable=False,
     )
-    NODE_TYPE_CHOICES = [("control", "Control plane node"), ("execution", "Execution plane node"), ("hybrid", "Controller and execution")]
+    NODE_TYPE_CHOICES = [
+        ("control", "Control plane node"),
+        ("execution", "Execution plane node"),
+        ("hybrid", "Controller and execution"),
+        ("hop", "Message-passing node, no execution capability"),
+    ]
     node_type = models.CharField(default='hybrid', choices=NODE_TYPE_CHOICES, max_length=16)
 
+    peers = models.ManyToManyField('self', symmetrical=False, through=InstanceLink, through_fields=('source', 'target'))
+
     class Meta:
         app_label = 'main'
         ordering = ("hostname",)
@@ -130,7 +147,14 @@ class Instance(HasPolicyEditsMixin, BaseModel):
 
     @property
     def consumed_capacity(self):
-        return sum(x.task_impact for x in UnifiedJob.objects.filter(execution_node=self.hostname, status__in=('running', 'waiting')))
+        capacity_consumed = 0
+        if self.node_type in ('hybrid', 'execution'):
+            capacity_consumed += sum(x.task_impact for x in UnifiedJob.objects.filter(execution_node=self.hostname, status__in=('running', 'waiting')))
+        if self.node_type in ('hybrid', 'control'):
+            capacity_consumed += sum(
+                settings.AWX_CONTROL_NODE_TASK_IMPACT for x in UnifiedJob.objects.filter(controller_node=self.hostname, status__in=('running', 'waiting'))
+            )
+        return capacity_consumed
 
     @property
     def remaining_capacity(self):
@@ -180,7 +204,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         if ref_time is None:
             ref_time = now()
         grace_period = settings.CLUSTER_NODE_HEARTBEAT_PERIOD * 2
-        if self.node_type == 'execution':
+        if self.node_type in ('execution', 'hop'):
             grace_period += settings.RECEPTOR_SERVICE_ADVERTISEMENT_PERIOD
         return self.last_seen < ref_time - timedelta(seconds=grace_period)
 
@@ -200,7 +224,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
 
     def set_capacity_value(self):
         """Sets capacity according to capacity adjustment rule (no save)"""
-        if self.enabled:
+        if self.enabled and self.node_type != 'hop':
             lower_cap = min(self.mem_capacity, self.cpu_capacity)
             higher_cap = max(self.mem_capacity, self.cpu_capacity)
             self.capacity = lower_cap + (higher_cap - lower_cap) * self.capacity_adjustment
@@ -248,7 +272,11 @@ class Instance(HasPolicyEditsMixin, BaseModel):
             self.mark_offline(perform_save=False, errors=errors)
         update_fields.extend(['cpu_capacity', 'mem_capacity', 'capacity', 'errors'])
 
-        self.save(update_fields=update_fields)
+        # disabling activity stream will avoid extra queries, which is important for heatbeat actions
+        from awx.main.signals import disable_activity_stream
+
+        with disable_activity_stream():
+            self.save(update_fields=update_fields)
 
     def local_health_check(self):
         """Only call this method on the instance that this record represents"""
@@ -305,7 +333,7 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
 
     @property
     def capacity(self):
-        return sum([inst.capacity for inst in self.instances.all()])
+        return sum(inst.capacity for inst in self.instances.all())
 
     @property
     def jobs_running(self):
@@ -326,15 +354,21 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
         app_label = 'main'
 
     @staticmethod
-    def fit_task_to_most_remaining_capacity_instance(task, instances):
+    def fit_task_to_most_remaining_capacity_instance(task, instances, impact=None, capacity_type=None, add_hybrid_control_cost=False):
+        impact = impact if impact else task.task_impact
+        capacity_type = capacity_type if capacity_type else task.capacity_type
         instance_most_capacity = None
+        most_remaining_capacity = -1
         for i in instances:
-            if i.node_type not in (task.capacity_type, 'hybrid'):
+            if i.node_type not in (capacity_type, 'hybrid'):
                 continue
-            if i.remaining_capacity >= task.task_impact and (
-                instance_most_capacity is None or i.remaining_capacity > instance_most_capacity.remaining_capacity
-            ):
+            would_be_remaining = i.remaining_capacity - impact
+            # hybrid nodes _always_ control their own tasks
+            if add_hybrid_control_cost and i.node_type == 'hybrid':
+                would_be_remaining -= settings.AWX_CONTROL_NODE_TASK_IMPACT
+            if would_be_remaining >= 0 and (instance_most_capacity is None or would_be_remaining > most_remaining_capacity):
                 instance_most_capacity = i
+                most_remaining_capacity = would_be_remaining
         return instance_most_capacity
 
     @staticmethod
@@ -361,7 +395,7 @@ class TowerScheduleState(SingletonModel):
 
 
 def schedule_policy_task():
-    from awx.main.tasks import apply_cluster_membership_policies
+    from awx.main.tasks.system import apply_cluster_membership_policies
 
     connection.on_commit(lambda: apply_cluster_membership_policies.apply_async())
 

awx/main/models/inventory.py

@@ -170,6 +170,12 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
         editable=False,
         help_text=_('Flag indicating the inventory is being deleted.'),
     )
+    labels = models.ManyToManyField(
+        "Label",
+        blank=True,
+        related_name='inventory_labels',
+        help_text=_('Labels associated with this inventory.'),
+    )
 
     def get_absolute_url(self, request=None):
         return reverse('api:inventory_detail', kwargs={'pk': self.pk}, request=request)
@@ -366,7 +372,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
 
     @transaction.atomic
     def schedule_deletion(self, user_id=None):
-        from awx.main.tasks import delete_inventory
+        from awx.main.tasks.system import delete_inventory
         from awx.main.signals import activity_stream_delete
 
         if self.pending_deletion is True:
@@ -382,7 +388,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
         if self.kind == 'smart' and settings.AWX_REBUILD_SMART_MEMBERSHIP:
 
             def on_commit():
-                from awx.main.tasks import update_host_smart_inventory_memberships
+                from awx.main.tasks.system import update_host_smart_inventory_memberships
 
                 update_host_smart_inventory_memberships.delay()
 
@@ -551,7 +557,7 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
         if settings.AWX_REBUILD_SMART_MEMBERSHIP:
 
             def on_commit():
-                from awx.main.tasks import update_host_smart_inventory_memberships
+                from awx.main.tasks.system import update_host_smart_inventory_memberships
 
                 update_host_smart_inventory_memberships.delay()
 
@@ -631,7 +637,7 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
     @transaction.atomic
     def delete_recursive(self):
         from awx.main.utils import ignore_inventory_computed_fields
-        from awx.main.tasks import update_inventory_computed_fields
+        from awx.main.tasks.system import update_inventory_computed_fields
         from awx.main.signals import disable_activity_stream, activity_stream_delete
 
         def mark_actual():
@@ -1219,7 +1225,7 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin,
 
     @classmethod
     def _get_task_class(cls):
-        from awx.main.tasks import RunInventoryUpdate
+        from awx.main.tasks.jobs import RunInventoryUpdate
 
         return RunInventoryUpdate
 

awx/main/models/jobs.py

@@ -583,7 +583,7 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
 
     @classmethod
     def _get_task_class(cls):
-        from awx.main.tasks import RunJob
+        from awx.main.tasks.jobs import RunJob
 
         return RunJob
 
@@ -1213,7 +1213,7 @@ class SystemJob(UnifiedJob, SystemJobOptions, JobNotificationMixin):
 
     @classmethod
     def _get_task_class(cls):
-        from awx.main.tasks import RunSystemJob
+        from awx.main.tasks.jobs import RunSystemJob
 
         return RunSystemJob
 

awx/main/models/label.py

@@ -9,6 +9,7 @@ from django.utils.translation import ugettext_lazy as _
 from awx.api.versioning import reverse
 from awx.main.models.base import CommonModelNameNotUnique
 from awx.main.models.unified_jobs import UnifiedJobTemplate, UnifiedJob
+from awx.main.models.inventory import Inventory
 
 __all__ = ('Label',)
 
@@ -35,15 +36,14 @@ class Label(CommonModelNameNotUnique):
 
     @staticmethod
     def get_orphaned_labels():
-        return Label.objects.filter(organization=None, unifiedjobtemplate_labels__isnull=True)
+        return Label.objects.filter(organization=None, unifiedjobtemplate_labels__isnull=True, inventory_labels__isnull=True)
 
     def is_detached(self):
-        return bool(Label.objects.filter(id=self.id, unifiedjob_labels__isnull=True, unifiedjobtemplate_labels__isnull=True).count())
+        return Label.objects.filter(id=self.id, unifiedjob_labels__isnull=True, unifiedjobtemplate_labels__isnull=True, inventory_labels__isnull=True).exists()
 
     def is_candidate_for_detach(self):
+
         c1 = UnifiedJob.objects.filter(labels__in=[self.id]).count()
         c2 = UnifiedJobTemplate.objects.filter(labels__in=[self.id]).count()
-        if (c1 + c2 - 1) == 0:
-            return True
-        else:
-            return False
+        c3 = Inventory.objects.filter(labels__in=[self.id]).count()
+        return (c1 + c2 + c3 - 1) == 0

awx/main/models/notifications.py

@@ -508,7 +508,7 @@ class JobNotificationMixin(object):
         return (msg, body)
 
     def send_notification_templates(self, status):
-        from awx.main.tasks import send_notifications  # avoid circular import
+        from awx.main.tasks.system import send_notifications  # avoid circular import
 
         if status not in ['running', 'succeeded', 'failed']:
             raise ValueError(_("status must be either running, succeeded or failed"))

awx/main/models/projects.py

@@ -471,7 +471,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
         r = super(Project, self).delete(*args, **kwargs)
         for path_to_delete in paths_to_delete:
             if self.scm_type and path_to_delete:  # non-manual, concrete path
-                from awx.main.tasks import delete_project_files
+                from awx.main.tasks.system import delete_project_files
 
                 delete_project_files.delay(path_to_delete)
         return r
@@ -532,7 +532,7 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
 
     @classmethod
     def _get_task_class(cls):
-        from awx.main.tasks import RunProjectUpdate
+        from awx.main.tasks.jobs import RunProjectUpdate
 
         return RunProjectUpdate
 
@@ -613,26 +613,6 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
     def get_notification_friendly_name(self):
         return "Project Update"
 
-    @property
-    def preferred_instance_groups(self):
-        '''
-        Project updates should pretty much always run on the control plane
-        however, we are not yet saying no to custom groupings within the control plane
-        Thus, we return custom groups and then unconditionally add the control plane
-        '''
-        if self.organization is not None:
-            organization_groups = [x for x in self.organization.instance_groups.all()]
-        else:
-            organization_groups = []
-        template_groups = [x for x in super(ProjectUpdate, self).preferred_instance_groups]
-        selected_groups = template_groups + organization_groups
-
-        controlplane_ig = self.control_plane_instance_group
-        if controlplane_ig and controlplane_ig[0] and controlplane_ig[0] not in selected_groups:
-            selected_groups += controlplane_ig
-
-        return selected_groups
-
     def save(self, *args, **kwargs):
         added_update_fields = []
         if not self.job_tags:

awx/main/models/unified_jobs.py

@@ -1046,7 +1046,7 @@ class UnifiedJob(
             fd = tempfile.NamedTemporaryFile(
                 mode='w', prefix='{}-{}-'.format(self.model_to_str(), self.pk), suffix='.out', dir=settings.JOBOUTPUT_ROOT, encoding='utf-8'
             )
-            from awx.main.tasks import purge_old_stdout_files  # circular import
+            from awx.main.tasks.system import purge_old_stdout_files  # circular import
 
             purge_old_stdout_files.apply_async()
 

awx/main/models/workflow.py

@@ -813,7 +813,7 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin):
         return True
 
     def send_approval_notification(self, approval_status):
-        from awx.main.tasks import send_notifications  # avoid circular import
+        from awx.main.tasks.system import send_notifications  # avoid circular import
 
         if self.workflow_job_template is None:
             return

awx/main/scheduler/kubernetes.py

@@ -9,29 +9,12 @@ from kubernetes import client, config
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 
-from awx.main.utils.common import parse_yaml_or_json
+from awx.main.utils.common import parse_yaml_or_json, deepmerge
 from awx.main.utils.execution_environments import get_default_pod_spec
 
 logger = logging.getLogger('awx.main.scheduler')
 
 
-def deepmerge(a, b):
-    """
-    Merge dict structures and return the result.
-
-    >>> a = {'first': {'all_rows': {'pass': 'dog', 'number': '1'}}}
-    >>> b = {'first': {'all_rows': {'fail': 'cat', 'number': '5'}}}
-    >>> import pprint; pprint.pprint(deepmerge(a, b))
-    {'first': {'all_rows': {'fail': 'cat', 'number': '5', 'pass': 'dog'}}}
-    """
-    if isinstance(a, dict) and isinstance(b, dict):
-        return dict([(k, deepmerge(a.get(k), b.get(k))) for k in set(a.keys()).union(b.keys())])
-    elif b is None:
-        return a
-    else:
-        return b
-
-
 class PodManager(object):
     def __init__(self, task=None):
         self.task = task
@@ -183,7 +166,7 @@ class PodManager(object):
         pod_spec_override = {}
         if self.task and self.task.instance_group.pod_spec_override:
             pod_spec_override = parse_yaml_or_json(self.task.instance_group.pod_spec_override)
-        pod_spec = {**default_pod_spec, **pod_spec_override}
+        pod_spec = deepmerge(default_pod_spec, pod_spec_override)
 
         if self.task:
             pod_spec['metadata'] = deepmerge(

awx/main/scheduler/task_manager.py

@@ -13,7 +13,6 @@ from django.db import transaction, connection
 from django.utils.translation import ugettext_lazy as _, gettext_noop
 from django.utils.timezone import now as tz_now
 from django.conf import settings
-from django.db.models import Q
 
 # AWX
 from awx.main.dispatch.reaper import reap_job
@@ -69,8 +68,10 @@ class TaskManager:
         """
         Init AFTER we know this instance of the task manager will run because the lock is acquired.
         """
-        instances = Instance.objects.filter(~Q(hostname=None), enabled=True)
+        instances = Instance.objects.filter(hostname__isnull=False, enabled=True).exclude(node_type='hop')
         self.real_instances = {i.hostname: i for i in instances}
+        self.controlplane_ig = None
+        self.dependency_graph = DependencyGraph()
 
         instances_partial = [
             SimpleNamespace(
@@ -87,33 +88,21 @@ class TaskManager:
         instances_by_hostname = {i.hostname: i for i in instances_partial}
 
         for rampart_group in InstanceGroup.objects.prefetch_related('instances'):
+            if rampart_group.name == settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME:
+                self.controlplane_ig = rampart_group
             self.graph[rampart_group.name] = dict(
-                graph=DependencyGraph(),
-                execution_capacity=0,
-                control_capacity=0,
-                consumed_capacity=0,
-                consumed_control_capacity=0,
-                consumed_execution_capacity=0,
-                instances=[],
+                instances=[
+                    instances_by_hostname[instance.hostname] for instance in rampart_group.instances.all() if instance.hostname in instances_by_hostname
+                ],
             )
-            for instance in rampart_group.instances.all():
-                if not instance.enabled:
-                    continue
-                for capacity_type in ('control', 'execution'):
-                    if instance.node_type in (capacity_type, 'hybrid'):
-                        self.graph[rampart_group.name][f'{capacity_type}_capacity'] += instance.capacity
-            for instance in rampart_group.instances.filter(enabled=True).order_by('hostname'):
-                if instance.hostname in instances_by_hostname:
-                    self.graph[rampart_group.name]['instances'].append(instances_by_hostname[instance.hostname])
 
     def job_blocked_by(self, task):
         # TODO: I'm not happy with this, I think blocking behavior should be decided outside of the dependency graph
         #       in the old task manager this was handled as a method on each task object outside of the graph and
         #       probably has the side effect of cutting down *a lot* of the logic from this task manager class
-        for g in self.graph:
-            blocked_by = self.graph[g]['graph'].task_blocked_by(task)
-            if blocked_by:
-                return blocked_by
+        blocked_by = self.dependency_graph.task_blocked_by(task)
+        if blocked_by:
+            return blocked_by
 
         if not task.dependent_jobs_finished():
             blocked_by = task.dependent_jobs.first()
@@ -239,7 +228,7 @@ class TaskManager:
                 update_fields = ['status', 'start_args']
                 workflow_job.status = new_status
                 if reason:
-                    logger.info(reason)
+                    logger.info(f'Workflow job {workflow_job.id} failed due to reason: {reason}')
                     workflow_job.job_explanation = gettext_noop("No error handling paths found, marking workflow as failed")
                     update_fields.append('job_explanation')
                 workflow_job.start_args = ''  # blank field to remove encrypted passwords
@@ -258,7 +247,7 @@ class TaskManager:
         if self.start_task_limit == 0:
             # schedule another run immediately after this task manager
             schedule_task_manager()
-        from awx.main.tasks import handle_work_error, handle_work_success
+        from awx.main.tasks.system import handle_work_error, handle_work_success
 
         dependent_tasks = dependent_tasks or []
 
@@ -284,47 +273,18 @@ class TaskManager:
                 task.send_notification_templates('running')
                 logger.debug('Transitioning %s to running status.', task.log_format)
                 schedule_task_manager()
-            elif rampart_group.is_container_group:
-                task.instance_group = rampart_group
-                if task.capacity_type == 'execution':
-                    # find one real, non-containerized instance with capacity to
-                    # act as the controller for k8s API interaction
-                    try:
-                        task.controller_node = Instance.choose_online_control_plane_node()
-                        task.log_lifecycle("controller_node_chosen")
-                    except IndexError:
-                        logger.warning("No control plane nodes available to run containerized job {}".format(task.log_format))
-                        return
-                else:
-                    # project updates and system jobs don't *actually* run in pods, so
-                    # just pick *any* non-containerized host and use it as the execution node
-                    task.execution_node = Instance.choose_online_control_plane_node()
-                    task.log_lifecycle("execution_node_chosen")
-                    logger.debug('Submitting containerized {} to queue {}.'.format(task.log_format, task.execution_node))
+            # at this point we already have control/execution nodes selected for the following cases
             else:
                 task.instance_group = rampart_group
-                task.execution_node = instance.hostname
-                task.log_lifecycle("execution_node_chosen")
-                if instance.node_type == 'execution':
-                    try:
-                        task.controller_node = Instance.choose_online_control_plane_node()
-                        task.log_lifecycle("controller_node_chosen")
-                    except IndexError:
-                        logger.warning("No control plane nodes available to manage {}".format(task.log_format))
-                        return
-                else:
-                    # control plane nodes will manage jobs locally for performance and resilience
-                    task.controller_node = task.execution_node
-                    task.log_lifecycle("controller_node_chosen")
-                logger.debug('Submitting job {} to queue {} controlled by {}.'.format(task.log_format, task.execution_node, task.controller_node))
+                execution_node_msg = f' and execution node {task.execution_node}' if task.execution_node else ''
+                logger.debug(
+                    f'Submitting job {task.log_format} controlled by {task.controller_node} to instance group {rampart_group.name}{execution_node_msg}.'
+                )
             with disable_activity_stream():
                 task.celery_task_id = str(uuid.uuid4())
                 task.save()
                 task.log_lifecycle("waiting")
 
-            if rampart_group is not None:
-                self.consume_capacity(task, rampart_group.name, instance=instance)
-
         def post_commit():
             if task.status != 'failed' and type(task) is not WorkflowJob:
                 # Before task is dispatched, ensure that job_event partitions exist
@@ -344,8 +304,7 @@ class TaskManager:
 
     def process_running_tasks(self, running_tasks):
         for task in running_tasks:
-            if task.instance_group:
-                self.graph[task.instance_group.name]['graph'].add_job(task)
+            self.dependency_graph.add_job(task)
 
     def create_project_update(self, task):
         project_task = Project.objects.get(id=task.project_id).create_project_update(_eager_fields=dict(launch_type='dependency'))
@@ -484,7 +443,7 @@ class TaskManager:
         return created_dependencies
 
     def process_pending_tasks(self, pending_tasks):
-        running_workflow_templates = set([wf.unified_job_template_id for wf in self.get_running_workflow_jobs()])
+        running_workflow_templates = {wf.unified_job_template_id for wf in self.get_running_workflow_jobs()}
         tasks_to_update_job_explanation = []
         for task in pending_tasks:
             if self.start_task_limit <= 0:
@@ -498,9 +457,10 @@ class TaskManager:
                         task.job_explanation = job_explanation
                         tasks_to_update_job_explanation.append(task)
                 continue
-            preferred_instance_groups = task.preferred_instance_groups
 
             found_acceptable_queue = False
+            preferred_instance_groups = task.preferred_instance_groups
+
             if isinstance(task, WorkflowJob):
                 if task.unified_job_template_id in running_workflow_templates:
                     if not task.allow_simultaneous:
@@ -511,9 +471,38 @@ class TaskManager:
                 self.start_task(task, None, task.get_jobs_fail_chain(), None)
                 continue
 
+            # Determine if there is control capacity for the task
+            if task.capacity_type == 'control':
+                control_impact = task.task_impact + settings.AWX_CONTROL_NODE_TASK_IMPACT
+            else:
+                control_impact = settings.AWX_CONTROL_NODE_TASK_IMPACT
+            control_instance = InstanceGroup.fit_task_to_most_remaining_capacity_instance(
+                task, self.graph[settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME]['instances'], impact=control_impact, capacity_type='control'
+            )
+            if not control_instance:
+                self.task_needs_capacity(task, tasks_to_update_job_explanation)
+                logger.debug(f"Skipping task {task.log_format} in pending, not enough capacity left on controlplane to control new tasks")
+                continue
+
+            task.controller_node = control_instance.hostname
+
+            # All task.capacity_type == 'control' jobs should run on control plane, no need to loop over instance groups
+            if task.capacity_type == 'control':
+                task.execution_node = control_instance.hostname
+                control_instance.remaining_capacity = max(0, control_instance.remaining_capacity - control_impact)
+                control_instance.jobs_running += 1
+                self.dependency_graph.add_job(task)
+                execution_instance = self.real_instances[control_instance.hostname]
+                task.log_lifecycle("controller_node_chosen")
+                task.log_lifecycle("execution_node_chosen")
+                self.start_task(task, self.controlplane_ig, task.get_jobs_fail_chain(), execution_instance)
+                found_acceptable_queue = True
+                continue
+
             for rampart_group in preferred_instance_groups:
-                if task.capacity_type == 'execution' and rampart_group.is_container_group:
-                    self.graph[rampart_group.name]['graph'].add_job(task)
+                if rampart_group.is_container_group:
+                    control_instance.jobs_running += 1
+                    self.dependency_graph.add_job(task)
                     self.start_task(task, rampart_group, task.get_jobs_fail_chain(), None)
                     found_acceptable_queue = True
                     break
@@ -522,29 +511,33 @@ class TaskManager:
                 if settings.IS_K8S and task.capacity_type == 'execution':
                     logger.debug("Skipping group {}, task cannot run on control plane".format(rampart_group.name))
                     continue
-
-                remaining_capacity = self.get_remaining_capacity(rampart_group.name, capacity_type=task.capacity_type)
-                if task.task_impact > 0 and remaining_capacity <= 0:
-                    logger.debug("Skipping group {}, remaining_capacity {} <= 0".format(rampart_group.name, remaining_capacity))
-                    continue
-
+                # at this point we know the instance group is NOT a container group
+                # because if it was, it would have started the task and broke out of the loop.
                 execution_instance = InstanceGroup.fit_task_to_most_remaining_capacity_instance(
-                    task, self.graph[rampart_group.name]['instances']
+                    task, self.graph[rampart_group.name]['instances'], add_hybrid_control_cost=True
                 ) or InstanceGroup.find_largest_idle_instance(self.graph[rampart_group.name]['instances'], capacity_type=task.capacity_type)
 
-                if execution_instance or rampart_group.is_container_group:
-                    if not rampart_group.is_container_group:
-                        execution_instance.remaining_capacity = max(0, execution_instance.remaining_capacity - task.task_impact)
-                        execution_instance.jobs_running += 1
-                        logger.debug(
-                            "Starting {} in group {} instance {} (remaining_capacity={})".format(
-                                task.log_format, rampart_group.name, execution_instance.hostname, remaining_capacity
-                            )
-                        )
+                if execution_instance:
+                    task.execution_node = execution_instance.hostname
+                    # If our execution instance is a hybrid, prefer to do control tasks there as well.
+                    if execution_instance.node_type == 'hybrid':
+                        control_instance = execution_instance
+                        task.controller_node = execution_instance.hostname
 
-                    if execution_instance:
-                        execution_instance = self.real_instances[execution_instance.hostname]
-                    self.graph[rampart_group.name]['graph'].add_job(task)
+                    control_instance.remaining_capacity = max(0, control_instance.remaining_capacity - settings.AWX_CONTROL_NODE_TASK_IMPACT)
+                    task.log_lifecycle("controller_node_chosen")
+                    if control_instance != execution_instance:
+                        control_instance.jobs_running += 1
+                    execution_instance.remaining_capacity = max(0, execution_instance.remaining_capacity - task.task_impact)
+                    execution_instance.jobs_running += 1
+                    task.log_lifecycle("execution_node_chosen")
+                    logger.debug(
+                        "Starting {} in group {} instance {} (remaining_capacity={})".format(
+                            task.log_format, rampart_group.name, execution_instance.hostname, execution_instance.remaining_capacity
+                        )
+                    )
+                    execution_instance = self.real_instances[execution_instance.hostname]
+                    self.dependency_graph.add_job(task)
                     self.start_task(task, rampart_group, task.get_jobs_fail_chain(), execution_instance)
                     found_acceptable_queue = True
                     break
@@ -555,18 +548,21 @@ class TaskManager:
                         )
                     )
             if not found_acceptable_queue:
-                task.log_lifecycle("needs_capacity")
-                job_explanation = gettext_noop("This job is not ready to start because there is not enough available capacity.")
-                if task.job_explanation != job_explanation:
-                    if task.created < (tz_now() - self.time_delta_job_explanation):
-                        # Many launched jobs are immediately blocked, but most blocks will resolve in a few seconds.
-                        # Therefore we should only update the job_explanation after some time has elapsed to
-                        # prevent excessive task saves.
-                        task.job_explanation = job_explanation
-                        tasks_to_update_job_explanation.append(task)
-                logger.debug("{} couldn't be scheduled on graph, waiting for next cycle".format(task.log_format))
+                self.task_needs_capacity(task, tasks_to_update_job_explanation)
         UnifiedJob.objects.bulk_update(tasks_to_update_job_explanation, ['job_explanation'])
 
+    def task_needs_capacity(self, task, tasks_to_update_job_explanation):
+        task.log_lifecycle("needs_capacity")
+        job_explanation = gettext_noop("This job is not ready to start because there is not enough available capacity.")
+        if task.job_explanation != job_explanation:
+            if task.created < (tz_now() - self.time_delta_job_explanation):
+                # Many launched jobs are immediately blocked, but most blocks will resolve in a few seconds.
+                # Therefore we should only update the job_explanation after some time has elapsed to
+                # prevent excessive task saves.
+                task.job_explanation = job_explanation
+                tasks_to_update_job_explanation.append(task)
+        logger.debug("{} couldn't be scheduled on graph, waiting for next cycle".format(task.log_format))
+
     def timeout_approval_node(self):
         workflow_approvals = WorkflowApproval.objects.filter(status='pending')
         now = tz_now()
@@ -593,33 +589,14 @@ class TaskManager:
         # elsewhere
         for j in UnifiedJob.objects.filter(
             status__in=['pending', 'waiting', 'running'],
-        ).exclude(execution_node__in=Instance.objects.values_list('hostname', flat=True)):
+        ).exclude(execution_node__in=Instance.objects.exclude(node_type='hop').values_list('hostname', flat=True)):
             if j.execution_node and not j.is_container_group_task:
                 logger.error(f'{j.execution_node} is not a registered instance; reaping {j.log_format}')
                 reap_job(j, 'failed')
 
-    def calculate_capacity_consumed(self, tasks):
-        self.graph = InstanceGroup.objects.capacity_values(tasks=tasks, graph=self.graph)
-
-    def consume_capacity(self, task, instance_group, instance=None):
-        logger.debug(
-            '{} consumed {} capacity units from {} with prior total of {}'.format(
-                task.log_format, task.task_impact, instance_group, self.graph[instance_group]['consumed_capacity']
-            )
-        )
-        self.graph[instance_group]['consumed_capacity'] += task.task_impact
-        for capacity_type in ('control', 'execution'):
-            if instance is None or instance.node_type in ('hybrid', capacity_type):
-                self.graph[instance_group][f'consumed_{capacity_type}_capacity'] += task.task_impact
-
-    def get_remaining_capacity(self, instance_group, capacity_type='execution'):
-        return self.graph[instance_group][f'{capacity_type}_capacity'] - self.graph[instance_group][f'consumed_{capacity_type}_capacity']
-
     def process_tasks(self, all_sorted_tasks):
         running_tasks = [t for t in all_sorted_tasks if t.status in ['waiting', 'running']]
 
-        self.calculate_capacity_consumed(running_tasks)
-
         self.process_running_tasks(running_tasks)
 
         pending_tasks = [t for t in all_sorted_tasks if t.status == 'pending']

awx/main/signals.py

@@ -57,7 +57,7 @@ from awx.main.models import (
 from awx.main.constants import CENSOR_VALUE
 from awx.main.utils import model_instance_diff, model_to_dict, camelcase_to_underscore, get_current_apps
 from awx.main.utils import ignore_inventory_computed_fields, ignore_inventory_group_removal, _inventory_updates
-from awx.main.tasks import update_inventory_computed_fields, handle_removed_image
+from awx.main.tasks.system import update_inventory_computed_fields, handle_removed_image
 from awx.main.fields import (
     is_implicit_parent,
     update_role_parentage_for_instance,

awx/main/tasks/callback.py

@@ -0,0 +1,257 @@
+import json
+import time
+import logging
+from collections import deque
+import os
+import stat
+
+# Django
+from django.utils.timezone import now
+from django.conf import settings
+from django_guid.middleware import GuidMiddleware
+
+# AWX
+from awx.main.redact import UriCleaner
+from awx.main.constants import MINIMAL_EVENTS
+from awx.main.utils.update_model import update_model
+from awx.main.queue import CallbackQueueDispatcher
+
+logger = logging.getLogger('awx.main.tasks.callback')
+
+
+class RunnerCallback:
+    event_data_key = 'job_id'
+
+    def __init__(self, model=None):
+        self.parent_workflow_job_id = None
+        self.host_map = {}
+        self.guid = GuidMiddleware.get_guid()
+        self.job_created = None
+        self.recent_event_timings = deque(maxlen=settings.MAX_WEBSOCKET_EVENT_RATE)
+        self.dispatcher = CallbackQueueDispatcher()
+        self.safe_env = {}
+        self.event_ct = 0
+        self.model = model
+
+    def update_model(self, pk, _attempt=0, **updates):
+        return update_model(self.model, pk, _attempt=0, **updates)
+
+    def event_handler(self, event_data):
+        #
+        # ⚠️  D-D-D-DANGER ZONE ⚠️
+        # This method is called once for *every event* emitted by Ansible
+        # Runner as a playbook runs.  That means that changes to the code in
+        # this method are _very_ likely to introduce performance regressions.
+        #
+        # Even if this function is made on average .05s slower, it can have
+        # devastating performance implications for playbooks that emit
+        # tens or hundreds of thousands of events.
+        #
+        # Proceed with caution!
+        #
+        """
+        Ansible runner puts a parent_uuid on each event, no matter what the type.
+        AWX only saves the parent_uuid if the event is for a Job.
+        """
+        # cache end_line locally for RunInventoryUpdate tasks
+        # which generate job events from two 'streams':
+        # ansible-inventory and the awx.main.commands.inventory_import
+        # logger
+
+        if event_data.get(self.event_data_key, None):
+            if self.event_data_key != 'job_id':
+                event_data.pop('parent_uuid', None)
+        if self.parent_workflow_job_id:
+            event_data['workflow_job_id'] = self.parent_workflow_job_id
+        event_data['job_created'] = self.job_created
+        if self.host_map:
+            host = event_data.get('event_data', {}).get('host', '').strip()
+            if host:
+                event_data['host_name'] = host
+                if host in self.host_map:
+                    event_data['host_id'] = self.host_map[host]
+            else:
+                event_data['host_name'] = ''
+                event_data['host_id'] = ''
+            if event_data.get('event') == 'playbook_on_stats':
+                event_data['host_map'] = self.host_map
+
+        if isinstance(self, RunnerCallbackForProjectUpdate):
+            # need a better way to have this check.
+            # it's common for Ansible's SCM modules to print
+            # error messages on failure that contain the plaintext
+            # basic auth credentials (username + password)
+            # it's also common for the nested event data itself (['res']['...'])
+            # to contain unredacted text on failure
+            # this is a _little_ expensive to filter
+            # with regex, but project updates don't have many events,
+            # so it *should* have a negligible performance impact
+            task = event_data.get('event_data', {}).get('task_action')
+            try:
+                if task in ('git', 'svn'):
+                    event_data_json = json.dumps(event_data)
+                    event_data_json = UriCleaner.remove_sensitive(event_data_json)
+                    event_data = json.loads(event_data_json)
+            except json.JSONDecodeError:
+                pass
+
+        if 'event_data' in event_data:
+            event_data['event_data']['guid'] = self.guid
+
+        # To prevent overwhelming the broadcast queue, skip some websocket messages
+        if self.recent_event_timings:
+            cpu_time = time.time()
+            first_window_time = self.recent_event_timings[0]
+            last_window_time = self.recent_event_timings[-1]
+
+            if event_data.get('event') in MINIMAL_EVENTS:
+                should_emit = True  # always send some types like playbook_on_stats
+            elif event_data.get('stdout') == '' and event_data['start_line'] == event_data['end_line']:
+                should_emit = False  # exclude events with no output
+            else:
+                should_emit = any(
+                    [
+                        # if 30the most recent websocket message was sent over 1 second ago
+                        cpu_time - first_window_time > 1.0,
+                        # if the very last websocket message came in over 1/30 seconds ago
+                        self.recent_event_timings.maxlen * (cpu_time - last_window_time) > 1.0,
+                        # if the queue is not yet full
+                        len(self.recent_event_timings) != self.recent_event_timings.maxlen,
+                    ]
+                )
+
+            if should_emit:
+                self.recent_event_timings.append(cpu_time)
+            else:
+                event_data.setdefault('event_data', {})
+                event_data['skip_websocket_message'] = True
+
+        elif self.recent_event_timings.maxlen:
+            self.recent_event_timings.append(time.time())
+
+        event_data.setdefault(self.event_data_key, self.instance.id)
+        self.dispatcher.dispatch(event_data)
+        self.event_ct += 1
+
+        '''
+        Handle artifacts
+        '''
+        if event_data.get('event_data', {}).get('artifact_data', {}):
+            self.instance.artifacts = event_data['event_data']['artifact_data']
+            self.instance.save(update_fields=['artifacts'])
+
+        return False
+
+    def cancel_callback(self):
+        """
+        Ansible runner callback to tell the job when/if it is canceled
+        """
+        unified_job_id = self.instance.pk
+        self.instance.refresh_from_db()
+        if not self.instance:
+            logger.error('unified job {} was deleted while running, canceling'.format(unified_job_id))
+            return True
+        if self.instance.cancel_flag or self.instance.status == 'canceled':
+            cancel_wait = (now() - self.instance.modified).seconds if self.instance.modified else 0
+            if cancel_wait > 5:
+                logger.warn('Request to cancel {} took {} seconds to complete.'.format(self.instance.log_format, cancel_wait))
+            return True
+        return False
+
+    def finished_callback(self, runner_obj):
+        """
+        Ansible runner callback triggered on finished run
+        """
+        event_data = {
+            'event': 'EOF',
+            'final_counter': self.event_ct,
+            'guid': self.guid,
+        }
+        event_data.setdefault(self.event_data_key, self.instance.id)
+        self.dispatcher.dispatch(event_data)
+
+    def status_handler(self, status_data, runner_config):
+        """
+        Ansible runner callback triggered on status transition
+        """
+        if status_data['status'] == 'starting':
+            job_env = dict(runner_config.env)
+            '''
+            Take the safe environment variables and overwrite
+            '''
+            for k, v in self.safe_env.items():
+                if k in job_env:
+                    job_env[k] = v
+            from awx.main.signals import disable_activity_stream  # Circular import
+
+            with disable_activity_stream():
+                self.instance = self.update_model(self.instance.pk, job_args=json.dumps(runner_config.command), job_cwd=runner_config.cwd, job_env=job_env)
+        elif status_data['status'] == 'failed':
+            # For encrypted ssh_key_data, ansible-runner worker will open and write the
+            # ssh_key_data to a named pipe. Then, once the podman container starts, ssh-agent will
+            # read from this named pipe so that the key can be used in ansible-playbook.
+            # Once the podman container exits, the named pipe is deleted.
+            # However, if the podman container fails to start in the first place, e.g. the image
+            # name is incorrect, then this pipe is not cleaned up. Eventually ansible-runner
+            # processor will attempt to write artifacts to the private data dir via unstream_dir, requiring
+            # that it open this named pipe. This leads to a hang. Thus, before any artifacts
+            # are written by the processor, it's important to remove this ssh_key_data pipe.
+            private_data_dir = self.instance.job_env.get('AWX_PRIVATE_DATA_DIR', None)
+            if private_data_dir:
+                key_data_file = os.path.join(private_data_dir, 'artifacts', str(self.instance.id), 'ssh_key_data')
+                if os.path.exists(key_data_file) and stat.S_ISFIFO(os.stat(key_data_file).st_mode):
+                    os.remove(key_data_file)
+        elif status_data['status'] == 'error':
+            result_traceback = status_data.get('result_traceback', None)
+            if result_traceback:
+                from awx.main.signals import disable_activity_stream  # Circular import
+
+                with disable_activity_stream():
+                    self.instance = self.update_model(self.instance.pk, result_traceback=result_traceback)
+
+
+class RunnerCallbackForProjectUpdate(RunnerCallback):
+
+    event_data_key = 'project_update_id'
+
+    def __init__(self, *args, **kwargs):
+        super(RunnerCallbackForProjectUpdate, self).__init__(*args, **kwargs)
+        self.playbook_new_revision = None
+        self.host_map = {}
+
+    def event_handler(self, event_data):
+        super_return_value = super(RunnerCallbackForProjectUpdate, self).event_handler(event_data)
+        returned_data = event_data.get('event_data', {})
+        if returned_data.get('task_action', '') == 'set_fact':
+            returned_facts = returned_data.get('res', {}).get('ansible_facts', {})
+            if 'scm_version' in returned_facts:
+                self.playbook_new_revision = returned_facts['scm_version']
+        return super_return_value
+
+
+class RunnerCallbackForInventoryUpdate(RunnerCallback):
+
+    event_data_key = 'inventory_update_id'
+
+    def __init__(self, *args, **kwargs):
+        super(RunnerCallbackForInventoryUpdate, self).__init__(*args, **kwargs)
+        self.end_line = 0
+
+    def event_handler(self, event_data):
+        self.end_line = event_data['end_line']
+
+        return super(RunnerCallbackForInventoryUpdate, self).event_handler(event_data)
+
+
+class RunnerCallbackForAdHocCommand(RunnerCallback):
+
+    event_data_key = 'ad_hoc_command_id'
+
+    def __init__(self, *args, **kwargs):
+        super(RunnerCallbackForAdHocCommand, self).__init__(*args, **kwargs)
+        self.host_map = {}
+
+
+class RunnerCallbackForSystemJob(RunnerCallback):
+
+    event_data_key = 'system_job_id'

awx/main/tasks/receptor.py

@@ -0,0 +1,575 @@
+# Python
+from base64 import b64encode
+from collections import namedtuple
+import concurrent.futures
+from enum import Enum
+import logging
+import os
+import shutil
+import socket
+import time
+import yaml
+
+# Django
+from django.conf import settings
+
+# Runner
+import ansible_runner
+
+# AWX
+from awx.main.utils.execution_environments import get_default_pod_spec
+from awx.main.exceptions import ReceptorNodeNotFound
+from awx.main.utils.common import (
+    deepmerge,
+    parse_yaml_or_json,
+    cleanup_new_process,
+)
+from awx.main.constants import MAX_ISOLATED_PATH_COLON_DELIMITER
+
+
+# Receptorctl
+from receptorctl.socket_interface import ReceptorControl
+
+logger = logging.getLogger('awx.main.tasks.receptor')
+__RECEPTOR_CONF = '/etc/receptor/receptor.conf'
+RECEPTOR_ACTIVE_STATES = ('Pending', 'Running')
+
+
+class ReceptorConnectionType(Enum):
+    DATAGRAM = 0
+    STREAM = 1
+    STREAMTLS = 2
+
+
+def get_receptor_sockfile():
+    with open(__RECEPTOR_CONF, 'r') as f:
+        data = yaml.safe_load(f)
+    for section in data:
+        for entry_name, entry_data in section.items():
+            if entry_name == 'control-service':
+                if 'filename' in entry_data:
+                    return entry_data['filename']
+                else:
+                    raise RuntimeError(f'Receptor conf {__RECEPTOR_CONF} control-service entry does not have a filename parameter')
+    else:
+        raise RuntimeError(f'Receptor conf {__RECEPTOR_CONF} does not have control-service entry needed to get sockfile')
+
+
+def get_tls_client(use_stream_tls=None):
+    if not use_stream_tls:
+        return None
+
+    with open(__RECEPTOR_CONF, 'r') as f:
+        data = yaml.safe_load(f)
+    for section in data:
+        for entry_name, entry_data in section.items():
+            if entry_name == 'tls-client':
+                if 'name' in entry_data:
+                    return entry_data['name']
+    return None
+
+
+def get_receptor_ctl():
+    receptor_sockfile = get_receptor_sockfile()
+    try:
+        return ReceptorControl(receptor_sockfile, config=__RECEPTOR_CONF, tlsclient=get_tls_client(True))
+    except RuntimeError:
+        return ReceptorControl(receptor_sockfile)
+
+
+def get_conn_type(node_name, receptor_ctl):
+    all_nodes = receptor_ctl.simple_command("status").get('Advertisements', None)
+    for node in all_nodes:
+        if node.get('NodeID') == node_name:
+            return ReceptorConnectionType(node.get('ConnType'))
+    raise ReceptorNodeNotFound(f'Instance {node_name} is not in the receptor mesh')
+
+
+def administrative_workunit_reaper(work_list=None):
+    """
+    This releases completed work units that were spawned by actions inside of this module
+    specifically, this should catch any completed work unit left by
+     - worker_info
+     - worker_cleanup
+    These should ordinarily be released when the method finishes, but this is a
+    cleanup of last-resort, in case something went awry
+    """
+    receptor_ctl = get_receptor_ctl()
+    if work_list is None:
+        work_list = receptor_ctl.simple_command("work list")
+
+    for unit_id, work_data in work_list.items():
+        extra_data = work_data.get('ExtraData')
+        if (extra_data is None) or (extra_data.get('RemoteWorkType') != 'ansible-runner'):
+            continue  # if this is not ansible-runner work, we do not want to touch it
+        params = extra_data.get('RemoteParams', {}).get('params')
+        if not params:
+            continue
+        if not (params == '--worker-info' or params.startswith('cleanup')):
+            continue  # if this is not a cleanup or health check, we do not want to touch it
+        if work_data.get('StateName') in RECEPTOR_ACTIVE_STATES:
+            continue  # do not want to touch active work units
+        logger.info(f'Reaping orphaned work unit {unit_id} with params {params}')
+        receptor_ctl.simple_command(f"work release {unit_id}")
+
+
+class RemoteJobError(RuntimeError):
+    pass
+
+
+def run_until_complete(node, timing_data=None, **kwargs):
+    """
+    Runs an ansible-runner work_type on remote node, waits until it completes, then returns stdout.
+    """
+    receptor_ctl = get_receptor_ctl()
+
+    use_stream_tls = getattr(get_conn_type(node, receptor_ctl), 'name', None) == "STREAMTLS"
+    kwargs.setdefault('tlsclient', get_tls_client(use_stream_tls))
+    kwargs.setdefault('ttl', '20s')
+    kwargs.setdefault('payload', '')
+
+    transmit_start = time.time()
+    sign_work = False if settings.IS_K8S else True
+    result = receptor_ctl.submit_work(worktype='ansible-runner', node=node, signwork=sign_work, **kwargs)
+
+    unit_id = result['unitid']
+    run_start = time.time()
+    if timing_data:
+        timing_data['transmit_timing'] = run_start - transmit_start
+    run_timing = 0.0
+    stdout = ''
+
+    try:
+
+        resultfile = receptor_ctl.get_work_results(unit_id)
+
+        while run_timing < 20.0:
+            status = receptor_ctl.simple_command(f'work status {unit_id}')
+            state_name = status.get('StateName')
+            if state_name not in RECEPTOR_ACTIVE_STATES:
+                break
+            run_timing = time.time() - run_start
+            time.sleep(0.5)
+        else:
+            raise RemoteJobError(f'Receptor job timeout on {node} after {run_timing} seconds, state remains in {state_name}')
+
+        if timing_data:
+            timing_data['run_timing'] = run_timing
+
+        stdout = resultfile.read()
+        stdout = str(stdout, encoding='utf-8')
+
+    finally:
+
+        if settings.RECEPTOR_RELEASE_WORK:
+            res = receptor_ctl.simple_command(f"work release {unit_id}")
+            if res != {'released': unit_id}:
+                logger.warn(f'Could not confirm release of receptor work unit id {unit_id} from {node}, data: {res}')
+
+        receptor_ctl.close()
+
+    if state_name.lower() == 'failed':
+        work_detail = status.get('Detail', '')
+        if work_detail:
+            raise RemoteJobError(f'Receptor error from {node}, detail:\n{work_detail}')
+        else:
+            raise RemoteJobError(f'Unknown ansible-runner error on node {node}, stdout:\n{stdout}')
+
+    return stdout
+
+
+def worker_info(node_name, work_type='ansible-runner'):
+    error_list = []
+    data = {'errors': error_list, 'transmit_timing': 0.0}
+
+    try:
+        stdout = run_until_complete(node=node_name, timing_data=data, params={"params": "--worker-info"})
+
+        yaml_stdout = stdout.strip()
+        remote_data = {}
+        try:
+            remote_data = yaml.safe_load(yaml_stdout)
+        except Exception as json_e:
+            error_list.append(f'Failed to parse node {node_name} --worker-info output as YAML, error: {json_e}, data:\n{yaml_stdout}')
+
+        if not isinstance(remote_data, dict):
+            error_list.append(f'Remote node {node_name} --worker-info output is not a YAML dict, output:{stdout}')
+        else:
+            error_list.extend(remote_data.pop('errors', []))  # merge both error lists
+            data.update(remote_data)
+
+    except RemoteJobError as exc:
+        details = exc.args[0]
+        if 'unrecognized arguments: --worker-info' in details:
+            error_list.append(f'Old version (2.0.1 or earlier) of ansible-runner on node {node_name} without --worker-info')
+        else:
+            error_list.append(details)
+
+    except (ReceptorNodeNotFound, RuntimeError) as exc:
+        error_list.append(str(exc))
+
+    # If we have a connection error, missing keys would be trivial consequence of that
+    if not data['errors']:
+        # see tasks.py usage of keys
+        missing_keys = set(('runner_version', 'mem_in_bytes', 'cpu_count')) - set(data.keys())
+        if missing_keys:
+            data['errors'].append('Worker failed to return keys {}'.format(' '.join(missing_keys)))
+
+    return data
+
+
+def _convert_args_to_cli(vargs):
+    """
+    For the ansible-runner worker cleanup command
+    converts the dictionary (parsed argparse variables) used for python interface
+    into a string of CLI options, which has to be used on execution nodes.
+    """
+    args = ['cleanup']
+    for option in ('exclude_strings', 'remove_images'):
+        if vargs.get(option):
+            args.append('--{}={}'.format(option.replace('_', '-'), ' '.join(vargs.get(option))))
+    for option in ('file_pattern', 'image_prune', 'process_isolation_executable', 'grace_period'):
+        if vargs.get(option) is True:
+            args.append('--{}'.format(option.replace('_', '-')))
+        elif vargs.get(option) not in (None, ''):
+            args.append('--{}={}'.format(option.replace('_', '-'), vargs.get(option)))
+    return args
+
+
+def worker_cleanup(node_name, vargs, timeout=300.0):
+    args = _convert_args_to_cli(vargs)
+
+    remote_command = ' '.join(args)
+    logger.debug(f'Running command over receptor mesh on {node_name}: ansible-runner worker {remote_command}')
+
+    stdout = run_until_complete(node=node_name, params={"params": remote_command})
+
+    return stdout
+
+
+class AWXReceptorJob:
+    def __init__(self, task, runner_params=None):
+        self.task = task
+        self.runner_params = runner_params
+        self.unit_id = None
+
+        if self.task and not self.task.instance.is_container_group_task:
+            execution_environment_params = self.task.build_execution_environment_params(self.task.instance, runner_params['private_data_dir'])
+            self.runner_params.update(execution_environment_params)
+
+        if not settings.IS_K8S and self.work_type == 'local' and 'only_transmit_kwargs' not in self.runner_params:
+            self.runner_params['only_transmit_kwargs'] = True
+
+    def run(self):
+        # We establish a connection to the Receptor socket
+        receptor_ctl = get_receptor_ctl()
+
+        res = None
+        try:
+            res = self._run_internal(receptor_ctl)
+            return res
+        finally:
+            # Make sure to always release the work unit if we established it
+            if self.unit_id is not None and settings.RECEPTOR_RELEASE_WORK:
+                try:
+                    receptor_ctl.simple_command(f"work release {self.unit_id}")
+                except Exception:
+                    logger.exception(f"Error releasing work unit {self.unit_id}.")
+
+    @property
+    def sign_work(self):
+        return False if settings.IS_K8S else True
+
+    def _run_internal(self, receptor_ctl):
+        # Create a socketpair. Where the left side will be used for writing our payload
+        # (private data dir, kwargs). The right side will be passed to Receptor for
+        # reading.
+        sockin, sockout = socket.socketpair()
+
+        # Prepare the submit_work kwargs before creating threads, because references to settings are not thread-safe
+        work_submit_kw = dict(worktype=self.work_type, params=self.receptor_params, signwork=self.sign_work)
+        if self.work_type == 'ansible-runner':
+            work_submit_kw['node'] = self.task.instance.execution_node
+            use_stream_tls = get_conn_type(work_submit_kw['node'], receptor_ctl).name == "STREAMTLS"
+            work_submit_kw['tlsclient'] = get_tls_client(use_stream_tls)
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+            transmitter_future = executor.submit(self.transmit, sockin)
+
+            # submit our work, passing in the right side of our socketpair for reading.
+            result = receptor_ctl.submit_work(payload=sockout.makefile('rb'), **work_submit_kw)
+
+            sockin.close()
+            sockout.close()
+
+            self.unit_id = result['unitid']
+            # Update the job with the work unit in-memory so that the log_lifecycle
+            # will print out the work unit that is to be associated with the job in the database
+            # via the update_model() call.
+            # We want to log the work_unit_id as early as possible. A failure can happen in between
+            # when we start the job in receptor and when we associate the job <-> work_unit_id.
+            # In that case, there will be work running in receptor and Controller will not know
+            # which Job it is associated with.
+            # We do not programatically handle this case. Ideally, we would handle this with a reaper case.
+            # The two distinct job lifecycle log events below allow for us to at least detect when this
+            # edge case occurs. If the lifecycle event work_unit_id_received occurs without the
+            # work_unit_id_assigned event then this case may have occured.
+            self.task.instance.work_unit_id = result['unitid']  # Set work_unit_id in-memory only
+            self.task.instance.log_lifecycle("work_unit_id_received")
+            self.task.update_model(self.task.instance.pk, work_unit_id=result['unitid'])
+            self.task.instance.log_lifecycle("work_unit_id_assigned")
+
+        # Throws an exception if the transmit failed.
+        # Will be caught by the try/except in BaseTask#run.
+        transmitter_future.result()
+
+        # Artifacts are an output, but sometimes they are an input as well
+        # this is the case with fact cache, where clearing facts deletes a file, and this must be captured
+        artifact_dir = os.path.join(self.runner_params['private_data_dir'], 'artifacts')
+        if os.path.exists(artifact_dir):
+            shutil.rmtree(artifact_dir)
+
+        resultsock, resultfile = receptor_ctl.get_work_results(self.unit_id, return_socket=True, return_sockfile=True)
+        # Both "processor" and "cancel_watcher" are spawned in separate threads.
+        # We wait for the first one to return. If cancel_watcher returns first,
+        # we yank the socket out from underneath the processor, which will cause it
+        # to exit. A reference to the processor_future is passed into the cancel_watcher_future,
+        # Which exits if the job has finished normally. The context manager ensures we do not
+        # leave any threads laying around.
+        with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+            processor_future = executor.submit(self.processor, resultfile)
+            cancel_watcher_future = executor.submit(self.cancel_watcher, processor_future)
+            futures = [processor_future, cancel_watcher_future]
+            first_future = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED)
+
+            res = list(first_future.done)[0].result()
+            if res.status == 'canceled':
+                receptor_ctl.simple_command(f"work cancel {self.unit_id}")
+                resultsock.shutdown(socket.SHUT_RDWR)
+                resultfile.close()
+            elif res.status == 'error':
+                try:
+                    unit_status = receptor_ctl.simple_command(f'work status {self.unit_id}')
+                    detail = unit_status.get('Detail', None)
+                    state_name = unit_status.get('StateName', None)
+                except Exception:
+                    detail = ''
+                    state_name = ''
+                    logger.exception(f'An error was encountered while getting status for work unit {self.unit_id}')
+
+                if 'exceeded quota' in detail:
+                    logger.warn(detail)
+                    log_name = self.task.instance.log_format
+                    logger.warn(f"Could not launch pod for {log_name}. Exceeded quota.")
+                    self.task.update_model(self.task.instance.pk, status='pending')
+                    return
+                # If ansible-runner ran, but an error occured at runtime, the traceback information
+                # is saved via the status_handler passed in to the processor.
+                if state_name == 'Succeeded':
+                    return res
+
+                if not self.task.instance.result_traceback:
+                    try:
+                        resultsock = receptor_ctl.get_work_results(self.unit_id, return_sockfile=True)
+                        lines = resultsock.readlines()
+                        receptor_output = b"".join(lines).decode()
+                        if receptor_output:
+                            self.task.instance.result_traceback = receptor_output
+                            self.task.instance.save(update_fields=['result_traceback'])
+                        elif detail:
+                            self.task.instance.result_traceback = detail
+                            self.task.instance.save(update_fields=['result_traceback'])
+                        else:
+                            logger.warn(f'No result details or output from {self.task.instance.log_format}, status:\n{state_name}')
+                    except Exception:
+                        raise RuntimeError(detail)
+
+        return res
+
+    # Spawned in a thread so Receptor can start reading before we finish writing, we
+    # write our payload to the left side of our socketpair.
+    @cleanup_new_process
+    def transmit(self, _socket):
+        try:
+            ansible_runner.interface.run(streamer='transmit', _output=_socket.makefile('wb'), **self.runner_params)
+        finally:
+            # Socket must be shutdown here, or the reader will hang forever.
+            _socket.shutdown(socket.SHUT_WR)
+
+    @cleanup_new_process
+    def processor(self, resultfile):
+        return ansible_runner.interface.run(
+            streamer='process',
+            quiet=True,
+            _input=resultfile,
+            event_handler=self.task.runner_callback.event_handler,
+            finished_callback=self.task.runner_callback.finished_callback,
+            status_handler=self.task.runner_callback.status_handler,
+            **self.runner_params,
+        )
+
+    @property
+    def receptor_params(self):
+        if self.task.instance.is_container_group_task:
+            spec_yaml = yaml.dump(self.pod_definition, explicit_start=True)
+
+            receptor_params = {
+                "secret_kube_pod": spec_yaml,
+                "pod_pending_timeout": getattr(settings, 'AWX_CONTAINER_GROUP_POD_PENDING_TIMEOUT', "5m"),
+            }
+
+            if self.credential:
+                kubeconfig_yaml = yaml.dump(self.kube_config, explicit_start=True)
+                receptor_params["secret_kube_config"] = kubeconfig_yaml
+        else:
+            private_data_dir = self.runner_params['private_data_dir']
+            if self.work_type == 'ansible-runner' and settings.AWX_CLEANUP_PATHS:
+                # on execution nodes, we rely on the private data dir being deleted
+                cli_params = f"--private-data-dir={private_data_dir} --delete"
+            else:
+                # on hybrid nodes, we rely on the private data dir NOT being deleted
+                cli_params = f"--private-data-dir={private_data_dir}"
+            receptor_params = {"params": cli_params}
+
+        return receptor_params
+
+    @property
+    def work_type(self):
+        if self.task.instance.is_container_group_task:
+            if self.credential:
+                return 'kubernetes-runtime-auth'
+            return 'kubernetes-incluster-auth'
+        if self.task.instance.execution_node == settings.CLUSTER_HOST_ID or self.task.instance.execution_node == self.task.instance.controller_node:
+            return 'local'
+        return 'ansible-runner'
+
+    @cleanup_new_process
+    def cancel_watcher(self, processor_future):
+        while True:
+            if processor_future.done():
+                return processor_future.result()
+
+            if self.task.runner_callback.cancel_callback():
+                result = namedtuple('result', ['status', 'rc'])
+                return result('canceled', 1)
+
+            time.sleep(1)
+
+    @property
+    def pod_definition(self):
+        ee = self.task.instance.execution_environment
+
+        default_pod_spec = get_default_pod_spec()
+
+        pod_spec_override = {}
+        if self.task and self.task.instance.instance_group.pod_spec_override:
+            pod_spec_override = parse_yaml_or_json(self.task.instance.instance_group.pod_spec_override)
+        # According to the deepmerge docstring, the second dictionary will override when
+        # they share keys, which is the desired behavior.
+        # This allows user to only provide elements they want to override, and for us to still provide any
+        # defaults they don't want to change
+        pod_spec = deepmerge(default_pod_spec, pod_spec_override)
+
+        pod_spec['spec']['containers'][0]['image'] = ee.image
+        pod_spec['spec']['containers'][0]['args'] = ['ansible-runner', 'worker', '--private-data-dir=/runner']
+
+        # Enforce EE Pull Policy
+        pull_options = {"always": "Always", "missing": "IfNotPresent", "never": "Never"}
+        if self.task and self.task.instance.execution_environment:
+            if self.task.instance.execution_environment.pull:
+                pod_spec['spec']['containers'][0]['imagePullPolicy'] = pull_options[self.task.instance.execution_environment.pull]
+
+        # This allows the user to also expose the isolated path list
+        # to EEs running in k8s/ocp environments, i.e. container groups.
+        # This assumes the node and SA supports hostPath volumes
+        # type is not passed due to backward compatibility,
+        # which means that no checks will be performed before mounting the hostPath volume.
+        if settings.AWX_MOUNT_ISOLATED_PATHS_ON_K8S and settings.AWX_ISOLATION_SHOW_PATHS:
+            spec_volume_mounts = []
+            spec_volumes = []
+
+            for idx, this_path in enumerate(settings.AWX_ISOLATION_SHOW_PATHS):
+                mount_option = None
+                if this_path.count(':') == MAX_ISOLATED_PATH_COLON_DELIMITER:
+                    src, dest, mount_option = this_path.split(':')
+                elif this_path.count(':') == MAX_ISOLATED_PATH_COLON_DELIMITER - 1:
+                    src, dest = this_path.split(':')
+                else:
+                    src = dest = this_path
+
+                # Enforce read-only volume if 'ro' has been explicitly passed
+                # We do this so we can use the same configuration for regular scenarios and k8s
+                # Since flags like ':O', ':z' or ':Z' are not valid in the k8s realm
+                # Example: /data:/data:ro
+                read_only = bool('ro' == mount_option)
+
+                # Since type is not being passed, k8s by default will not perform any checks if the
+                # hostPath volume exists on the k8s node itself.
+                spec_volumes.append({'name': f'volume-{idx}', 'hostPath': {'path': src}})
+
+                spec_volume_mounts.append({'name': f'volume-{idx}', 'mountPath': f'{dest}', 'readOnly': read_only})
+
+            # merge any volumes definition already present in the pod_spec
+            if 'volumes' in pod_spec['spec']:
+                pod_spec['spec']['volumes'] += spec_volumes
+            else:
+                pod_spec['spec']['volumes'] = spec_volumes
+
+            # merge any volumesMounts definition already present in the pod_spec
+            if 'volumeMounts' in pod_spec['spec']['containers'][0]:
+                pod_spec['spec']['containers'][0]['volumeMounts'] += spec_volume_mounts
+            else:
+                pod_spec['spec']['containers'][0]['volumeMounts'] = spec_volume_mounts
+
+        if self.task and self.task.instance.is_container_group_task:
+            # If EE credential is passed, create an imagePullSecret
+            if self.task.instance.execution_environment and self.task.instance.execution_environment.credential:
+                # Create pull secret in k8s cluster based on ee cred
+                from awx.main.scheduler.kubernetes import PodManager  # prevent circular import
+
+                pm = PodManager(self.task.instance)
+                secret_name = pm.create_secret(job=self.task.instance)
+
+                # Inject secret name into podspec
+                pod_spec['spec']['imagePullSecrets'] = [{"name": secret_name}]
+
+        if self.task:
+            pod_spec['metadata'] = deepmerge(
+                pod_spec.get('metadata', {}),
+                dict(name=self.pod_name, labels={'ansible-awx': settings.INSTALL_UUID, 'ansible-awx-job-id': str(self.task.instance.id)}),
+            )
+
+        return pod_spec
+
+    @property
+    def pod_name(self):
+        return f"automation-job-{self.task.instance.id}"
+
+    @property
+    def credential(self):
+        return self.task.instance.instance_group.credential
+
+    @property
+    def namespace(self):
+        return self.pod_definition['metadata']['namespace']
+
+    @property
+    def kube_config(self):
+        host_input = self.credential.get_input('host')
+        config = {
+            "apiVersion": "v1",
+            "kind": "Config",
+            "preferences": {},
+            "clusters": [{"name": host_input, "cluster": {"server": host_input}}],
+            "users": [{"name": host_input, "user": {"token": self.credential.get_input('bearer_token')}}],
+            "contexts": [{"name": host_input, "context": {"cluster": host_input, "user": host_input, "namespace": self.namespace}}],
+            "current-context": host_input,
+        }
+
+        if self.credential.get_input('verify_ssl') and 'ssl_ca_cert' in self.credential.inputs:
+            config["clusters"][0]["cluster"]["certificate-authority-data"] = b64encode(
+                self.credential.get_input('ssl_ca_cert').encode()  # encode to bytes
+            ).decode()  # decode the base64 data into a str
+        else:
+            config["clusters"][0]["cluster"]["insecure-skip-tls-verify"] = True
+        return config

awx/main/tasks/system.py

@@ -0,0 +1,906 @@
+# Python
+from collections import namedtuple
+import functools
+import importlib
+import json
+import logging
+import os
+from io import StringIO
+from contextlib import redirect_stdout
+import shutil
+import time
+from distutils.version import LooseVersion as Version
+
+# Django
+from django.conf import settings
+from django.db import transaction, DatabaseError, IntegrityError
+from django.db.models.fields.related import ForeignKey
+from django.utils.timezone import now
+from django.utils.encoding import smart_str
+from django.contrib.auth.models import User
+from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_noop
+from django.core.cache import cache
+from django.core.exceptions import ObjectDoesNotExist
+
+# Django-CRUM
+from crum import impersonate
+
+
+# Runner
+import ansible_runner.cleanup
+
+# dateutil
+from dateutil.parser import parse as parse_date
+
+# AWX
+from awx import __version__ as awx_application_version
+from awx.main.access import access_registry
+from awx.main.models import (
+    Schedule,
+    TowerScheduleState,
+    Instance,
+    InstanceGroup,
+    UnifiedJob,
+    Notification,
+    Inventory,
+    SmartInventoryMembership,
+    Job,
+)
+from awx.main.constants import ACTIVE_STATES
+from awx.main.dispatch.publish import task
+from awx.main.dispatch import get_local_queuename, reaper
+from awx.main.utils.common import (
+    ignore_inventory_computed_fields,
+    ignore_inventory_group_removal,
+    schedule_task_manager,
+)
+
+from awx.main.utils.external_logging import reconfigure_rsyslog
+from awx.main.utils.reload import stop_local_services
+from awx.main.utils.pglock import advisory_lock
+from awx.main.tasks.receptor import get_receptor_ctl, worker_info, worker_cleanup, administrative_workunit_reaper
+from awx.main.consumers import emit_channel_notification
+from awx.main import analytics
+from awx.conf import settings_registry
+from awx.main.analytics.subsystem_metrics import Metrics
+
+from rest_framework.exceptions import PermissionDenied
+
+logger = logging.getLogger('awx.main.tasks.system')
+
+OPENSSH_KEY_ERROR = u'''\
+It looks like you're trying to use a private key in OpenSSH format, which \
+isn't supported by the installed version of OpenSSH on this instance. \
+Try upgrading OpenSSH or providing your private key in an different format. \
+'''
+
+
+def dispatch_startup():
+    startup_logger = logging.getLogger('awx.main.tasks')
+    startup_logger.debug("Syncing Schedules")
+    for sch in Schedule.objects.all():
+        try:
+            sch.update_computed_fields()
+        except Exception:
+            logger.exception("Failed to rebuild schedule {}.".format(sch))
+
+    #
+    # When the dispatcher starts, if the instance cannot be found in the database,
+    # automatically register it.  This is mostly useful for openshift-based
+    # deployments where:
+    #
+    # 2 Instances come online
+    # Instance B encounters a network blip, Instance A notices, and
+    # deprovisions it
+    # Instance B's connectivity is restored, the dispatcher starts, and it
+    # re-registers itself
+    #
+    # In traditional container-less deployments, instances don't get
+    # deprovisioned when they miss their heartbeat, so this code is mostly a
+    # no-op.
+    #
+    apply_cluster_membership_policies()
+    cluster_node_heartbeat()
+    Metrics().clear_values()
+
+    # Update Tower's rsyslog.conf file based on loggins settings in the db
+    reconfigure_rsyslog()
+
+
+def inform_cluster_of_shutdown():
+    try:
+        this_inst = Instance.objects.get(hostname=settings.CLUSTER_HOST_ID)
+        this_inst.mark_offline(update_last_seen=True, errors=_('Instance received normal shutdown signal'))
+        try:
+            reaper.reap(this_inst)
+        except Exception:
+            logger.exception('failed to reap jobs for {}'.format(this_inst.hostname))
+        logger.warning('Normal shutdown signal for instance {}, ' 'removed self from capacity pool.'.format(this_inst.hostname))
+    except Exception:
+        logger.exception('Encountered problem with normal shutdown signal.')
+
+
+@task(queue=get_local_queuename)
+def apply_cluster_membership_policies():
+    from awx.main.signals import disable_activity_stream
+
+    started_waiting = time.time()
+    with advisory_lock('cluster_policy_lock', wait=True):
+        lock_time = time.time() - started_waiting
+        if lock_time > 1.0:
+            to_log = logger.info
+        else:
+            to_log = logger.debug
+        to_log('Waited {} seconds to obtain lock name: cluster_policy_lock'.format(lock_time))
+        started_compute = time.time()
+        # Hop nodes should never get assigned to an InstanceGroup.
+        all_instances = list(Instance.objects.exclude(node_type='hop').order_by('id'))
+        all_groups = list(InstanceGroup.objects.prefetch_related('instances'))
+
+        total_instances = len(all_instances)
+        actual_groups = []
+        actual_instances = []
+        Group = namedtuple('Group', ['obj', 'instances', 'prior_instances'])
+        Node = namedtuple('Instance', ['obj', 'groups'])
+
+        # Process policy instance list first, these will represent manually managed memberships
+        instance_hostnames_map = {inst.hostname: inst for inst in all_instances}
+        for ig in all_groups:
+            group_actual = Group(obj=ig, instances=[], prior_instances=[instance.pk for instance in ig.instances.all()])  # obtained in prefetch
+            for hostname in ig.policy_instance_list:
+                if hostname not in instance_hostnames_map:
+                    logger.info("Unknown instance {} in {} policy list".format(hostname, ig.name))
+                    continue
+                inst = instance_hostnames_map[hostname]
+                group_actual.instances.append(inst.id)
+                # NOTE: arguable behavior: policy-list-group is not added to
+                # instance's group count for consideration in minimum-policy rules
+            if group_actual.instances:
+                logger.debug("Policy List, adding Instances {} to Group {}".format(group_actual.instances, ig.name))
+
+            actual_groups.append(group_actual)
+
+        # Process Instance minimum policies next, since it represents a concrete lower bound to the
+        # number of instances to make available to instance groups
+        actual_instances = [Node(obj=i, groups=[]) for i in all_instances if i.managed_by_policy]
+        logger.debug("Total instances: {}, available for policy: {}".format(total_instances, len(actual_instances)))
+        for g in sorted(actual_groups, key=lambda x: len(x.instances)):
+            exclude_type = 'execution' if g.obj.name == settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME else 'control'
+            policy_min_added = []
+            for i in sorted(actual_instances, key=lambda x: len(x.groups)):
+                if i.obj.node_type == exclude_type:
+                    continue  # never place execution instances in controlplane group or control instances in other groups
+                if len(g.instances) >= g.obj.policy_instance_minimum:
+                    break
+                if i.obj.id in g.instances:
+                    # If the instance is already _in_ the group, it was
+                    # applied earlier via the policy list
+                    continue
+                g.instances.append(i.obj.id)
+                i.groups.append(g.obj.id)
+                policy_min_added.append(i.obj.id)
+            if policy_min_added:
+                logger.debug("Policy minimum, adding Instances {} to Group {}".format(policy_min_added, g.obj.name))
+
+        # Finally, process instance policy percentages
+        for g in sorted(actual_groups, key=lambda x: len(x.instances)):
+            exclude_type = 'execution' if g.obj.name == settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME else 'control'
+            candidate_pool_ct = sum(1 for i in actual_instances if i.obj.node_type != exclude_type)
+            if not candidate_pool_ct:
+                continue
+            policy_per_added = []
+            for i in sorted(actual_instances, key=lambda x: len(x.groups)):
+                if i.obj.node_type == exclude_type:
+                    continue
+                if i.obj.id in g.instances:
+                    # If the instance is already _in_ the group, it was
+                    # applied earlier via a minimum policy or policy list
+                    continue
+                if 100 * float(len(g.instances)) / candidate_pool_ct >= g.obj.policy_instance_percentage:
+                    break
+                g.instances.append(i.obj.id)
+                i.groups.append(g.obj.id)
+                policy_per_added.append(i.obj.id)
+            if policy_per_added:
+                logger.debug("Policy percentage, adding Instances {} to Group {}".format(policy_per_added, g.obj.name))
+
+        # Determine if any changes need to be made
+        needs_change = False
+        for g in actual_groups:
+            if set(g.instances) != set(g.prior_instances):
+                needs_change = True
+                break
+        if not needs_change:
+            logger.debug('Cluster policy no-op finished in {} seconds'.format(time.time() - started_compute))
+            return
+
+        # On a differential basis, apply instances to groups
+        with transaction.atomic():
+            with disable_activity_stream():
+                for g in actual_groups:
+                    if g.obj.is_container_group:
+                        logger.debug('Skipping containerized group {} for policy calculation'.format(g.obj.name))
+                        continue
+                    instances_to_add = set(g.instances) - set(g.prior_instances)
+                    instances_to_remove = set(g.prior_instances) - set(g.instances)
+                    if instances_to_add:
+                        logger.debug('Adding instances {} to group {}'.format(list(instances_to_add), g.obj.name))
+                        g.obj.instances.add(*instances_to_add)
+                    if instances_to_remove:
+                        logger.debug('Removing instances {} from group {}'.format(list(instances_to_remove), g.obj.name))
+                        g.obj.instances.remove(*instances_to_remove)
+        logger.debug('Cluster policy computation finished in {} seconds'.format(time.time() - started_compute))
+
+
+@task(queue='tower_broadcast_all')
+def handle_setting_changes(setting_keys):
+    orig_len = len(setting_keys)
+    for i in range(orig_len):
+        for dependent_key in settings_registry.get_dependent_settings(setting_keys[i]):
+            setting_keys.append(dependent_key)
+    cache_keys = set(setting_keys)
+    logger.debug('cache delete_many(%r)', cache_keys)
+    cache.delete_many(cache_keys)
+
+    if any([setting.startswith('LOG_AGGREGATOR') for setting in setting_keys]):
+        reconfigure_rsyslog()
+
+
+@task(queue='tower_broadcast_all')
+def delete_project_files(project_path):
+    # TODO: possibly implement some retry logic
+    lock_file = project_path + '.lock'
+    if os.path.exists(project_path):
+        try:
+            shutil.rmtree(project_path)
+            logger.debug('Success removing project files {}'.format(project_path))
+        except Exception:
+            logger.exception('Could not remove project directory {}'.format(project_path))
+    if os.path.exists(lock_file):
+        try:
+            os.remove(lock_file)
+            logger.debug('Success removing {}'.format(lock_file))
+        except Exception:
+            logger.exception('Could not remove lock file {}'.format(lock_file))
+
+
+@task(queue='tower_broadcast_all')
+def profile_sql(threshold=1, minutes=1):
+    if threshold <= 0:
+        cache.delete('awx-profile-sql-threshold')
+        logger.error('SQL PROFILING DISABLED')
+    else:
+        cache.set('awx-profile-sql-threshold', threshold, timeout=minutes * 60)
+        logger.error('SQL QUERIES >={}s ENABLED FOR {} MINUTE(S)'.format(threshold, minutes))
+
+
+@task(queue=get_local_queuename)
+def send_notifications(notification_list, job_id=None):
+    if not isinstance(notification_list, list):
+        raise TypeError("notification_list should be of type list")
+    if job_id is not None:
+        job_actual = UnifiedJob.objects.get(id=job_id)
+
+    notifications = Notification.objects.filter(id__in=notification_list)
+    if job_id is not None:
+        job_actual.notifications.add(*notifications)
+
+    for notification in notifications:
+        update_fields = ['status', 'notifications_sent']
+        try:
+            sent = notification.notification_template.send(notification.subject, notification.body)
+            notification.status = "successful"
+            notification.notifications_sent = sent
+            if job_id is not None:
+                job_actual.log_lifecycle("notifications_sent")
+        except Exception as e:
+            logger.exception("Send Notification Failed {}".format(e))
+            notification.status = "failed"
+            notification.error = smart_str(e)
+            update_fields.append('error')
+        finally:
+            try:
+                notification.save(update_fields=update_fields)
+            except Exception:
+                logger.exception('Error saving notification {} result.'.format(notification.id))
+
+
+@task(queue=get_local_queuename)
+def gather_analytics():
+    from awx.conf.models import Setting
+    from rest_framework.fields import DateTimeField
+
+    last_gather = Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_GATHER').first()
+    last_time = DateTimeField().to_internal_value(last_gather.value) if last_gather and last_gather.value else None
+    gather_time = now()
+
+    if not last_time or ((gather_time - last_time).total_seconds() > settings.AUTOMATION_ANALYTICS_GATHER_INTERVAL):
+        analytics.gather()
+
+
+@task(queue=get_local_queuename)
+def purge_old_stdout_files():
+    nowtime = time.time()
+    for f in os.listdir(settings.JOBOUTPUT_ROOT):
+        if os.path.getctime(os.path.join(settings.JOBOUTPUT_ROOT, f)) < nowtime - settings.LOCAL_STDOUT_EXPIRE_TIME:
+            os.unlink(os.path.join(settings.JOBOUTPUT_ROOT, f))
+            logger.debug("Removing {}".format(os.path.join(settings.JOBOUTPUT_ROOT, f)))
+
+
+def _cleanup_images_and_files(**kwargs):
+    if settings.IS_K8S:
+        return
+    this_inst = Instance.objects.me()
+    runner_cleanup_kwargs = this_inst.get_cleanup_task_kwargs(**kwargs)
+    if runner_cleanup_kwargs:
+        stdout = ''
+        with StringIO() as buffer:
+            with redirect_stdout(buffer):
+                ansible_runner.cleanup.run_cleanup(runner_cleanup_kwargs)
+                stdout = buffer.getvalue()
+        if '(changed: True)' in stdout:
+            logger.info(f'Performed local cleanup with kwargs {kwargs}, output:\n{stdout}')
+
+    # if we are the first instance alphabetically, then run cleanup on execution nodes
+    checker_instance = Instance.objects.filter(node_type__in=['hybrid', 'control'], enabled=True, capacity__gt=0).order_by('-hostname').first()
+    if checker_instance and this_inst.hostname == checker_instance.hostname:
+        for inst in Instance.objects.filter(node_type='execution', enabled=True, capacity__gt=0):
+            runner_cleanup_kwargs = inst.get_cleanup_task_kwargs(**kwargs)
+            if not runner_cleanup_kwargs:
+                continue
+            try:
+                stdout = worker_cleanup(inst.hostname, runner_cleanup_kwargs)
+                if '(changed: True)' in stdout:
+                    logger.info(f'Performed cleanup on execution node {inst.hostname} with output:\n{stdout}')
+            except RuntimeError:
+                logger.exception(f'Error running cleanup on execution node {inst.hostname}')
+
+
+@task(queue='tower_broadcast_all')
+def handle_removed_image(remove_images=None):
+    """Special broadcast invocation of this method to handle case of deleted EE"""
+    _cleanup_images_and_files(remove_images=remove_images, file_pattern='')
+
+
+@task(queue=get_local_queuename)
+def cleanup_images_and_files():
+    _cleanup_images_and_files()
+
+
+@task(queue=get_local_queuename)
+def cluster_node_health_check(node):
+    """
+    Used for the health check endpoint, refreshes the status of the instance, but must be ran on target node
+    """
+    if node == '':
+        logger.warn('Local health check incorrectly called with blank string')
+        return
+    elif node != settings.CLUSTER_HOST_ID:
+        logger.warn(f'Local health check for {node} incorrectly sent to {settings.CLUSTER_HOST_ID}')
+        return
+    try:
+        this_inst = Instance.objects.me()
+    except Instance.DoesNotExist:
+        logger.warn(f'Instance record for {node} missing, could not check capacity.')
+        return
+    this_inst.local_health_check()
+
+
+@task(queue=get_local_queuename)
+def execution_node_health_check(node):
+    if node == '':
+        logger.warn('Remote health check incorrectly called with blank string')
+        return
+    try:
+        instance = Instance.objects.get(hostname=node)
+    except Instance.DoesNotExist:
+        logger.warn(f'Instance record for {node} missing, could not check capacity.')
+        return
+
+    if instance.node_type != 'execution':
+        raise RuntimeError(f'Execution node health check ran against {instance.node_type} node {instance.hostname}')
+
+    data = worker_info(node)
+
+    prior_capacity = instance.capacity
+
+    instance.save_health_data(
+        version='ansible-runner-' + data.get('runner_version', '???'),
+        cpu=data.get('cpu_count', 0),
+        memory=data.get('mem_in_bytes', 0),
+        uuid=data.get('uuid'),
+        errors='\n'.join(data.get('errors', [])),
+    )
+
+    if data['errors']:
+        formatted_error = "\n".join(data["errors"])
+        if prior_capacity:
+            logger.warn(f'Health check marking execution node {node} as lost, errors:\n{formatted_error}')
+        else:
+            logger.info(f'Failed to find capacity of new or lost execution node {node}, errors:\n{formatted_error}')
+    else:
+        logger.info('Set capacity of execution node {} to {}, worker info data:\n{}'.format(node, instance.capacity, json.dumps(data, indent=2)))
+
+    return data
+
+
+def inspect_execution_nodes(instance_list):
+    with advisory_lock('inspect_execution_nodes_lock', wait=False):
+        node_lookup = {inst.hostname: inst for inst in instance_list}
+
+        ctl = get_receptor_ctl()
+        mesh_status = ctl.simple_command('status')
+
+        nowtime = now()
+        workers = mesh_status['Advertisements']
+        for ad in workers:
+            hostname = ad['NodeID']
+            changed = False
+
+            if hostname in node_lookup:
+                instance = node_lookup[hostname]
+            else:
+                logger.warn(f"Unrecognized node advertising on mesh: {hostname}")
+                continue
+
+            # Control-plane nodes are dealt with via local_health_check instead.
+            if instance.node_type in ('control', 'hybrid'):
+                continue
+
+            was_lost = instance.is_lost(ref_time=nowtime)
+            last_seen = parse_date(ad['Time'])
+
+            if instance.last_seen and instance.last_seen >= last_seen:
+                continue
+            instance.last_seen = last_seen
+            instance.save(update_fields=['last_seen'])
+
+            # Only execution nodes should be dealt with by execution_node_health_check
+            if instance.node_type == 'hop':
+                continue
+
+            if changed:
+                execution_node_health_check.apply_async([hostname])
+            elif was_lost:
+                # if the instance *was* lost, but has appeared again,
+                # attempt to re-establish the initial capacity and version
+                # check
+                logger.warn(f'Execution node attempting to rejoin as instance {hostname}.')
+                execution_node_health_check.apply_async([hostname])
+            elif instance.capacity == 0 and instance.enabled:
+                # nodes with proven connection but need remediation run health checks are reduced frequency
+                if not instance.last_health_check or (nowtime - instance.last_health_check).total_seconds() >= settings.EXECUTION_NODE_REMEDIATION_CHECKS:
+                    # Periodically re-run the health check of errored nodes, in case someone fixed it
+                    # TODO: perhaps decrease the frequency of these checks
+                    logger.debug(f'Restarting health check for execution node {hostname} with known errors.')
+                    execution_node_health_check.apply_async([hostname])
+
+
+@task(queue=get_local_queuename)
+def cluster_node_heartbeat():
+    logger.debug("Cluster node heartbeat task.")
+    nowtime = now()
+    instance_list = list(Instance.objects.all())
+    this_inst = None
+    lost_instances = []
+
+    for inst in instance_list:
+        if inst.hostname == settings.CLUSTER_HOST_ID:
+            this_inst = inst
+            break
+    else:
+        (changed, this_inst) = Instance.objects.get_or_register()
+        if changed:
+            logger.info("Registered tower control node '{}'".format(this_inst.hostname))
+
+    inspect_execution_nodes(instance_list)
+
+    for inst in list(instance_list):
+        if inst == this_inst:
+            continue
+        if inst.is_lost(ref_time=nowtime):
+            lost_instances.append(inst)
+            instance_list.remove(inst)
+
+    if this_inst:
+        startup_event = this_inst.is_lost(ref_time=nowtime)
+        this_inst.local_health_check()
+        if startup_event and this_inst.capacity != 0:
+            logger.warning('Rejoining the cluster as instance {}.'.format(this_inst.hostname))
+            return
+    else:
+        raise RuntimeError("Cluster Host Not Found: {}".format(settings.CLUSTER_HOST_ID))
+    # IFF any node has a greater version than we do, then we'll shutdown services
+    for other_inst in instance_list:
+        if other_inst.node_type in ('execution', 'hop'):
+            continue
+        if other_inst.version == "" or other_inst.version.startswith('ansible-runner'):
+            continue
+        if Version(other_inst.version.split('-', 1)[0]) > Version(awx_application_version.split('-', 1)[0]) and not settings.DEBUG:
+            logger.error(
+                "Host {} reports version {}, but this node {} is at {}, shutting down".format(
+                    other_inst.hostname, other_inst.version, this_inst.hostname, this_inst.version
+                )
+            )
+            # Shutdown signal will set the capacity to zero to ensure no Jobs get added to this instance.
+            # The heartbeat task will reset the capacity to the system capacity after upgrade.
+            stop_local_services(communicate=False)
+            raise RuntimeError("Shutting down.")
+
+    for other_inst in lost_instances:
+        try:
+            reaper.reap(other_inst)
+        except Exception:
+            logger.exception('failed to reap jobs for {}'.format(other_inst.hostname))
+        try:
+            # Capacity could already be 0 because:
+            #  * It's a new node and it never had a heartbeat
+            #  * It was set to 0 by another tower node running this method
+            #  * It was set to 0 by this node, but auto deprovisioning is off
+            #
+            # If auto deprovisioning is on, don't bother setting the capacity to 0
+            # since we will delete the node anyway.
+            if other_inst.capacity != 0 and not settings.AWX_AUTO_DEPROVISION_INSTANCES:
+                other_inst.mark_offline(errors=_('Another cluster node has determined this instance to be unresponsive'))
+                logger.error("Host {} last checked in at {}, marked as lost.".format(other_inst.hostname, other_inst.last_seen))
+            elif settings.AWX_AUTO_DEPROVISION_INSTANCES:
+                deprovision_hostname = other_inst.hostname
+                other_inst.delete()
+                logger.info("Host {} Automatically Deprovisioned.".format(deprovision_hostname))
+        except DatabaseError as e:
+            if 'did not affect any rows' in str(e):
+                logger.debug('Another instance has marked {} as lost'.format(other_inst.hostname))
+            else:
+                logger.exception('Error marking {} as lost'.format(other_inst.hostname))
+
+
+@task(queue=get_local_queuename)
+def awx_receptor_workunit_reaper():
+    """
+    When an AWX job is launched via receptor, files such as status, stdin, and stdout are created
+    in a specific receptor directory. This directory on disk is a random 8 character string, e.g. qLL2JFNT
+    This is also called the work Unit ID in receptor, and is used in various receptor commands,
+    e.g. "work results qLL2JFNT"
+    After an AWX job executes, the receptor work unit directory is cleaned up by
+    issuing the work release command. In some cases the release process might fail, or
+    if AWX crashes during a job's execution, the work release command is never issued to begin with.
+    As such, this periodic task will obtain a list of all receptor work units, and find which ones
+    belong to AWX jobs that are in a completed state (status is canceled, error, or succeeded).
+    This task will call "work release" on each of these work units to clean up the files on disk.
+
+    Note that when we call "work release" on a work unit that actually represents remote work
+    both the local and remote work units are cleaned up.
+
+    Since we are cleaning up jobs that controller considers to be inactive, we take the added
+    precaution of calling "work cancel" in case the work unit is still active.
+    """
+    if not settings.RECEPTOR_RELEASE_WORK:
+        return
+    logger.debug("Checking for unreleased receptor work units")
+    receptor_ctl = get_receptor_ctl()
+    receptor_work_list = receptor_ctl.simple_command("work list")
+
+    unit_ids = [id for id in receptor_work_list]
+    jobs_with_unreleased_receptor_units = UnifiedJob.objects.filter(work_unit_id__in=unit_ids).exclude(status__in=ACTIVE_STATES)
+    for job in jobs_with_unreleased_receptor_units:
+        logger.debug(f"{job.log_format} is not active, reaping receptor work unit {job.work_unit_id}")
+        receptor_ctl.simple_command(f"work cancel {job.work_unit_id}")
+        receptor_ctl.simple_command(f"work release {job.work_unit_id}")
+
+    administrative_workunit_reaper(receptor_work_list)
+
+
+@task(queue=get_local_queuename)
+def awx_k8s_reaper():
+    if not settings.RECEPTOR_RELEASE_WORK:
+        return
+
+    from awx.main.scheduler.kubernetes import PodManager  # prevent circular import
+
+    for group in InstanceGroup.objects.filter(is_container_group=True).iterator():
+        logger.debug("Checking for orphaned k8s pods for {}.".format(group))
+        pods = PodManager.list_active_jobs(group)
+        for job in UnifiedJob.objects.filter(pk__in=pods.keys()).exclude(status__in=ACTIVE_STATES):
+            logger.debug('{} is no longer active, reaping orphaned k8s pod'.format(job.log_format))
+            try:
+                pm = PodManager(job)
+                pm.kube_api.delete_namespaced_pod(name=pods[job.id], namespace=pm.namespace, _request_timeout=settings.AWX_CONTAINER_GROUP_K8S_API_TIMEOUT)
+            except Exception:
+                logger.exception("Failed to delete orphaned pod {} from {}".format(job.log_format, group))
+
+
+@task(queue=get_local_queuename)
+def awx_periodic_scheduler():
+    with advisory_lock('awx_periodic_scheduler_lock', wait=False) as acquired:
+        if acquired is False:
+            logger.debug("Not running periodic scheduler, another task holds lock")
+            return
+        logger.debug("Starting periodic scheduler")
+
+        run_now = now()
+        state = TowerScheduleState.get_solo()
+        last_run = state.schedule_last_run
+        logger.debug("Last scheduler run was: %s", last_run)
+        state.schedule_last_run = run_now
+        state.save()
+
+        old_schedules = Schedule.objects.enabled().before(last_run)
+        for schedule in old_schedules:
+            schedule.update_computed_fields()
+        schedules = Schedule.objects.enabled().between(last_run, run_now)
+
+        invalid_license = False
+        try:
+            access_registry[Job](None).check_license(quiet=True)
+        except PermissionDenied as e:
+            invalid_license = e
+
+        for schedule in schedules:
+            template = schedule.unified_job_template
+            schedule.update_computed_fields()  # To update next_run timestamp.
+            if template.cache_timeout_blocked:
+                logger.warn("Cache timeout is in the future, bypassing schedule for template %s" % str(template.id))
+                continue
+            try:
+                job_kwargs = schedule.get_job_kwargs()
+                new_unified_job = schedule.unified_job_template.create_unified_job(**job_kwargs)
+                logger.debug('Spawned {} from schedule {}-{}.'.format(new_unified_job.log_format, schedule.name, schedule.pk))
+
+                if invalid_license:
+                    new_unified_job.status = 'failed'
+                    new_unified_job.job_explanation = str(invalid_license)
+                    new_unified_job.save(update_fields=['status', 'job_explanation'])
+                    new_unified_job.websocket_emit_status("failed")
+                    raise invalid_license
+                can_start = new_unified_job.signal_start()
+            except Exception:
+                logger.exception('Error spawning scheduled job.')
+                continue
+            if not can_start:
+                new_unified_job.status = 'failed'
+                new_unified_job.job_explanation = gettext_noop(
+                    "Scheduled job could not start because it \
+                    was not in the right state or required manual credentials"
+                )
+                new_unified_job.save(update_fields=['status', 'job_explanation'])
+                new_unified_job.websocket_emit_status("failed")
+            emit_channel_notification('schedules-changed', dict(id=schedule.id, group_name="schedules"))
+        state.save()
+
+
+@task(queue=get_local_queuename)
+def handle_work_success(task_actual):
+    try:
+        instance = UnifiedJob.get_instance_by_type(task_actual['type'], task_actual['id'])
+    except ObjectDoesNotExist:
+        logger.warning('Missing {} `{}` in success callback.'.format(task_actual['type'], task_actual['id']))
+        return
+    if not instance:
+        return
+
+    schedule_task_manager()
+
+
+@task(queue=get_local_queuename)
+def handle_work_error(task_id, *args, **kwargs):
+    subtasks = kwargs.get('subtasks', None)
+    logger.debug('Executing error task id %s, subtasks: %s' % (task_id, str(subtasks)))
+    first_instance = None
+    first_instance_type = ''
+    if subtasks is not None:
+        for each_task in subtasks:
+            try:
+                instance = UnifiedJob.get_instance_by_type(each_task['type'], each_task['id'])
+                if not instance:
+                    # Unknown task type
+                    logger.warn("Unknown task type: {}".format(each_task['type']))
+                    continue
+            except ObjectDoesNotExist:
+                logger.warning('Missing {} `{}` in error callback.'.format(each_task['type'], each_task['id']))
+                continue
+
+            if first_instance is None:
+                first_instance = instance
+                first_instance_type = each_task['type']
+
+            if instance.celery_task_id != task_id and not instance.cancel_flag and not instance.status == 'successful':
+                instance.status = 'failed'
+                instance.failed = True
+                if not instance.job_explanation:
+                    instance.job_explanation = 'Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (
+                        first_instance_type,
+                        first_instance.name,
+                        first_instance.id,
+                    )
+                instance.save()
+                instance.websocket_emit_status("failed")
+
+    # We only send 1 job complete message since all the job completion message
+    # handling does is trigger the scheduler. If we extend the functionality of
+    # what the job complete message handler does then we may want to send a
+    # completion event for each job here.
+    if first_instance:
+        schedule_task_manager()
+        pass
+
+
+@task(queue=get_local_queuename)
+def handle_success_and_failure_notifications(job_id):
+    uj = UnifiedJob.objects.get(pk=job_id)
+    retries = 0
+    while retries < 5:
+        if uj.finished:
+            uj.send_notification_templates('succeeded' if uj.status == 'successful' else 'failed')
+            return
+        else:
+            # wait a few seconds to avoid a race where the
+            # events are persisted _before_ the UJ.status
+            # changes from running -> successful
+            retries += 1
+            time.sleep(1)
+            uj = UnifiedJob.objects.get(pk=job_id)
+
+    logger.warn(f"Failed to even try to send notifications for job '{uj}' due to job not being in finished state.")
+
+
+@task(queue=get_local_queuename)
+def update_inventory_computed_fields(inventory_id):
+    """
+    Signal handler and wrapper around inventory.update_computed_fields to
+    prevent unnecessary recursive calls.
+    """
+    i = Inventory.objects.filter(id=inventory_id)
+    if not i.exists():
+        logger.error("Update Inventory Computed Fields failed due to missing inventory: " + str(inventory_id))
+        return
+    i = i[0]
+    try:
+        i.update_computed_fields()
+    except DatabaseError as e:
+        if 'did not affect any rows' in str(e):
+            logger.debug('Exiting duplicate update_inventory_computed_fields task.')
+            return
+        raise
+
+
+def update_smart_memberships_for_inventory(smart_inventory):
+    current = set(SmartInventoryMembership.objects.filter(inventory=smart_inventory).values_list('host_id', flat=True))
+    new = set(smart_inventory.hosts.values_list('id', flat=True))
+    additions = new - current
+    removals = current - new
+    if additions or removals:
+        with transaction.atomic():
+            if removals:
+                SmartInventoryMembership.objects.filter(inventory=smart_inventory, host_id__in=removals).delete()
+            if additions:
+                add_for_inventory = [SmartInventoryMembership(inventory_id=smart_inventory.id, host_id=host_id) for host_id in additions]
+                SmartInventoryMembership.objects.bulk_create(add_for_inventory, ignore_conflicts=True)
+        logger.debug(
+            'Smart host membership cached for {}, {} additions, {} removals, {} total count.'.format(
+                smart_inventory.pk, len(additions), len(removals), len(new)
+            )
+        )
+        return True  # changed
+    return False
+
+
+@task(queue=get_local_queuename)
+def update_host_smart_inventory_memberships():
+    smart_inventories = Inventory.objects.filter(kind='smart', host_filter__isnull=False, pending_deletion=False)
+    changed_inventories = set([])
+    for smart_inventory in smart_inventories:
+        try:
+            changed = update_smart_memberships_for_inventory(smart_inventory)
+            if changed:
+                changed_inventories.add(smart_inventory)
+        except IntegrityError:
+            logger.exception('Failed to update smart inventory memberships for {}'.format(smart_inventory.pk))
+    # Update computed fields for changed inventories outside atomic action
+    for smart_inventory in changed_inventories:
+        smart_inventory.update_computed_fields()
+
+
+@task(queue=get_local_queuename)
+def delete_inventory(inventory_id, user_id, retries=5):
+    # Delete inventory as user
+    if user_id is None:
+        user = None
+    else:
+        try:
+            user = User.objects.get(id=user_id)
+        except Exception:
+            user = None
+    with ignore_inventory_computed_fields(), ignore_inventory_group_removal(), impersonate(user):
+        try:
+            i = Inventory.objects.get(id=inventory_id)
+            for host in i.hosts.iterator():
+                host.job_events_as_primary_host.update(host=None)
+            i.delete()
+            emit_channel_notification('inventories-status_changed', {'group_name': 'inventories', 'inventory_id': inventory_id, 'status': 'deleted'})
+            logger.debug('Deleted inventory {} as user {}.'.format(inventory_id, user_id))
+        except Inventory.DoesNotExist:
+            logger.exception("Delete Inventory failed due to missing inventory: " + str(inventory_id))
+            return
+        except DatabaseError:
+            logger.exception('Database error deleting inventory {}, but will retry.'.format(inventory_id))
+            if retries > 0:
+                time.sleep(10)
+                delete_inventory(inventory_id, user_id, retries=retries - 1)
+
+
+def with_path_cleanup(f):
+    @functools.wraps(f)
+    def _wrapped(self, *args, **kwargs):
+        try:
+            return f(self, *args, **kwargs)
+        finally:
+            for p in self.cleanup_paths:
+                try:
+                    if os.path.isdir(p):
+                        shutil.rmtree(p, ignore_errors=True)
+                    elif os.path.exists(p):
+                        os.remove(p)
+                except OSError:
+                    logger.exception("Failed to remove tmp file: {}".format(p))
+            self.cleanup_paths = []
+
+    return _wrapped
+
+
+def _reconstruct_relationships(copy_mapping):
+    for old_obj, new_obj in copy_mapping.items():
+        model = type(old_obj)
+        for field_name in getattr(model, 'FIELDS_TO_PRESERVE_AT_COPY', []):
+            field = model._meta.get_field(field_name)
+            if isinstance(field, ForeignKey):
+                if getattr(new_obj, field_name, None):
+                    continue
+                related_obj = getattr(old_obj, field_name)
+                related_obj = copy_mapping.get(related_obj, related_obj)
+                setattr(new_obj, field_name, related_obj)
+            elif field.many_to_many:
+                for related_obj in getattr(old_obj, field_name).all():
+                    logger.debug('Deep copy: Adding {} to {}({}).{} relationship'.format(related_obj, new_obj, model, field_name))
+                    getattr(new_obj, field_name).add(copy_mapping.get(related_obj, related_obj))
+        new_obj.save()
+
+
+@task(queue=get_local_queuename)
+def deep_copy_model_obj(model_module, model_name, obj_pk, new_obj_pk, user_pk, uuid, permission_check_func=None):
+    sub_obj_list = cache.get(uuid)
+    if sub_obj_list is None:
+        logger.error('Deep copy {} from {} to {} failed unexpectedly.'.format(model_name, obj_pk, new_obj_pk))
+        return
+
+    logger.debug('Deep copy {} from {} to {}.'.format(model_name, obj_pk, new_obj_pk))
+    from awx.api.generics import CopyAPIView
+    from awx.main.signals import disable_activity_stream
+
+    model = getattr(importlib.import_module(model_module), model_name, None)
+    if model is None:
+        return
+    try:
+        obj = model.objects.get(pk=obj_pk)
+        new_obj = model.objects.get(pk=new_obj_pk)
+        creater = User.objects.get(pk=user_pk)
+    except ObjectDoesNotExist:
+        logger.warning("Object or user no longer exists.")
+        return
+    with transaction.atomic(), ignore_inventory_computed_fields(), disable_activity_stream():
+        copy_mapping = {}
+        for sub_obj_setup in sub_obj_list:
+            sub_model = getattr(importlib.import_module(sub_obj_setup[0]), sub_obj_setup[1], None)
+            if sub_model is None:
+                continue
+            try:
+                sub_obj = sub_model.objects.get(pk=sub_obj_setup[2])
+            except ObjectDoesNotExist:
+                continue
+            copy_mapping.update(CopyAPIView.copy_model_obj(obj, new_obj, sub_model, sub_obj, creater))
+        _reconstruct_relationships(copy_mapping)
+        if permission_check_func:
+            permission_check_func = getattr(getattr(importlib.import_module(permission_check_func[0]), permission_check_func[1]), permission_check_func[2])
+            permission_check_func(creater, copy_mapping.values())
+    if isinstance(new_obj, Inventory):
+        update_inventory_computed_fields.delay(new_obj.id)

awx/main/tests/conftest.py

@@ -15,6 +15,7 @@ from awx.main.tests.factories import (
 )
 
 from django.core.cache import cache
+from django.conf import settings
 
 
 def pytest_addoption(parser):
@@ -80,13 +81,44 @@ def instance_group_factory():
 
 
 @pytest.fixture
-def default_instance_group(instance_factory, instance_group_factory):
-    return create_instance_group("default", instances=[create_instance("hostA")])
+def controlplane_instance_group(instance_factory, instance_group_factory):
+    """There always has to be a controlplane instancegroup and at least one instance in it"""
+    return create_instance_group(settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME, create_instance('hybrid-1', node_type='hybrid', capacity=500))
 
 
 @pytest.fixture
-def controlplane_instance_group(instance_factory, instance_group_factory):
-    return create_instance_group("controlplane", instances=[create_instance("hostA")])
+def default_instance_group(instance_factory, instance_group_factory):
+    return create_instance_group("default", instances=[create_instance("hostA", node_type='execution')])
+
+
+@pytest.fixture
+def control_instance():
+    '''Control instance in the controlplane automatic IG'''
+    inst = create_instance('control-1', node_type='control', capacity=500)
+    return inst
+
+
+@pytest.fixture
+def control_instance_low_capacity():
+    '''Control instance in the controlplane automatic IG that has low capacity'''
+    inst = create_instance('control-1', node_type='control', capacity=5)
+    return inst
+
+
+@pytest.fixture
+def execution_instance():
+    '''Execution node in the automatic default IG'''
+    ig = create_instance_group('default')
+    inst = create_instance('receptor-1', node_type='execution', capacity=500)
+    ig.instances.add(inst)
+    return inst
+
+
+@pytest.fixture
+def hybrid_instance():
+    '''Hybrid node in the default controlplane IG'''
+    inst = create_instance('hybrid-1', node_type='hybrid', capacity=500)
+    return inst
 
 
 @pytest.fixture

awx/main/tests/factories/fixtures.py

@@ -28,12 +28,15 @@ from awx.main.models import (
 #
 
 
-def mk_instance(persisted=True, hostname='instance.example.org'):
+def mk_instance(persisted=True, hostname='instance.example.org', node_type='hybrid', capacity=100):
     if not persisted:
         raise RuntimeError('creating an Instance requires persisted=True')
     from django.conf import settings
 
-    return Instance.objects.get_or_create(uuid=settings.SYSTEM_UUID, hostname=hostname)[0]
+    instance = Instance.objects.get_or_create(uuid=settings.SYSTEM_UUID, hostname=hostname, node_type=node_type, capacity=capacity)[0]
+    if node_type in ('control', 'hybrid'):
+        mk_instance_group(name=settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME, instance=instance)
+    return instance
 
 
 def mk_instance_group(name='default', instance=None, minimum=0, percentage=0):
@@ -52,7 +55,9 @@ def mk_organization(name, description=None, persisted=True):
     description = description or '{}-description'.format(name)
     org = Organization(name=name, description=description)
     if persisted:
-        mk_instance(persisted)
+        instances = Instance.objects.all()
+        if not instances:
+            mk_instance(persisted)
         org.save()
     return org
 

awx/main/tests/factories/tower.py

@@ -132,8 +132,8 @@ def generate_teams(organization, persisted, **kwargs):
     return teams
 
 
-def create_instance(name, instance_groups=None):
-    return mk_instance(hostname=name)
+def create_instance(name, instance_groups=None, node_type='hybrid', capacity=200):
+    return mk_instance(hostname=name, node_type=node_type, capacity=capacity)
 
 
 def create_instance_group(name, instances=None, minimum=0, percentage=0):

awx/main/tests/functional/api/test_instance.py

@@ -62,7 +62,7 @@ def test_health_check_throws_error(post, admin_user):
     # we will simulate a receptor error, similar to this one
     # https://github.com/ansible/receptor/blob/156e6e24a49fbf868734507f9943ac96208ed8f5/receptorctl/receptorctl/socket_interface.py#L204
     # related to issue https://github.com/ansible/tower/issues/5315
-    with mock.patch('awx.main.utils.receptor.run_until_complete', side_effect=RuntimeError('Remote error: foobar')):
+    with mock.patch('awx.main.tasks.receptor.run_until_complete', side_effect=RuntimeError('Remote error: foobar')):
         post(url=url, user=admin_user, expect=200)
     instance.refresh_from_db()
     assert 'Remote error: foobar' in instance.errors

awx/main/tests/functional/api/test_workflow_node.py

@@ -127,7 +127,7 @@ class TestApprovalNodes:
         ]
 
     @pytest.mark.django_db
-    def test_approval_node_approve(self, post, admin_user, job_template):
+    def test_approval_node_approve(self, post, admin_user, job_template, controlplane_instance_group):
         # This test ensures that a user (with permissions to do so) can APPROVE
         # workflow approvals.  Also asserts that trying to APPROVE approvals
         # that have already been dealt with will throw an error.
@@ -152,7 +152,7 @@ class TestApprovalNodes:
         post(reverse('api:workflow_approval_approve', kwargs={'pk': approval.pk}), user=admin_user, expect=400)
 
     @pytest.mark.django_db
-    def test_approval_node_deny(self, post, admin_user, job_template):
+    def test_approval_node_deny(self, post, admin_user, job_template, controlplane_instance_group):
         # This test ensures that a user (with permissions to do so) can DENY
         # workflow approvals.  Also asserts that trying to DENY approvals
         # that have already been dealt with will throw an error.

awx/main/tests/functional/commands/test_secret_key_regeneration.py

@@ -3,6 +3,8 @@ import json
 from cryptography.fernet import InvalidToken
 from django.test.utils import override_settings
 from django.conf import settings
+from django.core.management import call_command
+import os
 import pytest
 
 from awx.main import models
@@ -158,3 +160,25 @@ class TestKeyRegeneration:
         # verify that the new SECRET_KEY *does* work
         with override_settings(SECRET_KEY=new_key):
             assert models.OAuth2Application.objects.get(pk=oauth_application.pk).client_secret == secret
+
+    def test_use_custom_key_with_tower_secret_key_env_var(self):
+        custom_key = 'MXSq9uqcwezBOChl/UfmbW1k4op+bC+FQtwPqgJ1u9XV'
+        os.environ['TOWER_SECRET_KEY'] = custom_key
+        new_key = call_command('regenerate_secret_key', '--use-custom-key')
+        assert custom_key == new_key
+
+    def test_use_custom_key_with_empty_tower_secret_key_env_var(self):
+        os.environ['TOWER_SECRET_KEY'] = ''
+        new_key = call_command('regenerate_secret_key', '--use-custom-key')
+        assert settings.SECRET_KEY != new_key
+
+    def test_use_custom_key_with_no_tower_secret_key_env_var(self):
+        os.environ.pop('TOWER_SECRET_KEY', None)
+        new_key = call_command('regenerate_secret_key', '--use-custom-key')
+        assert settings.SECRET_KEY != new_key
+
+    def test_with_tower_secret_key_env_var(self):
+        custom_key = 'MXSq9uqcwezBOChl/UfmbW1k4op+bC+FQtwPqgJ1u9XV'
+        os.environ['TOWER_SECRET_KEY'] = custom_key
+        new_key = call_command('regenerate_secret_key')
+        assert custom_key != new_key

awx/main/tests/functional/models/test_schedule.py

@@ -308,7 +308,7 @@ def test_beginning_of_time(job_template):
     'rrule, tz',
     [
         ['DTSTART:20300112T210000Z RRULE:FREQ=DAILY;INTERVAL=1', 'UTC'],
-        ['DTSTART;TZID=America/New_York:20300112T210000 RRULE:FREQ=DAILY;INTERVAL=1', 'America/New_York'],
+        ['DTSTART;TZID=US/Eastern:20300112T210000 RRULE:FREQ=DAILY;INTERVAL=1', 'US/Eastern'],
     ],
 )
 def test_timezone_property(job_template, rrule, tz):

awx/main/tests/functional/task_management/test_container_groups.py

@@ -5,7 +5,7 @@ from collections import namedtuple
 from unittest import mock  # noqa
 import pytest
 
-from awx.main.tasks import AWXReceptorJob
+from awx.main.tasks.receptor import AWXReceptorJob
 from awx.main.utils import (
     create_temporary_fifo,
 )

awx/main/tests/functional/task_management/test_rampart_groups.py

@@ -3,11 +3,11 @@ from unittest import mock
 from datetime import timedelta
 from awx.main.scheduler import TaskManager
 from awx.main.models import InstanceGroup, WorkflowJob
-from awx.main.tasks import apply_cluster_membership_policies
+from awx.main.tasks.system import apply_cluster_membership_policies
 
 
 @pytest.mark.django_db
-def test_multi_group_basic_job_launch(instance_factory, default_instance_group, mocker, instance_group_factory, job_template_factory):
+def test_multi_group_basic_job_launch(instance_factory, controlplane_instance_group, mocker, instance_group_factory, job_template_factory):
     i1 = instance_factory("i1")
     i2 = instance_factory("i2")
     ig1 = instance_group_factory("ig1", instances=[i1])
@@ -67,7 +67,7 @@ def test_multi_group_with_shared_dependency(instance_factory, controlplane_insta
 
 
 @pytest.mark.django_db
-def test_workflow_job_no_instancegroup(workflow_job_template_factory, default_instance_group, mocker):
+def test_workflow_job_no_instancegroup(workflow_job_template_factory, controlplane_instance_group, mocker):
     wfjt = workflow_job_template_factory('anicedayforawalk').workflow_job_template
     wfj = WorkflowJob.objects.create(workflow_job_template=wfjt)
     wfj.status = "pending"
@@ -79,9 +79,10 @@ def test_workflow_job_no_instancegroup(workflow_job_template_factory, default_in
 
 
 @pytest.mark.django_db
-def test_overcapacity_blocking_other_groups_unaffected(instance_factory, default_instance_group, mocker, instance_group_factory, job_template_factory):
+def test_overcapacity_blocking_other_groups_unaffected(instance_factory, controlplane_instance_group, mocker, instance_group_factory, job_template_factory):
     i1 = instance_factory("i1")
-    i1.capacity = 1000
+    # need to account a little extra for controller node capacity impact
+    i1.capacity = 1020
     i1.save()
     i2 = instance_factory("i2")
     ig1 = instance_group_factory("ig1", instances=[i1])
@@ -120,7 +121,7 @@ def test_overcapacity_blocking_other_groups_unaffected(instance_factory, default
 
 
 @pytest.mark.django_db
-def test_failover_group_run(instance_factory, default_instance_group, mocker, instance_group_factory, job_template_factory):
+def test_failover_group_run(instance_factory, controlplane_instance_group, mocker, instance_group_factory, job_template_factory):
     i1 = instance_factory("i1")
     i2 = instance_factory("i2")
     ig1 = instance_group_factory("ig1", instances=[i1])

awx/main/tests/functional/task_management/test_scheduler.py

@@ -7,19 +7,20 @@ from awx.main.scheduler import TaskManager
 from awx.main.scheduler.dependency_graph import DependencyGraph
 from awx.main.utils import encrypt_field
 from awx.main.models import WorkflowJobTemplate, JobTemplate, Job
-from awx.main.models.ha import Instance, InstanceGroup
+from awx.main.models.ha import Instance
+from django.conf import settings
 
 
 @pytest.mark.django_db
-def test_single_job_scheduler_launch(default_instance_group, job_template_factory, mocker):
-    instance = default_instance_group.instances.all()[0]
+def test_single_job_scheduler_launch(hybrid_instance, controlplane_instance_group, job_template_factory, mocker):
+    instance = controlplane_instance_group.instances.all()[0]
     objects = job_template_factory('jt', organization='org1', project='proj', inventory='inv', credential='cred', jobs=["job_should_start"])
     j = objects.jobs["job_should_start"]
     j.status = 'pending'
     j.save()
     with mocker.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
-        TaskManager.start_task.assert_called_once_with(j, default_instance_group, [], instance)
+        TaskManager.start_task.assert_called_once_with(j, controlplane_instance_group, [], instance)
 
 
 @pytest.mark.django_db
@@ -47,7 +48,7 @@ class TestJobLifeCycle:
                     if expect_commit is not None:
                         assert mock_commit.mock_calls == expect_commit
 
-    def test_task_manager_workflow_rescheduling(self, job_template_factory, inventory, project, default_instance_group):
+    def test_task_manager_workflow_rescheduling(self, job_template_factory, inventory, project, controlplane_instance_group):
         jt = JobTemplate.objects.create(allow_simultaneous=True, inventory=inventory, project=project, playbook='helloworld.yml')
         wfjt = WorkflowJobTemplate.objects.create(name='foo')
         for i in range(2):
@@ -80,7 +81,7 @@ class TestJobLifeCycle:
         # no further action is necessary, so rescheduling should not happen
         self.run_tm(tm, [mock.call('successful')], [])
 
-    def test_task_manager_workflow_workflow_rescheduling(self):
+    def test_task_manager_workflow_workflow_rescheduling(self, controlplane_instance_group):
         wfjts = [WorkflowJobTemplate.objects.create(name='foo')]
         for i in range(5):
             wfjt = WorkflowJobTemplate.objects.create(name='foo{}'.format(i))
@@ -100,22 +101,6 @@ class TestJobLifeCycle:
                 self.run_tm(tm, expect_schedule=[mock.call()])
             wfjts[0].refresh_from_db()
 
-    @pytest.fixture
-    def control_instance(self):
-        '''Control instance in the controlplane automatic IG'''
-        ig = InstanceGroup.objects.create(name='controlplane')
-        inst = Instance.objects.create(hostname='control-1', node_type='control', capacity=500)
-        ig.instances.add(inst)
-        return inst
-
-    @pytest.fixture
-    def execution_instance(self):
-        '''Execution node in the automatic default IG'''
-        ig = InstanceGroup.objects.create(name='default')
-        inst = Instance.objects.create(hostname='receptor-1', node_type='execution', capacity=500)
-        ig.instances.add(inst)
-        return inst
-
     def test_control_and_execution_instance(self, project, system_job_template, job_template, inventory_source, control_instance, execution_instance):
         assert Instance.objects.count() == 2
 
@@ -142,10 +127,78 @@ class TestJobLifeCycle:
             assert uj.capacity_type == 'execution'
             assert [uj.execution_node, uj.controller_node] == [execution_instance.hostname, control_instance.hostname], uj
 
+    @pytest.mark.django_db
+    def test_job_fails_to_launch_when_no_control_capacity(self, job_template, control_instance_low_capacity, execution_instance):
+        enough_capacity = job_template.create_unified_job()
+        insufficient_capacity = job_template.create_unified_job()
+        all_ujs = [enough_capacity, insufficient_capacity]
+        for uj in all_ujs:
+            uj.signal_start()
+
+        # There is only enough control capacity to run one of the jobs so one should end up in pending and the other in waiting
+        tm = TaskManager()
+        self.run_tm(tm)
+
+        for uj in all_ujs:
+            uj.refresh_from_db()
+        assert enough_capacity.status == 'waiting'
+        assert insufficient_capacity.status == 'pending'
+        assert [enough_capacity.execution_node, enough_capacity.controller_node] == [
+            execution_instance.hostname,
+            control_instance_low_capacity.hostname,
+        ], enough_capacity
+
+    @pytest.mark.django_db
+    def test_hybrid_capacity(self, job_template, hybrid_instance):
+        enough_capacity = job_template.create_unified_job()
+        insufficient_capacity = job_template.create_unified_job()
+        expected_task_impact = enough_capacity.task_impact + settings.AWX_CONTROL_NODE_TASK_IMPACT
+        all_ujs = [enough_capacity, insufficient_capacity]
+        for uj in all_ujs:
+            uj.signal_start()
+
+        # There is only enough control capacity to run one of the jobs so one should end up in pending and the other in waiting
+        tm = TaskManager()
+        self.run_tm(tm)
+
+        for uj in all_ujs:
+            uj.refresh_from_db()
+        assert enough_capacity.status == 'waiting'
+        assert insufficient_capacity.status == 'pending'
+        assert [enough_capacity.execution_node, enough_capacity.controller_node] == [
+            hybrid_instance.hostname,
+            hybrid_instance.hostname,
+        ], enough_capacity
+        assert expected_task_impact == hybrid_instance.consumed_capacity
+
+    @pytest.mark.django_db
+    def test_project_update_capacity(self, project, hybrid_instance, instance_group_factory, controlplane_instance_group):
+        pu = project.create_unified_job()
+        instance_group_factory(name='second_ig', instances=[hybrid_instance])
+        expected_task_impact = pu.task_impact + settings.AWX_CONTROL_NODE_TASK_IMPACT
+        pu.signal_start()
+
+        tm = TaskManager()
+        self.run_tm(tm)
+
+        pu.refresh_from_db()
+        assert pu.status == 'waiting'
+        assert [pu.execution_node, pu.controller_node] == [
+            hybrid_instance.hostname,
+            hybrid_instance.hostname,
+        ], pu
+        assert expected_task_impact == hybrid_instance.consumed_capacity
+        # The hybrid node is in both instance groups, but the project update should
+        # always get assigned to the controlplane
+        assert pu.instance_group.name == settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME
+        pu.status = 'successful'
+        pu.save()
+        assert hybrid_instance.consumed_capacity == 0
+
 
 @pytest.mark.django_db
-def test_single_jt_multi_job_launch_blocks_last(default_instance_group, job_template_factory, mocker):
-    instance = default_instance_group.instances.all()[0]
+def test_single_jt_multi_job_launch_blocks_last(controlplane_instance_group, job_template_factory, mocker):
+    instance = controlplane_instance_group.instances.all()[0]
     objects = job_template_factory(
         'jt', organization='org1', project='proj', inventory='inv', credential='cred', jobs=["job_should_start", "job_should_not_start"]
     )
@@ -157,17 +210,17 @@ def test_single_jt_multi_job_launch_blocks_last(default_instance_group, job_temp
     j2.save()
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
-        TaskManager.start_task.assert_called_once_with(j1, default_instance_group, [], instance)
+        TaskManager.start_task.assert_called_once_with(j1, controlplane_instance_group, [], instance)
         j1.status = "successful"
         j1.save()
     with mocker.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
-        TaskManager.start_task.assert_called_once_with(j2, default_instance_group, [], instance)
+        TaskManager.start_task.assert_called_once_with(j2, controlplane_instance_group, [], instance)
 
 
 @pytest.mark.django_db
-def test_single_jt_multi_job_launch_allow_simul_allowed(default_instance_group, job_template_factory, mocker):
-    instance = default_instance_group.instances.all()[0]
+def test_single_jt_multi_job_launch_allow_simul_allowed(controlplane_instance_group, job_template_factory, mocker):
+    instance = controlplane_instance_group.instances.all()[0]
     objects = job_template_factory(
         'jt', organization='org1', project='proj', inventory='inv', credential='cred', jobs=["job_should_start", "job_should_not_start"]
     )
@@ -184,12 +237,15 @@ def test_single_jt_multi_job_launch_allow_simul_allowed(default_instance_group,
     j2.save()
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
-        TaskManager.start_task.assert_has_calls([mock.call(j1, default_instance_group, [], instance), mock.call(j2, default_instance_group, [], instance)])
+        TaskManager.start_task.assert_has_calls(
+            [mock.call(j1, controlplane_instance_group, [], instance), mock.call(j2, controlplane_instance_group, [], instance)]
+        )
 
 
 @pytest.mark.django_db
-def test_multi_jt_capacity_blocking(default_instance_group, job_template_factory, mocker):
-    instance = default_instance_group.instances.all()[0]
+def test_multi_jt_capacity_blocking(hybrid_instance, job_template_factory, mocker):
+    instance = hybrid_instance
+    controlplane_instance_group = instance.rampart_groups.first()
     objects1 = job_template_factory('jt1', organization='org1', project='proj1', inventory='inv1', credential='cred1', jobs=["job_should_start"])
     objects2 = job_template_factory('jt2', organization='org2', project='proj2', inventory='inv2', credential='cred2', jobs=["job_should_not_start"])
     j1 = objects1.jobs["job_should_start"]
@@ -200,15 +256,15 @@ def test_multi_jt_capacity_blocking(default_instance_group, job_template_factory
     j2.save()
     tm = TaskManager()
     with mock.patch('awx.main.models.Job.task_impact', new_callable=mock.PropertyMock) as mock_task_impact:
-        mock_task_impact.return_value = 500
+        mock_task_impact.return_value = 505
         with mock.patch.object(TaskManager, "start_task", wraps=tm.start_task) as mock_job:
             tm.schedule()
-            mock_job.assert_called_once_with(j1, default_instance_group, [], instance)
+            mock_job.assert_called_once_with(j1, controlplane_instance_group, [], instance)
             j1.status = "successful"
             j1.save()
     with mock.patch.object(TaskManager, "start_task", wraps=tm.start_task) as mock_job:
         tm.schedule()
-        mock_job.assert_called_once_with(j2, default_instance_group, [], instance)
+        mock_job.assert_called_once_with(j2, controlplane_instance_group, [], instance)
 
 
 @pytest.mark.django_db
@@ -240,9 +296,9 @@ def test_single_job_dependencies_project_launch(controlplane_instance_group, job
 
 
 @pytest.mark.django_db
-def test_single_job_dependencies_inventory_update_launch(default_instance_group, job_template_factory, mocker, inventory_source_factory):
+def test_single_job_dependencies_inventory_update_launch(controlplane_instance_group, job_template_factory, mocker, inventory_source_factory):
     objects = job_template_factory('jt', organization='org1', project='proj', inventory='inv', credential='cred', jobs=["job_should_start"])
-    instance = default_instance_group.instances.all()[0]
+    instance = controlplane_instance_group.instances.all()[0]
     j = objects.jobs["job_should_start"]
     j.status = 'pending'
     j.save()
@@ -260,18 +316,18 @@ def test_single_job_dependencies_inventory_update_launch(default_instance_group,
             mock_iu.assert_called_once_with(j, ii)
             iu = [x for x in ii.inventory_updates.all()]
             assert len(iu) == 1
-            TaskManager.start_task.assert_called_once_with(iu[0], default_instance_group, [j], instance)
+            TaskManager.start_task.assert_called_once_with(iu[0], controlplane_instance_group, [j], instance)
             iu[0].status = "successful"
             iu[0].save()
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
-        TaskManager.start_task.assert_called_once_with(j, default_instance_group, [], instance)
+        TaskManager.start_task.assert_called_once_with(j, controlplane_instance_group, [], instance)
 
 
 @pytest.mark.django_db
-def test_job_dependency_with_already_updated(default_instance_group, job_template_factory, mocker, inventory_source_factory):
+def test_job_dependency_with_already_updated(controlplane_instance_group, job_template_factory, mocker, inventory_source_factory):
     objects = job_template_factory('jt', organization='org1', project='proj', inventory='inv', credential='cred', jobs=["job_should_start"])
-    instance = default_instance_group.instances.all()[0]
+    instance = controlplane_instance_group.instances.all()[0]
     j = objects.jobs["job_should_start"]
     j.status = 'pending'
     j.save()
@@ -293,7 +349,7 @@ def test_job_dependency_with_already_updated(default_instance_group, job_templat
             mock_iu.assert_not_called()
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
-        TaskManager.start_task.assert_called_once_with(j, default_instance_group, [], instance)
+        TaskManager.start_task.assert_called_once_with(j, controlplane_instance_group, [], instance)
 
 
 @pytest.mark.django_db
@@ -349,10 +405,10 @@ def test_shared_dependencies_launch(controlplane_instance_group, job_template_fa
 
 
 @pytest.mark.django_db
-def test_job_not_blocking_project_update(default_instance_group, job_template_factory):
+def test_job_not_blocking_project_update(controlplane_instance_group, job_template_factory):
     objects = job_template_factory('jt', organization='org1', project='proj', inventory='inv', credential='cred', jobs=["job"])
     job = objects.jobs["job"]
-    job.instance_group = default_instance_group
+    job.instance_group = controlplane_instance_group
     job.status = "running"
     job.save()
 
@@ -362,7 +418,7 @@ def test_job_not_blocking_project_update(default_instance_group, job_template_fa
 
         proj = objects.project
         project_update = proj.create_project_update()
-        project_update.instance_group = default_instance_group
+        project_update.instance_group = controlplane_instance_group
         project_update.status = "pending"
         project_update.save()
         assert not task_manager.job_blocked_by(project_update)
@@ -373,10 +429,10 @@ def test_job_not_blocking_project_update(default_instance_group, job_template_fa
 
 
 @pytest.mark.django_db
-def test_job_not_blocking_inventory_update(default_instance_group, job_template_factory, inventory_source_factory):
+def test_job_not_blocking_inventory_update(controlplane_instance_group, job_template_factory, inventory_source_factory):
     objects = job_template_factory('jt', organization='org1', project='proj', inventory='inv', credential='cred', jobs=["job"])
     job = objects.jobs["job"]
-    job.instance_group = default_instance_group
+    job.instance_group = controlplane_instance_group
     job.status = "running"
     job.save()
 
@@ -389,7 +445,7 @@ def test_job_not_blocking_inventory_update(default_instance_group, job_template_
         inv_source.source = "ec2"
         inv.inventory_sources.add(inv_source)
         inventory_update = inv_source.create_inventory_update()
-        inventory_update.instance_group = default_instance_group
+        inventory_update.instance_group = controlplane_instance_group
         inventory_update.status = "pending"
         inventory_update.save()
 

awx/main/tests/functional/test_api_generics.py

@@ -0,0 +1,31 @@
+import pytest
+
+from django.test.utils import override_settings
+from awx.api.versioning import reverse
+
+
+@pytest.mark.django_db
+def test_change_400_error_log(caplog, post, admin_user):
+    with override_settings(API_400_ERROR_LOG_FORMAT='Test'):
+        post(url=reverse('api:setting_logging_test'), data={}, user=admin_user, expect=409)
+        assert 'Test' in caplog.text
+
+
+@pytest.mark.django_db
+def test_bad_400_error_log(caplog, post, admin_user):
+    with override_settings(API_400_ERROR_LOG_FORMAT="Not good {junk}"):
+        post(url=reverse('api:setting_logging_test'), data={}, user=admin_user, expect=409)
+        assert "Unable to format API_400_ERROR_LOG_FORMAT setting, defaulting log message: 'junk'" in caplog.text
+        assert 'status 409 received by user admin attempting to access /api/v2/settings/logging/test/ from 127.0.0.1' in caplog.text
+
+
+@pytest.mark.django_db
+def test_custom_400_error_log(caplog, post, admin_user):
+    with override_settings(API_400_ERROR_LOG_FORMAT="{status_code} {error}"):
+        post(url=reverse('api:setting_logging_test'), data={}, user=admin_user, expect=409)
+        assert '409 Logging not enabled' in caplog.text
+
+
+# The above tests the generation function with a dict/object.
+# The tower-qa test tests.api.inventories.test_inventory_update.TestInventoryUpdate.test_update_all_inventory_sources_with_nonfunctional_sources tests the function with a list
+# Someday it would be nice to test the else condition (not a dict/list) but we need to find an API test which will do this. For now it was added just as a catch all

awx/main/tests/functional/test_copy.py

@@ -5,7 +5,7 @@ from awx.api.versioning import reverse
 from awx.main.utils import decrypt_field
 from awx.main.models.workflow import WorkflowJobTemplate, WorkflowJobTemplateNode, WorkflowApprovalTemplate
 from awx.main.models.jobs import JobTemplate
-from awx.main.tasks import deep_copy_model_obj
+from awx.main.tasks.system import deep_copy_model_obj
 
 
 @pytest.mark.django_db

awx/main/tests/functional/test_instances.py

@@ -1,10 +1,10 @@
 import pytest
 from unittest import mock
 
-from awx.main.models import AdHocCommand, InventoryUpdate, JobTemplate, ProjectUpdate
+from awx.main.models import AdHocCommand, InventoryUpdate, JobTemplate
 from awx.main.models.activity_stream import ActivityStream
 from awx.main.models.ha import Instance, InstanceGroup
-from awx.main.tasks import apply_cluster_membership_policies
+from awx.main.tasks.system import apply_cluster_membership_policies
 from awx.api.versioning import reverse
 
 from django.utils.timezone import now
@@ -77,21 +77,24 @@ def test_instance_dup(org_admin, organization, project, instance_factory, instan
     ig_all = instance_group_factory("all", instances=[i1, i2, i3])
     ig_dup = instance_group_factory("duplicates", instances=[i1])
     project.organization.instance_groups.add(ig_all, ig_dup)
-    actual_num_instances = Instance.objects.active_count()
+    actual_num_instances = Instance.objects.count()
     list_response = get(reverse('api:instance_list'), user=system_auditor)
     api_num_instances_auditor = list(list_response.data.items())[0][1]
 
     list_response2 = get(reverse('api:instance_list'), user=org_admin)
     api_num_instances_oa = list(list_response2.data.items())[0][1]
 
-    assert actual_num_instances == api_num_instances_auditor
-    # Note: The org_admin will not see the default 'tower' node (instance fixture) because it is not in its group, as expected
+    assert api_num_instances_auditor == actual_num_instances
+    # Note: The org_admin will not see the default 'tower' node
+    # (instance fixture) because it is not in its group, as expected
     assert api_num_instances_oa == (actual_num_instances - 1)
 
 
 @pytest.mark.django_db
 def test_policy_instance_few_instances(instance_factory, instance_group_factory):
-    i1 = instance_factory("i1")
+    # we need to use node_type=execution because node_type=hybrid will implicitly
+    # create the controlplane execution group if it doesn't already exist
+    i1 = instance_factory("i1", node_type='execution')
     ig_1 = instance_group_factory("ig1", percentage=25)
     ig_2 = instance_group_factory("ig2", percentage=25)
     ig_3 = instance_group_factory("ig3", percentage=25)
@@ -112,7 +115,7 @@ def test_policy_instance_few_instances(instance_factory, instance_group_factory)
     assert len(ig_4.instances.all()) == 1
     assert i1 in ig_4.instances.all()
 
-    i2 = instance_factory("i2")
+    i2 = instance_factory("i2", node_type='execution')
     count += 1
     apply_cluster_membership_policies()
     assert ActivityStream.objects.count() == count
@@ -333,13 +336,14 @@ def test_mixed_group_membership(instance_factory, instance_group_factory):
 
 @pytest.mark.django_db
 def test_instance_group_capacity(instance_factory, instance_group_factory):
-    i1 = instance_factory("i1")
-    i2 = instance_factory("i2")
-    i3 = instance_factory("i3")
+    node_capacity = 100
+    i1 = instance_factory("i1", capacity=node_capacity)
+    i2 = instance_factory("i2", capacity=node_capacity)
+    i3 = instance_factory("i3", capacity=node_capacity)
     ig_all = instance_group_factory("all", instances=[i1, i2, i3])
-    assert ig_all.capacity == 300
+    assert ig_all.capacity == node_capacity * 3
     ig_single = instance_group_factory("single", instances=[i1])
-    assert ig_single.capacity == 100
+    assert ig_single.capacity == node_capacity
 
 
 @pytest.mark.django_db
@@ -384,16 +388,6 @@ class TestInstanceGroupOrdering:
         # API does not allow setting IGs on inventory source, so ignore those
         assert iu.preferred_instance_groups == [ig_inv, ig_org]
 
-    def test_project_update_instance_groups(self, instance_group_factory, project, controlplane_instance_group):
-        pu = ProjectUpdate.objects.create(project=project, organization=project.organization)
-        assert pu.preferred_instance_groups == [controlplane_instance_group]
-        ig_org = instance_group_factory("OrgIstGrp", [controlplane_instance_group.instances.first()])
-        ig_tmp = instance_group_factory("TmpIstGrp", [controlplane_instance_group.instances.first()])
-        project.organization.instance_groups.add(ig_org)
-        assert pu.preferred_instance_groups == [ig_org, controlplane_instance_group]
-        project.instance_groups.add(ig_tmp)
-        assert pu.preferred_instance_groups == [ig_tmp, ig_org, controlplane_instance_group]
-
     def test_job_instance_groups(self, instance_group_factory, inventory, project, default_instance_group):
         jt = JobTemplate.objects.create(inventory=inventory, project=project)
         job = jt.create_unified_job()

awx/main/tests/functional/test_inventory_source_injectors.py

@@ -5,7 +5,7 @@ import json
 import re
 from collections import namedtuple
 
-from awx.main.tasks import RunInventoryUpdate
+from awx.main.tasks.jobs import RunInventoryUpdate
 from awx.main.models import InventorySource, Credential, CredentialType, UnifiedJob, ExecutionEnvironment
 from awx.main.constants import CLOUD_PROVIDERS, STANDARD_INVENTORY_UPDATE_ENV
 from awx.main.tests import data
@@ -257,6 +257,6 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential
         # Also do not send websocket status updates
         with mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()):
             # The point of this test is that we replace run with assertions
-            with mock.patch('awx.main.tasks.AWXReceptorJob.run', substitute_run):
+            with mock.patch('awx.main.tasks.receptor.AWXReceptorJob.run', substitute_run):
                 # so this sets up everything for a run and then yields control over to substitute_run
                 task.run(inventory_update.pk)

awx/main/tests/functional/test_jobs.py

@@ -4,7 +4,7 @@ from unittest import mock
 import json
 
 from awx.main.models import Job, Instance, JobHostSummary, InventoryUpdate, InventorySource, Project, ProjectUpdate, SystemJob, AdHocCommand
-from awx.main.tasks import cluster_node_heartbeat
+from awx.main.tasks.system import cluster_node_heartbeat
 from django.test.utils import override_settings
 
 
@@ -20,7 +20,7 @@ def test_orphan_unified_job_creation(instance, inventory):
 
 
 @pytest.mark.django_db
-@mock.patch('awx.main.tasks.inspect_execution_nodes', lambda *args, **kwargs: None)
+@mock.patch('awx.main.tasks.system.inspect_execution_nodes', lambda *args, **kwargs: None)
 @mock.patch('awx.main.models.ha.get_cpu_effective_capacity', lambda cpu: 8)
 @mock.patch('awx.main.models.ha.get_mem_effective_capacity', lambda mem: 62)
 def test_job_capacity_and_with_inactive_node():

awx/main/tests/functional/test_tasks.py

@@ -2,7 +2,8 @@ import pytest
 from unittest import mock
 import os
 
-from awx.main.tasks import RunProjectUpdate, RunInventoryUpdate, execution_node_health_check
+from awx.main.tasks.jobs import RunProjectUpdate, RunInventoryUpdate
+from awx.main.tasks.system import execution_node_health_check
 from awx.main.models import ProjectUpdate, InventoryUpdate, InventorySource, Instance
 
 
@@ -49,7 +50,7 @@ class TestDependentInventoryUpdate:
         scm_inventory_source.scm_last_revision = ''
         proj_update = ProjectUpdate.objects.create(project=scm_inventory_source.source_project)
         with mock.patch.object(RunInventoryUpdate, 'run') as iu_run_mock:
-            with mock.patch('awx.main.tasks.create_partition'):
+            with mock.patch('awx.main.tasks.jobs.create_partition'):
                 task._update_dependent_inventories(proj_update, [scm_inventory_source])
                 assert InventoryUpdate.objects.count() == 1
                 inv_update = InventoryUpdate.objects.first()
@@ -73,7 +74,7 @@ class TestDependentInventoryUpdate:
             ProjectUpdate.objects.all().update(cancel_flag=True)
 
         with mock.patch.object(RunInventoryUpdate, 'run') as iu_run_mock:
-            with mock.patch('awx.main.tasks.create_partition'):
+            with mock.patch('awx.main.tasks.jobs.create_partition'):
                 iu_run_mock.side_effect = user_cancels_project
                 task._update_dependent_inventories(proj_update, [is1, is2])
                 # Verify that it bails after 1st update, detecting a cancel

awx/main/tests/unit/models/test_label.py

@@ -3,6 +3,7 @@ from unittest import mock
 
 from awx.main.models.label import Label
 from awx.main.models.unified_jobs import UnifiedJobTemplate, UnifiedJob
+from awx.main.models.inventory import Inventory
 
 
 mock_query_set = mock.MagicMock()
@@ -10,43 +11,45 @@ mock_query_set = mock.MagicMock()
 mock_objects = mock.MagicMock(filter=mock.MagicMock(return_value=mock_query_set))
 
 
+@pytest.mark.django_db
 @mock.patch('awx.main.models.label.Label.objects', mock_objects)
 class TestLabelFilterMocked:
     def test_get_orphaned_labels(self, mocker):
         ret = Label.get_orphaned_labels()
 
         assert mock_query_set == ret
-        Label.objects.filter.assert_called_with(organization=None, unifiedjobtemplate_labels__isnull=True)
+        Label.objects.filter.assert_called_with(organization=None, unifiedjobtemplate_labels__isnull=True, inventory_labels__isnull=True)
 
     def test_is_detached(self, mocker):
-        mock_query_set.count.return_value = 1
+        mock_query_set.exists.return_value = True
 
         label = Label(id=37)
         ret = label.is_detached()
 
         assert ret is True
-        Label.objects.filter.assert_called_with(id=37, unifiedjob_labels__isnull=True, unifiedjobtemplate_labels__isnull=True)
-        mock_query_set.count.assert_called_with()
+        Label.objects.filter.assert_called_with(id=37, unifiedjob_labels__isnull=True, unifiedjobtemplate_labels__isnull=True, inventory_labels__isnull=True)
+        mock_query_set.exists.assert_called_with()
 
     def test_is_detached_not(self, mocker):
-        mock_query_set.count.return_value = 0
+        mock_query_set.exists.return_value = False
 
         label = Label(id=37)
         ret = label.is_detached()
 
         assert ret is False
-        Label.objects.filter.assert_called_with(id=37, unifiedjob_labels__isnull=True, unifiedjobtemplate_labels__isnull=True)
-        mock_query_set.count.assert_called_with()
+        Label.objects.filter.assert_called_with(id=37, unifiedjob_labels__isnull=True, unifiedjobtemplate_labels__isnull=True, inventory_labels__isnull=True)
+        mock_query_set.exists.assert_called_with()
 
     @pytest.mark.parametrize(
-        "jt_count,j_count,expected",
+        "jt_count,j_count,inv_count,expected",
         [
-            (1, 0, True),
-            (0, 1, True),
-            (1, 1, False),
+            (1, 0, 0, True),
+            (0, 1, 0, True),
+            (0, 0, 1, True),
+            (1, 1, 1, False),
         ],
     )
-    def test_is_candidate_for_detach(self, mocker, jt_count, j_count, expected):
+    def test_is_candidate_for_detach(self, mocker, jt_count, j_count, inv_count, expected):
         mock_job_qs = mocker.MagicMock()
         mock_job_qs.count = mocker.MagicMock(return_value=j_count)
         mocker.patch.object(UnifiedJob, 'objects', mocker.MagicMock(filter=mocker.MagicMock(return_value=mock_job_qs)))
@@ -55,12 +58,18 @@ class TestLabelFilterMocked:
         mock_jt_qs.count = mocker.MagicMock(return_value=jt_count)
         mocker.patch.object(UnifiedJobTemplate, 'objects', mocker.MagicMock(filter=mocker.MagicMock(return_value=mock_jt_qs)))
 
+        mock_inv_qs = mocker.MagicMock()
+        mock_inv_qs.count = mocker.MagicMock(return_value=inv_count)
+        mocker.patch.object(Inventory, 'objects', mocker.MagicMock(filter=mocker.MagicMock(return_value=mock_inv_qs)))
+
         label = Label(id=37)
         ret = label.is_candidate_for_detach()
 
         UnifiedJob.objects.filter.assert_called_with(labels__in=[label.id])
         UnifiedJobTemplate.objects.filter.assert_called_with(labels__in=[label.id])
+        Inventory.objects.filter.assert_called_with(labels__in=[label.id])
         mock_job_qs.count.assert_called_with()
         mock_jt_qs.count.assert_called_with()
+        mock_inv_qs.count.assert_called_with()
 
         assert ret is expected

awx/main/tests/unit/models/test_survey_models.py

@@ -59,6 +59,38 @@ class SurveyVariableValidation:
         assert accepted == {}
         assert str(errors[0]) == "Value 5 for 'a' expected to be a string."
 
+    def test_job_template_survey_default_variable_validation(self, job_template_factory):
+        objects = job_template_factory(
+            "survey_variable_validation",
+            organization="org1",
+            inventory="inventory1",
+            credential="cred1",
+            persisted=False,
+        )
+        obj = objects.job_template
+        obj.survey_spec = {
+            "description": "",
+            "spec": [
+                {
+                    "required": True,
+                    "min": 0,
+                    "default": "2",
+                    "max": 1024,
+                    "question_description": "",
+                    "choices": "",
+                    "variable": "a",
+                    "question_name": "float_number",
+                    "type": "float",
+                }
+            ],
+            "name": "",
+        }
+
+        obj.survey_enabled = True
+        accepted, _, errors = obj.accept_or_ignore_variables({"a": 2})
+        assert accepted == {{"a": 2.0}}
+        assert not errors
+
 
 @pytest.fixture
 def job(mocker):

awx/main/tests/unit/settings/test_defaults.py

@@ -7,7 +7,7 @@ from datetime import timedelta
 @pytest.mark.parametrize(
     "job_name,function_path",
     [
-        ('tower_scheduler', 'awx.main.tasks.awx_periodic_scheduler'),
+        ('tower_scheduler', 'awx.main.tasks.system.awx_periodic_scheduler'),
     ],
 )
 def test_CELERYBEAT_SCHEDULE(mocker, job_name, function_path):

awx/main/tests/unit/settings/test_k8s_resource_setttings.py

@@ -0,0 +1,61 @@
+import pytest
+
+from unittest import mock
+
+from awx.main.utils.common import (
+    convert_mem_str_to_bytes,
+    get_mem_effective_capacity,
+    get_corrected_memory,
+    convert_cpu_str_to_decimal_cpu,
+    get_cpu_effective_capacity,
+    get_corrected_cpu,
+)
+
+
+@pytest.mark.parametrize(
+    "value,converted_value,mem_capacity",
+    [
+        ('2G', 2000000000, 19),
+        ('4G', 4000000000, 38),
+        ('2Gi', 2147483648, 20),
+        ('2.1G', 1, 1),  # expressing memory with non-integers is not supported, and we'll fall back to 1 fork for memory capacity.
+        ('4Gi', 4294967296, 40),
+        ('2M', 2000000, 1),
+        ('3M', 3000000, 1),
+        ('2Mi', 2097152, 1),
+        ('2048Mi', 2147483648, 20),
+        ('4096Mi', 4294967296, 40),
+        ('64G', 64000000000, 610),
+        ('64Garbage', 1, 1),
+    ],
+)
+def test_SYSTEM_TASK_ABS_MEM_conversion(value, converted_value, mem_capacity):
+    with mock.patch('django.conf.settings') as mock_settings:
+        mock_settings.SYSTEM_TASK_ABS_MEM = value
+        mock_settings.SYSTEM_TASK_FORKS_MEM = 100
+        mock_settings.IS_K8S = True
+        assert convert_mem_str_to_bytes(value) == converted_value
+        assert get_corrected_memory(-1) == converted_value
+        assert get_mem_effective_capacity(-1) == mem_capacity
+
+
+@pytest.mark.parametrize(
+    "value,converted_value,cpu_capacity",
+    [
+        ('2', 2.0, 8),
+        ('1.5', 1.5, 6),
+        ('100m', 0.1, 1),
+        ('2000m', 2.0, 8),
+        ('4MillionCPUm', 1.0, 4),  # Any suffix other than 'm' is not supported, we fall back to 1 CPU
+        ('Random', 1.0, 4),  # Any setting value other than integers, floats millicores (e.g 1, 1.0, or 1000m) is not supported, fall back to 1 CPU
+        ('2505m', 2.5, 10),
+        ('1.55', 1.6, 6),
+    ],
+)
+def test_SYSTEM_TASK_ABS_CPU_conversion(value, converted_value, cpu_capacity):
+    with mock.patch('django.conf.settings') as mock_settings:
+        mock_settings.SYSTEM_TASK_ABS_CPU = value
+        mock_settings.SYSTEM_TASK_FORKS_CPU = 4
+        assert convert_cpu_str_to_decimal_cpu(value) == converted_value
+        assert get_corrected_cpu(-1) == converted_value
+        assert get_cpu_effective_capacity(-1) == cpu_capacity

awx/main/tests/unit/test_capacity.py

@@ -18,6 +18,8 @@ class FakeObject(object):
 class Job(FakeObject):
     task_impact = 43
     is_container_group_task = False
+    controller_node = ''
+    execution_node = ''
 
     def log_format(self):
         return 'job 382 (fake)'

awx/main/tests/unit/test_tasks.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-
 import configparser
 import json
 import os
@@ -32,9 +31,9 @@ from awx.main.models import (
     User,
     build_safe_env,
 )
-from awx.main.models.credential import ManagedCredentialType
+from awx.main.models.credential import HIDDEN_PASSWORD, ManagedCredentialType
 
-from awx.main import tasks
+from awx.main.tasks import jobs, system
 from awx.main.utils import encrypt_field, encrypt_value
 from awx.main.utils.safe_yaml import SafeLoader
 from awx.main.utils.execution_environments import CONTAINER_ROOT, to_host_path
@@ -113,12 +112,12 @@ def adhoc_update_model_wrapper(adhoc_job):
 
 def test_send_notifications_not_list():
     with pytest.raises(TypeError):
-        tasks.send_notifications(None)
+        system.send_notifications(None)
 
 
 def test_send_notifications_job_id(mocker):
     with mocker.patch('awx.main.models.UnifiedJob.objects.get'):
-        tasks.send_notifications([], job_id=1)
+        system.send_notifications([], job_id=1)
         assert UnifiedJob.objects.get.called
         assert UnifiedJob.objects.get.called_with(id=1)
 
@@ -127,7 +126,7 @@ def test_work_success_callback_missing_job():
     task_data = {'type': 'project_update', 'id': 9999}
     with mock.patch('django.db.models.query.QuerySet.get') as get_mock:
         get_mock.side_effect = ProjectUpdate.DoesNotExist()
-        assert tasks.handle_work_success(task_data) is None
+        assert system.handle_work_success(task_data) is None
 
 
 @mock.patch('awx.main.models.UnifiedJob.objects.get')
@@ -138,7 +137,7 @@ def test_send_notifications_list(mock_notifications_filter, mock_job_get, mocker
     mock_notifications = [mocker.MagicMock(spec=Notification, subject="test", body={'hello': 'world'})]
     mock_notifications_filter.return_value = mock_notifications
 
-    tasks.send_notifications([1, 2], job_id=1)
+    system.send_notifications([1, 2], job_id=1)
     assert Notification.objects.filter.call_count == 1
     assert mock_notifications[0].status == "successful"
     assert mock_notifications[0].save.called
@@ -158,7 +157,7 @@ def test_send_notifications_list(mock_notifications_filter, mock_job_get, mocker
     ],
 )
 def test_safe_env_filtering(key, value):
-    assert build_safe_env({key: value})[key] == tasks.HIDDEN_PASSWORD
+    assert build_safe_env({key: value})[key] == HIDDEN_PASSWORD
 
 
 def test_safe_env_returns_new_copy():
@@ -168,7 +167,7 @@ def test_safe_env_returns_new_copy():
 
 @pytest.mark.parametrize("source,expected", [(None, True), (False, False), (True, True)])
 def test_openstack_client_config_generation(mocker, source, expected, private_data_dir):
-    update = tasks.RunInventoryUpdate()
+    update = jobs.RunInventoryUpdate()
     credential_type = CredentialType.defaults['openstack']()
     inputs = {
         'host': 'https://keystone.openstack.example.org',
@@ -208,7 +207,7 @@ def test_openstack_client_config_generation(mocker, source, expected, private_da
 
 @pytest.mark.parametrize("source,expected", [(None, True), (False, False), (True, True)])
 def test_openstack_client_config_generation_with_project_domain_name(mocker, source, expected, private_data_dir):
-    update = tasks.RunInventoryUpdate()
+    update = jobs.RunInventoryUpdate()
     credential_type = CredentialType.defaults['openstack']()
     inputs = {
         'host': 'https://keystone.openstack.example.org',
@@ -250,7 +249,7 @@ def test_openstack_client_config_generation_with_project_domain_name(mocker, sou
 
 @pytest.mark.parametrize("source,expected", [(None, True), (False, False), (True, True)])
 def test_openstack_client_config_generation_with_region(mocker, source, expected, private_data_dir):
-    update = tasks.RunInventoryUpdate()
+    update = jobs.RunInventoryUpdate()
     credential_type = CredentialType.defaults['openstack']()
     inputs = {
         'host': 'https://keystone.openstack.example.org',
@@ -294,7 +293,7 @@ def test_openstack_client_config_generation_with_region(mocker, source, expected
 
 @pytest.mark.parametrize("source,expected", [(False, False), (True, True)])
 def test_openstack_client_config_generation_with_private_source_vars(mocker, source, expected, private_data_dir):
-    update = tasks.RunInventoryUpdate()
+    update = jobs.RunInventoryUpdate()
     credential_type = CredentialType.defaults['openstack']()
     inputs = {
         'host': 'https://keystone.openstack.example.org',
@@ -357,7 +356,7 @@ class TestExtraVarSanitation(TestJobExecution):
         job.created_by = User(pk=123, username='angry-spud')
         job.inventory = Inventory(pk=123, name='example-inv')
 
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         task.build_extra_vars_file(job, private_data_dir)
 
         fd = open(os.path.join(private_data_dir, 'env', 'extravars'))
@@ -393,7 +392,7 @@ class TestExtraVarSanitation(TestJobExecution):
 
     def test_launchtime_vars_unsafe(self, job, private_data_dir):
         job.extra_vars = json.dumps({'msg': self.UNSAFE})
-        task = tasks.RunJob()
+        task = jobs.RunJob()
 
         task.build_extra_vars_file(job, private_data_dir)
 
@@ -404,7 +403,7 @@ class TestExtraVarSanitation(TestJobExecution):
 
     def test_nested_launchtime_vars_unsafe(self, job, private_data_dir):
         job.extra_vars = json.dumps({'msg': {'a': [self.UNSAFE]}})
-        task = tasks.RunJob()
+        task = jobs.RunJob()
 
         task.build_extra_vars_file(job, private_data_dir)
 
@@ -415,7 +414,7 @@ class TestExtraVarSanitation(TestJobExecution):
 
     def test_allowed_jt_extra_vars(self, job, private_data_dir):
         job.job_template.extra_vars = job.extra_vars = json.dumps({'msg': self.UNSAFE})
-        task = tasks.RunJob()
+        task = jobs.RunJob()
 
         task.build_extra_vars_file(job, private_data_dir)
 
@@ -427,7 +426,7 @@ class TestExtraVarSanitation(TestJobExecution):
     def test_nested_allowed_vars(self, job, private_data_dir):
         job.extra_vars = json.dumps({'msg': {'a': {'b': [self.UNSAFE]}}})
         job.job_template.extra_vars = job.extra_vars
-        task = tasks.RunJob()
+        task = jobs.RunJob()
 
         task.build_extra_vars_file(job, private_data_dir)
 
@@ -441,7 +440,7 @@ class TestExtraVarSanitation(TestJobExecution):
         # `other_var=SENSITIVE`
         job.job_template.extra_vars = json.dumps({'msg': self.UNSAFE})
         job.extra_vars = json.dumps({'msg': 'other-value', 'other_var': self.UNSAFE})
-        task = tasks.RunJob()
+        task = jobs.RunJob()
 
         task.build_extra_vars_file(job, private_data_dir)
 
@@ -456,7 +455,7 @@ class TestExtraVarSanitation(TestJobExecution):
     def test_overwritten_jt_extra_vars(self, job, private_data_dir):
         job.job_template.extra_vars = json.dumps({'msg': 'SAFE'})
         job.extra_vars = json.dumps({'msg': self.UNSAFE})
-        task = tasks.RunJob()
+        task = jobs.RunJob()
 
         task.build_extra_vars_file(job, private_data_dir)
 
@@ -472,13 +471,13 @@ class TestGenericRun:
         job.websocket_emit_status = mock.Mock()
         job.execution_environment = execution_environment
 
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         task.instance = job
         task.update_model = mock.Mock(return_value=job)
         task.model.objects.get = mock.Mock(return_value=job)
         task.build_private_data_files = mock.Mock(side_effect=OSError())
 
-        with mock.patch('awx.main.tasks.copy_tree'):
+        with mock.patch('awx.main.tasks.jobs.copy_tree'):
             with pytest.raises(Exception):
                 task.run(1)
 
@@ -494,13 +493,13 @@ class TestGenericRun:
         job.send_notification_templates = mock.Mock()
         job.execution_environment = execution_environment
 
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         task.instance = job
         task.update_model = mock.Mock(wraps=update_model_wrapper)
         task.model.objects.get = mock.Mock(return_value=job)
         task.build_private_data_files = mock.Mock()
 
-        with mock.patch('awx.main.tasks.copy_tree'):
+        with mock.patch('awx.main.tasks.jobs.copy_tree'):
             with pytest.raises(Exception):
                 task.run(1)
 
@@ -508,45 +507,45 @@ class TestGenericRun:
             assert c in task.update_model.call_args_list
 
     def test_event_count(self):
-        task = tasks.RunJob()
-        task.dispatcher = mock.MagicMock()
-        task.instance = Job()
-        task.event_ct = 0
+        task = jobs.RunJob()
+        task.runner_callback.dispatcher = mock.MagicMock()
+        task.runner_callback.instance = Job()
+        task.runner_callback.event_ct = 0
         event_data = {}
 
-        [task.event_handler(event_data) for i in range(20)]
-        assert 20 == task.event_ct
+        [task.runner_callback.event_handler(event_data) for i in range(20)]
+        assert 20 == task.runner_callback.event_ct
 
     def test_finished_callback_eof(self):
-        task = tasks.RunJob()
-        task.dispatcher = mock.MagicMock()
-        task.instance = Job(pk=1, id=1)
-        task.event_ct = 17
-        task.finished_callback(None)
-        task.dispatcher.dispatch.assert_called_with({'event': 'EOF', 'final_counter': 17, 'job_id': 1, 'guid': None})
+        task = jobs.RunJob()
+        task.runner_callback.dispatcher = mock.MagicMock()
+        task.runner_callback.instance = Job(pk=1, id=1)
+        task.runner_callback.event_ct = 17
+        task.runner_callback.finished_callback(None)
+        task.runner_callback.dispatcher.dispatch.assert_called_with({'event': 'EOF', 'final_counter': 17, 'job_id': 1, 'guid': None})
 
     def test_save_job_metadata(self, job, update_model_wrapper):
         class MockMe:
             pass
 
-        task = tasks.RunJob()
-        task.instance = job
-        task.safe_env = {'secret_key': 'redacted_value'}
-        task.update_model = mock.Mock(wraps=update_model_wrapper)
+        task = jobs.RunJob()
+        task.runner_callback.instance = job
+        task.runner_callback.safe_env = {'secret_key': 'redacted_value'}
+        task.runner_callback.update_model = mock.Mock(wraps=update_model_wrapper)
         runner_config = MockMe()
         runner_config.command = {'foo': 'bar'}
         runner_config.cwd = '/foobar'
         runner_config.env = {'switch': 'blade', 'foot': 'ball', 'secret_key': 'secret_value'}
-        task.status_handler({'status': 'starting'}, runner_config)
+        task.runner_callback.status_handler({'status': 'starting'}, runner_config)
 
-        task.update_model.assert_called_with(
+        task.runner_callback.update_model.assert_called_with(
             1, job_args=json.dumps({'foo': 'bar'}), job_cwd='/foobar', job_env={'switch': 'blade', 'foot': 'ball', 'secret_key': 'redacted_value'}
         )
 
     def test_created_by_extra_vars(self):
         job = Job(created_by=User(pk=123, username='angry-spud'))
 
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         task._write_extra_vars_file = mock.Mock()
         task.build_extra_vars_file(job, None)
 
@@ -563,7 +562,7 @@ class TestGenericRun:
         job.extra_vars = json.dumps({'super_secret': encrypt_value('CLASSIFIED', pk=None)})
         job.survey_passwords = {'super_secret': '$encrypted$'}
 
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         task._write_extra_vars_file = mock.Mock()
         task.build_extra_vars_file(job, None)
 
@@ -576,11 +575,11 @@ class TestGenericRun:
         job = Job(project=Project(), inventory=Inventory())
         job.execution_environment = execution_environment
 
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         task.instance = job
         task._write_extra_vars_file = mock.Mock()
 
-        with mock.patch('awx.main.tasks.settings.AWX_TASK_ENV', {'FOO': 'BAR'}):
+        with mock.patch('awx.main.tasks.jobs.settings.AWX_TASK_ENV', {'FOO': 'BAR'}):
             env = task.build_env(job, private_data_dir)
         assert env['FOO'] == 'BAR'
 
@@ -595,7 +594,7 @@ class TestAdhocRun(TestJobExecution):
         adhoc_job.websocket_emit_status = mock.Mock()
         adhoc_job.send_notification_templates = mock.Mock()
 
-        task = tasks.RunAdHocCommand()
+        task = jobs.RunAdHocCommand()
         task.update_model = mock.Mock(wraps=adhoc_update_model_wrapper)
         task.model.objects.get = mock.Mock(return_value=adhoc_job)
         task.build_inventory = mock.Mock()
@@ -619,7 +618,7 @@ class TestAdhocRun(TestJobExecution):
         })
         #adhoc_job.websocket_emit_status = mock.Mock()
 
-        task = tasks.RunAdHocCommand()
+        task = jobs.RunAdHocCommand()
         #task.update_model = mock.Mock(wraps=adhoc_update_model_wrapper)
         #task.build_inventory = mock.Mock(return_value='/tmp/something.inventory')
         task._write_extra_vars_file = mock.Mock()
@@ -634,7 +633,7 @@ class TestAdhocRun(TestJobExecution):
     def test_created_by_extra_vars(self):
         adhoc_job = AdHocCommand(created_by=User(pk=123, username='angry-spud'))
 
-        task = tasks.RunAdHocCommand()
+        task = jobs.RunAdHocCommand()
         task._write_extra_vars_file = mock.Mock()
         task.build_extra_vars_file(adhoc_job, None)
 
@@ -693,7 +692,7 @@ class TestJobCredentials(TestJobExecution):
     }
 
     def test_username_jinja_usage(self, job, private_data_dir):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         ssh = CredentialType.defaults['ssh']()
         credential = Credential(pk=1, credential_type=ssh, inputs={'username': '{{ ansible_ssh_pass }}'})
         job.credentials.add(credential)
@@ -704,7 +703,7 @@ class TestJobCredentials(TestJobExecution):
 
     @pytest.mark.parametrize("flag", ['become_username', 'become_method'])
     def test_become_jinja_usage(self, job, private_data_dir, flag):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         ssh = CredentialType.defaults['ssh']()
         credential = Credential(pk=1, credential_type=ssh, inputs={'username': 'joe', flag: '{{ ansible_ssh_pass }}'})
         job.credentials.add(credential)
@@ -715,7 +714,7 @@ class TestJobCredentials(TestJobExecution):
         assert 'Jinja variables are not allowed' in str(e.value)
 
     def test_ssh_passwords(self, job, private_data_dir, field, password_name, expected_flag):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         ssh = CredentialType.defaults['ssh']()
         credential = Credential(pk=1, credential_type=ssh, inputs={'username': 'bob', field: 'secret'})
         credential.inputs[field] = encrypt_field(credential, field)
@@ -732,7 +731,7 @@ class TestJobCredentials(TestJobExecution):
             assert expected_flag in ' '.join(args)
 
     def test_net_ssh_key_unlock(self, job):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         net = CredentialType.defaults['net']()
         credential = Credential(pk=1, credential_type=net, inputs={'ssh_key_unlock': 'secret'})
         credential.inputs['ssh_key_unlock'] = encrypt_field(credential, 'ssh_key_unlock')
@@ -745,7 +744,7 @@ class TestJobCredentials(TestJobExecution):
         assert 'secret' in expect_passwords.values()
 
     def test_net_first_ssh_key_unlock_wins(self, job):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         for i in range(3):
             net = CredentialType.defaults['net']()
             credential = Credential(pk=i, credential_type=net, inputs={'ssh_key_unlock': 'secret{}'.format(i)})
@@ -759,7 +758,7 @@ class TestJobCredentials(TestJobExecution):
         assert 'secret0' in expect_passwords.values()
 
     def test_prefer_ssh_over_net_ssh_key_unlock(self, job):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         net = CredentialType.defaults['net']()
         net_credential = Credential(pk=1, credential_type=net, inputs={'ssh_key_unlock': 'net_secret'})
         net_credential.inputs['ssh_key_unlock'] = encrypt_field(net_credential, 'ssh_key_unlock')
@@ -778,7 +777,7 @@ class TestJobCredentials(TestJobExecution):
         assert 'ssh_secret' in expect_passwords.values()
 
     def test_vault_password(self, private_data_dir, job):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         vault = CredentialType.defaults['vault']()
         credential = Credential(pk=1, credential_type=vault, inputs={'vault_password': 'vault-me'})
         credential.inputs['vault_password'] = encrypt_field(credential, 'vault_password')
@@ -793,7 +792,7 @@ class TestJobCredentials(TestJobExecution):
         assert '--ask-vault-pass' in ' '.join(args)
 
     def test_vault_password_ask(self, private_data_dir, job):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         vault = CredentialType.defaults['vault']()
         credential = Credential(pk=1, credential_type=vault, inputs={'vault_password': 'ASK'})
         credential.inputs['vault_password'] = encrypt_field(credential, 'vault_password')
@@ -808,7 +807,7 @@ class TestJobCredentials(TestJobExecution):
         assert '--ask-vault-pass' in ' '.join(args)
 
     def test_multi_vault_password(self, private_data_dir, job):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         vault = CredentialType.defaults['vault']()
         for i, label in enumerate(['dev', 'prod', 'dotted.name']):
             credential = Credential(pk=i, credential_type=vault, inputs={'vault_password': 'pass@{}'.format(label), 'vault_id': label})
@@ -831,7 +830,7 @@ class TestJobCredentials(TestJobExecution):
         assert '--vault-id dotted.name@prompt' in ' '.join(args)
 
     def test_multi_vault_id_conflict(self, job):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         vault = CredentialType.defaults['vault']()
         for i in range(2):
             credential = Credential(pk=i, credential_type=vault, inputs={'vault_password': 'some-pass', 'vault_id': 'conflict'})
@@ -844,7 +843,7 @@ class TestJobCredentials(TestJobExecution):
         assert 'multiple vault credentials were specified with --vault-id' in str(e.value)
 
     def test_multi_vault_password_ask(self, private_data_dir, job):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         vault = CredentialType.defaults['vault']()
         for i, label in enumerate(['dev', 'prod']):
             credential = Credential(pk=i, credential_type=vault, inputs={'vault_password': 'ASK', 'vault_id': label})
@@ -897,7 +896,7 @@ class TestJobCredentials(TestJobExecution):
             assert env['K8S_AUTH_VERIFY_SSL'] == 'False'
             assert 'K8S_AUTH_SSL_CA_CERT' not in env
 
-        assert safe_env['K8S_AUTH_API_KEY'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['K8S_AUTH_API_KEY'] == HIDDEN_PASSWORD
 
     def test_aws_cloud_credential(self, job, private_data_dir):
         aws = CredentialType.defaults['aws']()
@@ -912,7 +911,7 @@ class TestJobCredentials(TestJobExecution):
         assert env['AWS_ACCESS_KEY_ID'] == 'bob'
         assert env['AWS_SECRET_ACCESS_KEY'] == 'secret'
         assert 'AWS_SECURITY_TOKEN' not in env
-        assert safe_env['AWS_SECRET_ACCESS_KEY'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['AWS_SECRET_ACCESS_KEY'] == HIDDEN_PASSWORD
 
     def test_aws_cloud_credential_with_sts_token(self, private_data_dir, job):
         aws = CredentialType.defaults['aws']()
@@ -928,7 +927,7 @@ class TestJobCredentials(TestJobExecution):
         assert env['AWS_ACCESS_KEY_ID'] == 'bob'
         assert env['AWS_SECRET_ACCESS_KEY'] == 'secret'
         assert env['AWS_SECURITY_TOKEN'] == 'token'
-        assert safe_env['AWS_SECRET_ACCESS_KEY'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['AWS_SECRET_ACCESS_KEY'] == HIDDEN_PASSWORD
 
     def test_gce_credentials(self, private_data_dir, job):
         gce = CredentialType.defaults['gce']()
@@ -963,7 +962,7 @@ class TestJobCredentials(TestJobExecution):
         assert env['AZURE_SECRET'] == 'some-secret'
         assert env['AZURE_TENANT'] == 'some-tenant'
         assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription'
-        assert safe_env['AZURE_SECRET'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['AZURE_SECRET'] == HIDDEN_PASSWORD
 
     def test_azure_rm_with_password(self, private_data_dir, job):
         azure = CredentialType.defaults['azure_rm']()
@@ -981,7 +980,7 @@ class TestJobCredentials(TestJobExecution):
         assert env['AZURE_AD_USER'] == 'bob'
         assert env['AZURE_PASSWORD'] == 'secret'
         assert env['AZURE_CLOUD_ENVIRONMENT'] == 'foobar'
-        assert safe_env['AZURE_PASSWORD'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['AZURE_PASSWORD'] == HIDDEN_PASSWORD
 
     def test_vmware_credentials(self, private_data_dir, job):
         vmware = CredentialType.defaults['vmware']()
@@ -996,10 +995,10 @@ class TestJobCredentials(TestJobExecution):
         assert env['VMWARE_USER'] == 'bob'
         assert env['VMWARE_PASSWORD'] == 'secret'
         assert env['VMWARE_HOST'] == 'https://example.org'
-        assert safe_env['VMWARE_PASSWORD'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['VMWARE_PASSWORD'] == HIDDEN_PASSWORD
 
     def test_openstack_credentials(self, private_data_dir, job):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         task.instance = job
         openstack = CredentialType.defaults['openstack']()
         credential = Credential(
@@ -1067,7 +1066,7 @@ class TestJobCredentials(TestJobExecution):
         ],
     )
     def test_net_credentials(self, authorize, expected_authorize, job, private_data_dir):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         task.instance = job
         net = CredentialType.defaults['net']()
         inputs = {'username': 'bob', 'password': 'secret', 'ssh_key_data': self.EXAMPLE_PRIVATE_KEY, 'authorize_password': 'authorizeme'}
@@ -1089,7 +1088,7 @@ class TestJobCredentials(TestJobExecution):
         if authorize:
             assert env['ANSIBLE_NET_AUTH_PASS'] == 'authorizeme'
         assert open(env['ANSIBLE_NET_SSH_KEYFILE'], 'r').read() == self.EXAMPLE_PRIVATE_KEY
-        assert safe_env['ANSIBLE_NET_PASSWORD'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['ANSIBLE_NET_PASSWORD'] == HIDDEN_PASSWORD
 
     def test_custom_environment_injectors_with_jinja_syntax_error(self, private_data_dir):
         some_cloud = CredentialType(
@@ -1135,7 +1134,7 @@ class TestJobCredentials(TestJobExecution):
         assert env['TURBO_BUTTON'] == str(True)
 
     def test_custom_environment_injectors_with_reserved_env_var(self, private_data_dir, job):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         task.instance = job
         some_cloud = CredentialType(
             kind='cloud',
@@ -1168,10 +1167,10 @@ class TestJobCredentials(TestJobExecution):
 
         assert env['MY_CLOUD_PRIVATE_VAR'] == 'SUPER-SECRET-123'
         assert 'SUPER-SECRET-123' not in safe_env.values()
-        assert safe_env['MY_CLOUD_PRIVATE_VAR'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['MY_CLOUD_PRIVATE_VAR'] == HIDDEN_PASSWORD
 
     def test_custom_environment_injectors_with_extra_vars(self, private_data_dir, job):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         some_cloud = CredentialType(
             kind='cloud',
             name='SomeCloud',
@@ -1190,7 +1189,7 @@ class TestJobCredentials(TestJobExecution):
         assert hasattr(extra_vars["api_token"], '__UNSAFE__')
 
     def test_custom_environment_injectors_with_boolean_extra_vars(self, job, private_data_dir):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         some_cloud = CredentialType(
             kind='cloud',
             name='SomeCloud',
@@ -1209,7 +1208,7 @@ class TestJobCredentials(TestJobExecution):
         return ['successful', 0]
 
     def test_custom_environment_injectors_with_complicated_boolean_template(self, job, private_data_dir):
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         some_cloud = CredentialType(
             kind='cloud',
             name='SomeCloud',
@@ -1230,7 +1229,7 @@ class TestJobCredentials(TestJobExecution):
         """
         extra_vars that contain secret field values should be censored in the DB
         """
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         some_cloud = CredentialType(
             kind='cloud',
             name='SomeCloud',
@@ -1331,11 +1330,11 @@ class TestJobCredentials(TestJobExecution):
         assert json_data['client_email'] == 'bob'
         assert json_data['project_id'] == 'some-project'
 
-        assert safe_env['AZURE_PASSWORD'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['AZURE_PASSWORD'] == HIDDEN_PASSWORD
 
     def test_awx_task_env(self, settings, private_data_dir, job):
         settings.AWX_TASK_ENV = {'FOO': 'BAR'}
-        task = tasks.RunJob()
+        task = jobs.RunJob()
         task.instance = job
         env = task.build_env(job, private_data_dir)
 
@@ -1362,7 +1361,7 @@ class TestProjectUpdateGalaxyCredentials(TestJobExecution):
 
     def test_galaxy_credentials_ignore_certs(self, private_data_dir, project_update, ignore):
         settings.GALAXY_IGNORE_CERTS = ignore
-        task = tasks.RunProjectUpdate()
+        task = jobs.RunProjectUpdate()
         task.instance = project_update
         env = task.build_env(project_update, private_data_dir)
         if ignore:
@@ -1371,7 +1370,7 @@ class TestProjectUpdateGalaxyCredentials(TestJobExecution):
             assert 'ANSIBLE_GALAXY_IGNORE' not in env
 
     def test_galaxy_credentials_empty(self, private_data_dir, project_update):
-        class RunProjectUpdate(tasks.RunProjectUpdate):
+        class RunProjectUpdate(jobs.RunProjectUpdate):
             __vars__ = {}
 
             def _write_extra_vars_file(self, private_data_dir, extra_vars, *kw):
@@ -1390,7 +1389,7 @@ class TestProjectUpdateGalaxyCredentials(TestJobExecution):
             assert not k.startswith('ANSIBLE_GALAXY_SERVER')
 
     def test_single_public_galaxy(self, private_data_dir, project_update):
-        class RunProjectUpdate(tasks.RunProjectUpdate):
+        class RunProjectUpdate(jobs.RunProjectUpdate):
             __vars__ = {}
 
             def _write_extra_vars_file(self, private_data_dir, extra_vars, *kw):
@@ -1439,7 +1438,7 @@ class TestProjectUpdateGalaxyCredentials(TestJobExecution):
         )
         project_update.project.organization.galaxy_credentials.add(public_galaxy)
         project_update.project.organization.galaxy_credentials.add(rh)
-        task = tasks.RunProjectUpdate()
+        task = jobs.RunProjectUpdate()
         task.instance = project_update
         env = task.build_env(project_update, private_data_dir)
         assert sorted([(k, v) for k, v in env.items() if k.startswith('ANSIBLE_GALAXY')]) == [
@@ -1481,7 +1480,7 @@ class TestProjectUpdateCredentials(TestJobExecution):
     }
 
     def test_username_and_password_auth(self, project_update, scm_type):
-        task = tasks.RunProjectUpdate()
+        task = jobs.RunProjectUpdate()
         ssh = CredentialType.defaults['ssh']()
         project_update.scm_type = scm_type
         project_update.credential = Credential(pk=1, credential_type=ssh, inputs={'username': 'bob', 'password': 'secret'})
@@ -1495,7 +1494,7 @@ class TestProjectUpdateCredentials(TestJobExecution):
         assert 'secret' in expect_passwords.values()
 
     def test_ssh_key_auth(self, project_update, scm_type):
-        task = tasks.RunProjectUpdate()
+        task = jobs.RunProjectUpdate()
         ssh = CredentialType.defaults['ssh']()
         project_update.scm_type = scm_type
         project_update.credential = Credential(pk=1, credential_type=ssh, inputs={'username': 'bob', 'ssh_key_data': self.EXAMPLE_PRIVATE_KEY})
@@ -1509,7 +1508,7 @@ class TestProjectUpdateCredentials(TestJobExecution):
     def test_awx_task_env(self, project_update, settings, private_data_dir, scm_type, execution_environment):
         project_update.execution_environment = execution_environment
         settings.AWX_TASK_ENV = {'FOO': 'BAR'}
-        task = tasks.RunProjectUpdate()
+        task = jobs.RunProjectUpdate()
         task.instance = project_update
         project_update.scm_type = scm_type
 
@@ -1524,7 +1523,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
         return InventoryUpdate(pk=1, execution_environment=execution_environment, inventory_source=InventorySource(pk=1, inventory=Inventory(pk=1)))
 
     def test_source_without_credential(self, mocker, inventory_update, private_data_dir):
-        task = tasks.RunInventoryUpdate()
+        task = jobs.RunInventoryUpdate()
         task.instance = inventory_update
         inventory_update.source = 'ec2'
         inventory_update.get_cloud_credential = mocker.Mock(return_value=None)
@@ -1537,7 +1536,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
         assert 'AWS_SECRET_ACCESS_KEY' not in env
 
     def test_ec2_source(self, private_data_dir, inventory_update, mocker):
-        task = tasks.RunInventoryUpdate()
+        task = jobs.RunInventoryUpdate()
         task.instance = inventory_update
         aws = CredentialType.defaults['aws']()
         inventory_update.source = 'ec2'
@@ -1558,10 +1557,10 @@ class TestInventoryUpdateCredentials(TestJobExecution):
         assert env['AWS_ACCESS_KEY_ID'] == 'bob'
         assert env['AWS_SECRET_ACCESS_KEY'] == 'secret'
 
-        assert safe_env['AWS_SECRET_ACCESS_KEY'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['AWS_SECRET_ACCESS_KEY'] == HIDDEN_PASSWORD
 
     def test_vmware_source(self, inventory_update, private_data_dir, mocker):
-        task = tasks.RunInventoryUpdate()
+        task = jobs.RunInventoryUpdate()
         task.instance = inventory_update
         vmware = CredentialType.defaults['vmware']()
         inventory_update.source = 'vmware'
@@ -1589,7 +1588,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
         env["VMWARE_VALIDATE_CERTS"] == "False",
 
     def test_azure_rm_source_with_tenant(self, private_data_dir, inventory_update, mocker):
-        task = tasks.RunInventoryUpdate()
+        task = jobs.RunInventoryUpdate()
         task.instance = inventory_update
         azure_rm = CredentialType.defaults['azure_rm']()
         inventory_update.source = 'azure_rm'
@@ -1622,10 +1621,10 @@ class TestInventoryUpdateCredentials(TestJobExecution):
         assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription'
         assert env['AZURE_CLOUD_ENVIRONMENT'] == 'foobar'
 
-        assert safe_env['AZURE_SECRET'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['AZURE_SECRET'] == HIDDEN_PASSWORD
 
     def test_azure_rm_source_with_password(self, private_data_dir, inventory_update, mocker):
-        task = tasks.RunInventoryUpdate()
+        task = jobs.RunInventoryUpdate()
         task.instance = inventory_update
         azure_rm = CredentialType.defaults['azure_rm']()
         inventory_update.source = 'azure_rm'
@@ -1651,10 +1650,10 @@ class TestInventoryUpdateCredentials(TestJobExecution):
         assert env['AZURE_PASSWORD'] == 'secret'
         assert env['AZURE_CLOUD_ENVIRONMENT'] == 'foobar'
 
-        assert safe_env['AZURE_PASSWORD'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['AZURE_PASSWORD'] == HIDDEN_PASSWORD
 
     def test_gce_source(self, inventory_update, private_data_dir, mocker):
-        task = tasks.RunInventoryUpdate()
+        task = jobs.RunInventoryUpdate()
         task.instance = inventory_update
         gce = CredentialType.defaults['gce']()
         inventory_update.source = 'gce'
@@ -1684,7 +1683,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
             assert json_data['project_id'] == 'some-project'
 
     def test_openstack_source(self, inventory_update, private_data_dir, mocker):
-        task = tasks.RunInventoryUpdate()
+        task = jobs.RunInventoryUpdate()
         task.instance = inventory_update
         openstack = CredentialType.defaults['openstack']()
         inventory_update.source = 'openstack'
@@ -1724,7 +1723,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
         )
 
     def test_satellite6_source(self, inventory_update, private_data_dir, mocker):
-        task = tasks.RunInventoryUpdate()
+        task = jobs.RunInventoryUpdate()
         task.instance = inventory_update
         satellite6 = CredentialType.defaults['satellite6']()
         inventory_update.source = 'satellite6'
@@ -1744,10 +1743,10 @@ class TestInventoryUpdateCredentials(TestJobExecution):
         assert env["FOREMAN_SERVER"] == "https://example.org"
         assert env["FOREMAN_USER"] == "bob"
         assert env["FOREMAN_PASSWORD"] == "secret"
-        assert safe_env["FOREMAN_PASSWORD"] == tasks.HIDDEN_PASSWORD
+        assert safe_env["FOREMAN_PASSWORD"] == HIDDEN_PASSWORD
 
     def test_insights_source(self, inventory_update, private_data_dir, mocker):
-        task = tasks.RunInventoryUpdate()
+        task = jobs.RunInventoryUpdate()
         task.instance = inventory_update
         insights = CredentialType.defaults['insights']()
         inventory_update.source = 'insights'
@@ -1772,11 +1771,11 @@ class TestInventoryUpdateCredentials(TestJobExecution):
 
         assert env["INSIGHTS_USER"] == "bob"
         assert env["INSIGHTS_PASSWORD"] == "secret"
-        assert safe_env['INSIGHTS_PASSWORD'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['INSIGHTS_PASSWORD'] == HIDDEN_PASSWORD
 
     @pytest.mark.parametrize('verify', [True, False])
     def test_tower_source(self, verify, inventory_update, private_data_dir, mocker):
-        task = tasks.RunInventoryUpdate()
+        task = jobs.RunInventoryUpdate()
         task.instance = inventory_update
         tower = CredentialType.defaults['controller']()
         inventory_update.source = 'controller'
@@ -1801,10 +1800,10 @@ class TestInventoryUpdateCredentials(TestJobExecution):
             assert env['CONTROLLER_VERIFY_SSL'] == 'True'
         else:
             assert env['CONTROLLER_VERIFY_SSL'] == 'False'
-        assert safe_env['CONTROLLER_PASSWORD'] == tasks.HIDDEN_PASSWORD
+        assert safe_env['CONTROLLER_PASSWORD'] == HIDDEN_PASSWORD
 
     def test_tower_source_ssl_verify_empty(self, inventory_update, private_data_dir, mocker):
-        task = tasks.RunInventoryUpdate()
+        task = jobs.RunInventoryUpdate()
         task.instance = inventory_update
         tower = CredentialType.defaults['controller']()
         inventory_update.source = 'controller'
@@ -1832,7 +1831,7 @@ class TestInventoryUpdateCredentials(TestJobExecution):
         assert env['TOWER_VERIFY_SSL'] == 'False'
 
     def test_awx_task_env(self, inventory_update, private_data_dir, settings, mocker):
-        task = tasks.RunInventoryUpdate()
+        task = jobs.RunInventoryUpdate()
         task.instance = inventory_update
         gce = CredentialType.defaults['gce']()
         inventory_update.source = 'gce'
@@ -1883,7 +1882,7 @@ def test_aquire_lock_open_fail_logged(logging_getLogger, os_open):
     logger = mock.Mock()
     logging_getLogger.return_value = logger
 
-    ProjectUpdate = tasks.RunProjectUpdate()
+    ProjectUpdate = jobs.RunProjectUpdate()
 
     with pytest.raises(OSError):
         ProjectUpdate.acquire_lock(instance)
@@ -1910,7 +1909,7 @@ def test_aquire_lock_acquisition_fail_logged(fcntl_lockf, logging_getLogger, os_
 
     fcntl_lockf.side_effect = err
 
-    ProjectUpdate = tasks.RunProjectUpdate()
+    ProjectUpdate = jobs.RunProjectUpdate()
     with pytest.raises(IOError):
         ProjectUpdate.acquire_lock(instance)
     os_close.assert_called_with(3)
@@ -1920,7 +1919,7 @@ def test_aquire_lock_acquisition_fail_logged(fcntl_lockf, logging_getLogger, os_
 @pytest.mark.parametrize('injector_cls', [cls for cls in ManagedCredentialType.registry.values() if cls.injectors])
 def test_managed_injector_redaction(injector_cls):
     """See awx.main.models.inventory.PluginFileInjector._get_shared_env
-    The ordering within awx.main.tasks.BaseTask and contract with build_env
+    The ordering within awx.main.tasks.jobs.BaseTask and contract with build_env
     requires that all managed injectors are safely redacted by the
     static method build_safe_env without having to employ the safe namespace
     as in inject_credential
@@ -1947,7 +1946,7 @@ def test_notification_job_not_finished(logging_getLogger, mocker):
     logging_getLogger.return_value = logger
 
     with mocker.patch('awx.main.models.UnifiedJob.objects.get', uj):
-        tasks.handle_success_and_failure_notifications(1)
+        system.handle_success_and_failure_notifications(1)
         assert logger.warn.called_with(f"Failed to even try to send notifications for job '{uj}' due to job not being in finished state.")
 
 
@@ -1955,7 +1954,7 @@ def test_notification_job_finished(mocker):
     uj = mocker.MagicMock(send_notification_templates=mocker.MagicMock(), finished=True)
 
     with mocker.patch('awx.main.models.UnifiedJob.objects.get', mocker.MagicMock(return_value=uj)):
-        tasks.handle_success_and_failure_notifications(1)
+        system.handle_success_and_failure_notifications(1)
         uj.send_notification_templates.assert_called()
 
 
@@ -1964,12 +1963,12 @@ def test_job_run_no_ee():
     proj = Project(pk=1, organization=org)
     job = Job(project=proj, organization=org, inventory=Inventory(pk=1))
     job.execution_environment = None
-    task = tasks.RunJob()
+    task = jobs.RunJob()
     task.instance = job
     task.update_model = mock.Mock(return_value=job)
     task.model.objects.get = mock.Mock(return_value=job)
 
-    with mock.patch('awx.main.tasks.copy_tree'):
+    with mock.patch('awx.main.tasks.jobs.copy_tree'):
         with pytest.raises(RuntimeError) as e:
             task.pre_run_hook(job, private_data_dir)
 
@@ -1983,7 +1982,7 @@ def test_project_update_no_ee():
     proj = Project(pk=1, organization=org)
     project_update = ProjectUpdate(pk=1, project=proj, scm_type='git')
     project_update.execution_environment = None
-    task = tasks.RunProjectUpdate()
+    task = jobs.RunProjectUpdate()
     task.instance = project_update
 
     with pytest.raises(RuntimeError) as e:

awx/main/tests/unit/utils/test_receptor.py

@@ -1,4 +1,4 @@
-from awx.main.utils.receptor import _convert_args_to_cli
+from awx.main.tasks.receptor import _convert_args_to_cli
 
 
 def test_file_cleanup_scenario():

awx/main/utils/common.py

@@ -10,7 +10,6 @@ import os
 import subprocess
 import re
 import stat
-import subprocess
 import urllib.parse
 import threading
 import contextlib
@@ -211,20 +210,6 @@ def get_event_partition_epoch():
     return MigrationRecorder.Migration.objects.filter(app='main', name='0144_event_partitions').first().applied
 
 
-@memoize()
-def get_ansible_version():
-    """
-    Return Ansible version installed.
-    Ansible path needs to be provided to account for custom virtual environments
-    """
-    try:
-        proc = subprocess.Popen(['ansible', '--version'], stdout=subprocess.PIPE)
-        result = smart_str(proc.communicate()[0])
-        return result.split('\n')[0].replace('ansible', '').strip()
-    except Exception:
-        return 'unknown'
-
-
 def get_awx_version():
     """
     Return AWX version as reported by setuptools.
@@ -707,28 +692,33 @@ def parse_yaml_or_json(vars_str, silent_failure=True):
     return vars_dict
 
 
-def get_cpu_effective_capacity(cpu_count):
-    from django.conf import settings
+def convert_cpu_str_to_decimal_cpu(cpu_str):
+    """Convert a string indicating cpu units to decimal.
 
-    settings_abscpu = getattr(settings, 'SYSTEM_TASK_ABS_CPU', None)
-    env_abscpu = os.getenv('SYSTEM_TASK_ABS_CPU', None)
+    Useful for dealing with cpu setting that may be expressed in units compatible with
+    kubernetes.
 
-    if env_abscpu is not None:
-        return int(env_abscpu)
-    elif settings_abscpu is not None:
-        return int(settings_abscpu)
+    See https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units
+    """
+    cpu = cpu_str
+    millicores = False
 
-    settings_forkcpu = getattr(settings, 'SYSTEM_TASK_FORKS_CPU', None)
-    env_forkcpu = os.getenv('SYSTEM_TASK_FORKS_CPU', None)
+    if cpu_str[-1] == 'm':
+        cpu = cpu_str[:-1]
+        millicores = True
 
-    if env_forkcpu:
-        forkcpu = int(env_forkcpu)
-    elif settings_forkcpu:
-        forkcpu = int(settings_forkcpu)
-    else:
-        forkcpu = 4
+    try:
+        cpu = float(cpu)
+    except ValueError:
+        cpu = 1.0
+        millicores = False
+        logger.warning(f"Could not convert SYSTEM_TASK_ABS_CPU {cpu_str} to a decimal number, falling back to default of 1 cpu")
 
-    return cpu_count * forkcpu
+    if millicores:
+        cpu = cpu / 1000
+
+    # Per kubernetes docs, fractional CPU less than .1 are not allowed
+    return max(0.1, round(cpu, 1))
 
 
 def get_corrected_cpu(cpu_count):  # formerlly get_cpu_capacity
@@ -740,34 +730,70 @@ def get_corrected_cpu(cpu_count):  # formerlly get_cpu_capacity
     settings_abscpu = getattr(settings, 'SYSTEM_TASK_ABS_CPU', None)
     env_abscpu = os.getenv('SYSTEM_TASK_ABS_CPU', None)
 
-    if env_abscpu is not None or settings_abscpu is not None:
-        return 0
+    if env_abscpu is not None:
+        return convert_cpu_str_to_decimal_cpu(env_abscpu)
+    elif settings_abscpu is not None:
+        return convert_cpu_str_to_decimal_cpu(settings_abscpu)
 
     return cpu_count  # no correction
 
 
-def get_mem_effective_capacity(mem_mb):
+def get_cpu_effective_capacity(cpu_count):
     from django.conf import settings
 
-    settings_absmem = getattr(settings, 'SYSTEM_TASK_ABS_MEM', None)
-    env_absmem = os.getenv('SYSTEM_TASK_ABS_MEM', None)
+    cpu_count = get_corrected_cpu(cpu_count)
 
-    if env_absmem is not None:
-        return int(env_absmem)
-    elif settings_absmem is not None:
-        return int(settings_absmem)
+    settings_forkcpu = getattr(settings, 'SYSTEM_TASK_FORKS_CPU', None)
+    env_forkcpu = os.getenv('SYSTEM_TASK_FORKS_CPU', None)
 
-    settings_forkmem = getattr(settings, 'SYSTEM_TASK_FORKS_MEM', None)
-    env_forkmem = os.getenv('SYSTEM_TASK_FORKS_MEM', None)
-
-    if env_forkmem:
-        forkmem = int(env_forkmem)
-    elif settings_forkmem:
-        forkmem = int(settings_forkmem)
+    if env_forkcpu:
+        forkcpu = int(env_forkcpu)
+    elif settings_forkcpu:
+        forkcpu = int(settings_forkcpu)
     else:
-        forkmem = 100
+        forkcpu = 4
 
-    return max(1, ((mem_mb // 1024 // 1024) - 2048) // forkmem)
+    return max(1, int(cpu_count * forkcpu))
+
+
+def convert_mem_str_to_bytes(mem_str):
+    """Convert string with suffix indicating units to memory in bytes (base 2)
+
+    Useful for dealing with memory setting that may be expressed in units compatible with
+    kubernetes.
+
+    See https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory
+    """
+    # If there is no suffix, the memory sourced from the request is in bytes
+    if mem_str.isdigit():
+        return int(mem_str)
+
+    conversions = {
+        'Ei': lambda x: x * 2**60,
+        'E': lambda x: x * 10**18,
+        'Pi': lambda x: x * 2**50,
+        'P': lambda x: x * 10**15,
+        'Ti': lambda x: x * 2**40,
+        'T': lambda x: x * 10**12,
+        'Gi': lambda x: x * 2**30,
+        'G': lambda x: x * 10**9,
+        'Mi': lambda x: x * 2**20,
+        'M': lambda x: x * 10**6,
+        'Ki': lambda x: x * 2**10,
+        'K': lambda x: x * 10**3,
+    }
+    mem = 0
+    mem_unit = None
+    for i, char in enumerate(mem_str):
+        if not char.isdigit():
+            mem_unit = mem_str[i:]
+            mem = int(mem_str[:i])
+            break
+    if not mem_unit or mem_unit not in conversions.keys():
+        error = f"Unsupported value for SYSTEM_TASK_ABS_MEM: {mem_str}, memory must be expressed in bytes or with known suffix: {conversions.keys()}. Falling back to 1 byte"
+        logger.warning(error)
+        return 1
+    return max(1, conversions[mem_unit](mem))
 
 
 def get_corrected_memory(memory):
@@ -776,12 +802,48 @@ def get_corrected_memory(memory):
     settings_absmem = getattr(settings, 'SYSTEM_TASK_ABS_MEM', None)
     env_absmem = os.getenv('SYSTEM_TASK_ABS_MEM', None)
 
-    if env_absmem is not None or settings_absmem is not None:
-        return 0
+    # Runner returns memory in bytes
+    # so we convert memory from settings to bytes as well.
+    if env_absmem is not None:
+        return convert_mem_str_to_bytes(env_absmem)
+    elif settings_absmem is not None:
+        return convert_mem_str_to_bytes(settings_absmem)
 
     return memory
 
 
+def get_mem_effective_capacity(mem_bytes):
+    from django.conf import settings
+
+    mem_bytes = get_corrected_memory(mem_bytes)
+
+    settings_mem_mb_per_fork = getattr(settings, 'SYSTEM_TASK_FORKS_MEM', None)
+    env_mem_mb_per_fork = os.getenv('SYSTEM_TASK_FORKS_MEM', None)
+
+    if env_mem_mb_per_fork:
+        mem_mb_per_fork = int(env_mem_mb_per_fork)
+    elif settings_mem_mb_per_fork:
+        mem_mb_per_fork = int(settings_mem_mb_per_fork)
+    else:
+        mem_mb_per_fork = 100
+
+    # Per docs, deduct 2GB of memory from the available memory
+    # to cover memory consumption of background tasks when redis/web etc are colocated with
+    # the other control processes
+    memory_penalty_bytes = 2147483648
+    if settings.IS_K8S:
+        # In k8s, this is dealt with differently because
+        # redis and the web containers have their own memory allocation
+        memory_penalty_bytes = 0
+
+    # convert memory to megabytes because our setting of how much memory we
+    # should allocate per fork is in megabytes
+    mem_mb = (mem_bytes - memory_penalty_bytes) // 2**20
+    max_forks_based_on_memory = mem_mb // mem_mb_per_fork
+
+    return max(1, max_forks_based_on_memory)
+
+
 _inventory_updates = threading.local()
 _task_manager = threading.local()
 

awx/main/utils/execution_environments.py

@@ -31,11 +31,14 @@ def get_default_pod_spec():
         "kind": "Pod",
         "metadata": {"namespace": settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE},
         "spec": {
+            "serviceAccountName": "default",
+            "automountServiceAccountToken": False,
             "containers": [
                 {
                     "image": ee.image,
                     "name": 'worker',
                     "args": ['ansible-runner', 'worker', '--private-data-dir=/runner'],
+                    "resources": {"requests": {"cpu": "250m", "memory": "100Mi"}},
                 }
             ],
         },

awx/main/utils/handlers.py

@@ -10,6 +10,7 @@ from datetime import datetime
 # Django
 from django.conf import settings
 from django.utils.timezone import now
+from django.utils.encoding import force_str
 
 # AWX
 from awx.main.exceptions import PostRunError
@@ -42,7 +43,7 @@ class RSysLogHandler(logging.handlers.SysLogHandler):
             msg += exc.splitlines()[-1]
         except Exception:
             msg += exc
-        msg = '\n'.join([msg, record.msg, ''])
+        msg = '\n'.join([msg, force_str(record.msg), ''])  # force_str used in case of translated strings
         sys.stderr.write(msg)
 
     def emit(self, msg):