Merge pull request #11570 from AlanCoding/keycloak_docs

Minor docs tweaks for keycloak setup

.dockerignore

@@ -1,2 +1,3 @@
 awx/ui/node_modules
 Dockerfile
+.git

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,11 @@
+<!--- changelog-entry
+# Fill in 'msg' below to have an entry automatically added to the next release changelog.
+# Leaving 'msg' blank will not generate a changelog entry for this PR.
+# Please ensure this is a simple (and readable) one-line string.
+---
+msg: ""
+-->
+
 ##### SUMMARY
 <!--- Describe the change, including rationale and design decisions -->
 

.github/workflows/ci.yml

@@ -1,5 +1,7 @@
 ---
 name: CI
+env:
+  BRANCH: ${{ github.base_ref || 'devel' }}
 on:
   pull_request:
 jobs:
@@ -17,16 +19,16 @@ jobs:
 
       - name: Pre-pull image to warm build cache
         run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }}
+          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || :
 
       - name: Build image
         run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build
+          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build
 
       - name: Run API Tests
         run: |
           docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} /start_tests.sh
+            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /start_tests.sh
   api-lint:
     runs-on: ubuntu-latest
     permissions:
@@ -41,16 +43,16 @@ jobs:
 
       - name: Pre-pull image to warm build cache
         run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }}
+          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || :
 
       - name: Build image
         run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build
+          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build
 
       - name: Run API Linters
         run: |
           docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} /var/lib/awx/venv/awx/bin/tox -e linters
+            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /var/lib/awx/venv/awx/bin/tox -e linters
   api-swagger:
     runs-on: ubuntu-latest
     permissions:
@@ -65,16 +67,16 @@ jobs:
 
       - name: Pre-pull image to warm build cache
         run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} || :
+          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || : || :
 
       - name: Build image
         run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }}  make docker-compose-build
+          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }}  make docker-compose-build
 
       - name: Generate API Reference
         run: |
           docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} /start_tests.sh swagger
+            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /start_tests.sh swagger
   awx-collection:
     runs-on: ubuntu-latest
     permissions:
@@ -89,16 +91,16 @@ jobs:
 
       - name: Pre-pull image to warm build cache
         run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }}
+          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || :
 
       - name: Build image
         run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }}  make docker-compose-build
+          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }}  make docker-compose-build
 
       - name: Run Collection Tests
         run: |
           docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} /start_tests.sh test_collection_all
+            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /start_tests.sh test_collection_all
   api-schema:
     runs-on: ubuntu-latest
     permissions:
@@ -113,16 +115,16 @@ jobs:
 
       - name: Pre-pull image to warm build cache
         run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }}
+          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || :
 
       - name: Build image
         run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }}  make docker-compose-build
+          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }}  make docker-compose-build
 
       - name: Check API Schema
         run: |
           docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} /start_tests.sh detect-schema-change
+            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} /start_tests.sh detect-schema-change
   ui-lint:
     runs-on: ubuntu-latest
     permissions:
@@ -137,16 +139,16 @@ jobs:
 
       - name: Pre-pull image to warm build cache
         run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }}
+          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || :
 
       - name: Build image
         run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build
+          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build
 
       - name: Run UI Linters
         run: |
           docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} make ui-lint
+            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} make ui-lint
   ui-test:
     runs-on: ubuntu-latest
     permissions:
@@ -161,13 +163,51 @@ jobs:
 
       - name: Pre-pull image to warm build cache
         run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }}
+          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} || :
 
       - name: Build image
         run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose-build
+          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ env.BRANCH }} make docker-compose-build
 
       - name: Run UI Tests
         run: |
           docker run -u $(id -u) --rm -v ${{ github.workspace}}:/awx_devel/:Z \
-            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }} make ui-test
+            --workdir=/awx_devel ghcr.io/${{ github.repository_owner }}/awx_devel:${{ env.BRANCH }} make ui-test
+  awx-operator:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout awx
+        uses: actions/checkout@v2
+        with:
+          path: awx
+
+      - name: Checkout awx-operator
+        uses: actions/checkout@v2
+        with:
+          repository: ansible/awx-operator
+          path: awx-operator
+
+      - name: Install playbook dependencies
+        run: |
+          python3 -m pip install docker
+
+      - name: Build AWX image
+        working-directory: awx
+        run: |
+          ansible-playbook -v tools/ansible/build.yml \
+            -e headless=yes \
+            -e awx_image=awx \
+            -e awx_image_tag=ci \
+            -e ansible_python_interpreter=$(which python3)
+
+      - name: Run test deployment with awx-operator
+        working-directory: awx-operator
+        run: |
+          python3 -m pip install -r molecule/requirements.txt
+          ansible-galaxy collection install -r molecule/requirements.yml
+          sudo rm -f $(which kustomize)
+          make kustomize
+          KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind
+        env:
+          AWX_TEST_IMAGE: awx
+          AWX_TEST_VERSION: ci

.github/workflows/e2e_test.yml

@@ -85,7 +85,7 @@ jobs:
           -e CYPRESS_baseUrl="https://$AWX_IP:8043" \
           -e CYPRESS_AWX_E2E_USERNAME=admin \
           -e CYPRESS_AWX_E2E_PASSWORD='password' \
-          -e COMMAND="npm run cypress-gha" \
+          -e COMMAND="npm run cypress-concurrently-gha" \
           -v /dev/shm:/dev/shm \
           -v $PWD:/e2e \
           -w /e2e \

.github/workflows/promote.yml

@@ -0,0 +1,26 @@
+---
+name: Promote Release
+on:
+  release:
+    types: [published]
+
+jobs:
+  promote:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Log in to GHCR
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Log in to Quay
+        run: |
+          echo ${{ secrets.QUAY_TOKEN }} | docker login quay.io -u ${{ secrets.QUAY_USER }} --password-stdin
+
+      - name: Re-tag and promote awx image
+        run: |
+          docker pull ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name }}
+          docker tag ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name }} quay.io/${{ github.repository }}:${{ github.event.release.tag_name }}
+          docker tag ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name }} quay.io/${{ github.repository }}:latest
+          docker push quay.io/${{ github.repository }}:${{ github.event.release.tag_name }}
+          docker push quay.io/${{ github.repository }}:latest
+          

.github/workflows/stage.yml

@@ -0,0 +1,123 @@
+---
+name: Stage Release
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'AWX version.'
+        required: true
+        default: ''
+      operator_version:
+        description: 'Operator version. Leave blank to skip staging awx-operator.'
+        default: ''
+      confirm:
+        description: 'Are you sure? Set this to yes.'
+        required: true
+        default: 'no'
+
+jobs:
+  stage:
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: write
+    steps:
+      - name: Verify inputs
+        run: |
+          set -e
+
+          if [[ ${{ github.event.inputs.confirm }} != "yes" ]]; then
+            >&2 echo "Confirm must be 'yes'"
+            exit 1
+          fi
+
+          if [[ ${{ github.event.inputs.version }} == "" ]]; then
+            >&2 echo "Set version to continue."
+            exit 1
+          fi
+
+          exit 0
+
+      - name: Checkout awx
+        uses: actions/checkout@v2
+        with:
+          path: awx
+
+      - name: Checkout awx-logos
+        uses: actions/checkout@v2
+        with:
+          repository: ansible/awx-logos
+          path: awx-logos
+
+      - name: Checkout awx-operator
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ github.repository_owner }}/awx-operator
+          path: awx-operator
+
+      - name: Install playbook dependencies
+        run: |
+          python3 -m pip install docker
+
+      - name: Build and stage AWX
+        working-directory: awx
+        run: |
+          ansible-playbook -v tools/ansible/build.yml \
+            -e registry=ghcr.io \
+            -e registry_username=${{ github.actor }} \
+            -e registry_password=${{ secrets.GITHUB_TOKEN }} \
+            -e awx_image=${{ github.repository }} \
+            -e awx_version=${{ github.event.inputs.version }} \
+            -e ansible_python_interpreter=$(which python3) \
+            -e push=yes \
+            -e awx_official=yes
+
+      - name: Build and stage awx-operator
+        working-directory: awx-operator
+        run: |
+          BUILD_ARGS="--build-arg DEFAULT_AWX_VERSION=${{ github.event.inputs.version }}" \
+          IMAGE_TAG_BASE=ghcr.io/${{ github.repository_owner }}/awx-operator \
+          VERSION=${{ github.event.inputs.operator_version }} make docker-build docker-push
+
+      - name: Run test deployment with awx-operator
+        working-directory: awx-operator
+        run: |
+          python3 -m pip install -r molecule/requirements.txt
+          ansible-galaxy collection install -r molecule/requirements.yml
+          sudo rm -f $(which kustomize)
+          make kustomize
+          KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind
+        env:
+          AWX_TEST_IMAGE: ${{ github.repository }}
+          AWX_TEST_VERSION: ${{ github.event.inputs.version }}
+
+      - name: Generate changelog
+        uses: shanemcd/simple-changelog-generator@v1
+        id: changelog
+        with:
+          repo: "${{ github.repository }}"
+
+      - name: Write changelog to file
+        run: |
+          cat << 'EOF' > /tmp/awx-changelog
+          ${{ steps.changelog.outputs.changelog }}
+          EOF
+
+      - name: Create draft release for AWX
+        working-directory: awx
+        run: |
+          ansible-playbook -v tools/ansible/stage.yml \
+            -e changelog_path=/tmp/awx-changelog \
+            -e repo=${{ github.repository }} \
+            -e awx_image=ghcr.io/${{ github.repository }} \
+            -e version=${{ github.event.inputs.version }} \
+            -e github_token=${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create draft release for awx-operator
+        if: ${{ github.event.inputs.operator_version != '' }}
+        working-directory: awx
+        run: |
+          ansible-playbook tools/ansible/stage.yml \
+            -e version=${{ github.event.inputs.operator_version }} \
+            -e repo=${{ github.repository_owner }}/awx-operator \
+            -e github_token=${{ secrets.AWX_OPERATOR_RELEASE_TOKEN }}

.github/workflows/upload_schema.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - devel
+      - release_4.1
 jobs:
   push:
     runs-on: ubuntu-latest
@@ -19,7 +20,7 @@ jobs:
 
       - name: Pre-pull image to warm build cache
         run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${GITHUB_REF##*/}
+          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${GITHUB_REF##*/} || :
 
       - name: Build image
         run: |

.gitignore

@@ -42,6 +42,7 @@ tools/docker-compose/_build
 tools/docker-compose/_sources
 tools/docker-compose/overrides/
 tools/docker-compose-minikube/_sources
+tools/docker-compose/keycloak.awx.realm.json
 
 # Tower setup playbook testing
 setup/test/roles/postgresql
@@ -58,6 +59,7 @@ __pycache__
 /dist
 /*.egg-info
 *.py[c,o]
+/.eggs
 
 # JavaScript
 /Gruntfile.js

.yamllint

@@ -6,8 +6,11 @@ ignore: |
   # vault files
   awx/main/tests/data/ansible_utils/playbooks/valid/vault.yml
   awx/ui/test/e2e/tests/smoke-vars.yml
+  awx/ui/node_modules
+  tools/docker-compose/_sources
 
 extends: default
 
 rules:
   line-length: disable
+  truthy: disable

CHANGELOG.md

@@ -1,520 +1,7 @@
 # Changelog
 
-# 19.3.0 (August 12, 2021)
+**Note:** This file is deprecated and will be removed at some point in a future release.
 
-- Fixed threading bug that would sometimes cause jobs to randomly fail (https://github.com/ansible/awx/pull/10537)
-- Fixed race where app would crash when postgres is not available (https://github.com/ansible/awx/pull/10583)
-- Add support for workflow node aliasing via identifier field (https://github.com/ansible/awx/pull/10592)
-- Add UI support for management jobs in workflows (https://github.com/ansible/awx/pull/10572)
-- Show PAT as part of bulk delete list (https://github.com/ansible/awx/pull/10794)
-- Return 404 for ad_hoc_command_events list api. Remove api endtpoint (https://github.com/ansible/awx/pull/10716)
-- Fix multiple accessibility violations (https://github.com/ansible/awx/pull/10713)
-- Fix ignoring --no-color for awx-manage list_instances command (https://github.com/ansible/awx/pull/10668)
-- Fix to handle ask_* parameters correctly when set false (https://github.com/ansible/awx/pull/10108)
-- Default source_project to organization for inventory source (https://github.com/ansible/awx/pull/10573)
-- Fix headers missing in webhook notification request (https://github.com/ansible/awx/pull/10566)
-- Avoid double LDAP updates (https://github.com/ansible/awx/pull/9703)
-- introduced a pre-flight check for postgres 12 (https://github.com/ansible/awx/pull/10425)
-- Fix Job Settings Page Break on Firefox (https://github.com/ansible/awx/pull/10523)
-- bumped django version to 2.2.20 (https://github.com/ansible/awx/pull/10564)
-- Add Thycotic SecretServer support (https://github.com/ansible/awx/pull/10632)
+Starting with AWX 20, release notes are published to [GitHub Releases](https://github.com/ansible/awx/releases).
 
-# 19.2.2 (June 28, 2021)
-
-- Fixed bug where symlinks pointing to directories were not preserved (https://github.com/ansible/ansible-runner/pull/736)
-- Various bugfixes found during testing (https://github.com/ansible/awx/pull/10532)
-
-# 19.2.1 (June 17, 2021)
-
-- There are now 2 default Instance Groups: 'controlplane' and 'default' (https://github.com/ansible/awx/pull/10324)
-- Removed deprecated modules: `tower_send`, `tower_receive`, `tower_workflow_template` (https://github.com/ansible/awx/pull/9980)
-- Improved UI performance when a large amount of events are being emitted by jobs (https://github.com/ansible/awx/pull/10053)
-- Settings UI Revert All button now issues a DELETE instead of PATCHing all fields (https://github.com/ansible/awx/pull/10376)
-- Fixed a bug with the schedule date/time picker in Firefox (https://github.com/ansible/awx/pull/10291)
-- UI now preselects the system default Galaxy credential when creating a new organization (https://github.com/ansible/awx/pull/10395)
-- Added favicon (https://github.com/ansible/awx/pull/10388)
-- Removed `not` option from smart inventory host filter search as it's not supported by the API (https://github.com/ansible/awx/pull/10380)
-- Added button to allow user to refetch project revision after project sync has finished (https://github.com/ansible/awx/pull/10334)
-- Fixed bug where extraneous CONFIG requests were made on logout (https://github.com/ansible/awx/pull/10379)
-- Fixed bug where users were unable to cancel inventory syncs (https://github.com/ansible/awx/pull/10346)
-- Added missing dashboard graph filters (https://github.com/ansible/awx/pull/10349)
-- Added support for typing in to single select lookup form fields (https://github.com/ansible/awx/pull/10257)
-- Fixed various bugs related to user sessions (https://github.com/ansible/awx/pull/9908)
-- Fixed bug where sorting in modals would close the modal (https://github.com/ansible/awx/pull/10215)
-- Added support for Red Hat Insights as an inventory source (https://github.com/ansible/awx/pull/8650)
-- Fixed bugs when selecting items in a list then sorting/paginating (https://github.com/ansible/awx/pull/10329)
-
-# 19.2.0 (June 1, 2021)
-- Fixed race condition that would sometimes cause jobs to error out at the very end of an otherwise successful run (https://github.com/ansible/receptor/pull/328)
-- Fixes bug where users were unable to click on text next to checkboxes in modals (https://github.com/ansible/awx/pull/10279)
-- Have the project update playbook warn if role/collection syncing is disabled. (https://github.com/ansible/awx/pull/10068)
-- Move irc references to point to irc.libera.chat (https://github.com/ansible/awx/pull/10295)
-- Fixes bug where activity stream changes were displaying as [object object] (https://github.com/ansible/awx/pull/10267)
-- Update awxkit to enable export of Galaxy credentials associated to organizations (https://github.com/ansible/awx/pull/10271)
-- Bump receptor and receptorctl versions to 1.0.0a2 (https://github.com/ansible/awx/pull/10261)
-- Add the ability to disable local authentication (https://github.com/ansible/awx/pull/10102)
-- Show error if no Execution Environment is found on project sync/job run (https://github.com/ansible/awx/pull/10183)
-- Allow for editing and deleting managed_by_tower EEs from API/UI (https://github.com/ansible/awx/pull/10173)
-
-
-# 19.1.0 (May 1, 2021)
-
-- Custom inventory scripts have been removed from the API https://github.com/ansible/awx/pull/9822
-  - Old scripts can be exported via `awx-manage export_custom_scripts`
-- Fixed a bug where ad-hoc commands targeted against multiple hosts would run against only 1 host https://github.com/ansible/awx/pull/9973
-- AWX will now look for a top-level requirements.yml when installing collections / roles in project updates https://github.com/ansible/awx/pull/9945
-- Improved error handling when Container Group pods fail to launch https://github.com/ansible/awx/pull/10025
-- Added ability to set server-side password policies using Django's AUTH_PASSWORD_VALIDATORS setting https://github.com/ansible/awx/pull/9999
-- Bumped versions of Ansible Runner & AWX EE https://github.com/ansible/awx/pull/10013
-  - If you have built any custom EEs on top of awx-ee 0.1.0, you will need to rebuild on top of 0.2.0.
-- Remove legacy resource profiling code https://github.com/ansible/awx/pull/9883
-
-# 19.0.0 (April 7, 2021)
-
-- AWX now runs on Python 3.8 (https://github.com/ansible/awx/pull/8778/)
-- Fixed inventories-from-projects when running in Kubernetes (https://github.com/ansible/awx/pull/9741)
-- Fixed a bug where a slash was appended to invetory file paths in UI dropdown (https://github.com/ansible/awx/pull/9713)
-- Fix a bug with large file parsing in project sync (https://github.com/ansible/awx/pull/9627)
-- Fix k8s credentials that use a custom ca cert (https://github.com/ansible/awx/pull/9744)
-- Fix a bug that allowed a user to attempt deleting a running job (https://github.com/ansible/awx/pull/9758)
-- Fixed the Kubernetes Pod reaper to properly delete Pods launched by Receptor (https://github.com/ansible/awx/pull/9819)
-- AWX Collection Modules: added ability to set instance groups for organization, job templates, and inventories. (https://github.com/ansible/awx/pull/9804)
-- Fixed CSP violation errors on job details and job settings views (https://github.com/ansible/awx/pull/9818)
-- Added support for convergence any/all on workflow nodes (https://github.com/ansible/awx/pull/9737)
-- Fixed race condition that causes InvalidGitRepositoryError (https://github.com/ansible/awx/pull/9754)
-- Added support for Execution Environments to the Activity Stream (https://github.com/ansible/awx/issues/9308)
-- Fixed a bug that improperly formats OpenSSH keys specified in custom Credential Types (https://github.com/ansible/awx/issues/9361)
-- Fixed an HTTP 500 error for unauthenticated users (https://github.com/ansible/awx/pull/9725)
-- Added subscription wizard: https://github.com/ansible/awx/pull/9496
-
-# 18.0.0 (March 23, 2021)
-
-**IMPORTANT INSTALL AND UPGRADE NOTES**
-
-Starting in version 18.0, the [AWX Operator](https://github.com/ansible/awx-operator) is the preferred way to install AWX: https://github.com/ansible/awx/blob/devel/INSTALL.md#installing-awx
-
-If you have a pre-existing installation of AWX that utilizes the Docker-based installation method, this install method has ** notably changed** from 17.x to 18.x.  For details, please see:
-
-- https://groups.google.com/g/awx-project/c/47MjWSUQaOc/m/bCjSDn0eBQAJ
-- https://github.com/ansible/awx/blob/devel/tools/docker-compose
-- https://github.com/ansible/awx/blob/devel/tools/docker-compose/docs/data_migration.md
-
-### Introducing Execution Environments
-
-After a herculean effort from a number of contributors, we're excited to announce that AWX 18.0.0 introduces a new concept called Execution Environments.
-
-Execution Environments are container images which consist of everything necessary to run a playbook within AWX, and which drive the entire management and lifecycle of playbook execution runtime in AWX: https://github.com/ansible/awx/issues/5157.  This means that going forward, AWX no longer utilizes the [bubblewrap](https://github.com/containers/bubblewrap) project for playbook isolation, but instead utilizes a container per playbook run.
-
-Much like custom virtualenvs, custom Execution Environments can be crafted to specify additional Python or system-level dependencies.  [Ansible Builder](https://github.com/ansible/ansible-builder) outputs images you can upload to your registry which can *then* be defined in AWX and utilized for playbook runs.
-
-To learn more about Ansible Builder and Execution Environments, see: https://www.ansible.com/blog/introduction-to-ansible-builder
-
-### Other Notable Changes
-
-- Removed `installer` directory.
-  - The Kubernetes installer has been removed in favor of [AWX Operator](https://github.com/ansible/awx-operator).  Official images for Operator-based installs are no longer hosted on Docker Hub, but are instead available on [Quay](https://quay.io/repository/ansible/awx?tab=tags).
-  - The "Local Docker" install method has been removed in favor of the development environment. Details can be found at: https://github.com/ansible/awx/blob/devel/tools/docker-compose/README.md
-- Removal of custom virtual environments https://github.com/ansible/awx/pull/9498
-  - Custom virtual environments have been replaced by Execution Environments https://github.com/ansible/awx/pull/9570
-- The default Container Group Pod definition has changed. All custom Pod specs have been reset. https://github.com/ansible/awx/commit/05ef51f710dad8f8036bc5acee4097db4adc0d71
-- Added user interface for the activity stream: https://github.com/ansible/awx/pull/9083
-- Converted many of the top-level list views (Jobs, Teams, Hosts, Inventories, Projects, and more) to a new, permanent table component for substantially increased responsiveness, usability, maintainability, and other 'ility's: https://github.com/ansible/awx/pull/8970, https://github.com/ansible/awx/pull/9182 and many others!
-- Added support for Centrify Vault (https://www.centrify.com) as a credential lookup plugin (https://github.com/ansible/awx/pull/9542)
-- Added support for namespaces in Hashicorp Vault credential plugin (https://github.com/ansible/awx/pull/9590)
-- Added click-to-expand details for job tables
-- Added search filtering to job output https://github.com/ansible/awx/pull/9208
-- Added the new migration, update, and "installation in progress" page https://github.com/ansible/awx/pull/9123
-- Added the user interface for job settings https://github.com/ansible/awx/pull/8661
-- Runtime errors from jobs are now displayed, along with an explanation for what went wrong, on the output page https://github.com/ansible/awx/pull/8726
-- You can now cancel a running job from its output and details panel https://github.com/ansible/awx/pull/9199
-- Fixed a bug where launch prompt inputs were unexpectedly deposited in the url: https://github.com/ansible/awx/pull/9231
-- Playbook, credential type, and inventory file inputs now support type-ahead and manual type-in! https://github.com/ansible/awx/pull/9120
-- Added ability to relaunch against failed hosts: https://github.com/ansible/awx/pull/9225
-- Added pending workflow approval count to the application header https://github.com/ansible/awx/pull/9334
-- Added user interface for management jobs: https://github.com/ansible/awx/pull/9224
-- Added toast message to show notification template test result to notification templates list https://github.com/ansible/awx/pull/9318
-- Replaced CodeMirror with AceEditor for editing template variables and notification templates https://github.com/ansible/awx/pull/9281
-- Added support for filtering and pagination on job output https://github.com/ansible/awx/pull/9208
-- Added support for html in custom login text https://github.com/ansible/awx/pull/9519
-
-# 17.1.0 (March 9, 2021)
-- Addressed a security issue in AWX (CVE-2021-20253)
-- Fixed a bug permissions error related to redis in K8S-based deployments: https://github.com/ansible/awx/issues/9401
-
-# 17.0.1 (January 26, 2021)
-- Fixed pgdocker directory permissions issue with Local Docker installer: https://github.com/ansible/awx/pull/9152
-- Fixed a bug in the UI which caused toggle settings to not be changed when clicked: https://github.com/ansible/awx/pull/9093
-
-# 17.0.0 (January 22, 2021)
-- AWX now requires PostgreSQL 12 by default: https://github.com/ansible/awx/pull/8943
-  **Note:** users who encounter permissions errors at upgrade time should `chown -R ~/.awx/pgdocker` to ensure it's owned by the user running the install playbook
-- Added support for region name for OpenStack inventory: https://github.com/ansible/awx/issues/5080
-- Added the ability to chain undefined attributes in custom notification templates: https://github.com/ansible/awx/issues/8677
-- Dramatically simplified the `image_build` role: https://github.com/ansible/awx/pull/8980
-- Fixed a bug which can cause schema migrations to fail at install time: https://github.com/ansible/awx/issues/9077
-- Fixed a bug which caused the `is_superuser` user property to be out of date in certain circumstances: https://github.com/ansible/awx/pull/8833
-- Fixed a bug which sometimes results in race conditions on setting access: https://github.com/ansible/awx/pull/8580
-- Fixed a bug which sometimes causes an unexpected delay in stdout for some playbooks: https://github.com/ansible/awx/issues/9085
-- (UI) Added support for credential password prompting on job launch: https://github.com/ansible/awx/pull/9028
-- (UI) Added the ability to configure LDAP settings in the UI: https://github.com/ansible/awx/issues/8291
-- (UI) Added a sync button to the Project detail view: https://github.com/ansible/awx/issues/8847
-- (UI) Added a form for configuring Google Outh 2.0 settings: https://github.com/ansible/awx/pull/8762
-- (UI) Added searchable keys and related keys to the Credentials list: https://github.com/ansible/awx/issues/8603
-- (UI) Added support for advanced search and copying to Notification Templates: https://github.com/ansible/awx/issues/7879
-- (UI) Added support for prompting on workflow nodes: https://github.com/ansible/awx/issues/5913
-- (UI) Added support for session timeouts: https://github.com/ansible/awx/pull/8250
-- (UI) Fixed a bug that broke websocket streaming for the insecure ws:// protocol: https://github.com/ansible/awx/pull/8877
-- (UI) Fixed a bug in the user interface when a translation for the browser's preferred locale isn't available: https://github.com/ansible/awx/issues/8884
-- (UI) Fixed bug where navigating from one survey question form directly to another wasn't reloading the form: https://github.com/ansible/awx/issues/7522
-- (UI) Fixed a bug which can cause an uncaught error while launching a Job Template: https://github.com/ansible/awx/issues/8936
-- Updated autobahn to address CVE-2020-35678
-
-## 16.0.0 (December 10, 2020)
-- AWX now ships with a reimagined user interface.  **Please read this before upgrading:** https://groups.google.com/g/awx-project/c/KuT5Ao92HWo
-- Removed support for syncing inventory from Red Hat CloudForms - https://github.com/ansible/awx/commit/0b701b3b2
-- Removed support for Mercurial-based project updates - https://github.com/ansible/awx/issues/7932
-- Upgraded NodeJS to actively maintained LTS 14.15.1 - https://github.com/ansible/awx/pull/8766
-- Added Git-LFS to the default image build - https://github.com/ansible/awx/pull/8700
-- Added the ability to specify `metadata.labels` in the podspec for container groups - https://github.com/ansible/awx/issues/8486
-- Added support for Kubernetes pod annotations - https://github.com/ansible/awx/pull/8434
-- Added the ability to label the web container in local Docker installs - https://github.com/ansible/awx/pull/8449
-- Added additional metadata (as an extra var) to playbook runs to report the SCM branch name - https://github.com/ansible/awx/pull/8433
-- Fixed a bug that caused k8s installations to fail due to an incorrect Helm repo - https://github.com/ansible/awx/issues/8715
-- Fixed a bug that prevented certain Workflow Approval resources from being deleted - https://github.com/ansible/awx/pull/8612
-- Fixed a bug that prevented the deletion of inventories stuck in "pending deletion" state - https://github.com/ansible/awx/issues/8525
-- Fixed a display bug in webhook notifications with certain unicode characters - https://github.com/ansible/awx/issues/7400
-- Improved support for exporting dependent objects (Inventory Hosts and Groups) in the `awx export` CLI tool - https://github.com/ansible/awx/commit/607bc0788
-
-## 15.0.1 (October 20, 2020)
-- Added several optimizations to improve performance for a variety of high-load simultaneous job launch use cases https://github.com/ansible/awx/pull/8403
-- Added the ability to source roles and collections from requirements.yaml files (not just requirements.yml) - https://github.com/ansible/awx/issues/4540
-- awx.awx collection modules now provide a clearer error message for incompatible versions of awxkit - https://github.com/ansible/awx/issues/8127
-- Fixed a bug in notification messages that contain certain unicode characters - https://github.com/ansible/awx/issues/7400
-- Fixed a bug that prevents the deletion of Workflow Approval records - https://github.com/ansible/awx/issues/8305
-- Fixed a bug that broke the selection of webhook credentials - https://github.com/ansible/awx/issues/7892
-- Fixed a bug which can cause confusing behavior for social auth logins across distinct browser tabs - https://github.com/ansible/awx/issues/8154
-- Fixed several bugs in the output of Workflow Job Templates using the `awx export` tool - https://github.com/ansible/awx/issues/7798 https://github.com/ansible/awx/pull/7847
-- Fixed a race condition that can lead to missing hosts when running parallel inventory syncs - https://github.com/ansible/awx/issues/5571
-- Fixed an HTTP 500 error when certain LDAP group parameters aren't properly set - https://github.com/ansible/awx/issues/7622
-- Updated a few dependencies in response to several CVEs:
-    * CVE-2020-7720
-    * CVE-2020-7743
-    * CVE-2020-7676
-
-## 15.0.0 (September 30, 2020)
-- Added improved support for fetching Ansible collections from private Galaxy content sources (such as https://github.com/ansible/galaxy_ng) - https://github.com/ansible/awx/issues/7813
-  **Note:** as part of this change, new Organizations created in the AWX API will _no longer_ automatically synchronize roles and collections from galaxy.ansible.com by default.  More details on this change can be found at:  https://github.com/ansible/awx/issues/8341#issuecomment-707310633
-- AWX now utilizes a version of certifi that auto-discovers certificates in the system certificate store - https://github.com/ansible/awx/pull/8242
-- Added support for arbitrary custom inventory plugin configuration: https://github.com/ansible/awx/issues/5150
-- Added an optional setting to disable the auto-creation of organizations and teams on successful SAML login. - https://github.com/ansible/awx/pull/8069
-- Added a number of optimizations to AWX's callback receiver to improve the speed of stdout processing for simultaneous playbooks runs - https://github.com/ansible/awx/pull/8193 https://github.com/ansible/awx/pull/8191
-- Added the ability to use `!include` and `!import` constructors when constructing YAML for use with the AWX CLI - https://github.com/ansible/awx/issues/8135
-- Fixed a bug that prevented certain users from being able to edit approval nodes in Workflows - https://github.com/ansible/awx/pull/8253
-- Fixed a bug that broke password prompting for credentials in certain cases - https://github.com/ansible/awx/issues/8202
-- Fixed a bug which can cause PostgreSQL deadlocks when running many parallel playbooks against large shared inventories - https://github.com/ansible/awx/issues/8145
-- Fixed a bug which can cause delays in AWX's task manager when large numbers of simultaneous jobs are scheduled - https://github.com/ansible/awx/issues/7655
-- Fixed a bug which can cause certain scheduled jobs - those that run every X minute(s) or hour(s) - to fail to run at the proper time - https://github.com/ansible/awx/issues/8071
-- Fixed a performance issue for playbooks that store large amounts of data using the `set_stats` module - https://github.com/ansible/awx/issues/8006
-- Fixed a bug related to AWX's handling of the auth_path argument for the HashiVault KeyValue credential plugin - https://github.com/ansible/awx/pull/7991
-- Fixed a bug that broke support for Remote Archive SCM Type project syncs on platforms that utilize Python2 - https://github.com/ansible/awx/pull/8057
-- Updated to the latest version of Django Rest Framework to address CVE-2020-25626
-- Updated to the latest version of Django to address CVE-2020-24583 and CVE-2020-24584
-- Updated to the latest verson of channels_redis to address a bug that slowly causes Daphne processes to leak memory over time - https://github.com/django/channels_redis/issues/212
-
-## 14.1.0 (Aug 25, 2020)
-- AWX images can now be built on ARM64 - https://github.com/ansible/awx/pull/7607
-- Added the Remote Archive SCM Type to support using immutable artifacts and releases (such as tarballs and zip files) as projects - https://github.com/ansible/awx/issues/7954
-- Deprecated official support for Mercurial-based project updates - https://github.com/ansible/awx/issues/7932
-- Added resource import/export support to the official AWX collection - https://github.com/ansible/awx/issues/7329
-- Added the ability to import YAML-based resources (instead of just JSON) when using the AWX CLI - https://github.com/ansible/awx/pull/7808
-- Users upgrading from older versions of AWX may encounter an issue that causes their postgres container to restart in a loop (https://github.com/ansible/awx/issues/7854) - if you encounter this, bring your containers down and then back up (e.g., `docker-compose down && docker-compose up -d`) after upgrading to 14.1.0.
-- Updated the AWX CLI to export labels associated with Workflow Job Templates - https://github.com/ansible/awx/pull/7847
-- Updated to the latest python-ldap to address a bug - https://github.com/ansible/awx/issues/7868
-- Upgraded git-python to fix a bug that caused workflows to sometimes fail - https://github.com/ansible/awx/issues/6119
-- Worked around a bug in the channels_redis library that slowly causes Daphne processes to leak memory over time - https://github.com/django/channels_redis/issues/212
-- Fixed a bug in the AWX CLI that prevented Workflow nodes from importing properly - https://github.com/ansible/awx/issues/7793
-- Fixed a bug in the awx.awx collection release process that templated the wrong version - https://github.com/ansible/awx/issues/7870
-- Fixed a bug that caused errors rendering stdout that contained UTF-16 surrogate pairs - https://github.com/ansible/awx/pull/7918
-
-## 14.0.0 (Aug 6, 2020)
-- As part of our commitment to inclusivity in open source, we recently took some time to audit AWX's source code and user interface and replace certain terminology with more inclusive language.  Strictly speaking, this isn't a bug or a feature, but we think it's important and worth calling attention to:
-    * https://github.com/ansible/awx/commit/78229f58715fbfbf88177e54031f532543b57acc
-    * https://www.redhat.com/en/blog/making-open-source-more-inclusive-eradicating-problematic-language
-- Installing roles and collections via requirements.yml as part of Project Updates now requires at least Ansible 2.9 - https://github.com/ansible/awx/issues/7769
-- Deprecated the use of the `PRIMARY_GALAXY_USERNAME` and `PRIMARY_GALAXY_PASSWORD` settings. We recommend using tokens to access Galaxy or Automation Hub.
-- Added local caching for downloaded roles and collections so they are not re-downloaded on nodes where they are up to date with the project - https://github.com/ansible/awx/issues/5518
-- Added the ability to associate K8S/OpenShift credentials to Job Template for playbook interaction with the `community.kubernetes` collection - https://github.com/ansible/awx/issues/5735
-- Added the ability to include HTML in the Custom Login Info presented on the login page - https://github.com/ansible/awx/issues/7600
-- Fixed https://access.redhat.com/security/cve/cve-2020-14327 - Server-side request forgery on credentials
-- Fixed https://access.redhat.com/security/cve/cve-2020-14328 - Server-side request forgery on webhooks
-- Fixed https://access.redhat.com/security/cve/cve-2020-14329 - Sensitive data exposure on labels
-- Fixed https://access.redhat.com/security/cve/cve-2020-14337 - Named URLs allow for testing the presence or absence of objects
-- Fixed a number of bugs in the user interface related to an upgrade of jQuery:
-     * https://github.com/ansible/awx/issues/7530
-     * https://github.com/ansible/awx/issues/7546
-     * https://github.com/ansible/awx/issues/7534
-     * https://github.com/ansible/awx/issues/7606
-- Fixed a bug that caused the `-f yaml` flag of the AWX CLI to not print properly formatted YAML - https://github.com/ansible/awx/issues/7795
-- Fixed a bug in the installer that caused errors when `docker_registry_password` was set - https://github.com/ansible/awx/issues/7695
-- Fixed a permissions error that prevented certain users from starting AWX services - https://github.com/ansible/awx/issues/7545
-- Fixed a bug that allows superusers to run unsafe Jinja code when defining custom Credential Types - https://github.com/ansible/awx/pull/7584/
-- Fixed a bug that prevented users from creating (or editing) custom Credential Types containing boolean fields - https://github.com/ansible/awx/issues/7483
-- Fixed a bug that prevented users with postgres usernames containing uppercase letters from restoring backups succesfully - https://github.com/ansible/awx/pull/7519
-- Fixed a bug which allowed the creation (in the Tower API) of Groups and Hosts with the same name - https://github.com/ansible/awx/issues/4680
-
-## 13.0.0 (Jun 23, 2020)
-- Added import and export commands to the official AWX CLI, replacing send and receive from the old tower-cli (https://github.com/ansible/awx/pull/6125).
-- Removed scripts as a means of running inventory updates of built-in types (https://github.com/ansible/awx/pull/6911)
-- Ansible 2.8 is now partially unsupported; some inventory source types are known to no longer work.
-- Fixed an issue where the vmware inventory source ssl_verify source variable was not recognized (https://github.com/ansible/awx/pull/7360)
-- Fixed a bug that caused redis' listen socket to have too-permissive file permissions (https://github.com/ansible/awx/pull/7317)
-- Fixed a bug that caused rsyslogd's configuration file to have world-readable file permissions, potentially leaking secrets (CVE-2020-10782)
-
-## 12.0.0 (Jun 9, 2020)
-- Removed memcached as a dependency of AWX (https://github.com/ansible/awx/pull/7240)
-- Moved to a single container image build instead of separate awx_web and awx_task images. The container image is just `awx` (https://github.com/ansible/awx/pull/7228)
-- Official AWX container image builds now use a two-stage container build process that notably reduces the size of our published images (https://github.com/ansible/awx/pull/7017)
-- Removed support for HipChat notifications ([EoL announcement](https://www.atlassian.com/partnerships/slack/faq#faq-98b17ca3-247f-423b-9a78-70a91681eff0)); all previously-created HipChat notification templates will be deleted due to this removal.
-- Fixed a bug which broke AWX installations with oc version 4.3 (https://github.com/ansible/awx/pull/6948/)
-- Fixed a performance issue that caused notable delay of stdout processing for playbooks run against large numbers of hosts (https://github.com/ansible/awx/issues/6991)
-- Fixed a bug that caused CyberArk AIM credential plugin looks to hang forever in some environments (https://github.com/ansible/awx/issues/6986)
-- Fixed a bug that caused ANY/ALL converage settings not to properly save when editing approval nodes in the UI (https://github.com/ansible/awx/issues/6998)
-- Fixed a bug that broke support for the satellite6_group_prefix source variable (https://github.com/ansible/awx/issues/7031)
-- Fixed a bug that prevented changes to workflow node convergence settings when approval nodes were in use (https://github.com/ansible/awx/issues/7063)
-- Fixed a bug that caused notifications to fail on newer version of Mattermost (https://github.com/ansible/awx/issues/7264)
-- Fixed a bug (by upgrading to 0.8.1 of the foreman collection) that prevented host_filters from working properly with Foreman-based inventory (https://github.com/ansible/awx/issues/7225)
-- Fixed a bug that prevented the usage of the Conjur credential plugin with secrets that contain spaces (https://github.com/ansible/awx/issues/7191)
-- Fixed a bug in awx-manage run_wsbroadcast --status in kubernetes (https://github.com/ansible/awx/pull/7009)
-- Fixed a bug that broke notification toggles for system jobs in the UI (https://github.com/ansible/awx/pull/7042)
-- Fixed a bug that broke local pip installs of awxkit (https://github.com/ansible/awx/issues/7107)
-- Fixed a bug that prevented PagerDuty notifications from sending for workflow job template approvals (https://github.com/ansible/awx/issues/7094)
-- Fixed a bug that broke external log aggregation support for URL paths that include the = character (such as the tokens for SumoLogic) (https://github.com/ansible/awx/issues/7139)
-- Fixed a bug that prevented organization admins from removing labels from workflow job templates (https://github.com/ansible/awx/pull/7143)
-
-## 11.2.0 (Apr 29, 2020)
-
-- Inventory updates now use collection-based plugins by default (in Ansible 2.9+):
-    - amazon.aws.aws_ec2
-    - community.vmware.vmware_vm_inventory
-    - azure.azcollection.azure_rm
-    - google.cloud.gcp_compute
-    - theforeman.foreman.foreman
-    - openstack.cloud.openstack
-    - ovirt.ovirt_collection.ovirt
-    - awx.awx.tower
-- Added support for Approle and LDAP/AD mechanisms to the Hashicorp Vault credential plugin (https://github.com/ansible/awx/issues/5076)
-- Added Project (Domain Name) support for the OpenStack Keystone v3 API (https://github.com/ansible/awx/issues/6831)
-- Added a new setting for raising log verbosity for rsyslogd (https://github.com/ansible/awx/pull/6818)
-- Added the ability to monitor stdout in the CLI for running jobs and workflow jobs (https://github.com/ansible/awx/issues/6165)
-- Fixed a bug which prevented the AWX CLI from properly installing with newer versions of pip (https://github.com/ansible/awx/issues/6870)
-- Fixed a bug which broke AWX's external logging support when configured with HTTPS endpoints that utilize self-signed certificates (https://github.com/ansible/awx/issues/6851)
-- Fixed a local docker installer bug that mistakenly attempted to upgrade PostgreSQL when an external pg_hostname is specified (https://github.com/ansible/awx/pull/5398)
-- Fixed a race condition that caused task container crashes when pods are quickly brought down and back up (https://github.com/ansible/awx/issues/6750)
-- Fixed a bug that caused 404 errors when attempting to view the second page of the workflow approvals view (https://github.com/ansible/awx/issues/6803)
-- Fixed a bug that prevented the use of ANSIBLE_SSH_ARGS for ad-hoc-commands (https://github.com/ansible/awx/pull/6811)
-- Fixed a bug that broke AWX installs/upgrades on Red Hat OpenShift (https://github.com/ansible/awx/issues/6791)
-
-
-## 11.1.0 (Apr 22, 2020)
-- Changed rsyslogd to persist queued events to disk (to prevent a risk of out-of-memory errors) (https://github.com/ansible/awx/issues/6746)
-- Added the ability to configure the destination and maximum disk size of rsyslogd spool (in the event of a log aggregator outage) (https://github.com/ansible/awx/pull/6763)
-- Added the ability to discover playbooks in project clones from symlinked directories (https://github.com/ansible/awx/pull/6773)
-- Fixed a bug that caused certain log aggregator settings to break logging integration (https://github.com/ansible/awx/issues/6760)
-- Fixed a bug that caused playbook execution in container groups to sometimes unexpectedly deadlock (https://github.com/ansible/awx/issues/6692)
-- Improved stability of the new redis clustering implementation (https://github.com/ansible/awx/pull/6739 https://github.com/ansible/awx/pull/6720)
-- Improved stability of the new rsyslogd-based logging implementation (https://github.com/ansible/awx/pull/6796)
-
-## 11.0.0 (Apr 16, 2020)
-- As of AWX 11.0.0, Kubernetes-based deployments use a Deployment rather than a StatefulSet.
-- Reimplemented external logging support using rsyslogd to improve reliability and address a number of issues (https://github.com/ansible/awx/issues/5155)
-- Changed activity stream logs to include summary fields for related objects (https://github.com/ansible/awx/issues/1761)
-- Added code to more gracefully attempt to reconnect to redis if it restarts/becomes unavailable (https://github.com/ansible/awx/pull/6670)
-- Fixed a bug that caused REFRESH_TOKEN_EXPIRE_SECONDS to not properly be respected for OAuth2.0 refresh tokens generated by AWX (https://github.com/ansible/awx/issues/6630)
-- Fixed a bug that broke schedules containing RRULES with very old DTSTART dates (https://github.com/ansible/awx/pull/6550)
-- Fixed a bug that broke installs on older versions of Ansible packaged with certain Linux distributions (https://github.com/ansible/awx/issues/5501)
-- Fixed a bug that caused the activity stream to sometimes report the incorrect actor when associating user membership on SAML login (https://github.com/ansible/awx/pull/6525)
-- Fixed a bug in AWX's Grafana notification support when annotation tags are omitted (https://github.com/ansible/awx/issues/6580)
-- Fixed a bug that prevented some users from searching for Source Control credentials in the AWX user interface (https://github.com/ansible/awx/issues/6600)
-- Fixed a bug that prevented disassociating orphaned users from credentials (https://github.com/ansible/awx/pull/6554)
-- Updated Twisted to address CVE-2020-10108 and CVE-2020-10109.
-
-## 10.0.0 (Mar 30, 2020)
-- As of AWX 10.0.0, the official AWX CLI no longer supports Python 2 (it requires at least Python 3.6) (https://github.com/ansible/awx/pull/6327)
-- AWX no longer relies on RabbitMQ; Redis is added as a new dependency (https://github.com/ansible/awx/issues/5443)
-- Altered AWX's event tables to allow more than ~2 billion total events (https://github.com/ansible/awx/issues/6010)
-- Improved the performance (time to execute, and memory consumption) of the periodic job cleanup system job (https://github.com/ansible/awx/pull/6166)
-- Updated Job Templates so they now have an explicit Organization field (it is no longer inferred from the associated Project) (https://github.com/ansible/awx/issues/3903)
-- Updated social-auth-core to address an upcoming GitHub API deprecation (https://github.com/ansible/awx/issues/5970)
-- Updated to ansible-runner 1.4.6 to address various bugs.
-- Updated Django to address CVE-2020-9402
-- Updated pyyaml version to address CVE-2017-18342
-- Fixed a bug which prevented the new `scm_branch` field from being used in custom notification templates (https://github.com/ansible/awx/issues/6258)
-- Fixed a race condition that sometimes causes success/failure notifications to include an incomplete list of hosts (https://github.com/ansible/awx/pull/6290)
-- Fixed a bug that can cause certain setting pages to lose unsaved form edits when a playbook is launched (https://github.com/ansible/awx/issues/5265)
-- Fixed a bug that can prevent the "Use TLS/SSL" field from properly saving when editing email notification templates (https://github.com/ansible/awx/issues/6383)
-- Fixed a race condition that sometimes broke event/stdout processing for jobs launched in container groups (https://github.com/ansible/awx/issues/6280)
-
-## 9.3.0 (Mar 12, 2020)
-- Added the ability to specify an OAuth2 token description in the AWX CLI (https://github.com/ansible/awx/issues/6122)
-- Added support for K8S service account annotations to the installer (https://github.com/ansible/awx/pull/6007)
-- Added support for K8S imagePullSecrets to the installer (https://github.com/ansible/awx/pull/5989)
-- Launching jobs (and workflows) using the --monitor flag in the AWX CLI now returns a non-zero exit code on job failure (https://github.com/ansible/awx/issues/5920)
-- Improved UI performance for various job views when many simultaneous users are logged into AWX (https://github.com/ansible/awx/issues/5883)
-- Updated to the latest version of Django to address a few open CVEs (https://github.com/ansible/awx/pull/6080)
-- Fixed a critical bug which can cause AWX to hang and stop launching playbooks after a periodic of time (https://github.com/ansible/awx/issues/5617)
-- Fixed a bug which caused delays in project update stdout for certain large SCM clones (as of Ansible 2.9+) (https://github.com/ansible/awx/pull/6254)
-- Fixed a bug which caused certain smart inventory filters to mistakenly return duplicate hosts (https://github.com/ansible/awx/pull/5972)
-- Fixed an unclear server error when creating smart inventories with the AWX collection (https://github.com/ansible/awx/issues/6250)
-- Fixed a bug that broke Grafana notification support (https://github.com/ansible/awx/issues/6137)
-- Fixed a UI bug which prevent users with read access to an organization from editing credentials for that organization (https://github.com/ansible/awx/pull/6241)
-- Fixed a bug which prevent workflow approval records from recording a `started` and `elapsed` date (https://github.com/ansible/awx/issues/6202)
-- Fixed a bug which caused workflow nodes to have a confusing option for `verbosity` (https://github.com/ansible/awx/issues/6196)
-- Fixed an RBAC bug which prevented projects and inventory schedules from being created by certain users in certain contexts (https://github.com/ansible/awx/issues/5717)
-- Fixed a bug that caused `role_path` in a project's config to not be respected due to an error processing `/etc/ansible/ansible.cfg` (https://github.com/ansible/awx/pull/6038)
-- Fixed a bug that broke inventory updates for installs with custom home directories for the awx user (https://github.com/ansible/awx/pull/6152)
-- Fixed a bug that broke fact data collection when AWX encounters invalid/unexpected fact data (https://github.com/ansible/awx/issues/5935)
-
-
-## 9.2.0 (Feb 12, 2020)
-- Added the ability to configure the convergence behavior of workflow nodes https://github.com/ansible/awx/issues/3054
-- AWX now allows for a configurable global limit for fork count (per-job run).  The default maximum is 200. https://github.com/ansible/awx/pull/5604
-- Added the ability to specify AZURE_PUBLIC_CLOUD (for e.g., Azure Government KeyVault support) for the Azure credential plugin https://github.com/ansible/awx/issues/5138
-- Added support for several additional parameters for Satellite dynamic inventory https://github.com/ansible/awx/pull/5598
-- Added a new field to jobs for tracking the date/time a job is cancelled https://github.com/ansible/awx/pull/5610
-- Made a series of additional optimizations to the callback receiver to further improve stdout write speed for running playbooks https://github.com/ansible/awx/pull/5677 https://github.com/ansible/awx/pull/5739
-- Updated AWX to be compatible with Helm 3.x (https://github.com/ansible/awx/pull/5776)
-- Optimized AWX's job dependency/scheduling code to drastically improve processing time in scenarios where there are many pending jobs scheduled simultaneously https://github.com/ansible/awx/issues/5154
-- Fixed a bug which could cause SCM authentication details (basic auth passwords) to be reported to external loggers in certain failure scenarios (e.g., when a git clone fails and ansible itself prints an error message to stdout) https://github.com/ansible/awx/pull/5812
-- Fixed a k8s installer bug that caused installs to fail in certain situations https://github.com/ansible/awx/issues/5574
-- Fixed a number of issues that caused analytics gathering and reporting to run more often than necessary https://github.com/ansible/awx/pull/5721
-- Fixed a bug in the AWX CLI that prevented JSON-type settings from saving properly https://github.com/ansible/awx/issues/5528
-- Improved support for fetching custom virtualenv dependencies when AWX is installed behind a proxy https://github.com/ansible/awx/pull/5805
-- Updated the bundled version of openstacksdk to address a known issue https://github.com/ansible/awx/issues/5821
-- Updated the bundled vmware_inventory plugin to the latest version to address a bug https://github.com/ansible/awx/pull/5668
-- Fixed a bug that can cause inventory updates to fail to properly save their output when run within a workflow https://github.com/ansible/awx/pull/5666
-- Removed a number of pre-computed fields from the Host and Group models to improve AWX performance.  As part of this change, inventory group UIs throughout the interface no longer display status icons https://github.com/ansible/awx/pull/5448
-
-## 9.1.1 (Jan 14, 2020)
-
-- Fixed a bug that caused database migrations on Kubernetes installs to hang https://github.com/ansible/awx/pull/5579
-- Upgraded Python-level app dependencies in AWX virtual environment https://github.com/ansible/awx/pull/5407
-- Running jobs no longer block associated inventory updates https://github.com/ansible/awx/pull/5519
-- Fixed invalid_response SAML error https://github.com/ansible/awx/pull/5577
-- Optimized the callback receiver to drastically improve the write speed of stdout for parallel jobs (https://github.com/ansible/awx/pull/5618)
-
-## 9.1.0 (Dec 17, 2019)
-- Added a command to generate a new SECRET_KEY and rekey the secrets in the database
-- Removed project update locking when jobs using it are running
-- Fixed slow queries for /api/v2/instances and /api/v2/instance_groups when smart inventories are used
-- Fixed a partial password disclosure when special characters existed in the RabbitMQ password (CVE-2019-19342)
-- Fixed hang in error handling for source control checkouts
-- Fixed an error on subsequent job runs that override the branch of a project on an instance that did not have a prior project checkout
-- Fixed an issue where jobs launched in isolated or container groups would incorrectly timeout
-- Fixed an incorrect link to instance groups documentation in the user interface
-- Fixed editing of inventory on Workflow templates
-- Fixed multiple issues with OAuth2 token cleanup system jobs
-- Fixed a bug that broke email notifications for workflow approval/deny https://github.com/ansible/awx/issues/5401
-- Updated SAML implementation to automatically login if authorization already exists
-- Updated AngularJS to 1.7.9 for CVE-2019-10768
-
-## 9.0.1 (Nov 4, 2019)
-
-- Fixed a bug in the installer that broke certain types of k8s installs https://github.com/ansible/awx/issues/5205
-
-## 9.0.0 (Oct 31, 2019)
-
-- Updated AWX images to use centos:8 as the parent image.
-- Updated to ansible-runner 1.4.4 to address various bugs.
-- Added oc and kubectl to the AWX images to support new container-based execution introduced in 8.0.0.
-- Added some optimizations to speed up the deletion of large Inventory Groups.
-- Fixed a bug that broke webhook launches for Job Templates that define a survey (https://github.com/ansible/awx/issues/5062).
-- Fixed a bug in the CLI which incorrectly parsed launch time arguments for `awx job_templates launch` and `awx workflow_job_templates launch` (https://github.com/ansible/awx/issues/5093).
-- Fixed a bug that caused inventory updates using "sourced from a project" to stop working (https://github.com/ansible/awx/issues/4750).
-- Fixed a bug that caused Slack notifications to sometimes show the wrong bot avatar (https://github.com/ansible/awx/pull/5125).
-- Fixed a bug that prevented the use of digits in AWX's URL settings (https://github.com/ansible/awx/issues/5081).
-
-## 8.0.0 (Oct 21, 2019)
-
-- The Ansible Tower Ansible modules have been migrated to a new official Ansible AWX collection: https://galaxy.ansible.com/awx/AWX
-  Please note that this functionality is only supported in Ansible 2.9+
-- AWX now supports the ability to launch jobs from external webhooks (GitHub and GitLab integration are supported).
-- AWX now supports Container Groups, a new feature that allows you to schedule and run playbooks on single-use kubernetes pods on-demand.
-- AWX now supports sending notifications when Workflow steps are approved, denied, or time out.
-- AWX now records the user who approved or denied Workflow steps.
-- AWX now supports fetching Ansible Collections from private galaxy servers.
-- AWX now checks the user's ansible.cfg for paths where role/collections may live when running project updates.
-- AWX now uses PostgreSQL 10 by default.
-- AWX now warns more loudly about underlying AMQP connectivity issues (https://github.com/ansible/awx/pull/4857).
-- Added a few optimizations to drastically improve dashboard performance for larger AWX installs (installs with several hundred thousand jobs or more).
-- Updated to the latest version of Ansible's VMWare inventory script (which adds support for vmware_guest_facts).
-- Deprecated /api/v2/inventory_scripts/ (this endpoint - and the Custom Inventory Script feature - will be removed in a future release of AWX).
-- Fixed a bug which prevented Organization Admins from removing users from their own Organization (https://github.com/ansible/awx/issues/2979)
-- Fixed a bug which sometimes caused cluster nodes to fail to re-join with a cryptic error, "No instance found with the current cluster host id" (https://github.com/ansible/awx/issues/4294)
-- Fixed a bug that prevented the use of launch-time passphrases when using credential plugins (https://github.com/ansible/awx/pull/4807)
-- Fixed a bug that caused notifications assigned at the Organization level not to take effect for Workflows in that Organization (https://github.com/ansible/awx/issues/4712)
-- Fixed a bug which caused a notable amount of CPU overhead on RabbitMQ health checks (https://github.com/ansible/awx/pull/5009)
-- Fixed a bug which sometimes caused the <return> key to stop functioning in <textarea> elements (https://github.com/ansible/awx/issues/4192)
-- Fixed a bug which caused request contention when the same OAuth2.0 token was used in multiple simultaneous requests (https://github.com/ansible/awx/issues/4694)
-- Fixed a bug related to parsing multiple choice survey options (https://github.com/ansible/awx/issues/4452).
-- Fixed a bug that caused single-sign-on icons on the login page to fail to render in certain Windows browsers (https://github.com/ansible/awx/issues/3924)
-- Fixed a number of bugs that caused certain OAuth2 settings to not be properly respected, such as REFRESH_TOKEN_EXPIRE_SECONDS.
-- Fixed a number of bugs in the AWX CLI, including a bug which sometimes caused long lines of stdout output to be unexpectedly truncated.
-- Fixed a number of bugs on the job details UI which sometimes caused auto-scrolling stdout to become stuck.
-- Fixed a bug which caused LDAP authentication to fail if the TLD of the server URL contained digits (https://github.com/ansible/awx/issues/3646)
-- Fixed a bug which broke HashiCorp Vault integration on older versions of HashiCorp Vault.
-
-## 7.0.0 (Sept 4, 2019)
-
-- AWX now detects and installs Ansible Collections defined in your project (note - this feature only works in Ansible 2.9+) (https://github.com/ansible/awx/issues/2534)
-- AWX now includes an official command line client.  Keep an eye out for a follow-up email on this mailing list for information on how to install it and try it out.
-- Added the ability to provide a specific SCM branch on jobs (https://github.com/ansible/awx/issues/282)
-- Added support for Workflow Approval Nodes, a new feature which allows you to add "pause and wait for approval" steps into your workflows (https://github.com/ansible/awx/issues/1206)
-- Added the ability to specify a specific HTTP method for webhook notifications (POST vs PUT) (https://github.com/ansible/awx/pull/4124)
-- Added the ability to specify a username and password for HTTP Basic Authorization for webhook notifications (https://github.com/ansible/awx/pull/4124)
-- Added support for customizing the text content of notifications (https://github.com/ansible/awx/issues/79)
-- Added the ability to enable and disable hosts in dynamic inventory (https://github.com/ansible/awx/pull/4420)
-- Added the description (if any) to the Job Template list (https://github.com/ansible/awx/issues/4359)
-- Added new metrics for instance hostnames and pending jobs to the /api/v2/metrics/ endpoint (https://github.com/ansible/awx/pull/4375)
-- Changed AWX's on/off toggle buttons to a non-text based style to simplify internationalization (https://github.com/ansible/awx/pull/4425)
-- Events emitted by ansible for adhoc commands are now sent to the external log aggregrator (https://github.com/ansible/awx/issues/4545)
-- Fixed a bug which allowed a user to make an organization credential in another organization without permissions to that organization (https://github.com/ansible/awx/pull/4483)
-- Fixed a bug that caused `extra_vars` on workflows to break when edited (https://github.com/ansible/awx/issues/4293)
-- Fixed a slow SQL query that caused performance issues when large numbers of groups exist (https://github.com/ansible/awx/issues/4461)
-- Fixed a few minor bugs in survey field validation (https://github.com/ansible/awx/pull/4509) (https://github.com/ansible/awx/pull/4479)
-- Fixed a bug that sometimes resulted in orphaned `ansible_runner_pi` directories in `/tmp` after playbook execution (https://github.com/ansible/awx/pull/4409)
-- Fixed a bug that caused the `is_system_auditor` flag in LDAP configuration to not work (https://github.com/ansible/awx/pull/4396)
-- Fixed a bug which caused schedules to disappear from the UI when toggled off (https://github.com/ansible/awx/pull/4378)
-- Fixed a bug that sometimes caused stdout content to contain extraneous blank lines in newer versions of Ansible (https://github.com/ansible/awx/pull/4391)
-- Updated to the latest Django security release, 2.2.4 (https://github.com/ansible/awx/pull/4410) (https://www.djangoproject.com/weblog/2019/aug/01/security-releases/)
-- Updated the default version of git to a version that includes support for x509 certificates (https://github.com/ansible/awx/issues/4362)
-- Removed the deprecated `credential` field from `/api/v2/workflow_job_templates/N/` (as part of the `/api/v1/` removal in prior AWX versions - https://github.com/ansible/awx/pull/4490).
-
-## 6.1.0 (Jul 18, 2019)
-
-- Updated AWX to use Django 2.2.2.
-- Updated the provided openstacksdk version to support new functionality (such as Nova scheduler_hints)
-- Added the ability to specify a custom cacert for the HashiCorp Vault credential plugin
-- Fixed a number of bugs related to path lookups for the HashiCorp Vault credential plugin
-- Fixed a bug which prevented signed SSH certificates from working, including the HashiCorp Vault Signed SSH backend
-- Fixed a bug which prevented custom logos from displaying on the login page (as a result of a new Content Security Policy in 6.0.0)
-- Fixed a bug which broke websocket connectivity in Apple Safari (as a result of a new Content Security Policy in 6.0.0)
-- Fixed a bug on the job output page that occasionally caused the "up" and "down" buttons to not load additional output
-- Fixed a bug on the job output page that caused quoted task names to display incorrectly
-
-## 6.0.0 (Jul 1, 2019)
-
-- Removed support for "Any" notification templates and their API endpoints e.g., /api/v2/job_templates/N/notification_templates/any/ (https://github.com/ansible/awx/issues/4022)
-- Fixed a bug which prevented credentials from properly being applied to inventory sources (https://github.com/ansible/awx/issues/4059)
-- Fixed a bug which can cause the task dispatcher to hang indefinitely when external logging support (e.g., Splunk, Logstash) is enabled (https://github.com/ansible/awx/issues/4181)
-- Fixed a bug which causes slow stdout display when running jobs against smart inventories. (https://github.com/ansible/awx/issues/3106)
-- Fixed a bug that caused SSL verification flags to fail to be respected for LDAP authentication in certain environments. (https://github.com/ansible/awx/pull/4190)
-- Added a simple Content Security Policy (https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP) to restrict access to third-party resources in the browser. (https://github.com/ansible/awx/pull/4167)
-- Updated ovirt4 library dependencies to work with newer versions of oVirt (https://github.com/ansible/awx/issues/4138)
-
-## 5.0.0 (Jun 21, 2019)
-
-- Bump Django Rest Framework from 3.7.7 to 3.9.4
-- Bump setuptools / pip dependencies
-- Fixed bug where Recent Notification list would not appear
-- Added notifications on job start
-- Default to Ansible 2.8
+For older release notes, see https://github.com/ansible/awx/blob/19.3.0/CHANGELOG.md.

CONTRIBUTING.md

@@ -110,7 +110,7 @@ For feature work, take a look at the current [Enhancements](https://github.com/a
 
 If it has someone assigned to it then that person is the person responsible for working the enhancement. If you feel like you could contribute then reach out to that person.
 
-Fixing bugs, adding translations, and updating the documentation are always appreciated, so reviewing the backlog of issues is always a good place to start. For extra information on debugging tools, see [Debugging](https://github.com/ansible/awx/blob/devel/docs/debugging.md).
+Fixing bugs, adding translations, and updating the documentation are always appreciated, so reviewing the backlog of issues is always a good place to start. For extra information on debugging tools, see [Debugging](./docs/debugging/).
 
 **NOTE**
 

Makefile

@@ -1,61 +1,41 @@
 PYTHON ?= python3.8
 PYTHON_VERSION = $(shell $(PYTHON) -c "from distutils.sysconfig import get_python_version; print(get_python_version())")
-SITELIB=$(shell $(PYTHON) -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
 OFFICIAL ?= no
-PACKER ?= packer
-PACKER_BUILD_OPTS ?= -var 'official=$(OFFICIAL)' -var 'aw_repo_url=$(AW_REPO_URL)'
 NODE ?= node
 NPM_BIN ?= npm
 CHROMIUM_BIN=/tmp/chrome-linux/chrome
-DEPS_SCRIPT ?= packaging/bundle/deps.py
 GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 MANAGEMENT_COMMAND ?= awx-manage
-IMAGE_REPOSITORY_AUTH ?=
-IMAGE_REPOSITORY_BASE ?= https://gcr.io
-VERSION := $(shell cat VERSION)
+VERSION := $(shell $(PYTHON) setup.py --version)
+COLLECTION_VERSION := $(shell $(PYTHON) setup.py --version | cut -d . -f 1-3)
 
 # NOTE: This defaults the container image version to the branch that's active
 COMPOSE_TAG ?= $(GIT_BRANCH)
-COMPOSE_HOST ?= $(shell hostname)
+MAIN_NODE_TYPE ?= hybrid
+# If set to true docker-compose will also start a keycloak instance
+KEYCLOAK ?= false
 
-VENV_BASE ?= /var/lib/awx/venv/
-SCL_PREFIX ?=
-CELERY_SCHEDULE_FILE ?= /var/lib/awx/beat.db
+VENV_BASE ?= /var/lib/awx/venv
 
 DEV_DOCKER_TAG_BASE ?= quay.io/awx
 DEVEL_IMAGE_NAME ?= $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG)
 
+RECEPTOR_IMAGE ?= quay.io/ansible/receptor:devel
+
 # Python packages to install only from source (not from binary wheels)
 # Comma separated list
 SRC_ONLY_PKGS ?= cffi,pycparser,psycopg2,twilio
 # These should be upgraded in the AWX and Ansible venv before attempting
 # to install the actual requirements
-VENV_BOOTSTRAP ?= pip==19.3.1 setuptools==41.6.0 wheel==0.36.2
-
-# Determine appropriate shasum command
-UNAME_S := $(shell uname -s)
-ifeq ($(UNAME_S),Linux)
-    SHASUM_BIN ?= sha256sum
-endif
-ifeq ($(UNAME_S),Darwin)
-    SHASUM_BIN ?= shasum -a 256
-endif
-
-# Get the branch information from git
-GIT_DATE := $(shell git log -n 1 --format="%ai")
-DATE := $(shell date -u +%Y%m%d%H%M)
+VENV_BOOTSTRAP ?= pip==21.2.4 setuptools==58.2.0 wheel==0.36.2
 
 NAME ?= awx
-GIT_REMOTE_URL = $(shell git config --get remote.origin.url)
 
 # TAR build parameters
 SDIST_TAR_NAME=$(NAME)-$(VERSION)
-WHEEL_NAME=$(NAME)-$(VERSION)
 
 SDIST_COMMAND ?= sdist
-WHEEL_COMMAND ?= bdist_wheel
 SDIST_TAR_FILE ?= $(SDIST_TAR_NAME).tar.gz
-WHEEL_FILE ?= $(WHEEL_NAME)-py2-none-any.whl
 
 I18N_FLAG_FILE = .i18n_built
 
@@ -166,15 +146,6 @@ version_file:
 	fi; \
 	$(PYTHON) -c "import awx; print(awx.__version__)" > /var/lib/awx/.awx_version; \
 
-# Do any one-time init tasks.
-comma := ,
-init:
-	if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/awx/bin/activate; \
-	fi; \
-	$(MANAGEMENT_COMMAND) provision_instance --hostname=$(COMPOSE_HOST); \
-	$(MANAGEMENT_COMMAND) register_queue --queuename=controlplane --instance_percent=100;
-
 # Refresh development environment after pulling new code.
 refresh: clean requirements_dev version_file develop migrate
 
@@ -305,7 +276,6 @@ test:
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
 	PYTHONDONTWRITEBYTECODE=1 py.test -p no:cacheprovider -n auto $(TEST_DIRS)
-	cmp VERSION awxkit/VERSION || "VERSION and awxkit/VERSION *must* match"
 	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py3
 	awx-manage check_migrations --dry-run --check  -n 'missing_migration_file'
 
@@ -337,12 +307,16 @@ symlink_collection:
 	ln -s $(shell pwd)/awx_collection $(COLLECTION_INSTALL)
 
 build_collection:
-	ansible-playbook -i localhost, awx_collection/tools/template_galaxy.yml -e collection_package=$(COLLECTION_PACKAGE) -e collection_namespace=$(COLLECTION_NAMESPACE) -e collection_version=$(VERSION) -e '{"awx_template_version":false}'
+	ansible-playbook -i localhost, awx_collection/tools/template_galaxy.yml \
+	  -e collection_package=$(COLLECTION_PACKAGE) \
+	  -e collection_namespace=$(COLLECTION_NAMESPACE) \
+	  -e collection_version=$(COLLECTION_VERSION) \
+	  -e '{"awx_template_version":false}'
 	ansible-galaxy collection build awx_collection_build --force --output-path=awx_collection_build
 
 install_collection: build_collection
 	rm -rf $(COLLECTION_INSTALL)
-	ansible-galaxy collection install awx_collection_build/$(COLLECTION_NAMESPACE)-$(COLLECTION_PACKAGE)-$(VERSION).tar.gz
+	ansible-galaxy collection install awx_collection_build/$(COLLECTION_NAMESPACE)-$(COLLECTION_PACKAGE)-$(COLLECTION_VERSION).tar.gz
 
 test_collection_sanity: install_collection
 	cd $(COLLECTION_INSTALL) && ansible-test sanity
@@ -395,7 +369,7 @@ clean-ui:
 awx/ui/node_modules:
 	NODE_OPTIONS=--max-old-space-size=4096 $(NPM_BIN) --prefix awx/ui --loglevel warn ci
 
-$(UI_BUILD_FLAG_FILE):
+$(UI_BUILD_FLAG_FILE): awx/ui/node_modules
 	$(PYTHON) tools/scripts/compilemessages.py
 	$(NPM_BIN) --prefix awx/ui --loglevel warn run compile-strings
 	$(NPM_BIN) --prefix awx/ui --loglevel warn run build
@@ -407,7 +381,9 @@ $(UI_BUILD_FLAG_FILE):
 	cp -r awx/ui/build/static/media/* awx/public/static/media
 	touch $@
 
-ui-release: awx/ui/node_modules $(UI_BUILD_FLAG_FILE)
+
+
+ui-release: $(UI_BUILD_FLAG_FILE)
 
 ui-devel: awx/ui/node_modules
 	@$(MAKE) -B $(UI_BUILD_FLAG_FILE)
@@ -425,7 +401,7 @@ ui-lint:
 
 ui-test:
 	$(NPM_BIN) --prefix awx/ui install
-	$(NPM_BIN) run --prefix awx/ui test -- --coverage --maxWorkers=4 --watchAll=false
+	$(NPM_BIN) run --prefix awx/ui test 
 
 
 # Build a pip-installable package into dist/ with a timestamped version number.
@@ -436,33 +412,22 @@ dev_build:
 release_build:
 	$(PYTHON) setup.py release_build
 
-dist/$(SDIST_TAR_FILE): ui-release VERSION
+HEADLESS ?= no
+ifeq ($(HEADLESS), yes)
+dist/$(SDIST_TAR_FILE):
+else
+dist/$(SDIST_TAR_FILE): $(UI_BUILD_FLAG_FILE)
+endif
 	$(PYTHON) setup.py $(SDIST_COMMAND)
-
-dist/$(WHEEL_FILE): ui-release
-	$(PYTHON) setup.py $(WHEEL_COMMAND)
+	ln -sf $(SDIST_TAR_FILE) dist/awx.tar.gz
 
 sdist: dist/$(SDIST_TAR_FILE)
+	echo $(HEADLESS)
 	@echo "#############################################"
 	@echo "Artifacts:"
 	@echo dist/$(SDIST_TAR_FILE)
 	@echo "#############################################"
 
-wheel: dist/$(WHEEL_FILE)
-	@echo "#############################################"
-	@echo "Artifacts:"
-	@echo dist/$(WHEEL_FILE)
-	@echo "#############################################"
-
-# Build setup bundle tarball
-setup-bundle-build:
-	mkdir -p $@
-
-docker-auth:
-	@if [ "$(IMAGE_REPOSITORY_AUTH)" ]; then \
-		echo "$(IMAGE_REPOSITORY_AUTH)" | docker login -u oauth2accesstoken --password-stdin $(IMAGE_REPOSITORY_BASE); \
-	fi;
-
 # This directory is bind-mounted inside of the development container and
 # needs to be pre-created for permissions to be set correctly. Otherwise,
 # Docker will create this directory as root.
@@ -470,7 +435,9 @@ awx/projects:
 	@mkdir -p $@
 
 COMPOSE_UP_OPTS ?=
-CLUSTER_NODE_COUNT ?= 1
+COMPOSE_OPTS ?=
+CONTROL_PLANE_NODE_COUNT ?= 1
+EXECUTION_NODE_COUNT ?= 2
 MINIKUBE_CONTAINER_GROUP ?= false
 
 docker-compose-sources: .git/hooks/pre-commit
@@ -481,18 +448,21 @@ docker-compose-sources: .git/hooks/pre-commit
 	ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/sources.yml \
 	    -e awx_image=$(DEV_DOCKER_TAG_BASE)/awx_devel \
 	    -e awx_image_tag=$(COMPOSE_TAG) \
-	    -e cluster_node_count=$(CLUSTER_NODE_COUNT) \
-	    -e minikube_container_group=$(MINIKUBE_CONTAINER_GROUP)
+	    -e receptor_image=$(RECEPTOR_IMAGE) \
+	    -e control_plane_node_count=$(CONTROL_PLANE_NODE_COUNT) \
+	    -e execution_node_count=$(EXECUTION_NODE_COUNT) \
+	    -e minikube_container_group=$(MINIKUBE_CONTAINER_GROUP) \
+	    -e enable_keycloak=$(KEYCLOAK)
 
 
-docker-compose: docker-auth awx/projects docker-compose-sources
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml up $(COMPOSE_UP_OPTS)
+docker-compose: awx/projects docker-compose-sources
+	docker-compose -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans
 
-docker-compose-credential-plugins: docker-auth awx/projects docker-compose-sources
+docker-compose-credential-plugins: awx/projects docker-compose-sources
 	echo -e "\033[0;31mTo generate a CyberArk Conjur API key: docker exec -it tools_conjur_1 conjurctl account create quick-start\033[0m"
-	docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/docker-credential-plugins-override.yml up --no-recreate awx_1
+	docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/docker-credential-plugins-override.yml up --no-recreate awx_1 --remove-orphans
 
-docker-compose-test: docker-auth awx/projects docker-compose-sources
+docker-compose-test: awx/projects docker-compose-sources
 	docker-compose -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports awx_1 /bin/bash
 
 docker-compose-runtest: awx/projects docker-compose-sources
@@ -517,14 +487,16 @@ docker-compose-container-group-clean:
 
 # Base development image build
 docker-compose-build:
-	ansible-playbook tools/ansible/dockerfile.yml -e build_dev=True
+	ansible-playbook tools/ansible/dockerfile.yml -e build_dev=True -e receptor_image=$(RECEPTOR_IMAGE)
 	DOCKER_BUILDKIT=1 docker build -t $(DEVEL_IMAGE_NAME) \
 	    --build-arg BUILDKIT_INLINE_CACHE=1 \
 	    --cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) .
 
 docker-clean:
-	$(foreach container_id,$(shell docker ps -f name=tools_awx -aq),docker stop $(container_id); docker rm -f $(container_id);)
-	docker images | grep "awx_devel" | awk '{print $$1 ":" $$2}' | xargs docker rmi
+	$(foreach container_id,$(shell docker ps -f name=tools_awx -aq && docker ps -f name=tools_receptor -aq),docker stop $(container_id); docker rm -f $(container_id);)
+	if [ "$(shell docker images | grep awx_devel)" ]; then \
+	  docker images | grep awx_devel | awk '{print $$3}' | xargs docker rmi --force; \
+	fi
 
 docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean
 	docker volume rm tools_awx_db
@@ -532,10 +504,10 @@ docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean
 docker-refresh: docker-clean docker-compose
 
 # Docker Development Environment with Elastic Stack Connected
-docker-compose-elk: docker-auth awx/projects docker-compose-sources
+docker-compose-elk: awx/projects docker-compose-sources
 	docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
-docker-compose-cluster-elk: docker-auth awx/projects docker-compose-sources
+docker-compose-cluster-elk: awx/projects docker-compose-sources
 	docker-compose -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
 
 prometheus:
@@ -559,13 +531,14 @@ VERSION:
 	@echo "awx: $(VERSION)"
 
 Dockerfile: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
-	ansible-playbook tools/ansible/dockerfile.yml
+	ansible-playbook tools/ansible/dockerfile.yml -e receptor_image=$(RECEPTOR_IMAGE)
 
 Dockerfile.kube-dev: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
 	ansible-playbook tools/ansible/dockerfile.yml \
 	    -e dockerfile_name=Dockerfile.kube-dev \
 	    -e kube_dev=True \
-	    -e template_dest=_build_kube_dev
+	    -e template_dest=_build_kube_dev \
+	    -e receptor_image=$(RECEPTOR_IMAGE)
 
 awx-kube-dev-build: Dockerfile.kube-dev
 	docker build -f Dockerfile.kube-dev \

README.md

@@ -1,4 +1,4 @@
-[![.github/workflows/ci.yml](https://github.com/ansible/awx/actions/workflows/ci.yml/badge.svg)](https://github.com/ansible/awx/actions/workflows/ci.yml) [![Code of Conduct](https://img.shields.io/badge/code%20of%20conduct-Ansible-yellow.svg)](https://docs.ansible.com/ansible/latest/community/code_of_conduct.html) [![Apache v2 License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](https://github.com/ansible/awx/blob/devel/LICENSE.md) [![AWX Mailing List](https://img.shields.io/badge/mailing%20list-AWX-orange.svg)](https://groups.google.com/g/awx-project)
+[![CI](https://github.com/ansible/awx/actions/workflows/ci.yml/badge.svg?branch=devel)](https://github.com/ansible/awx/actions/workflows/ci.yml) [![Code of Conduct](https://img.shields.io/badge/code%20of%20conduct-Ansible-yellow.svg)](https://docs.ansible.com/ansible/latest/community/code_of_conduct.html) [![Apache v2 License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](https://github.com/ansible/awx/blob/devel/LICENSE.md) [![AWX Mailing List](https://img.shields.io/badge/mailing%20list-AWX-orange.svg)](https://groups.google.com/g/awx-project)
 [![IRC Chat - #ansible-awx](https://img.shields.io/badge/IRC-%23ansible--awx-blueviolet.svg)](https://libera.chat)
 
 <img src="https://raw.githubusercontent.com/ansible/awx-logos/master/awx/ui/client/assets/logo-login.svg?sanitize=true" width=200 alt="AWX" />

VERSION

@@ -1 +0,0 @@
-19.3.0

awx/__init__.py

@@ -151,7 +151,7 @@ def manage():
     from django.core.management import execute_from_command_line
 
     # enforce the postgres version is equal to 12. if not, then terminate program with exit code of 1
-    if not MODE == 'development':
+    if not os.getenv('SKIP_PG_VERSION_CHECK', False) and not MODE == 'development':
         if (connection.pg_version // 10000) < 12:
             sys.stderr.write("Postgres version 12 is required\n")
             sys.exit(1)

awx/api/generics.py

@@ -44,6 +44,7 @@ from awx.main.views import ApiErrorView
 from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer, UserSerializer
 from awx.api.versioning import URLPathVersioning
 from awx.api.metadata import SublistAttachDetatchMetadata, Metadata
+from awx.conf import settings_registry
 
 __all__ = [
     'APIView',
@@ -208,12 +209,20 @@ class APIView(views.APIView):
             return response
 
         if response.status_code >= 400:
-            status_msg = "status %s received by user %s attempting to access %s from %s" % (
-                response.status_code,
-                request.user,
-                request.path,
-                request.META.get('REMOTE_ADDR', None),
-            )
+            msg_data = {
+                'status_code': response.status_code,
+                'user_name': request.user,
+                'url_path': request.path,
+                'remote_addr': request.META.get('REMOTE_ADDR', None),
+                'error': response.data.get('error', response.status_text),
+            }
+            try:
+                status_msg = getattr(settings, 'API_400_ERROR_LOG_FORMAT').format(**msg_data)
+            except Exception as e:
+                if getattr(settings, 'API_400_ERROR_LOG_FORMAT', None):
+                    logger.error("Unable to format API_400_ERROR_LOG_FORMAT setting, defaulting log message: {}".format(e))
+                status_msg = settings_registry.get_setting_field('API_400_ERROR_LOG_FORMAT').get_default().format(**msg_data)
+
             if hasattr(self, '__init_request_error__'):
                 response = self.handle_exception(self.__init_request_error__)
             if response.status_code == 401:
@@ -221,6 +230,7 @@ class APIView(views.APIView):
                 logger.info(status_msg)
             else:
                 logger.warning(status_msg)
+
         response = super(APIView, self).finalize_response(request, response, *args, **kwargs)
         time_started = getattr(self, 'time_started', None)
         response['X-API-Product-Version'] = get_awx_version()
@@ -817,7 +827,7 @@ class ResourceAccessList(ParentMixin, ListAPIView):
 
 
 def trigger_delayed_deep_copy(*args, **kwargs):
-    from awx.main.tasks import deep_copy_model_obj
+    from awx.main.tasks.system import deep_copy_model_obj
 
     connection.on_commit(lambda: deep_copy_model_obj.delay(*args, **kwargs))
 

awx/api/permissions.py

@@ -25,7 +25,7 @@ __all__ = [
     'ProjectUpdatePermission',
     'InventoryInventorySourcesUpdatePermission',
     'UserPermission',
-    'IsSuperUser',
+    'IsSystemAdminOrAuditor',
     'InstanceGroupTowerPermission',
     'WorkflowApprovalPermission',
 ]
@@ -236,13 +236,18 @@ class UserPermission(ModelAccessPermission):
         raise PermissionDenied()
 
 
-class IsSuperUser(permissions.BasePermission):
+class IsSystemAdminOrAuditor(permissions.BasePermission):
     """
-    Allows access only to admin users.
+    Allows write access only to system admin users.
+    Allows read access only to system auditor users.
     """
 
     def has_permission(self, request, view):
-        return request.user and request.user.is_superuser
+        if not (request.user and request.user.is_authenticated):
+            return False
+        if request.method == 'GET':
+            return request.user.is_superuser or request.user.is_system_auditor
+        return request.user.is_superuser
 
 
 class InstanceGroupTowerPermission(ModelAccessPermission):

awx/api/serializers.py

@@ -57,6 +57,7 @@ from awx.main.models import (
     Host,
     Instance,
     InstanceGroup,
+    InstanceLink,
     Inventory,
     InventorySource,
     InventoryUpdate,
@@ -4767,6 +4768,28 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria
         return super(ScheduleSerializer, self).validate(attrs)
 
 
+class InstanceLinkSerializer(BaseSerializer):
+    class Meta:
+        model = InstanceLink
+        fields = ('source', 'target')
+
+    source = serializers.SlugRelatedField(slug_field="hostname", read_only=True)
+    target = serializers.SlugRelatedField(slug_field="hostname", read_only=True)
+
+
+class InstanceNodeSerializer(BaseSerializer):
+    class Meta:
+        model = Instance
+        fields = ('id', 'hostname', 'node_type', 'node_state')
+
+    node_state = serializers.SerializerMethodField()
+
+    def get_node_state(self, obj):
+        if not obj.enabled:
+            return "disabled"
+        return "unhealthy" if obj.errors else "healthy"
+
+
 class InstanceSerializer(BaseSerializer):
 
     consumed_capacity = serializers.SerializerMethodField()
@@ -4786,6 +4809,9 @@ class InstanceSerializer(BaseSerializer):
             "hostname",
             "created",
             "modified",
+            "last_seen",
+            "last_health_check",
+            "errors",
             'capacity_adjustment',
             "version",
             "capacity",
@@ -4806,6 +4832,8 @@ class InstanceSerializer(BaseSerializer):
         res = super(InstanceSerializer, self).get_related(obj)
         res['jobs'] = self.reverse('api:instance_unified_jobs_list', kwargs={'pk': obj.pk})
         res['instance_groups'] = self.reverse('api:instance_instance_groups_list', kwargs={'pk': obj.pk})
+        if self.context['request'].user.is_superuser or self.context['request'].user.is_system_auditor:
+            res['health_check'] = self.reverse('api:instance_health_check', kwargs={'pk': obj.pk})
         return res
 
     def get_consumed_capacity(self, obj):
@@ -4818,6 +4846,13 @@ class InstanceSerializer(BaseSerializer):
             return float("{0:.2f}".format(((float(obj.capacity) - float(obj.consumed_capacity)) / (float(obj.capacity))) * 100))
 
 
+class InstanceHealthCheckSerializer(BaseSerializer):
+    class Meta:
+        model = Instance
+        read_only_fields = ('uuid', 'hostname', 'version', 'last_health_check', 'errors', 'cpu', 'memory', 'cpu_capacity', 'mem_capacity', 'capacity')
+        fields = read_only_fields
+
+
 class InstanceGroupSerializer(BaseSerializer):
 
     show_capabilities = ['edit', 'delete']
@@ -4991,6 +5026,7 @@ class ActivityStreamSerializer(BaseSerializer):
             ('credential_type', ('id', 'name', 'description', 'kind', 'managed')),
             ('ad_hoc_command', ('id', 'name', 'status', 'limit')),
             ('workflow_approval', ('id', 'name', 'unified_job_id')),
+            ('instance', ('id', 'hostname')),
         ]
         return field_list
 

awx/api/templates/api/instance_health_check.md

@@ -0,0 +1,33 @@
+{% ifmeth GET %}
+# Health Check Data
+
+Health checks are used to obtain important data about an instance.
+Instance fields affected by the health check are shown in this view.
+Fundamentally, health checks require running code on the machine in question.
+
+ - For instances with `node_type` of "control" or "hybrid", health checks are
+performed as part of a periodic task that runs in the background.
+ - For instances with `node_type` of "execution", health checks are done by submitting
+a work unit through the receptor mesh.
+
+If ran through the receptor mesh, the invoked command is:
+
+```
+ansible-runner worker --worker-info
+```
+
+For execution nodes, these checks are _not_ performed on a regular basis.
+Health checks against functional nodes will be ran when the node is first discovered.
+Health checks against nodes with errors will be repeated at a reduced frequency.
+
+{% endifmeth %}
+
+{% ifmeth POST %}
+# Manually Initiate a Health Check
+For purposes of error remediation or debugging, a health check can be
+manually initiated by making a POST request to this endpoint.
+
+This will submit the work unit to the target node through the receptor mesh and wait for it to finish.
+The model will be updated with the result.
+Up-to-date values of the fields will be returned in the response data.
+{% endifmeth %}

awx/api/templates/api/mesh_visualizer.md

@@ -0,0 +1 @@
+Make a GET request to this resource to obtain a list all Receptor Nodes and their links.

awx/api/urls/instance.py

@@ -3,7 +3,7 @@
 
 from django.conf.urls import url
 
-from awx.api.views import InstanceList, InstanceDetail, InstanceUnifiedJobsList, InstanceInstanceGroupsList
+from awx.api.views import InstanceList, InstanceDetail, InstanceUnifiedJobsList, InstanceInstanceGroupsList, InstanceHealthCheck
 
 
 urls = [
@@ -11,6 +11,7 @@ urls = [
     url(r'^(?P<pk>[0-9]+)/$', InstanceDetail.as_view(), name='instance_detail'),
     url(r'^(?P<pk>[0-9]+)/jobs/$', InstanceUnifiedJobsList.as_view(), name='instance_unified_jobs_list'),
     url(r'^(?P<pk>[0-9]+)/instance_groups/$', InstanceInstanceGroupsList.as_view(), name='instance_instance_groups_list'),
+    url(r'^(?P<pk>[0-9]+)/health_check/$', InstanceHealthCheck.as_view(), name='instance_health_check'),
 ]
 
 __all__ = ['urls']

awx/api/urls/urls.py

@@ -28,6 +28,7 @@ from awx.api.views import (
     OAuth2TokenList,
     ApplicationOAuth2TokenList,
     OAuth2ApplicationDetail,
+    MeshVisualizer,
 )
 
 from awx.api.views.metrics import MetricsView
@@ -95,6 +96,7 @@ v2_urls = [
     url(r'^me/$', UserMeList.as_view(), name='user_me_list'),
     url(r'^dashboard/$', DashboardView.as_view(), name='dashboard_view'),
     url(r'^dashboard/graphs/jobs/$', DashboardJobsGraphView.as_view(), name='dashboard_jobs_graph_view'),
+    url(r'^mesh_visualizer/', MeshVisualizer.as_view(), name='mesh_visualizer_view'),
     url(r'^settings/', include('awx.conf.urls')),
     url(r'^instances/', include(instance_urls)),
     url(r'^instance_groups/', include(instance_group_urls)),

awx/api/views/__init__.py

@@ -62,7 +62,7 @@ import pytz
 from wsgiref.util import FileWrapper
 
 # AWX
-from awx.main.tasks import send_notifications, update_inventory_computed_fields
+from awx.main.tasks.system import send_notifications, update_inventory_computed_fields
 from awx.main.access import get_user_queryset, HostAccess
 from awx.api.generics import (
     APIView,
@@ -108,6 +108,7 @@ from awx.api.permissions import (
     InstanceGroupTowerPermission,
     VariableDataPermission,
     WorkflowApprovalPermission,
+    IsSystemAdminOrAuditor,
 )
 from awx.api import renderers
 from awx.api import serializers
@@ -158,6 +159,7 @@ from awx.api.views.inventory import (  # noqa
     InventoryJobTemplateList,
     InventoryCopy,
 )
+from awx.api.views.mesh_visualizer import MeshVisualizer  # noqa
 from awx.api.views.root import (  # noqa
     ApiRootView,
     ApiOAuthAuthorizationRootView,
@@ -363,19 +365,22 @@ class InstanceList(ListAPIView):
     serializer_class = serializers.InstanceSerializer
     search_fields = ('hostname',)
 
+    def get_queryset(self):
+        return super().get_queryset().exclude(node_type='hop')
+
 
 class InstanceDetail(RetrieveUpdateAPIView):
 
     name = _("Instance Detail")
-    model = models.Instance
+    queryset = models.Instance.objects.exclude(node_type='hop')
     serializer_class = serializers.InstanceSerializer
 
     def update(self, request, *args, **kwargs):
         r = super(InstanceDetail, self).update(request, *args, **kwargs)
         if status.is_success(r.status_code):
             obj = self.get_object()
-            obj.refresh_capacity()
-            obj.save()
+            obj.set_capacity_value()
+            obj.save(update_fields=['capacity'])
             r.data = serializers.InstanceSerializer(obj, context=self.get_serializer_context()).to_representation(obj)
         return r
 
@@ -402,6 +407,63 @@ class InstanceInstanceGroupsList(InstanceGroupMembershipMixin, SubListCreateAtta
     parent_model = models.Instance
     relationship = 'rampart_groups'
 
+    def is_valid_relation(self, parent, sub, created=False):
+        if parent.node_type == 'control':
+            return {'msg': _(f"Cannot change instance group membership of control-only node: {parent.hostname}.")}
+        if parent.node_type == 'hop':
+            return {'msg': _(f"Cannot change instance group membership of hop node: {parent.hostname}.")}
+        return None
+
+
+class InstanceHealthCheck(GenericAPIView):
+
+    name = _('Instance Health Check')
+    queryset = models.Instance.objects.exclude(node_type='hop')
+    serializer_class = serializers.InstanceHealthCheckSerializer
+    permission_classes = (IsSystemAdminOrAuditor,)
+
+    def get(self, request, *args, **kwargs):
+        obj = self.get_object()
+        data = self.get_serializer(data=request.data).to_representation(obj)
+        return Response(data, status=status.HTTP_200_OK)
+
+    def post(self, request, *args, **kwargs):
+        obj = self.get_object()
+
+        if obj.node_type == 'execution':
+            from awx.main.tasks.system import execution_node_health_check
+
+            runner_data = execution_node_health_check(obj.hostname)
+            obj.refresh_from_db()
+            data = self.get_serializer(data=request.data).to_representation(obj)
+            # Add in some extra unsaved fields
+            for extra_field in ('transmit_timing', 'run_timing'):
+                if extra_field in runner_data:
+                    data[extra_field] = runner_data[extra_field]
+        else:
+            from awx.main.tasks.system import cluster_node_health_check
+
+            if settings.CLUSTER_HOST_ID == obj.hostname:
+                cluster_node_health_check(obj.hostname)
+            else:
+                cluster_node_health_check.apply_async([obj.hostname], queue=obj.hostname)
+                start_time = time.time()
+                prior_check_time = obj.last_health_check
+                while time.time() - start_time < 50.0:
+                    obj.refresh_from_db(fields=['last_health_check'])
+                    if obj.last_health_check != prior_check_time:
+                        break
+                    if time.time() - start_time < 1.0:
+                        time.sleep(0.1)
+                    else:
+                        time.sleep(1.0)
+                else:
+                    obj.mark_offline(errors=_('Health check initiated by user determined this instance to be unresponsive'))
+            obj.refresh_from_db()
+            data = self.get_serializer(data=request.data).to_representation(obj)
+
+        return Response(data, status=status.HTTP_200_OK)
+
 
 class InstanceGroupList(ListCreateAPIView):
 
@@ -444,6 +506,13 @@ class InstanceGroupInstanceList(InstanceGroupMembershipMixin, SubListAttachDetac
     relationship = "instances"
     search_fields = ('hostname',)
 
+    def is_valid_relation(self, parent, sub, created=False):
+        if sub.node_type == 'control':
+            return {'msg': _(f"Cannot change instance group membership of control-only node: {sub.hostname}.")}
+        if sub.node_type == 'hop':
+            return {'msg': _(f"Cannot change instance group membership of hop node: {sub.hostname}.")}
+        return None
+
 
 class ScheduleList(ListCreateAPIView):
 

awx/api/views/mesh_visualizer.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2018 Red Hat, Inc.
+# All Rights Reserved.
+
+from django.utils.translation import ugettext_lazy as _
+
+from awx.api.generics import APIView, Response
+from awx.api.permissions import IsSystemAdminOrAuditor
+from awx.api.serializers import InstanceLinkSerializer, InstanceNodeSerializer
+from awx.main.models import InstanceLink, Instance
+
+
+class MeshVisualizer(APIView):
+
+    name = _("Mesh Visualizer")
+    permission_classes = (IsSystemAdminOrAuditor,)
+    swagger_topic = "System Configuration"
+
+    def get(self, request, format=None):
+
+        data = {
+            'nodes': InstanceNodeSerializer(Instance.objects.all(), many=True).data,
+            'links': InstanceLinkSerializer(InstanceLink.objects.all(), many=True).data,
+        }
+
+        return Response(data)

awx/api/views/mixin.py

@@ -68,13 +68,23 @@ class InstanceGroupMembershipMixin(object):
     membership.
     """
 
+    def attach_validate(self, request):
+        parent = self.get_parent_object()
+        sub_id, res = super().attach_validate(request)
+        if res:  # handle an error
+            return sub_id, res
+        sub = get_object_or_400(self.model, pk=sub_id)
+        attach_errors = self.is_valid_relation(parent, sub)
+        if attach_errors:
+            return sub_id, Response(attach_errors, status=status.HTTP_400_BAD_REQUEST)
+        return sub_id, res
+
     def attach(self, request, *args, **kwargs):
         response = super(InstanceGroupMembershipMixin, self).attach(request, *args, **kwargs)
         sub_id, res = self.attach_validate(request)
         if status.is_success(response.status_code):
             if self.parent_model is Instance:
-                ig_obj = get_object_or_400(self.model, pk=sub_id)
-                inst_name = ig_obj.hostname
+                inst_name = self.get_parent_object().hostname
             else:
                 inst_name = get_object_or_400(self.model, pk=sub_id).hostname
             with transaction.atomic():
@@ -91,11 +101,12 @@ class InstanceGroupMembershipMixin(object):
         return response
 
     def unattach_validate(self, request):
+        parent = self.get_parent_object()
         (sub_id, res) = super(InstanceGroupMembershipMixin, self).unattach_validate(request)
         if res:
             return (sub_id, res)
         sub = get_object_or_400(self.model, pk=sub_id)
-        attach_errors = self.is_valid_relation(None, sub)
+        attach_errors = self.is_valid_relation(parent, sub)
         if attach_errors:
             return (sub_id, Response(attach_errors, status=status.HTTP_400_BAD_REQUEST))
         return (sub_id, res)

awx/api/views/root.py

@@ -123,6 +123,7 @@ class ApiVersionRootView(APIView):
         data['workflow_approvals'] = reverse('api:workflow_approval_list', request=request)
         data['workflow_job_template_nodes'] = reverse('api:workflow_job_template_node_list', request=request)
         data['workflow_job_nodes'] = reverse('api:workflow_job_node_list', request=request)
+        data['mesh_visualizer'] = reverse('api:mesh_visualizer_view', request=request)
         return Response(data)
 
 
@@ -149,16 +150,24 @@ class ApiV2PingView(APIView):
         response = {'ha': is_ha_environment(), 'version': get_awx_version(), 'active_node': settings.CLUSTER_HOST_ID, 'install_uuid': settings.INSTALL_UUID}
 
         response['instances'] = []
-        for instance in Instance.objects.all():
+        for instance in Instance.objects.exclude(node_type='hop'):
             response['instances'].append(
-                dict(node=instance.hostname, uuid=instance.uuid, heartbeat=instance.modified, capacity=instance.capacity, version=instance.version)
+                dict(
+                    node=instance.hostname,
+                    node_type=instance.node_type,
+                    uuid=instance.uuid,
+                    heartbeat=instance.last_seen,
+                    capacity=instance.capacity,
+                    version=instance.version,
+                )
             )
-            sorted(response['instances'], key=operator.itemgetter('node'))
+            response['instances'] = sorted(response['instances'], key=operator.itemgetter('node'))
         response['instance_groups'] = []
         for instance_group in InstanceGroup.objects.prefetch_related('instances'):
             response['instance_groups'].append(
                 dict(name=instance_group.name, capacity=instance_group.capacity, instances=[x.hostname for x in instance_group.instances.all()])
             )
+            response['instance_groups'] = sorted(response['instance_groups'], key=lambda x: x['name'].lower())
         return Response(response)
 
 

awx/conf/views.py

@@ -23,10 +23,10 @@ from rest_framework import status
 
 # AWX
 from awx.api.generics import APIView, GenericAPIView, ListAPIView, RetrieveUpdateDestroyAPIView
-from awx.api.permissions import IsSuperUser
+from awx.api.permissions import IsSystemAdminOrAuditor
 from awx.api.versioning import reverse
 from awx.main.utils import camelcase_to_underscore
-from awx.main.tasks import handle_setting_changes
+from awx.main.tasks.system import handle_setting_changes
 from awx.conf.models import Setting
 from awx.conf.serializers import SettingCategorySerializer, SettingSingletonSerializer
 from awx.conf import settings_registry
@@ -150,7 +150,7 @@ class SettingLoggingTest(GenericAPIView):
     name = _('Logging Connectivity Test')
     model = Setting
     serializer_class = SettingSingletonSerializer
-    permission_classes = (IsSuperUser,)
+    permission_classes = (IsSystemAdminOrAuditor,)
     filter_backends = []
 
     def post(self, request, *args, **kwargs):

awx/locale/en-us/LC_MESSAGES/django.po

@@ -4856,7 +4856,7 @@ msgid "Exception connecting to PagerDuty: {}"
 msgstr ""
 
 #: awx/main/notifications/pagerduty_backend.py:87
-#: awx/main/notifications/slack_backend.py:48
+#: awx/main/notifications/slack_backend.py:49
 #: awx/main/notifications/twilio_backend.py:47
 msgid "Exception sending messages: {}"
 msgstr ""

awx/main/analytics/collectors.py

@@ -337,7 +337,11 @@ def _events_table(since, full_path, until, tbl, where_column, project_job_create
                           {tbl}.parent_uuid,
                           {tbl}.event,
                           task_action,
-                          (CASE WHEN event = 'playbook_on_stats' THEN event_data END) as playbook_on_stats,
+                          -- '-' operator listed here:
+                          -- https://www.postgresql.org/docs/12/functions-json.html
+                          -- note that operator is only supported by jsonb objects
+                          -- https://www.postgresql.org/docs/current/datatype-json.html
+                          (CASE WHEN event = 'playbook_on_stats' THEN {event_data} - 'artifact_data' END) as playbook_on_stats,
                           {tbl}.failed,
                           {tbl}.changed,
                           {tbl}.playbook,
@@ -352,14 +356,14 @@ def _events_table(since, full_path, until, tbl, where_column, project_job_create
                           x.duration AS duration,
                           x.res->'warnings' AS warnings,
                           x.res->'deprecations' AS deprecations
-                          FROM {tbl}, json_to_record({event_data}) AS x("res" json, "duration" text, "task_action" text, "start" text, "end" text)
+                          FROM {tbl}, jsonb_to_record({event_data}) AS x("res" json, "duration" text, "task_action" text, "start" text, "end" text)
                           WHERE ({tbl}.{where_column} > '{since.isoformat()}' AND {tbl}.{where_column} <= '{until.isoformat()}')) TO STDOUT WITH CSV HEADER'''
         return query
 
     try:
-        return _copy_table(table='events', query=query(f"{tbl}.event_data::json"), path=full_path)
+        return _copy_table(table='events', query=query(f"{tbl}.event_data::jsonb"), path=full_path)
     except UntranslatableCharacter:
-        return _copy_table(table='events', query=query(f"replace({tbl}.event_data::text, '\\u0000', '')::json"), path=full_path)
+        return _copy_table(table='events', query=query(f"replace({tbl}.event_data::text, '\\u0000', '')::jsonb"), path=full_path)
 
 
 @register('events_table', '1.3', format='csv', description=_('Automation task records'), expensive=four_hour_slicing)

awx/main/conf.py

@@ -408,6 +408,21 @@ register(
     unit=_('seconds'),
 )
 
+register(
+    'DEFAULT_JOB_IDLE_TIMEOUT',
+    field_class=fields.IntegerField,
+    min_value=0,
+    default=0,
+    label=_('Default Job Idle Timeout'),
+    help_text=_(
+        'If no output is detected from ansible in this number of seconds the execution will be terminated. '
+        'Use value of 0 to used default idle_timeout is 600s.'
+    ),
+    category=_('Jobs'),
+    category_slug='jobs',
+    unit=_('seconds'),
+)
+
 register(
     'DEFAULT_INVENTORY_UPDATE_TIMEOUT',
     field_class=fields.IntegerField,
@@ -659,6 +674,24 @@ register(
     category=_('Logging'),
     category_slug='logging',
 )
+register(
+    'API_400_ERROR_LOG_FORMAT',
+    field_class=fields.CharField,
+    default='status {status_code} received by user {user_name} attempting to access {url_path} from {remote_addr}',
+    label=_('Log Format For API 4XX Errors'),
+    help_text=_(
+        'The format of logged messages when an API 4XX error occurs, '
+        'the following variables will be substituted: \n'
+        'status_code - The HTTP status code of the error\n'
+        'user_name - The user name attempting to use the API\n'
+        'url_path - The URL path to the API endpoint called\n'
+        'remote_addr - The remote address seen for the user\n'
+        'error - The error set by the api endpoint\n'
+        'Variables need to be in the format {<variable name>}.'
+    ),
+    category=_('Logging'),
+    category_slug='logging',
+)
 
 
 register(
@@ -672,7 +705,7 @@ register(
 register(
     'AUTOMATION_ANALYTICS_LAST_ENTRIES',
     field_class=fields.CharField,
-    label=_('Last gathered entries for expensive collectors for Insights for Ansible Automation Platform.'),
+    label=_('Last gathered entries from the data collection service of Insights for Ansible Automation Platform'),
     default='',
     allow_blank=True,
     category=_('System'),

awx/main/constants.py

@@ -77,3 +77,11 @@ LOGGER_BLOCKLIST = (
     # loggers that may be called getting logging settings
     'awx.conf',
 )
+
+# Reported version for node seen in receptor mesh but for which capacity check
+# failed or is in progress
+RECEPTOR_PENDING = 'ansible-runner-???'
+
+# Naming pattern for AWX jobs in /tmp folder, like /tmp/awx_42_xiwm
+# also update awxkit.api.pages.unified_jobs if changed
+JOB_FOLDER_PREFIX = 'awx_%s_'

awx/main/dispatch/pool.py

@@ -248,7 +248,7 @@ class WorkerPool(object):
         except Exception:
             logger.exception('could not fork')
         else:
-            logger.warn('scaling up worker pid:{}'.format(worker.pid))
+            logger.debug('scaling up worker pid:{}'.format(worker.pid))
         return idx, worker
 
     def debug(self, *args, **kwargs):
@@ -387,7 +387,7 @@ class AutoscalePool(WorkerPool):
                 # more processes in the pool than we need (> min)
                 # send this process a message so it will exit gracefully
                 # at the next opportunity
-                logger.warn('scaling down worker pid:{}'.format(w.pid))
+                logger.debug('scaling down worker pid:{}'.format(w.pid))
                 w.quit()
                 self.workers.remove(w)
             if w.alive:

awx/main/dispatch/worker/base.py

@@ -60,7 +60,7 @@ class AWXConsumerBase(object):
         return f'listening on {self.queues}'
 
     def control(self, body):
-        logger.warn(body)
+        logger.warn(f'Received control signal:\n{body}')
         control = body.get('control')
         if control in ('status', 'running'):
             reply_queue = body['reply_to']
@@ -137,7 +137,7 @@ class AWXConsumerPG(AWXConsumerBase):
     def run(self, *args, **kwargs):
         super(AWXConsumerPG, self).run(*args, **kwargs)
 
-        logger.warn(f"Running worker {self.name} listening to queues {self.queues}")
+        logger.info(f"Running worker {self.name} listening to queues {self.queues}")
         init = False
 
         while True:
@@ -188,7 +188,7 @@ class BaseWorker(object):
                 if 'uuid' in body:
                     uuid = body['uuid']
                     finished.put(uuid)
-        logger.warn('worker exiting gracefully pid:{}'.format(os.getpid()))
+        logger.debug('worker exiting gracefully pid:{}'.format(os.getpid()))
 
     def perform_work(self, body):
         raise NotImplementedError()

awx/main/dispatch/worker/callback.py

@@ -17,7 +17,7 @@ import redis
 
 from awx.main.consumers import emit_channel_notification
 from awx.main.models import JobEvent, AdHocCommandEvent, ProjectUpdateEvent, InventoryUpdateEvent, SystemJobEvent, UnifiedJob, Job
-from awx.main.tasks import handle_success_and_failure_notifications
+from awx.main.tasks.system import handle_success_and_failure_notifications
 from awx.main.models.events import emit_event_detail
 from awx.main.utils.profiling import AWXProfiler
 import awx.main.analytics.subsystem_metrics as s_metrics

awx/main/dispatch/worker/task.py

@@ -9,7 +9,7 @@ from kubernetes.config import kube_config
 from django.conf import settings
 from django_guid.middleware import GuidMiddleware
 
-from awx.main.tasks import dispatch_startup, inform_cluster_of_shutdown
+from awx.main.tasks.system import dispatch_startup, inform_cluster_of_shutdown
 
 from .base import BaseWorker
 
@@ -30,8 +30,8 @@ class TaskWorker(BaseWorker):
         """
         Transform a dotted notation task into an imported, callable function, e.g.,
 
-        awx.main.tasks.delete_inventory
-        awx.main.tasks.RunProjectUpdate
+        awx.main.tasks.system.delete_inventory
+        awx.main.tasks.jobs.RunProjectUpdate
         """
         if not task.startswith('awx.'):
             raise ValueError('{} is not a valid awx task'.format(task))
@@ -73,15 +73,15 @@ class TaskWorker(BaseWorker):
             'callbacks': [{
                 'args': [],
                 'kwargs': {}
-                'task': u'awx.main.tasks.handle_work_success'
+                'task': u'awx.main.tasks.system.handle_work_success'
             }],
             'errbacks': [{
                 'args': [],
                 'kwargs': {},
-                'task': 'awx.main.tasks.handle_work_error'
+                'task': 'awx.main.tasks.system.handle_work_error'
             }],
             'kwargs': {},
-            'task': u'awx.main.tasks.RunProjectUpdate'
+            'task': u'awx.main.tasks.jobs.RunProjectUpdate'
         }
         """
         settings.__clean_on_fork__()

awx/main/exceptions.py

@@ -36,3 +36,7 @@ class PostRunError(Exception):
         self.status = status
         self.tb = tb
         super(PostRunError, self).__init__(msg)
+
+
+class ReceptorNodeNotFound(RuntimeError):
+    pass

awx/main/ha.py

@@ -10,6 +10,6 @@ def is_ha_environment():
     otherwise.
     """
     # If there are two or more instances, then we are in an HA environment.
-    if Instance.objects.count() > 1:
+    if Instance.objects.filter(node_type__in=('control', 'hybrid')).count() > 1:
         return True
     return False

awx/main/management/commands/create_preload_data.py

@@ -23,44 +23,50 @@ class Command(BaseCommand):
         with impersonate(superuser):
             with disable_computed_fields():
                 if not Organization.objects.exists():
-                    o = Organization.objects.create(name='Default')
+                    o, _ = Organization.objects.get_or_create(name='Default')
 
-                    p = Project(
+                    p, _ = Project.objects.get_or_create(
                         name='Demo Project',
                         scm_type='git',
                         scm_url='https://github.com/ansible/ansible-tower-samples',
                         scm_update_on_launch=True,
                         scm_update_cache_timeout=0,
-                        organization=o,
                     )
+                    p.organization = o
                     p.save(skip_update=True)
 
                     ssh_type = CredentialType.objects.filter(namespace='ssh').first()
-                    c = Credential.objects.create(
+                    c, _ = Credential.objects.get_or_create(
                         credential_type=ssh_type, name='Demo Credential', inputs={'username': superuser.username}, created_by=superuser
                     )
 
                     c.admin_role.members.add(superuser)
 
-                    public_galaxy_credential = Credential(
+                    public_galaxy_credential, _ = Credential.objects.get_or_create(
                         name='Ansible Galaxy',
                         managed=True,
                         credential_type=CredentialType.objects.get(kind='galaxy'),
                         inputs={'url': 'https://galaxy.ansible.com/'},
                     )
-                    public_galaxy_credential.save()
                     o.galaxy_credentials.add(public_galaxy_credential)
 
-                    i = Inventory.objects.create(name='Demo Inventory', organization=o, created_by=superuser)
+                    i, _ = Inventory.objects.get_or_create(name='Demo Inventory', organization=o, created_by=superuser)
 
-                    Host.objects.create(
+                    Host.objects.get_or_create(
                         name='localhost',
                         inventory=i,
                         variables="ansible_connection: local\nansible_python_interpreter: '{{ ansible_playbook_python }}'",
                         created_by=superuser,
                     )
 
-                    jt = JobTemplate.objects.create(name='Demo Job Template', playbook='hello_world.yml', project=p, inventory=i)
+                    jt = JobTemplate.objects.filter(name='Demo Job Template').first()
+                    if jt:
+                        jt.project = p
+                        jt.inventory = i
+                        jt.playbook = 'hello_world.yml'
+                        jt.save()
+                    else:
+                        jt, _ = JobTemplate.objects.get_or_create(name='Demo Job Template', playbook='hello_world.yml', project=p, inventory=i)
                     jt.credentials.add(c)
 
                     print('Default organization added.')

awx/main/management/commands/inventory_import.py

@@ -10,6 +10,7 @@ import subprocess
 import sys
 import time
 import traceback
+from collections import OrderedDict
 
 # Django
 from django.conf import settings
@@ -75,7 +76,24 @@ class AnsibleInventoryLoader(object):
         bargs.extend(['-v', '{0}:{0}:Z'.format(self.source)])
         for key, value in STANDARD_INVENTORY_UPDATE_ENV.items():
             bargs.extend(['-e', '{0}={1}'.format(key, value)])
-        bargs.extend([get_default_execution_environment().image])
+        ee = get_default_execution_environment()
+
+        if settings.IS_K8S:
+            logger.warn('This command is not able to run on kubernetes-based deployment. This action should be done using the API.')
+            sys.exit(1)
+
+        if ee.credential:
+            process = subprocess.run(['podman', 'image', 'exists', ee.image], capture_output=True)
+            if process.returncode != 0:
+                logger.warn(
+                    f'The default execution environment (id={ee.id}, name={ee.name}, image={ee.image}) is not available on this node. '
+                    'The image needs to be available locally before using this command, due to registry authentication. '
+                    'To pull this image, either run a job on this node or manually pull the image.'
+                )
+                sys.exit(1)
+
+        bargs.extend([ee.image])
+
         bargs.extend(['ansible-inventory', '-i', self.source])
         bargs.extend(['--playbook-dir', functioning_dir(self.source)])
         if self.verbosity:
@@ -110,9 +128,7 @@ class AnsibleInventoryLoader(object):
 
     def load(self):
         base_args = self.get_base_args()
-
         logger.info('Reading Ansible inventory source: %s', self.source)
-
         return self.command_to_json(base_args)
 
 
@@ -137,7 +153,7 @@ class Command(BaseCommand):
             type=str,
             default=None,
             metavar='v',
-            help='host variable used to ' 'set/clear enabled flag when host is online/offline, may ' 'be specified as "foo.bar" to traverse nested dicts.',
+            help='host variable used to set/clear enabled flag when host is online/offline, may be specified as "foo.bar" to traverse nested dicts.',
         )
         parser.add_argument(
             '--enabled-value',
@@ -145,7 +161,7 @@ class Command(BaseCommand):
             type=str,
             default=None,
             metavar='v',
-            help='value of host variable ' 'specified by --enabled-var that indicates host is ' 'enabled/online.',
+            help='value of host variable specified by --enabled-var that indicates host is enabled/online.',
         )
         parser.add_argument(
             '--group-filter',
@@ -153,7 +169,7 @@ class Command(BaseCommand):
             type=str,
             default=None,
             metavar='regex',
-            help='regular expression ' 'to filter group name(s); only matches are imported.',
+            help='regular expression to filter group name(s); only matches are imported.',
         )
         parser.add_argument(
             '--host-filter',
@@ -161,14 +177,14 @@ class Command(BaseCommand):
             type=str,
             default=None,
             metavar='regex',
-            help='regular expression ' 'to filter host name(s); only matches are imported.',
+            help='regular expression to filter host name(s); only matches are imported.',
         )
         parser.add_argument(
             '--exclude-empty-groups',
             dest='exclude_empty_groups',
             action='store_true',
             default=False,
-            help='when set, ' 'exclude all groups that have no child groups, hosts, or ' 'variables.',
+            help='when set, exclude all groups that have no child groups, hosts, or variables.',
         )
         parser.add_argument(
             '--instance-id-var',
@@ -176,7 +192,7 @@ class Command(BaseCommand):
             type=str,
             default=None,
             metavar='v',
-            help='host variable that ' 'specifies the unique, immutable instance ID, may be ' 'specified as "foo.bar" to traverse nested dicts.',
+            help='host variable that specifies the unique, immutable instance ID, may be specified as "foo.bar" to traverse nested dicts.',
         )
 
     def set_logging_level(self, verbosity):
@@ -269,12 +285,13 @@ class Command(BaseCommand):
         self.db_instance_id_map = {}
         if self.instance_id_var:
             host_qs = self.inventory_source.hosts.all()
-            host_qs = host_qs.filter(instance_id='', variables__contains=self.instance_id_var.split('.')[0])
-            for host in host_qs:
-                instance_id = self._get_instance_id(host.variables_dict)
-                if not instance_id:
-                    continue
-                self.db_instance_id_map[instance_id] = host.pk
+            for instance_id_part in reversed(self.instance_id_var.split(',')):
+                host_qs = host_qs.filter(instance_id='', variables__contains=instance_id_part.split('.')[0])
+                for host in host_qs:
+                    instance_id = self._get_instance_id(host.variables_dict)
+                    if not instance_id:
+                        continue
+                    self.db_instance_id_map[instance_id] = host.pk
 
     def _build_mem_instance_id_map(self):
         """
@@ -300,7 +317,7 @@ class Command(BaseCommand):
             self._cached_host_pk_set = frozenset(self.inventory_source.hosts.values_list('pk', flat=True))
         return self._cached_host_pk_set
 
-    def _delete_hosts(self):
+    def _delete_hosts(self, pk_mem_host_map):
         """
         For each host in the database that is NOT in the local list, delete
         it. When importing from a cloud inventory source attached to a
@@ -309,25 +326,10 @@ class Command(BaseCommand):
         """
         if settings.SQL_DEBUG:
             queries_before = len(connection.queries)
+
         hosts_qs = self.inventory_source.hosts
-        # Build list of all host pks, remove all that should not be deleted.
-        del_host_pks = set(self._existing_host_pks())  # makes mutable copy
-        if self.instance_id_var:
-            all_instance_ids = list(self.mem_instance_id_map.keys())
-            instance_ids = []
-            for offset in range(0, len(all_instance_ids), self._batch_size):
-                instance_ids = all_instance_ids[offset : (offset + self._batch_size)]
-                for host_pk in hosts_qs.filter(instance_id__in=instance_ids).values_list('pk', flat=True):
-                    del_host_pks.discard(host_pk)
-            for host_pk in set([v for k, v in self.db_instance_id_map.items() if k in instance_ids]):
-                del_host_pks.discard(host_pk)
-            all_host_names = list(set(self.mem_instance_id_map.values()) - set(self.all_group.all_hosts.keys()))
-        else:
-            all_host_names = list(self.all_group.all_hosts.keys())
-        for offset in range(0, len(all_host_names), self._batch_size):
-            host_names = all_host_names[offset : (offset + self._batch_size)]
-            for host_pk in hosts_qs.filter(name__in=host_names).values_list('pk', flat=True):
-                del_host_pks.discard(host_pk)
+        del_host_pks = hosts_qs.exclude(pk__in=pk_mem_host_map.keys()).values_list('pk', flat=True)
+
         # Now delete all remaining hosts in batches.
         all_del_pks = sorted(list(del_host_pks))
         for offset in range(0, len(all_del_pks), self._batch_size):
@@ -568,7 +570,63 @@ class Command(BaseCommand):
                 logger.debug('Host "%s" is now disabled', mem_host.name)
         self._batch_add_m2m(self.inventory_source.hosts, db_host)
 
-    def _create_update_hosts(self):
+    def _build_pk_mem_host_map(self):
+        """
+        Creates and returns a data structure that maps DB hosts to in-memory host that
+        they correspond to - meaning that those hosts will be updated to in-memory host values
+        """
+        mem_host_pk_map = OrderedDict()  # keys are mem_host name, values are matching DB host pk
+        host_pks_updated = set()  # same as items of mem_host_pk_map but used for efficiency
+        mem_host_pk_map_by_id = {}  # incomplete mapping by new instance_id to be sorted and pushed to mem_host_pk_map
+        mem_host_instance_id_map = {}
+        for k, v in self.all_group.all_hosts.items():
+            instance_id = self._get_instance_id(v.variables)
+            if instance_id in self.db_instance_id_map:
+                mem_host_pk_map_by_id[self.db_instance_id_map[instance_id]] = v
+            elif instance_id:
+                mem_host_instance_id_map[instance_id] = v
+
+        # Update all existing hosts where we know the PK based on instance_id.
+        all_host_pks = sorted(mem_host_pk_map_by_id.keys())
+        for offset in range(0, len(all_host_pks), self._batch_size):
+            host_pks = all_host_pks[offset : (offset + self._batch_size)]
+            for db_host in self.inventory.hosts.only('pk').filter(pk__in=host_pks):
+                if db_host.pk in host_pks_updated:
+                    continue
+                mem_host = mem_host_pk_map_by_id[db_host.pk]
+                mem_host_pk_map[mem_host.name] = db_host.pk
+                host_pks_updated.add(db_host.pk)
+
+        # Update all existing hosts where we know the DB (the prior) instance_id.
+        all_instance_ids = sorted(mem_host_instance_id_map.keys())
+        for offset in range(0, len(all_instance_ids), self._batch_size):
+            instance_ids = all_instance_ids[offset : (offset + self._batch_size)]
+            for db_host in self.inventory.hosts.only('pk', 'instance_id').filter(instance_id__in=instance_ids):
+                if db_host.pk in host_pks_updated:
+                    continue
+                mem_host = mem_host_instance_id_map[db_host.instance_id]
+                mem_host_pk_map[mem_host.name] = db_host.pk
+                host_pks_updated.add(db_host.pk)
+
+        # Update all existing hosts by name.
+        all_host_names = sorted(self.all_group.all_hosts.keys())
+        for offset in range(0, len(all_host_names), self._batch_size):
+            host_names = all_host_names[offset : (offset + self._batch_size)]
+            for db_host in self.inventory.hosts.only('pk', 'name').filter(name__in=host_names):
+                if db_host.pk in host_pks_updated:
+                    continue
+                mem_host = self.all_group.all_hosts[db_host.name]
+                mem_host_pk_map[mem_host.name] = db_host.pk
+                host_pks_updated.add(db_host.pk)
+
+        # Rotate the dictionary so that lookups are done by the host pk
+        pk_mem_host_map = OrderedDict()
+        for name, host_pk in mem_host_pk_map.items():
+            pk_mem_host_map[host_pk] = name
+
+        return pk_mem_host_map  # keys are DB host pk, keys are matching mem host name
+
+    def _create_update_hosts(self, pk_mem_host_map):
         """
         For each host in the local list, create it if it doesn't exist in the
         database.  Otherwise, update/replace database variables from the
@@ -577,57 +635,22 @@ class Command(BaseCommand):
         """
         if settings.SQL_DEBUG:
             queries_before = len(connection.queries)
-        host_pks_updated = set()
-        mem_host_pk_map = {}
-        mem_host_instance_id_map = {}
-        mem_host_name_map = {}
-        mem_host_names_to_update = set(self.all_group.all_hosts.keys())
-        for k, v in self.all_group.all_hosts.items():
-            mem_host_name_map[k] = v
-            instance_id = self._get_instance_id(v.variables)
-            if instance_id in self.db_instance_id_map:
-                mem_host_pk_map[self.db_instance_id_map[instance_id]] = v
-            elif instance_id:
-                mem_host_instance_id_map[instance_id] = v
 
-        # Update all existing hosts where we know the PK based on instance_id.
-        all_host_pks = sorted(mem_host_pk_map.keys())
+        updated_mem_host_names = set()
+
+        all_host_pks = sorted(pk_mem_host_map.keys())
         for offset in range(0, len(all_host_pks), self._batch_size):
             host_pks = all_host_pks[offset : (offset + self._batch_size)]
             for db_host in self.inventory.hosts.filter(pk__in=host_pks):
-                if db_host.pk in host_pks_updated:
-                    continue
-                mem_host = mem_host_pk_map[db_host.pk]
+                mem_host_name = pk_mem_host_map[db_host.pk]
+                mem_host = self.all_group.all_hosts[mem_host_name]
                 self._update_db_host_from_mem_host(db_host, mem_host)
-                host_pks_updated.add(db_host.pk)
-                mem_host_names_to_update.discard(mem_host.name)
+                updated_mem_host_names.add(mem_host.name)
 
-        # Update all existing hosts where we know the instance_id.
-        all_instance_ids = sorted(mem_host_instance_id_map.keys())
-        for offset in range(0, len(all_instance_ids), self._batch_size):
-            instance_ids = all_instance_ids[offset : (offset + self._batch_size)]
-            for db_host in self.inventory.hosts.filter(instance_id__in=instance_ids):
-                if db_host.pk in host_pks_updated:
-                    continue
-                mem_host = mem_host_instance_id_map[db_host.instance_id]
-                self._update_db_host_from_mem_host(db_host, mem_host)
-                host_pks_updated.add(db_host.pk)
-                mem_host_names_to_update.discard(mem_host.name)
-
-        # Update all existing hosts by name.
-        all_host_names = sorted(mem_host_name_map.keys())
-        for offset in range(0, len(all_host_names), self._batch_size):
-            host_names = all_host_names[offset : (offset + self._batch_size)]
-            for db_host in self.inventory.hosts.filter(name__in=host_names):
-                if db_host.pk in host_pks_updated:
-                    continue
-                mem_host = mem_host_name_map[db_host.name]
-                self._update_db_host_from_mem_host(db_host, mem_host)
-                host_pks_updated.add(db_host.pk)
-                mem_host_names_to_update.discard(mem_host.name)
+        mem_host_names_to_create = set(self.all_group.all_hosts.keys()) - updated_mem_host_names
 
         # Create any new hosts.
-        for mem_host_name in sorted(mem_host_names_to_update):
+        for mem_host_name in sorted(mem_host_names_to_create):
             mem_host = self.all_group.all_hosts[mem_host_name]
             import_vars = mem_host.variables
             host_desc = import_vars.pop('_awx_description', 'imported')
@@ -726,13 +749,14 @@ class Command(BaseCommand):
         self._batch_size = 500
         self._build_db_instance_id_map()
         self._build_mem_instance_id_map()
+        pk_mem_host_map = self._build_pk_mem_host_map()
         if self.overwrite:
-            self._delete_hosts()
+            self._delete_hosts(pk_mem_host_map)
             self._delete_groups()
             self._delete_group_children_and_hosts()
         self._update_inventory()
         self._create_update_groups()
-        self._create_update_hosts()
+        self._create_update_hosts(pk_mem_host_map)
         self._create_update_group_children()
         self._create_update_group_hosts()
 
@@ -1008,4 +1032,4 @@ class Command(BaseCommand):
             if settings.SQL_DEBUG:
                 queries_this_import = connection.queries[queries_before:]
                 sqltime = sum(float(x['time']) for x in queries_this_import)
-                logger.warning('Inventory import required %d queries ' 'taking %0.3fs', len(queries_this_import), sqltime)
+                logger.warning('Inventory import required %d queries taking %0.3fs', len(queries_this_import), sqltime)

awx/main/management/commands/list_instances.py

@@ -13,7 +13,7 @@ class Ungrouped(object):
 
     @property
     def instances(self):
-        return Instance.objects.filter(rampart_groups__isnull=True)
+        return Instance.objects.filter(rampart_groups__isnull=True).exclude(node_type='hop')
 
     @property
     def capacity(self):
@@ -47,7 +47,7 @@ class Command(BaseCommand):
                     color = '\033[90m[DISABLED] '
                 if no_color:
                     color = ''
-                fmt = '\t' + color + '{0.hostname} capacity={0.capacity} version={1}'
+                fmt = '\t' + color + '{0.hostname} capacity={0.capacity} node_type={0.node_type} version={1}'
                 if x.capacity:
                     fmt += ' heartbeat="{0.modified:%Y-%m-%d %H:%M:%S}"'
                 print((fmt + '\033[0m').format(x, x.version or '?'))

awx/main/management/commands/profile_sql.py

@@ -1,6 +1,6 @@
 from django.core.management.base import BaseCommand
 
-from awx.main.tasks import profile_sql
+from awx.main.tasks.system import profile_sql
 
 
 class Command(BaseCommand):

awx/main/management/commands/provision_instance.py

@@ -1,7 +1,6 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved
 
-from django.conf import settings
 from django.core.management.base import BaseCommand, CommandError
 from django.db import transaction
 
@@ -14,18 +13,19 @@ class Command(BaseCommand):
     Register this instance with the database for HA tracking.
     """
 
-    help = 'Add instance to the database. ' 'Specify `--hostname` to use this command.'
+    help = "Add instance to the database. Specify `--hostname` to use this command."
 
     def add_arguments(self, parser):
-        parser.add_argument('--hostname', dest='hostname', type=str, help='Hostname used during provisioning')
-        parser.add_argument('--node_type', type=str, default="hybrid", choices=["control", "execution", "hybrid"], help='Instance Node type')
+        parser.add_argument('--hostname', dest='hostname', type=str, help="Hostname used during provisioning")
+        parser.add_argument('--node_type', type=str, default='hybrid', choices=['control', 'execution', 'hop', 'hybrid'], help="Instance Node type")
+        parser.add_argument('--uuid', type=str, help="Instance UUID")
 
-    def _register_hostname(self, hostname, node_type):
+    def _register_hostname(self, hostname, node_type, uuid):
         if not hostname:
             return
-        (changed, instance) = Instance.objects.register(uuid=self.uuid, hostname=hostname, node_type=node_type)
+        (changed, instance) = Instance.objects.register(hostname=hostname, node_type=node_type, uuid=uuid)
         if changed:
-            print('Successfully registered instance {}'.format(hostname))
+            print("Successfully registered instance {}".format(hostname))
         else:
             print("Instance already registered {}".format(instance.hostname))
         self.changed = changed
@@ -34,8 +34,7 @@ class Command(BaseCommand):
     def handle(self, **options):
         if not options.get('hostname'):
             raise CommandError("Specify `--hostname` to use this command.")
-        self.uuid = settings.SYSTEM_UUID
         self.changed = False
-        self._register_hostname(options.get('hostname'), options.get('node_type'))
+        self._register_hostname(options.get('hostname'), options.get('node_type'), options.get('uuid'))
         if self.changed:
-            print('(changed: True)')
+            print("(changed: True)")

awx/main/management/commands/register_peers.py

@@ -0,0 +1,85 @@
+from django.core.management.base import BaseCommand, CommandError
+from django.db import transaction
+
+from awx.main.models import Instance, InstanceLink
+
+
+class Command(BaseCommand):
+    """
+    Internal tower command.
+    Register the peers of a receptor node.
+    """
+
+    help = "Register or remove links between Receptor nodes."
+
+    def add_arguments(self, parser):
+        parser.add_argument('source', type=str, help="Receptor node opening the connections.")
+        parser.add_argument('--peers', type=str, nargs='+', required=False, help="Nodes that the source node connects out to.")
+        parser.add_argument('--disconnect', type=str, nargs='+', required=False, help="Nodes that should no longer be connected to by the source node.")
+        parser.add_argument(
+            '--exact',
+            type=str,
+            nargs='*',
+            required=False,
+            help="The exact set of nodes the source node should connect out to. Any existing links registered in the database that do not match will be removed. May be empty.",
+        )
+
+    def handle(self, **options):
+        nodes = Instance.objects.in_bulk(field_name='hostname')
+        if options['source'] not in nodes:
+            raise CommandError(f"Host {options['source']} is not a registered instance.")
+        if not (options['peers'] or options['disconnect'] or options['exact'] is not None):
+            raise CommandError("One of the options --peers, --disconnect, or --exact is required.")
+        if options['exact'] is not None and options['peers']:
+            raise CommandError("The option --peers may not be used with --exact.")
+        if options['exact'] is not None and options['disconnect']:
+            raise CommandError("The option --disconnect may not be used with --exact.")
+
+        # No 1-cycles
+        for collection in ('peers', 'disconnect', 'exact'):
+            if options[collection] is not None and options['source'] in options[collection]:
+                raise CommandError(f"Source node {options['source']} may not also be in --{collection}.")
+
+        # No 2-cycles
+        if options['peers'] or options['exact'] is not None:
+            peers = set(options['peers'] or options['exact'])
+            incoming = set(InstanceLink.objects.filter(target=nodes[options['source']]).values_list('source__hostname', flat=True))
+            if peers & incoming:
+                raise CommandError(f"Source node {options['source']} cannot link to nodes already peering to it: {peers & incoming}.")
+
+        if options['peers']:
+            missing_peers = set(options['peers']) - set(nodes)
+            if missing_peers:
+                missing = ' '.join(missing_peers)
+                raise CommandError(f"Peers not currently registered as instances: {missing}")
+
+            results = 0
+            for target in options['peers']:
+                _, created = InstanceLink.objects.get_or_create(source=nodes[options['source']], target=nodes[target])
+                if created:
+                    results += 1
+
+            print(f"{results} new peer links added to the database.")
+
+        if options['disconnect']:
+            results = 0
+            for target in options['disconnect']:
+                if target not in nodes:  # Be permissive, the node might have already been de-registered.
+                    continue
+                n, _ = InstanceLink.objects.filter(source=nodes[options['source']], target=nodes[target]).delete()
+                results += n
+
+            print(f"{results} peer links removed from the database.")
+
+        if options['exact'] is not None:
+            additions = 0
+            with transaction.atomic():
+                peers = set(options['exact'])
+                links = set(InstanceLink.objects.filter(source=nodes[options['source']]).values_list('target__hostname', flat=True))
+                removals, _ = InstanceLink.objects.filter(source=nodes[options['source']], target__hostname__in=links - peers).delete()
+                for target in peers - links:
+                    _, created = InstanceLink.objects.get_or_create(source=nodes[options['source']], target=nodes[target])
+                    if created:
+                        additions += 1
+
+            print(f"{additions} peer links added and {removals} deleted from the database.")

awx/main/management/commands/register_queue.py

@@ -17,13 +17,14 @@ class InstanceNotFound(Exception):
 
 
 class RegisterQueue:
-    def __init__(self, queuename, instance_percent, inst_min, hostname_list, is_container_group=None):
+    def __init__(self, queuename, instance_percent, inst_min, hostname_list, is_container_group=None, pod_spec_override=None):
         self.instance_not_found_err = None
         self.queuename = queuename
         self.instance_percent = instance_percent
         self.instance_min = inst_min
         self.hostname_list = hostname_list
         self.is_container_group = is_container_group
+        self.pod_spec_override = pod_spec_override
 
     def get_create_update_instance_group(self):
         created = False
@@ -36,10 +37,14 @@ class RegisterQueue:
             ig.policy_instance_minimum = self.instance_min
             changed = True
 
-        if self.is_container_group:
+        if self.is_container_group and (ig.is_container_group != self.is_container_group):
             ig.is_container_group = self.is_container_group
             changed = True
 
+        if self.pod_spec_override and (ig.pod_spec_override != self.pod_spec_override):
+            ig.pod_spec_override = self.pod_spec_override
+            changed = True
+
         if changed:
             ig.save()
 

awx/main/managers.py

@@ -4,16 +4,18 @@
 import sys
 import logging
 import os
-
 from django.db import models
 from django.conf import settings
 
 from awx.main.utils.filters import SmartFilter
 from awx.main.utils.pglock import advisory_lock
+from awx.main.utils.common import get_capacity_type
+from awx.main.constants import RECEPTOR_PENDING
 
-___all__ = ['HostManager', 'InstanceManager', 'InstanceGroupManager', 'DeferJobCreatedManager']
+___all__ = ['HostManager', 'InstanceManager', 'InstanceGroupManager', 'DeferJobCreatedManager', 'UUID_DEFAULT']
 
 logger = logging.getLogger('awx.main.managers')
+UUID_DEFAULT = '00000000-0000-0000-0000-000000000000'
 
 
 class DeferJobCreatedManager(models.Manager):
@@ -104,20 +106,17 @@ class InstanceManager(models.Manager):
         """Return the currently active instance."""
         # If we are running unit tests, return a stub record.
         if settings.IS_TESTING(sys.argv) or hasattr(sys, '_called_from_test'):
-            return self.model(id=1, hostname='localhost', uuid='00000000-0000-0000-0000-000000000000')
+            return self.model(id=1, hostname=settings.CLUSTER_HOST_ID, uuid=UUID_DEFAULT)
 
         node = self.filter(hostname=settings.CLUSTER_HOST_ID)
         if node.exists():
             return node[0]
         raise RuntimeError("No instance found with the current cluster host id")
 
-    def register(self, uuid=None, hostname=None, ip_address=None, node_type=None):
-        if not uuid:
-            uuid = settings.SYSTEM_UUID
+    def register(self, uuid=None, hostname=None, ip_address=None, node_type='hybrid', defaults=None):
         if not hostname:
             hostname = settings.CLUSTER_HOST_ID
-        if not node_type:
-            node_type = "hybrid"
+
         with advisory_lock('instance_registration_%s' % hostname):
             if settings.AWX_AUTO_DEPROVISION_INSTANCES:
                 # detect any instances with the same IP address.
@@ -130,13 +129,25 @@ class InstanceManager(models.Manager):
                         other_inst.save(update_fields=['ip_address'])
                         logger.warning("IP address {0} conflict detected, ip address unset for host {1}.".format(ip_address, other_hostname))
 
-            instance = self.filter(hostname=hostname)
+            # Return existing instance that matches hostname or UUID (default to UUID)
+            if uuid is not None and uuid != UUID_DEFAULT and self.filter(uuid=uuid).exists():
+                instance = self.filter(uuid=uuid)
+            else:
+                # if instance was not retrieved by uuid and hostname was, use the hostname
+                instance = self.filter(hostname=hostname)
+
+            # Return existing instance
             if instance.exists():
-                instance = instance.get()
+                instance = instance.first()  # in the unusual occasion that there is more than one, only get one
                 update_fields = []
+                # if instance was retrieved by uuid and hostname has changed, update hostname
+                if instance.hostname != hostname:
+                    logger.warning("passed in hostname {0} is different from the original hostname {1}, updating to {0}".format(hostname, instance.hostname))
+                    instance.hostname = hostname
+                    update_fields.append('hostname')
+                # if any other fields are to be updated
                 if instance.ip_address != ip_address:
                     instance.ip_address = ip_address
-                    update_fields.append('ip_address')
                 if instance.node_type != node_type:
                     instance.node_type = node_type
                     update_fields.append('node_type')
@@ -145,7 +156,17 @@ class InstanceManager(models.Manager):
                     return (True, instance)
                 else:
                     return (False, instance)
-            instance = self.create(uuid=uuid, hostname=hostname, ip_address=ip_address, capacity=0, node_type=node_type)
+
+            # Create new instance, and fill in default values
+            create_defaults = dict(capacity=0)
+            if defaults is not None:
+                create_defaults.update(defaults)
+            uuid_option = {}
+            if uuid is not None:
+                uuid_option = dict(uuid=uuid)
+            if node_type == 'execution' and 'version' not in create_defaults:
+                create_defaults['version'] = RECEPTOR_PENDING
+            instance = self.create(hostname=hostname, ip_address=ip_address, node_type=node_type, **create_defaults, **uuid_option)
         return (True, instance)
 
     def get_or_register(self):
@@ -153,17 +174,18 @@ class InstanceManager(models.Manager):
             from awx.main.management.commands.register_queue import RegisterQueue
 
             pod_ip = os.environ.get('MY_POD_IP')
-            registered = self.register(ip_address=pod_ip)
+            if settings.IS_K8S:
+                registered = self.register(ip_address=pod_ip, node_type='control', uuid=settings.SYSTEM_UUID)
+            else:
+                registered = self.register(ip_address=pod_ip, uuid=settings.SYSTEM_UUID)
             RegisterQueue(settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME, 100, 0, [], is_container_group=False).register()
-            RegisterQueue(settings.DEFAULT_EXECUTION_QUEUE_NAME, 100, 0, [], is_container_group=True).register()
+            RegisterQueue(
+                settings.DEFAULT_EXECUTION_QUEUE_NAME, 100, 0, [], is_container_group=True, pod_spec_override=settings.DEFAULT_EXECUTION_QUEUE_POD_SPEC_OVERRIDE
+            ).register()
             return registered
         else:
             return (False, self.me())
 
-    def active_count(self):
-        """Return count of active Tower nodes for licensing."""
-        return self.all().count()
-
 
 class InstanceGroupManager(models.Manager):
     """A custom manager class for the Instance model.
@@ -197,6 +219,8 @@ class InstanceGroupManager(models.Manager):
         if name not in graph:
             graph[name] = {}
         graph[name]['consumed_capacity'] = 0
+        for capacity_type in ('execution', 'control'):
+            graph[name][f'consumed_{capacity_type}_capacity'] = 0
         if breakdown:
             graph[name]['committed_capacity'] = 0
             graph[name]['running_capacity'] = 0
@@ -232,6 +256,8 @@ class InstanceGroupManager(models.Manager):
                     if group_name not in graph:
                         self.zero_out_group(graph, group_name, breakdown)
                     graph[group_name]['consumed_capacity'] += impact
+                    capacity_type = get_capacity_type(t)
+                    graph[group_name][f'consumed_{capacity_type}_capacity'] += impact
                     if breakdown:
                         graph[group_name]['committed_capacity'] += impact
             elif t.status == 'running':
@@ -249,6 +275,8 @@ class InstanceGroupManager(models.Manager):
                     if group_name not in graph:
                         self.zero_out_group(graph, group_name, breakdown)
                     graph[group_name]['consumed_capacity'] += impact
+                    capacity_type = get_capacity_type(t)
+                    graph[group_name][f'consumed_{capacity_type}_capacity'] += impact
                     if breakdown:
                         graph[group_name]['running_capacity'] += impact
             else:

awx/main/migrations/0139_isolated_removal.py

@@ -9,12 +9,6 @@ def remove_iso_instances(apps, schema_editor):
         Instance.objects.filter(rampart_groups__controller__isnull=False).delete()
 
 
-def remove_iso_groups(apps, schema_editor):
-    InstanceGroup = apps.get_model('main', 'InstanceGroup')
-    with transaction.atomic():
-        InstanceGroup.objects.filter(controller__isnull=False).delete()
-
-
 class Migration(migrations.Migration):
     atomic = False
 
@@ -24,7 +18,6 @@ class Migration(migrations.Migration):
 
     operations = [
         migrations.RunPython(remove_iso_instances),
-        migrations.RunPython(remove_iso_groups),
         migrations.RemoveField(
             model_name='instance',
             name='last_isolated_check',

awx/main/migrations/0153_instance_last_seen.py

@@ -0,0 +1,27 @@
+# Generated by Django 2.2.20 on 2021-08-12 13:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0152_instance_node_type'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='instance',
+            name='last_seen',
+            field=models.DateTimeField(
+                editable=False,
+                help_text='Last time instance ran its heartbeat task for main cluster nodes. Last known connection to receptor mesh for execution nodes.',
+                null=True,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='instance',
+            name='memory',
+            field=models.BigIntegerField(default=0, editable=False, help_text='Total system memory of this instance in bytes.'),
+        ),
+    ]

awx/main/migrations/0154_set_default_uuid.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.20 on 2021-09-01 22:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0153_instance_last_seen'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='instance',
+            name='uuid',
+            field=models.CharField(default='00000000-0000-0000-0000-000000000000', max_length=40),
+        ),
+    ]

awx/main/migrations/0155_improved_health_check.py

@@ -0,0 +1,25 @@
+# Generated by Django 2.2.20 on 2021-08-31 17:41
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0154_set_default_uuid'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='instance',
+            name='errors',
+            field=models.TextField(blank=True, default='', editable=False, help_text='Any error details from the last health check.'),
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='last_health_check',
+            field=models.DateTimeField(
+                editable=False, help_text='Last time a health check was ran on this instance to refresh cpu, memory, and capacity.', null=True
+            ),
+        ),
+    ]

awx/main/migrations/0156_capture_mesh_topology.py

@@ -0,0 +1,44 @@
+# Generated by Django 2.2.20 on 2021-12-17 19:26
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0155_improved_health_check'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='instance',
+            name='node_type',
+            field=models.CharField(
+                choices=[
+                    ('control', 'Control plane node'),
+                    ('execution', 'Execution plane node'),
+                    ('hybrid', 'Controller and execution'),
+                    ('hop', 'Message-passing node, no execution capability'),
+                ],
+                default='hybrid',
+                max_length=16,
+            ),
+        ),
+        migrations.CreateModel(
+            name='InstanceLink',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='main.Instance')),
+                ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='reverse_peers', to='main.Instance')),
+            ],
+            options={
+                'unique_together': {('source', 'target')},
+            },
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='peers',
+            field=models.ManyToManyField(through='main.InstanceLink', to='main.Instance'),
+        ),
+    ]

awx/main/models/__init__.py

@@ -47,6 +47,7 @@ from awx.main.models.execution_environments import ExecutionEnvironment  # noqa
 from awx.main.models.activity_stream import ActivityStream  # noqa
 from awx.main.models.ha import (  # noqa
     Instance,
+    InstanceLink,
     InstanceGroup,
     TowerScheduleState,
 )
@@ -201,6 +202,8 @@ activity_stream_registrar.connect(Organization)
 activity_stream_registrar.connect(Inventory)
 activity_stream_registrar.connect(Host)
 activity_stream_registrar.connect(Group)
+activity_stream_registrar.connect(Instance)
+activity_stream_registrar.connect(InstanceGroup)
 activity_stream_registrar.connect(InventorySource)
 # activity_stream_registrar.connect(InventoryUpdate)
 activity_stream_registrar.connect(Credential)

awx/main/models/ad_hoc_commands.py

@@ -144,7 +144,7 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
 
     @classmethod
     def _get_task_class(cls):
-        from awx.main.tasks import RunAdHocCommand
+        from awx.main.tasks.jobs import RunAdHocCommand
 
         return RunAdHocCommand
 
@@ -152,10 +152,6 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
     def is_container_group_task(self):
         return bool(self.instance_group and self.instance_group.is_container_group)
 
-    @property
-    def can_run_containerized(self):
-        return True
-
     def get_absolute_url(self, request=None):
         return reverse('api:ad_hoc_command_detail', kwargs={'pk': self.pk}, request=request)
 

awx/main/models/credential/__init__.py

@@ -299,10 +299,7 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
 
     def has_inputs(self, field_names=()):
         for name in field_names:
-            if name in self.inputs:
-                if self.inputs[name] in ('', None):
-                    return False
-            else:
+            if not self.has_input(name):
                 raise ValueError('{} is not an input field'.format(name))
         return True
 

awx/main/models/events.py

@@ -388,7 +388,7 @@ class BasePlaybookEvent(CreatedModifiedModel):
                     job.get_event_queryset().filter(uuid__in=failed).update(failed=True)
 
                     # send success/failure notifications when we've finished handling the playbook_on_stats event
-                    from awx.main.tasks import handle_success_and_failure_notifications  # circular import
+                    from awx.main.tasks.system import handle_success_and_failure_notifications  # circular import
 
                     def _send_notifications():
                         handle_success_and_failure_notifications.apply_async([job.id])
@@ -541,8 +541,7 @@ class JobEvent(BasePlaybookEvent):
                 return
             job = self.job
 
-            from awx.main.models import Host, JobHostSummary  # circular import
-            from awx.main.models import Host, JobHostSummary, HostMetric
+            from awx.main.models import Host, JobHostSummary, HostMetric  # circular import
 
             all_hosts = Host.objects.filter(pk__in=self.host_map.values()).only('id', 'name')
             existing_host_ids = set(h.id for h in all_hosts)

awx/main/models/ha.py

@@ -2,6 +2,8 @@
 # All Rights Reserved.
 
 from decimal import Decimal
+import random
+import logging
 
 from django.core.validators import MinValueValidator
 from django.db import models, connection
@@ -16,14 +18,20 @@ from solo.models import SingletonModel
 
 from awx import __version__ as awx_application_version
 from awx.api.versioning import reverse
-from awx.main.managers import InstanceManager, InstanceGroupManager
+from awx.main.managers import InstanceManager, InstanceGroupManager, UUID_DEFAULT
 from awx.main.fields import JSONField
+from awx.main.constants import JOB_FOLDER_PREFIX
 from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search
 from awx.main.models.unified_jobs import UnifiedJob
-from awx.main.utils import get_cpu_capacity, get_mem_capacity, get_system_task_capacity
+from awx.main.utils.common import get_corrected_cpu, get_cpu_effective_capacity, get_corrected_memory, get_mem_effective_capacity
 from awx.main.models.mixins import RelatedJobsMixin
 
-__all__ = ('Instance', 'InstanceGroup', 'TowerScheduleState')
+# ansible-runner
+from ansible_runner.utils.capacity import get_cpu_count, get_mem_in_bytes
+
+__all__ = ('Instance', 'InstanceGroup', 'InstanceLink', 'TowerScheduleState')
+
+logger = logging.getLogger('awx.main.models.ha')
 
 
 class HasPolicyEditsMixin(HasEditsMixin):
@@ -46,12 +54,21 @@ class HasPolicyEditsMixin(HasEditsMixin):
         return self._values_have_edits(new_values)
 
 
+class InstanceLink(BaseModel):
+    source = models.ForeignKey('Instance', on_delete=models.CASCADE, related_name='+')
+    target = models.ForeignKey('Instance', on_delete=models.CASCADE, related_name='reverse_peers')
+
+    class Meta:
+        unique_together = ('source', 'target')
+
+
 class Instance(HasPolicyEditsMixin, BaseModel):
     """A model representing an AWX instance running against this database."""
 
     objects = InstanceManager()
 
-    uuid = models.CharField(max_length=40)
+    # Fields set in instance registration
+    uuid = models.CharField(max_length=40, default=UUID_DEFAULT)
     hostname = models.CharField(max_length=250, unique=True)
     ip_address = models.CharField(
         blank=True,
@@ -60,16 +77,11 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         max_length=50,
         unique=True,
     )
+    # Auto-fields, implementation is different from BaseModel
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
+    # Fields defined in health check or heartbeat
     version = models.CharField(max_length=120, blank=True)
-    capacity = models.PositiveIntegerField(
-        default=100,
-        editable=False,
-    )
-    capacity_adjustment = models.DecimalField(default=Decimal(1.0), max_digits=3, decimal_places=2, validators=[MinValueValidator(0)])
-    enabled = models.BooleanField(default=True)
-    managed_by_policy = models.BooleanField(default=True)
     cpu = models.IntegerField(
         default=0,
         editable=False,
@@ -77,7 +89,33 @@ class Instance(HasPolicyEditsMixin, BaseModel):
     memory = models.BigIntegerField(
         default=0,
         editable=False,
+        help_text=_('Total system memory of this instance in bytes.'),
     )
+    errors = models.TextField(
+        default='',
+        blank=True,
+        editable=False,
+        help_text=_('Any error details from the last health check.'),
+    )
+    last_seen = models.DateTimeField(
+        null=True,
+        editable=False,
+        help_text=_('Last time instance ran its heartbeat task for main cluster nodes. Last known connection to receptor mesh for execution nodes.'),
+    )
+    last_health_check = models.DateTimeField(
+        null=True,
+        editable=False,
+        help_text=_('Last time a health check was ran on this instance to refresh cpu, memory, and capacity.'),
+    )
+    # Capacity management
+    capacity = models.PositiveIntegerField(
+        default=100,
+        editable=False,
+    )
+    capacity_adjustment = models.DecimalField(default=Decimal(1.0), max_digits=3, decimal_places=2, validators=[MinValueValidator(0)])
+    enabled = models.BooleanField(default=True)
+    managed_by_policy = models.BooleanField(default=True)
+
     cpu_capacity = models.IntegerField(
         default=0,
         editable=False,
@@ -86,9 +124,16 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         default=0,
         editable=False,
     )
-    NODE_TYPE_CHOICES = [("control", "Control plane node"), ("execution", "Execution plane node"), ("hybrid", "Controller and execution")]
+    NODE_TYPE_CHOICES = [
+        ("control", "Control plane node"),
+        ("execution", "Execution plane node"),
+        ("hybrid", "Controller and execution"),
+        ("hop", "Message-passing node, no execution capability"),
+    ]
     node_type = models.CharField(default='hybrid', choices=NODE_TYPE_CHOICES, max_length=16)
 
+    peers = models.ManyToManyField('self', symmetrical=False, through=InstanceLink, through_fields=('source', 'target'))
+
     class Meta:
         app_label = 'main'
         ordering = ("hostname",)
@@ -106,11 +151,6 @@ class Instance(HasPolicyEditsMixin, BaseModel):
     def remaining_capacity(self):
         return self.capacity - self.consumed_capacity
 
-    @property
-    def role(self):
-        # NOTE: TODO: Likely to repurpose this once standalone ramparts are a thing
-        return "awx"
-
     @property
     def jobs_running(self):
         return UnifiedJob.objects.filter(
@@ -125,33 +165,117 @@ class Instance(HasPolicyEditsMixin, BaseModel):
     def jobs_total(self):
         return UnifiedJob.objects.filter(execution_node=self.hostname).count()
 
+    @staticmethod
+    def choose_online_control_plane_node():
+        return random.choice(
+            Instance.objects.filter(enabled=True, capacity__gt=0).filter(node_type__in=['control', 'hybrid']).values_list('hostname', flat=True)
+        )
+
+    def get_cleanup_task_kwargs(self, **kwargs):
+        """
+        Produce options to use for the command: ansible-runner worker cleanup
+        returns a dict that is passed to the python interface for the runner method corresponding to that command
+        any kwargs will override that key=value combination in the returned dict
+        """
+        vargs = dict()
+        if settings.AWX_CLEANUP_PATHS:
+            vargs['file_pattern'] = '/tmp/{}*'.format(JOB_FOLDER_PREFIX % '*')
+        vargs.update(kwargs)
+        if 'exclude_strings' not in vargs and vargs.get('file_pattern'):
+            active_pks = list(UnifiedJob.objects.filter(execution_node=self.hostname, status__in=('running', 'waiting')).values_list('pk', flat=True))
+            if active_pks:
+                vargs['exclude_strings'] = [JOB_FOLDER_PREFIX % job_id for job_id in active_pks]
+        if 'remove_images' in vargs or 'image_prune' in vargs:
+            vargs.setdefault('process_isolation_executable', 'podman')
+        return vargs
+
     def is_lost(self, ref_time=None):
+        if self.last_seen is None:
+            return True
         if ref_time is None:
             ref_time = now()
-        grace_period = 120
-        return self.modified < ref_time - timedelta(seconds=grace_period)
+        grace_period = settings.CLUSTER_NODE_HEARTBEAT_PERIOD * 2
+        if self.node_type == 'execution':
+            grace_period += settings.RECEPTOR_SERVICE_ADVERTISEMENT_PERIOD
+        return self.last_seen < ref_time - timedelta(seconds=grace_period)
 
-    def refresh_capacity(self):
-        cpu = get_cpu_capacity()
-        mem = get_mem_capacity()
-        if self.enabled:
-            self.capacity = get_system_task_capacity(self.capacity_adjustment)
+    def mark_offline(self, update_last_seen=False, perform_save=True, errors=''):
+        if self.cpu_capacity == 0 and self.mem_capacity == 0 and self.capacity == 0 and self.errors == errors and (not update_last_seen):
+            return
+        self.cpu_capacity = self.mem_capacity = self.capacity = 0
+        self.errors = errors
+        if update_last_seen:
+            self.last_seen = now()
+
+        if perform_save:
+            update_fields = ['capacity', 'cpu_capacity', 'mem_capacity', 'errors']
+            if update_last_seen:
+                update_fields += ['last_seen']
+            self.save(update_fields=update_fields)
+
+    def set_capacity_value(self):
+        """Sets capacity according to capacity adjustment rule (no save)"""
+        if self.enabled and self.node_type != 'hop':
+            lower_cap = min(self.mem_capacity, self.cpu_capacity)
+            higher_cap = max(self.mem_capacity, self.cpu_capacity)
+            self.capacity = lower_cap + (higher_cap - lower_cap) * self.capacity_adjustment
         else:
             self.capacity = 0
 
+    def refresh_capacity_fields(self):
+        """Update derived capacity fields from cpu and memory (no save)"""
+        self.cpu_capacity = get_cpu_effective_capacity(self.cpu)
+        self.mem_capacity = get_mem_effective_capacity(self.memory)
+        self.set_capacity_value()
+
+    def save_health_data(self, version, cpu, memory, uuid=None, update_last_seen=False, errors=''):
+        self.last_health_check = now()
+        update_fields = ['last_health_check']
+
+        if update_last_seen:
+            self.last_seen = self.last_health_check
+            update_fields.append('last_seen')
+
+        if uuid is not None and self.uuid != uuid:
+            if self.uuid is not None:
+                logger.warn(f'Self-reported uuid of {self.hostname} changed from {self.uuid} to {uuid}')
+            self.uuid = uuid
+            update_fields.append('uuid')
+
+        if self.version != version:
+            self.version = version
+            update_fields.append('version')
+
+        new_cpu = get_corrected_cpu(cpu)
+        if new_cpu != self.cpu:
+            self.cpu = new_cpu
+            update_fields.append('cpu')
+
+        new_memory = get_corrected_memory(memory)
+        if new_memory != self.memory:
+            self.memory = new_memory
+            update_fields.append('memory')
+
+        if not errors:
+            self.refresh_capacity_fields()
+            self.errors = ''
+        else:
+            self.mark_offline(perform_save=False, errors=errors)
+        update_fields.extend(['cpu_capacity', 'mem_capacity', 'capacity', 'errors'])
+
+        self.save(update_fields=update_fields)
+
+    def local_health_check(self):
+        """Only call this method on the instance that this record represents"""
+        errors = None
         try:
             # if redis is down for some reason, that means we can't persist
             # playbook event data; we should consider this a zero capacity event
             redis.Redis.from_url(settings.BROKER_URL).ping()
         except redis.ConnectionError:
-            self.capacity = 0
+            errors = _('Failed to connect ot Redis')
 
-        self.cpu = cpu[0]
-        self.memory = mem[0]
-        self.cpu_capacity = cpu[1]
-        self.mem_capacity = mem[1]
-        self.version = awx_application_version
-        self.save(update_fields=['capacity', 'version', 'modified', 'cpu', 'memory', 'cpu_capacity', 'mem_capacity'])
+        self.save_health_data(awx_application_version, get_cpu_count(), get_mem_in_bytes(), update_last_seen=True, errors=errors)
 
 
 class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
@@ -196,7 +320,7 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
 
     @property
     def capacity(self):
-        return sum([inst.capacity for inst in self.instances.all()])
+        return sum(inst.capacity for inst in self.instances.all())
 
     @property
     def jobs_running(self):
@@ -220,6 +344,8 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
     def fit_task_to_most_remaining_capacity_instance(task, instances):
         instance_most_capacity = None
         for i in instances:
+            if i.node_type not in (task.capacity_type, 'hybrid'):
+                continue
             if i.remaining_capacity >= task.task_impact and (
                 instance_most_capacity is None or i.remaining_capacity > instance_most_capacity.remaining_capacity
             ):
@@ -227,9 +353,11 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin):
         return instance_most_capacity
 
     @staticmethod
-    def find_largest_idle_instance(instances):
+    def find_largest_idle_instance(instances, capacity_type='execution'):
         largest_instance = None
         for i in instances:
+            if i.node_type not in (capacity_type, 'hybrid'):
+                continue
             if i.jobs_running == 0:
                 if largest_instance is None:
                     largest_instance = i
@@ -248,7 +376,7 @@ class TowerScheduleState(SingletonModel):
 
 
 def schedule_policy_task():
-    from awx.main.tasks import apply_cluster_membership_policies
+    from awx.main.tasks.system import apply_cluster_membership_policies
 
     connection.on_commit(lambda: apply_cluster_membership_policies.apply_async())
 

awx/main/models/inventory.py

@@ -366,7 +366,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
 
     @transaction.atomic
     def schedule_deletion(self, user_id=None):
-        from awx.main.tasks import delete_inventory
+        from awx.main.tasks.system import delete_inventory
         from awx.main.signals import activity_stream_delete
 
         if self.pending_deletion is True:
@@ -382,7 +382,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
         if self.kind == 'smart' and settings.AWX_REBUILD_SMART_MEMBERSHIP:
 
             def on_commit():
-                from awx.main.tasks import update_host_smart_inventory_memberships
+                from awx.main.tasks.system import update_host_smart_inventory_memberships
 
                 update_host_smart_inventory_memberships.delay()
 
@@ -551,7 +551,7 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
         if settings.AWX_REBUILD_SMART_MEMBERSHIP:
 
             def on_commit():
-                from awx.main.tasks import update_host_smart_inventory_memberships
+                from awx.main.tasks.system import update_host_smart_inventory_memberships
 
                 update_host_smart_inventory_memberships.delay()
 
@@ -631,7 +631,7 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
     @transaction.atomic
     def delete_recursive(self):
         from awx.main.utils import ignore_inventory_computed_fields
-        from awx.main.tasks import update_inventory_computed_fields
+        from awx.main.tasks.system import update_inventory_computed_fields
         from awx.main.signals import disable_activity_stream, activity_stream_delete
 
         def mark_actual():
@@ -1214,16 +1214,12 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin,
     def is_container_group_task(self):
         return bool(self.instance_group and self.instance_group.is_container_group)
 
-    @property
-    def can_run_containerized(self):
-        return True
-
     def _get_parent_field_name(self):
         return 'inventory_source'
 
     @classmethod
     def _get_task_class(cls):
-        from awx.main.tasks import RunInventoryUpdate
+        from awx.main.tasks.jobs import RunInventoryUpdate
 
         return RunInventoryUpdate
 

awx/main/models/jobs.py

@@ -583,7 +583,7 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
 
     @classmethod
     def _get_task_class(cls):
-        from awx.main.tasks import RunJob
+        from awx.main.tasks.jobs import RunJob
 
         return RunJob
 
@@ -743,10 +743,6 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
             return "$hidden due to Ansible no_log flag$"
         return artifacts
 
-    @property
-    def can_run_containerized(self):
-        return True
-
     @property
     def is_container_group_task(self):
         return bool(self.instance_group and self.instance_group.is_container_group)
@@ -1217,7 +1213,7 @@ class SystemJob(UnifiedJob, SystemJobOptions, JobNotificationMixin):
 
     @classmethod
     def _get_task_class(cls):
-        from awx.main.tasks import RunSystemJob
+        from awx.main.tasks.jobs import RunSystemJob
 
         return RunSystemJob
 
@@ -1236,10 +1232,6 @@ class SystemJob(UnifiedJob, SystemJobOptions, JobNotificationMixin):
             return UnpartitionedSystemJobEvent
         return SystemJobEvent
 
-    @property
-    def can_run_on_control_plane(self):
-        return True
-
     @property
     def task_impact(self):
         return 5

awx/main/models/notifications.py

@@ -508,7 +508,7 @@ class JobNotificationMixin(object):
         return (msg, body)
 
     def send_notification_templates(self, status):
-        from awx.main.tasks import send_notifications  # avoid circular import
+        from awx.main.tasks.system import send_notifications  # avoid circular import
 
         if status not in ['running', 'succeeded', 'failed']:
             raise ValueError(_("status must be either running, succeeded or failed"))

awx/main/models/organization.py

@@ -118,7 +118,7 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi
         from awx.main.models import Credential
 
         public_galaxy_credential = Credential.objects.filter(managed=True, name='Ansible Galaxy').first()
-        if public_galaxy_credential not in self.galaxy_credentials.all():
+        if public_galaxy_credential is not None and public_galaxy_credential not in self.galaxy_credentials.all():
             self.galaxy_credentials.add(public_galaxy_credential)
 
 

awx/main/models/projects.py

@@ -471,7 +471,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
         r = super(Project, self).delete(*args, **kwargs)
         for path_to_delete in paths_to_delete:
             if self.scm_type and path_to_delete:  # non-manual, concrete path
-                from awx.main.tasks import delete_project_files
+                from awx.main.tasks.system import delete_project_files
 
                 delete_project_files.delay(path_to_delete)
         return r
@@ -532,7 +532,7 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
 
     @classmethod
     def _get_task_class(cls):
-        from awx.main.tasks import RunProjectUpdate
+        from awx.main.tasks.jobs import RunProjectUpdate
 
         return RunProjectUpdate
 
@@ -553,10 +553,6 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
         websocket_data.update(dict(project_id=self.project.id))
         return websocket_data
 
-    @property
-    def can_run_on_control_plane(self):
-        return True
-
     @property
     def event_class(self):
         if self.has_unpartitioned_events:
@@ -619,16 +615,22 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManage
 
     @property
     def preferred_instance_groups(self):
+        '''
+        Project updates should pretty much always run on the control plane
+        however, we are not yet saying no to custom groupings within the control plane
+        Thus, we return custom groups and then unconditionally add the control plane
+        '''
         if self.organization is not None:
             organization_groups = [x for x in self.organization.instance_groups.all()]
         else:
             organization_groups = []
         template_groups = [x for x in super(ProjectUpdate, self).preferred_instance_groups]
         selected_groups = template_groups + organization_groups
-        if not any([not group.is_container_group for group in selected_groups]):
-            selected_groups = selected_groups + list(self.control_plane_instance_group)
-        if not selected_groups:
-            return self.global_instance_groups
+
+        controlplane_ig = self.control_plane_instance_group
+        if controlplane_ig and controlplane_ig[0] and controlplane_ig[0] not in selected_groups:
+            selected_groups += controlplane_ig
+
         return selected_groups
 
     def save(self, *args, **kwargs):

awx/main/models/unified_jobs.py

@@ -36,21 +36,21 @@ from awx.main.dispatch import get_local_queuename
 from awx.main.dispatch.control import Control as ControlDispatcher
 from awx.main.registrar import activity_stream_registrar
 from awx.main.models.mixins import ResourceMixin, TaskManagerUnifiedJobMixin, ExecutionEnvironmentMixin
-from awx.main.utils import (
+from awx.main.utils.common import (
     camelcase_to_underscore,
     get_model_for_type,
-    encrypt_dict,
-    decrypt_field,
     _inventory_updates,
     copy_model_by_class,
     copy_m2m_relationships,
     get_type_for_model,
     parse_yaml_or_json,
     getattr_dne,
-    polymorphic,
     schedule_task_manager,
     get_event_partition_epoch,
+    get_capacity_type,
 )
+from awx.main.utils.encryption import encrypt_dict, decrypt_field
+from awx.main.utils import polymorphic
 from awx.main.constants import ACTIVE_STATES, CAN_CANCEL
 from awx.main.redact import UriCleaner, REPLACE_STR
 from awx.main.consumers import emit_channel_notification
@@ -740,15 +740,8 @@ class UnifiedJob(
         raise NotImplementedError  # Implement in subclasses.
 
     @property
-    def can_run_on_control_plane(self):
-        if settings.IS_K8S:
-            return False
-
-        return True
-
-    @property
-    def can_run_containerized(self):
-        return False
+    def capacity_type(self):
+        return get_capacity_type(self)
 
     def _get_parent_field_name(self):
         return 'unified_job_template'  # Override in subclasses.
@@ -1053,7 +1046,7 @@ class UnifiedJob(
             fd = tempfile.NamedTemporaryFile(
                 mode='w', prefix='{}-{}-'.format(self.model_to_str(), self.pk), suffix='.out', dir=settings.JOBOUTPUT_ROOT, encoding='utf-8'
             )
-            from awx.main.tasks import purge_old_stdout_files  # circular import
+            from awx.main.tasks.system import purge_old_stdout_files  # circular import
 
             purge_old_stdout_files.apply_async()
 
@@ -1442,9 +1435,13 @@ class UnifiedJob(
         if not settings.IS_K8S:
             default_instance_group_names.append(settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME)
 
-        default_instance_groups = InstanceGroup.objects.filter(name__in=default_instance_group_names)
+        default_instance_groups = list(InstanceGroup.objects.filter(name__in=default_instance_group_names))
 
-        return list(default_instance_groups)
+        # assure deterministic precedence by making sure the default group is first
+        if (not settings.IS_K8S) and default_instance_groups and default_instance_groups[0].name != settings.DEFAULT_EXECUTION_QUEUE_NAME:
+            default_instance_groups.reverse()
+
+        return default_instance_groups
 
     def awx_meta_vars(self):
         """
@@ -1500,7 +1497,12 @@ class UnifiedJob(
         return False
 
     def log_lifecycle(self, state, blocked_by=None):
-        extra = {'type': self._meta.model_name, 'task_id': self.id, 'state': state}
+        extra = {
+            'type': self._meta.model_name,
+            'task_id': self.id,
+            'state': state,
+            'work_unit_id': self.work_unit_id,
+        }
         if self.unified_job_template:
             extra["template_name"] = self.unified_job_template.name
         if state == "blocked" and blocked_by:
@@ -1509,6 +1511,11 @@ class UnifiedJob(
             extra["blocked_by"] = blocked_by_msg
         else:
             msg = f"{self._meta.model_name}-{self.id} {state.replace('_', ' ')}"
+
+        if state == "controller_node_chosen":
+            extra["controller_node"] = self.controller_node or "NOT_SET"
+        elif state == "execution_node_chosen":
+            extra["execution_node"] = self.execution_node or "NOT_SET"
         logger_job_lifecycle.debug(msg, extra=extra)
 
     @property

awx/main/models/workflow.py

@@ -813,7 +813,7 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin):
         return True
 
     def send_approval_notification(self, approval_status):
-        from awx.main.tasks import send_notifications  # avoid circular import
+        from awx.main.tasks.system import send_notifications  # avoid circular import
 
         if self.workflow_job_template is None:
             return

awx/main/notifications/rocketchat_backend.py

@@ -9,6 +9,7 @@ from django.utils.encoding import smart_text
 from django.utils.translation import ugettext_lazy as _
 
 from awx.main.notifications.base import AWXBaseEmailBackend
+from awx.main.utils import get_awx_http_client_headers
 from awx.main.notifications.custom_notification_base import CustomNotificationBase
 
 logger = logging.getLogger('awx.main.notifications.rocketchat_backend')
@@ -38,7 +39,9 @@ class RocketChatBackend(AWXBaseEmailBackend, CustomNotificationBase):
                 if optvalue is not None:
                     payload[optval] = optvalue.strip()
 
-            r = requests.post("{}".format(m.recipients()[0]), data=json.dumps(payload), verify=(not self.rocketchat_no_verify_ssl))
+            r = requests.post(
+                "{}".format(m.recipients()[0]), data=json.dumps(payload), headers=get_awx_http_client_headers(), verify=(not self.rocketchat_no_verify_ssl)
+            )
 
             if r.status_code >= 400:
                 logger.error(smart_text(_("Error sending notification rocket.chat: {}").format(r.status_code)))

awx/main/notifications/slack_backend.py

@@ -2,7 +2,8 @@
 # All Rights Reserved.
 
 import logging
-from slackclient import SlackClient
+from slack_sdk import WebClient
+from slack_sdk.errors import SlackApiError
 
 from django.utils.encoding import smart_text
 from django.utils.translation import ugettext_lazy as _
@@ -28,23 +29,30 @@ class SlackBackend(AWXBaseEmailBackend, CustomNotificationBase):
             self.color = hex_color
 
     def send_messages(self, messages):
-        connection = SlackClient(self.token)
+        client = WebClient(self.token)
         sent_messages = 0
         for m in messages:
             try:
                 for r in m.recipients():
                     if r.startswith('#'):
                         r = r[1:]
+                    thread = None
+                    channel = r
+                    thread = None
+                    if ',' in r:
+                        channel, thread = r.split(',')
                     if self.color:
-                        ret = connection.api_call("chat.postMessage", channel=r, as_user=True, attachments=[{"color": self.color, "text": m.subject}])
+                        response = client.chat_postMessage(
+                            channel=channel, thread_ts=thread, as_user=True, attachments=[{"color": self.color, "text": m.subject}]
+                        )
                     else:
-                        ret = connection.api_call("chat.postMessage", channel=r, as_user=True, text=m.subject)
-                    logger.debug(ret)
-                    if ret['ok']:
+                        response = client.chat_postMessage(channel=channel, thread_ts=thread, as_user=True, text=m.subject)
+                    logger.debug(response)
+                    if response['ok']:
                         sent_messages += 1
                     else:
-                        raise RuntimeError("Slack Notification unable to send {}: {} ({})".format(r, m.subject, ret['error']))
-            except Exception as e:
+                        raise RuntimeError("Slack Notification unable to send {}: {} ({})".format(r, m.subject, response['error']))
+            except SlackApiError as e:
                 logger.error(smart_text(_("Exception sending messages: {}").format(e)))
                 if not self.fail_silently:
                     raise

awx/main/scheduler/kubernetes.py

@@ -9,29 +9,12 @@ from kubernetes import client, config
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 
-from awx.main.utils.common import parse_yaml_or_json
+from awx.main.utils.common import parse_yaml_or_json, deepmerge
 from awx.main.utils.execution_environments import get_default_pod_spec
 
 logger = logging.getLogger('awx.main.scheduler')
 
 
-def deepmerge(a, b):
-    """
-    Merge dict structures and return the result.
-
-    >>> a = {'first': {'all_rows': {'pass': 'dog', 'number': '1'}}}
-    >>> b = {'first': {'all_rows': {'fail': 'cat', 'number': '5'}}}
-    >>> import pprint; pprint.pprint(deepmerge(a, b))
-    {'first': {'all_rows': {'fail': 'cat', 'number': '5', 'pass': 'dog'}}}
-    """
-    if isinstance(a, dict) and isinstance(b, dict):
-        return dict([(k, deepmerge(a.get(k), b.get(k))) for k in set(a.keys()).union(b.keys())])
-    elif b is None:
-        return a
-    else:
-        return b
-
-
 class PodManager(object):
     def __init__(self, task=None):
         self.task = task
@@ -183,7 +166,7 @@ class PodManager(object):
         pod_spec_override = {}
         if self.task and self.task.instance_group.pod_spec_override:
             pod_spec_override = parse_yaml_or_json(self.task.instance_group.pod_spec_override)
-        pod_spec = {**default_pod_spec, **pod_spec_override}
+        pod_spec = deepmerge(default_pod_spec, pod_spec_override)
 
         if self.task:
             pod_spec['metadata'] = deepmerge(

awx/main/scheduler/task_manager.py

@@ -13,7 +13,6 @@ from django.db import transaction, connection
 from django.utils.translation import ugettext_lazy as _, gettext_noop
 from django.utils.timezone import now as tz_now
 from django.conf import settings
-from django.db.models import Q
 
 # AWX
 from awx.main.dispatch.reaper import reap_job
@@ -69,12 +68,13 @@ class TaskManager:
         """
         Init AFTER we know this instance of the task manager will run because the lock is acquired.
         """
-        instances = Instance.objects.filter(~Q(hostname=None), enabled=True)
+        instances = Instance.objects.filter(hostname__isnull=False, enabled=True).exclude(node_type='hop')
         self.real_instances = {i.hostname: i for i in instances}
 
         instances_partial = [
             SimpleNamespace(
                 obj=instance,
+                node_type=instance.node_type,
                 remaining_capacity=instance.remaining_capacity,
                 capacity=instance.capacity,
                 jobs_running=instance.jobs_running,
@@ -86,7 +86,21 @@ class TaskManager:
         instances_by_hostname = {i.hostname: i for i in instances_partial}
 
         for rampart_group in InstanceGroup.objects.prefetch_related('instances'):
-            self.graph[rampart_group.name] = dict(graph=DependencyGraph(), capacity_total=rampart_group.capacity, consumed_capacity=0, instances=[])
+            self.graph[rampart_group.name] = dict(
+                graph=DependencyGraph(),
+                execution_capacity=0,
+                control_capacity=0,
+                consumed_capacity=0,
+                consumed_control_capacity=0,
+                consumed_execution_capacity=0,
+                instances=[],
+            )
+            for instance in rampart_group.instances.all():
+                if not instance.enabled:
+                    continue
+                for capacity_type in ('control', 'execution'):
+                    if instance.node_type in (capacity_type, 'hybrid'):
+                        self.graph[rampart_group.name][f'{capacity_type}_capacity'] += instance.capacity
             for instance in rampart_group.instances.filter(enabled=True).order_by('hostname'):
                 if instance.hostname in instances_by_hostname:
                     self.graph[rampart_group.name]['instances'].append(instances_by_hostname[instance.hostname])
@@ -224,7 +238,7 @@ class TaskManager:
                 update_fields = ['status', 'start_args']
                 workflow_job.status = new_status
                 if reason:
-                    logger.info(reason)
+                    logger.info(f'Workflow job {workflow_job.id} failed due to reason: {reason}')
                     workflow_job.job_explanation = gettext_noop("No error handling paths found, marking workflow as failed")
                     update_fields.append('job_explanation')
                 workflow_job.start_args = ''  # blank field to remove encrypted passwords
@@ -243,7 +257,7 @@ class TaskManager:
         if self.start_task_limit == 0:
             # schedule another run immediately after this task manager
             schedule_task_manager()
-        from awx.main.tasks import handle_work_error, handle_work_success
+        from awx.main.tasks.system import handle_work_error, handle_work_success
 
         dependent_tasks = dependent_tasks or []
 
@@ -270,35 +284,45 @@ class TaskManager:
                 logger.debug('Transitioning %s to running status.', task.log_format)
                 schedule_task_manager()
             elif rampart_group.is_container_group:
-                # find one real, non-containerized instance with capacity to
-                # act as the controller for k8s API interaction
-                match = None
-                for group in InstanceGroup.objects.filter(is_container_group=False):
-                    match = group.fit_task_to_most_remaining_capacity_instance(task, group.instances.all())
-                    if match:
-                        break
                 task.instance_group = rampart_group
-                if match is None:
-                    logger.warn('No available capacity to run containerized <{}>.'.format(task.log_format))
-                elif task.can_run_containerized and any(ig.is_container_group for ig in task.preferred_instance_groups):
-                    task.controller_node = match.hostname
+                if task.capacity_type == 'execution':
+                    # find one real, non-containerized instance with capacity to
+                    # act as the controller for k8s API interaction
+                    try:
+                        task.controller_node = Instance.choose_online_control_plane_node()
+                        task.log_lifecycle("controller_node_chosen")
+                    except IndexError:
+                        logger.warning("No control plane nodes available to run containerized job {}".format(task.log_format))
+                        return
                 else:
-                    # project updates and inventory updates don't *actually* run in pods, so
+                    # project updates and system jobs don't *actually* run in pods, so
                     # just pick *any* non-containerized host and use it as the execution node
-                    task.execution_node = match.hostname
+                    task.execution_node = Instance.choose_online_control_plane_node()
+                    task.log_lifecycle("execution_node_chosen")
                     logger.debug('Submitting containerized {} to queue {}.'.format(task.log_format, task.execution_node))
             else:
                 task.instance_group = rampart_group
-                if instance is not None:
-                    task.execution_node = instance.hostname
-                logger.debug('Submitting {} to <instance group, instance> <{},{}>.'.format(task.log_format, task.instance_group_id, task.execution_node))
+                task.execution_node = instance.hostname
+                task.log_lifecycle("execution_node_chosen")
+                if instance.node_type == 'execution':
+                    try:
+                        task.controller_node = Instance.choose_online_control_plane_node()
+                        task.log_lifecycle("controller_node_chosen")
+                    except IndexError:
+                        logger.warning("No control plane nodes available to manage {}".format(task.log_format))
+                        return
+                else:
+                    # control plane nodes will manage jobs locally for performance and resilience
+                    task.controller_node = task.execution_node
+                    task.log_lifecycle("controller_node_chosen")
+                logger.debug('Submitting job {} to queue {} controlled by {}.'.format(task.log_format, task.execution_node, task.controller_node))
             with disable_activity_stream():
                 task.celery_task_id = str(uuid.uuid4())
                 task.save()
                 task.log_lifecycle("waiting")
 
             if rampart_group is not None:
-                self.consume_capacity(task, rampart_group.name)
+                self.consume_capacity(task, rampart_group.name, instance=instance)
 
         def post_commit():
             if task.status != 'failed' and type(task) is not WorkflowJob:
@@ -459,7 +483,7 @@ class TaskManager:
         return created_dependencies
 
     def process_pending_tasks(self, pending_tasks):
-        running_workflow_templates = set([wf.unified_job_template_id for wf in self.get_running_workflow_jobs()])
+        running_workflow_templates = {wf.unified_job_template_id for wf in self.get_running_workflow_jobs()}
         tasks_to_update_job_explanation = []
         for task in pending_tasks:
             if self.start_task_limit <= 0:
@@ -487,24 +511,25 @@ class TaskManager:
                 continue
 
             for rampart_group in preferred_instance_groups:
-                if task.can_run_containerized and rampart_group.is_container_group:
+                if task.capacity_type == 'execution' and rampart_group.is_container_group:
                     self.graph[rampart_group.name]['graph'].add_job(task)
                     self.start_task(task, rampart_group, task.get_jobs_fail_chain(), None)
                     found_acceptable_queue = True
                     break
 
-                if not task.can_run_on_control_plane:
+                # TODO: remove this after we have confidence that OCP control nodes are reporting node_type=control
+                if settings.IS_K8S and task.capacity_type == 'execution':
                     logger.debug("Skipping group {}, task cannot run on control plane".format(rampart_group.name))
                     continue
 
-                remaining_capacity = self.get_remaining_capacity(rampart_group.name)
-                if task.task_impact > 0 and self.get_remaining_capacity(rampart_group.name) <= 0:
+                remaining_capacity = self.get_remaining_capacity(rampart_group.name, capacity_type=task.capacity_type)
+                if task.task_impact > 0 and remaining_capacity <= 0:
                     logger.debug("Skipping group {}, remaining_capacity {} <= 0".format(rampart_group.name, remaining_capacity))
                     continue
 
                 execution_instance = InstanceGroup.fit_task_to_most_remaining_capacity_instance(
                     task, self.graph[rampart_group.name]['instances']
-                ) or InstanceGroup.find_largest_idle_instance(self.graph[rampart_group.name]['instances'])
+                ) or InstanceGroup.find_largest_idle_instance(self.graph[rampart_group.name]['instances'], capacity_type=task.capacity_type)
 
                 if execution_instance or rampart_group.is_container_group:
                     if not rampart_group.is_container_group:
@@ -567,7 +592,7 @@ class TaskManager:
         # elsewhere
         for j in UnifiedJob.objects.filter(
             status__in=['pending', 'waiting', 'running'],
-        ).exclude(execution_node__in=Instance.objects.values_list('hostname', flat=True)):
+        ).exclude(execution_node__in=Instance.objects.exclude(node_type='hop').values_list('hostname', flat=True)):
             if j.execution_node and not j.is_container_group_task:
                 logger.error(f'{j.execution_node} is not a registered instance; reaping {j.log_format}')
                 reap_job(j, 'failed')
@@ -575,16 +600,19 @@ class TaskManager:
     def calculate_capacity_consumed(self, tasks):
         self.graph = InstanceGroup.objects.capacity_values(tasks=tasks, graph=self.graph)
 
-    def consume_capacity(self, task, instance_group):
+    def consume_capacity(self, task, instance_group, instance=None):
         logger.debug(
             '{} consumed {} capacity units from {} with prior total of {}'.format(
                 task.log_format, task.task_impact, instance_group, self.graph[instance_group]['consumed_capacity']
             )
         )
         self.graph[instance_group]['consumed_capacity'] += task.task_impact
+        for capacity_type in ('control', 'execution'):
+            if instance is None or instance.node_type in ('hybrid', capacity_type):
+                self.graph[instance_group][f'consumed_{capacity_type}_capacity'] += task.task_impact
 
-    def get_remaining_capacity(self, instance_group):
-        return self.graph[instance_group]['capacity_total'] - self.graph[instance_group]['consumed_capacity']
+    def get_remaining_capacity(self, instance_group, capacity_type='execution'):
+        return self.graph[instance_group][f'{capacity_type}_capacity'] - self.graph[instance_group][f'consumed_{capacity_type}_capacity']
 
     def process_tasks(self, all_sorted_tasks):
         running_tasks = [t for t in all_sorted_tasks if t.status in ['waiting', 'running']]

awx/main/signals.py

@@ -34,7 +34,6 @@ from awx.main.models import (
     ExecutionEnvironment,
     Group,
     Host,
-    InstanceGroup,
     Inventory,
     InventorySource,
     Job,
@@ -58,7 +57,7 @@ from awx.main.models import (
 from awx.main.constants import CENSOR_VALUE
 from awx.main.utils import model_instance_diff, model_to_dict, camelcase_to_underscore, get_current_apps
 from awx.main.utils import ignore_inventory_computed_fields, ignore_inventory_group_removal, _inventory_updates
-from awx.main.tasks import update_inventory_computed_fields
+from awx.main.tasks.system import update_inventory_computed_fields, handle_removed_image
 from awx.main.fields import (
     is_implicit_parent,
     update_role_parentage_for_instance,
@@ -377,6 +376,7 @@ def model_serializer_mapping():
         models.Inventory: serializers.InventorySerializer,
         models.Host: serializers.HostSerializer,
         models.Group: serializers.GroupSerializer,
+        models.Instance: serializers.InstanceSerializer,
         models.InstanceGroup: serializers.InstanceGroupSerializer,
         models.InventorySource: serializers.InventorySourceSerializer,
         models.Credential: serializers.CredentialSerializer,
@@ -624,10 +624,26 @@ def deny_orphaned_approvals(sender, instance, **kwargs):
         approval.deny()
 
 
+def _handle_image_cleanup(removed_image, pk):
+    if (not removed_image) or ExecutionEnvironment.objects.filter(image=removed_image).exclude(pk=pk).exists():
+        return  # if other EE objects reference the tag, then do not purge it
+    handle_removed_image.delay(remove_images=[removed_image])
+
+
 @receiver(pre_delete, sender=ExecutionEnvironment)
 def remove_default_ee(sender, instance, **kwargs):
     if instance.id == getattr(settings.DEFAULT_EXECUTION_ENVIRONMENT, 'id', None):
         settings.DEFAULT_EXECUTION_ENVIRONMENT = None
+    _handle_image_cleanup(instance.image, instance.pk)
+
+
+@receiver(post_save, sender=ExecutionEnvironment)
+def remove_stale_image(sender, instance, created, **kwargs):
+    if created:
+        return
+    removed_image = instance._prior_values_store.get('image')
+    if removed_image and removed_image != instance.image:
+        _handle_image_cleanup(removed_image, instance.pk)
 
 
 @receiver(post_save, sender=Session)
@@ -659,9 +675,3 @@ def create_access_token_user_if_missing(sender, **kwargs):
         post_save.disconnect(create_access_token_user_if_missing, sender=OAuth2AccessToken)
         obj.save()
         post_save.connect(create_access_token_user_if_missing, sender=OAuth2AccessToken)
-
-
-# Connect the Instance Group to Activity Stream receivers.
-post_save.connect(activity_stream_create, sender=InstanceGroup, dispatch_uid=str(InstanceGroup) + "_create")
-pre_save.connect(activity_stream_update, sender=InstanceGroup, dispatch_uid=str(InstanceGroup) + "_update")
-pre_delete.connect(activity_stream_delete, sender=InstanceGroup, dispatch_uid=str(InstanceGroup) + "_delete")

awx/main/tasks/receptor.py

@@ -0,0 +1,542 @@
+# Python
+from base64 import b64encode
+from collections import namedtuple
+import concurrent.futures
+from enum import Enum
+import logging
+import os
+import shutil
+import socket
+import sys
+import threading
+import time
+import yaml
+
+# Django
+from django.conf import settings
+
+# Runner
+import ansible_runner
+
+# AWX
+from awx.main.utils.execution_environments import get_default_pod_spec
+from awx.main.exceptions import ReceptorNodeNotFound
+from awx.main.utils.common import (
+    deepmerge,
+    parse_yaml_or_json,
+    cleanup_new_process,
+)
+
+# Receptorctl
+from receptorctl.socket_interface import ReceptorControl
+
+logger = logging.getLogger('awx.main.tasks.receptor')
+__RECEPTOR_CONF = '/etc/receptor/receptor.conf'
+RECEPTOR_ACTIVE_STATES = ('Pending', 'Running')
+
+
+class ReceptorConnectionType(Enum):
+    DATAGRAM = 0
+    STREAM = 1
+    STREAMTLS = 2
+
+
+def get_receptor_sockfile():
+    with open(__RECEPTOR_CONF, 'r') as f:
+        data = yaml.safe_load(f)
+    for section in data:
+        for entry_name, entry_data in section.items():
+            if entry_name == 'control-service':
+                if 'filename' in entry_data:
+                    return entry_data['filename']
+                else:
+                    raise RuntimeError(f'Receptor conf {__RECEPTOR_CONF} control-service entry does not have a filename parameter')
+    else:
+        raise RuntimeError(f'Receptor conf {__RECEPTOR_CONF} does not have control-service entry needed to get sockfile')
+
+
+def get_tls_client(use_stream_tls=None):
+    if not use_stream_tls:
+        return None
+
+    with open(__RECEPTOR_CONF, 'r') as f:
+        data = yaml.safe_load(f)
+    for section in data:
+        for entry_name, entry_data in section.items():
+            if entry_name == 'tls-client':
+                if 'name' in entry_data:
+                    return entry_data['name']
+    return None
+
+
+def get_receptor_ctl():
+    receptor_sockfile = get_receptor_sockfile()
+    try:
+        return ReceptorControl(receptor_sockfile, config=__RECEPTOR_CONF, tlsclient=get_tls_client(True))
+    except RuntimeError:
+        return ReceptorControl(receptor_sockfile)
+
+
+def get_conn_type(node_name, receptor_ctl):
+    all_nodes = receptor_ctl.simple_command("status").get('Advertisements', None)
+    for node in all_nodes:
+        if node.get('NodeID') == node_name:
+            return ReceptorConnectionType(node.get('ConnType'))
+    raise ReceptorNodeNotFound(f'Instance {node_name} is not in the receptor mesh')
+
+
+def administrative_workunit_reaper(work_list=None):
+    """
+    This releases completed work units that were spawned by actions inside of this module
+    specifically, this should catch any completed work unit left by
+     - worker_info
+     - worker_cleanup
+    These should ordinarily be released when the method finishes, but this is a
+    cleanup of last-resort, in case something went awry
+    """
+    receptor_ctl = get_receptor_ctl()
+    if work_list is None:
+        work_list = receptor_ctl.simple_command("work list")
+
+    for unit_id, work_data in work_list.items():
+        extra_data = work_data.get('ExtraData')
+        if (extra_data is None) or (extra_data.get('RemoteWorkType') != 'ansible-runner'):
+            continue  # if this is not ansible-runner work, we do not want to touch it
+        params = extra_data.get('RemoteParams', {}).get('params')
+        if not params:
+            continue
+        if not (params == '--worker-info' or params.startswith('cleanup')):
+            continue  # if this is not a cleanup or health check, we do not want to touch it
+        if work_data.get('StateName') in RECEPTOR_ACTIVE_STATES:
+            continue  # do not want to touch active work units
+        logger.info(f'Reaping orphaned work unit {unit_id} with params {params}')
+        receptor_ctl.simple_command(f"work release {unit_id}")
+
+
+class RemoteJobError(RuntimeError):
+    pass
+
+
+def run_until_complete(node, timing_data=None, **kwargs):
+    """
+    Runs an ansible-runner work_type on remote node, waits until it completes, then returns stdout.
+    """
+    receptor_ctl = get_receptor_ctl()
+
+    use_stream_tls = getattr(get_conn_type(node, receptor_ctl), 'name', None) == "STREAMTLS"
+    kwargs.setdefault('tlsclient', get_tls_client(use_stream_tls))
+    kwargs.setdefault('ttl', '20s')
+    kwargs.setdefault('payload', '')
+
+    transmit_start = time.time()
+    sign_work = False if settings.IS_K8S else True
+    result = receptor_ctl.submit_work(worktype='ansible-runner', node=node, signwork=sign_work, **kwargs)
+
+    unit_id = result['unitid']
+    run_start = time.time()
+    if timing_data:
+        timing_data['transmit_timing'] = run_start - transmit_start
+    run_timing = 0.0
+    stdout = ''
+
+    try:
+
+        resultfile = receptor_ctl.get_work_results(unit_id)
+
+        while run_timing < 20.0:
+            status = receptor_ctl.simple_command(f'work status {unit_id}')
+            state_name = status.get('StateName')
+            if state_name not in RECEPTOR_ACTIVE_STATES:
+                break
+            run_timing = time.time() - run_start
+            time.sleep(0.5)
+        else:
+            raise RemoteJobError(f'Receptor job timeout on {node} after {run_timing} seconds, state remains in {state_name}')
+
+        if timing_data:
+            timing_data['run_timing'] = run_timing
+
+        stdout = resultfile.read()
+        stdout = str(stdout, encoding='utf-8')
+
+    finally:
+
+        if settings.RECEPTOR_RELEASE_WORK:
+            res = receptor_ctl.simple_command(f"work release {unit_id}")
+            if res != {'released': unit_id}:
+                logger.warn(f'Could not confirm release of receptor work unit id {unit_id} from {node}, data: {res}')
+
+        receptor_ctl.close()
+
+    if state_name.lower() == 'failed':
+        work_detail = status.get('Detail', '')
+        if work_detail:
+            raise RemoteJobError(f'Receptor error from {node}, detail:\n{work_detail}')
+        else:
+            raise RemoteJobError(f'Unknown ansible-runner error on node {node}, stdout:\n{stdout}')
+
+    return stdout
+
+
+def worker_info(node_name, work_type='ansible-runner'):
+    error_list = []
+    data = {'errors': error_list, 'transmit_timing': 0.0}
+
+    try:
+        stdout = run_until_complete(node=node_name, timing_data=data, params={"params": "--worker-info"})
+
+        yaml_stdout = stdout.strip()
+        remote_data = {}
+        try:
+            remote_data = yaml.safe_load(yaml_stdout)
+        except Exception as json_e:
+            error_list.append(f'Failed to parse node {node_name} --worker-info output as YAML, error: {json_e}, data:\n{yaml_stdout}')
+
+        if not isinstance(remote_data, dict):
+            error_list.append(f'Remote node {node_name} --worker-info output is not a YAML dict, output:{stdout}')
+        else:
+            error_list.extend(remote_data.pop('errors', []))  # merge both error lists
+            data.update(remote_data)
+
+    except RemoteJobError as exc:
+        details = exc.args[0]
+        if 'unrecognized arguments: --worker-info' in details:
+            error_list.append(f'Old version (2.0.1 or earlier) of ansible-runner on node {node_name} without --worker-info')
+        else:
+            error_list.append(details)
+
+    except (ReceptorNodeNotFound, RuntimeError) as exc:
+        error_list.append(str(exc))
+
+    # If we have a connection error, missing keys would be trivial consequence of that
+    if not data['errors']:
+        # see tasks.py usage of keys
+        missing_keys = set(('runner_version', 'mem_in_bytes', 'cpu_count')) - set(data.keys())
+        if missing_keys:
+            data['errors'].append('Worker failed to return keys {}'.format(' '.join(missing_keys)))
+
+    return data
+
+
+def _convert_args_to_cli(vargs):
+    """
+    For the ansible-runner worker cleanup command
+    converts the dictionary (parsed argparse variables) used for python interface
+    into a string of CLI options, which has to be used on execution nodes.
+    """
+    args = ['cleanup']
+    for option in ('exclude_strings', 'remove_images'):
+        if vargs.get(option):
+            args.append('--{}={}'.format(option.replace('_', '-'), ' '.join(vargs.get(option))))
+    for option in ('file_pattern', 'image_prune', 'process_isolation_executable', 'grace_period'):
+        if vargs.get(option) is True:
+            args.append('--{}'.format(option.replace('_', '-')))
+        elif vargs.get(option) not in (None, ''):
+            args.append('--{}={}'.format(option.replace('_', '-'), vargs.get(option)))
+    return args
+
+
+def worker_cleanup(node_name, vargs, timeout=300.0):
+    args = _convert_args_to_cli(vargs)
+
+    remote_command = ' '.join(args)
+    logger.debug(f'Running command over receptor mesh on {node_name}: ansible-runner worker {remote_command}')
+
+    stdout = run_until_complete(node=node_name, params={"params": remote_command})
+
+    return stdout
+
+
+class TransmitterThread(threading.Thread):
+    def run(self):
+        self.exc = None
+
+        try:
+            super().run()
+        except Exception:
+            self.exc = sys.exc_info()
+
+
+class AWXReceptorJob:
+    def __init__(self, task, runner_params=None):
+        self.task = task
+        self.runner_params = runner_params
+        self.unit_id = None
+
+        if self.task and not self.task.instance.is_container_group_task:
+            execution_environment_params = self.task.build_execution_environment_params(self.task.instance, runner_params['private_data_dir'])
+            self.runner_params.update(execution_environment_params)
+
+        if not settings.IS_K8S and self.work_type == 'local' and 'only_transmit_kwargs' not in self.runner_params:
+            self.runner_params['only_transmit_kwargs'] = True
+
+    def run(self):
+        # We establish a connection to the Receptor socket
+        receptor_ctl = get_receptor_ctl()
+
+        res = None
+        try:
+            res = self._run_internal(receptor_ctl)
+            return res
+        finally:
+            # Make sure to always release the work unit if we established it
+            if self.unit_id is not None and settings.RECEPTOR_RELEASE_WORK:
+                try:
+                    receptor_ctl.simple_command(f"work release {self.unit_id}")
+                except Exception:
+                    logger.exception(f"Error releasing work unit {self.unit_id}.")
+
+    @property
+    def sign_work(self):
+        return False if settings.IS_K8S else True
+
+    def _run_internal(self, receptor_ctl):
+        # Create a socketpair. Where the left side will be used for writing our payload
+        # (private data dir, kwargs). The right side will be passed to Receptor for
+        # reading.
+        sockin, sockout = socket.socketpair()
+
+        transmitter_thread = TransmitterThread(target=self.transmit, args=[sockin])
+        transmitter_thread.start()
+
+        # submit our work, passing
+        # in the right side of our socketpair for reading.
+        _kw = {}
+        if self.work_type == 'ansible-runner':
+            _kw['node'] = self.task.instance.execution_node
+            use_stream_tls = get_conn_type(_kw['node'], receptor_ctl).name == "STREAMTLS"
+            _kw['tlsclient'] = get_tls_client(use_stream_tls)
+        result = receptor_ctl.submit_work(worktype=self.work_type, payload=sockout.makefile('rb'), params=self.receptor_params, signwork=self.sign_work, **_kw)
+        self.unit_id = result['unitid']
+        # Update the job with the work unit in-memory so that the log_lifecycle
+        # will print out the work unit that is to be associated with the job in the database
+        # via the update_model() call.
+        # We want to log the work_unit_id as early as possible. A failure can happen in between
+        # when we start the job in receptor and when we associate the job <-> work_unit_id.
+        # In that case, there will be work running in receptor and Controller will not know
+        # which Job it is associated with.
+        # We do not programatically handle this case. Ideally, we would handle this with a reaper case.
+        # The two distinct job lifecycle log events below allow for us to at least detect when this
+        # edge case occurs. If the lifecycle event work_unit_id_received occurs without the
+        # work_unit_id_assigned event then this case may have occured.
+        self.task.instance.work_unit_id = result['unitid']  # Set work_unit_id in-memory only
+        self.task.instance.log_lifecycle("work_unit_id_received")
+        self.task.update_model(self.task.instance.pk, work_unit_id=result['unitid'])
+        self.task.instance.log_lifecycle("work_unit_id_assigned")
+
+        sockin.close()
+        sockout.close()
+
+        if transmitter_thread.exc:
+            raise transmitter_thread.exc[1].with_traceback(transmitter_thread.exc[2])
+
+        transmitter_thread.join()
+
+        # Artifacts are an output, but sometimes they are an input as well
+        # this is the case with fact cache, where clearing facts deletes a file, and this must be captured
+        artifact_dir = os.path.join(self.runner_params['private_data_dir'], 'artifacts')
+        if os.path.exists(artifact_dir):
+            shutil.rmtree(artifact_dir)
+
+        resultsock, resultfile = receptor_ctl.get_work_results(self.unit_id, return_socket=True, return_sockfile=True)
+        # Both "processor" and "cancel_watcher" are spawned in separate threads.
+        # We wait for the first one to return. If cancel_watcher returns first,
+        # we yank the socket out from underneath the processor, which will cause it
+        # to exit. A reference to the processor_future is passed into the cancel_watcher_future,
+        # Which exits if the job has finished normally. The context manager ensures we do not
+        # leave any threads laying around.
+        with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+            processor_future = executor.submit(self.processor, resultfile)
+            cancel_watcher_future = executor.submit(self.cancel_watcher, processor_future)
+            futures = [processor_future, cancel_watcher_future]
+            first_future = concurrent.futures.wait(futures, return_when=concurrent.futures.FIRST_COMPLETED)
+
+            res = list(first_future.done)[0].result()
+            if res.status == 'canceled':
+                receptor_ctl.simple_command(f"work cancel {self.unit_id}")
+                resultsock.shutdown(socket.SHUT_RDWR)
+                resultfile.close()
+            elif res.status == 'error':
+                try:
+                    unit_status = receptor_ctl.simple_command(f'work status {self.unit_id}')
+                    detail = unit_status.get('Detail', None)
+                    state_name = unit_status.get('StateName', None)
+                except Exception:
+                    detail = ''
+                    state_name = ''
+                    logger.exception(f'An error was encountered while getting status for work unit {self.unit_id}')
+
+                if 'exceeded quota' in detail:
+                    logger.warn(detail)
+                    log_name = self.task.instance.log_format
+                    logger.warn(f"Could not launch pod for {log_name}. Exceeded quota.")
+                    self.task.update_model(self.task.instance.pk, status='pending')
+                    return
+                # If ansible-runner ran, but an error occured at runtime, the traceback information
+                # is saved via the status_handler passed in to the processor.
+                if state_name == 'Succeeded':
+                    return res
+
+                if not self.task.instance.result_traceback:
+                    try:
+                        resultsock = receptor_ctl.get_work_results(self.unit_id, return_sockfile=True)
+                        lines = resultsock.readlines()
+                        receptor_output = b"".join(lines).decode()
+                        if receptor_output:
+                            self.task.instance.result_traceback = receptor_output
+                            self.task.instance.save(update_fields=['result_traceback'])
+                        elif detail:
+                            self.task.instance.result_traceback = detail
+                            self.task.instance.save(update_fields=['result_traceback'])
+                        else:
+                            logger.warn(f'No result details or output from {self.task.instance.log_format}, status:\n{state_name}')
+                    except Exception:
+                        raise RuntimeError(detail)
+
+        return res
+
+    # Spawned in a thread so Receptor can start reading before we finish writing, we
+    # write our payload to the left side of our socketpair.
+    @cleanup_new_process
+    def transmit(self, _socket):
+        try:
+            ansible_runner.interface.run(streamer='transmit', _output=_socket.makefile('wb'), **self.runner_params)
+        finally:
+            # Socket must be shutdown here, or the reader will hang forever.
+            _socket.shutdown(socket.SHUT_WR)
+
+    @cleanup_new_process
+    def processor(self, resultfile):
+        return ansible_runner.interface.run(
+            streamer='process',
+            quiet=True,
+            _input=resultfile,
+            event_handler=self.task.event_handler,
+            finished_callback=self.task.finished_callback,
+            status_handler=self.task.status_handler,
+            **self.runner_params,
+        )
+
+    @property
+    def receptor_params(self):
+        if self.task.instance.is_container_group_task:
+            spec_yaml = yaml.dump(self.pod_definition, explicit_start=True)
+
+            receptor_params = {
+                "secret_kube_pod": spec_yaml,
+                "pod_pending_timeout": getattr(settings, 'AWX_CONTAINER_GROUP_POD_PENDING_TIMEOUT', "5m"),
+            }
+
+            if self.credential:
+                kubeconfig_yaml = yaml.dump(self.kube_config, explicit_start=True)
+                receptor_params["secret_kube_config"] = kubeconfig_yaml
+        else:
+            private_data_dir = self.runner_params['private_data_dir']
+            if self.work_type == 'ansible-runner' and settings.AWX_CLEANUP_PATHS:
+                # on execution nodes, we rely on the private data dir being deleted
+                cli_params = f"--private-data-dir={private_data_dir} --delete"
+            else:
+                # on hybrid nodes, we rely on the private data dir NOT being deleted
+                cli_params = f"--private-data-dir={private_data_dir}"
+            receptor_params = {"params": cli_params}
+
+        return receptor_params
+
+    @property
+    def work_type(self):
+        if self.task.instance.is_container_group_task:
+            if self.credential:
+                return 'kubernetes-runtime-auth'
+            return 'kubernetes-incluster-auth'
+        if self.task.instance.execution_node == settings.CLUSTER_HOST_ID or self.task.instance.execution_node == self.task.instance.controller_node:
+            return 'local'
+        return 'ansible-runner'
+
+    @cleanup_new_process
+    def cancel_watcher(self, processor_future):
+        while True:
+            if processor_future.done():
+                return processor_future.result()
+
+            if self.task.cancel_callback():
+                result = namedtuple('result', ['status', 'rc'])
+                return result('canceled', 1)
+
+            time.sleep(1)
+
+    @property
+    def pod_definition(self):
+        ee = self.task.instance.execution_environment
+
+        default_pod_spec = get_default_pod_spec()
+
+        pod_spec_override = {}
+        if self.task and self.task.instance.instance_group.pod_spec_override:
+            pod_spec_override = parse_yaml_or_json(self.task.instance.instance_group.pod_spec_override)
+        # According to the deepmerge docstring, the second dictionary will override when
+        # they share keys, which is the desired behavior.
+        # This allows user to only provide elements they want to override, and for us to still provide any
+        # defaults they don't want to change
+        pod_spec = deepmerge(default_pod_spec, pod_spec_override)
+
+        pod_spec['spec']['containers'][0]['image'] = ee.image
+        pod_spec['spec']['containers'][0]['args'] = ['ansible-runner', 'worker', '--private-data-dir=/runner']
+
+        # Enforce EE Pull Policy
+        pull_options = {"always": "Always", "missing": "IfNotPresent", "never": "Never"}
+        if self.task and self.task.instance.execution_environment:
+            if self.task.instance.execution_environment.pull:
+                pod_spec['spec']['containers'][0]['imagePullPolicy'] = pull_options[self.task.instance.execution_environment.pull]
+
+        if self.task and self.task.instance.is_container_group_task:
+            # If EE credential is passed, create an imagePullSecret
+            if self.task.instance.execution_environment and self.task.instance.execution_environment.credential:
+                # Create pull secret in k8s cluster based on ee cred
+                from awx.main.scheduler.kubernetes import PodManager  # prevent circular import
+
+                pm = PodManager(self.task.instance)
+                secret_name = pm.create_secret(job=self.task.instance)
+
+                # Inject secret name into podspec
+                pod_spec['spec']['imagePullSecrets'] = [{"name": secret_name}]
+
+        if self.task:
+            pod_spec['metadata'] = deepmerge(
+                pod_spec.get('metadata', {}),
+                dict(name=self.pod_name, labels={'ansible-awx': settings.INSTALL_UUID, 'ansible-awx-job-id': str(self.task.instance.id)}),
+            )
+
+        return pod_spec
+
+    @property
+    def pod_name(self):
+        return f"automation-job-{self.task.instance.id}"
+
+    @property
+    def credential(self):
+        return self.task.instance.instance_group.credential
+
+    @property
+    def namespace(self):
+        return self.pod_definition['metadata']['namespace']
+
+    @property
+    def kube_config(self):
+        host_input = self.credential.get_input('host')
+        config = {
+            "apiVersion": "v1",
+            "kind": "Config",
+            "preferences": {},
+            "clusters": [{"name": host_input, "cluster": {"server": host_input}}],
+            "users": [{"name": host_input, "user": {"token": self.credential.get_input('bearer_token')}}],
+            "contexts": [{"name": host_input, "context": {"cluster": host_input, "user": host_input, "namespace": self.namespace}}],
+            "current-context": host_input,
+        }
+
+        if self.credential.get_input('verify_ssl') and 'ssl_ca_cert' in self.credential.inputs:
+            config["clusters"][0]["cluster"]["certificate-authority-data"] = b64encode(
+                self.credential.get_input('ssl_ca_cert').encode()  # encode to bytes
+            ).decode()  # decode the base64 data into a str
+        else:
+            config["clusters"][0]["cluster"]["insecure-skip-tls-verify"] = True
+        return config

awx/main/tasks/system.py

@@ -0,0 +1,897 @@
+# Python
+from collections import namedtuple
+import functools
+import importlib
+import json
+import logging
+import os
+from io import StringIO
+from contextlib import redirect_stdout
+import shutil
+import time
+from distutils.version import LooseVersion as Version
+
+# Django
+from django.conf import settings
+from django.db import transaction, DatabaseError, IntegrityError
+from django.db.models.fields.related import ForeignKey
+from django.utils.timezone import now
+from django.utils.encoding import smart_str
+from django.contrib.auth.models import User
+from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_noop
+from django.core.cache import cache
+from django.core.exceptions import ObjectDoesNotExist
+
+# Django-CRUM
+from crum import impersonate
+
+
+# Runner
+import ansible_runner.cleanup
+
+# dateutil
+from dateutil.parser import parse as parse_date
+
+# AWX
+from awx import __version__ as awx_application_version
+from awx.main.access import access_registry
+from awx.main.models import (
+    Schedule,
+    TowerScheduleState,
+    Instance,
+    InstanceGroup,
+    UnifiedJob,
+    Notification,
+    Inventory,
+    SmartInventoryMembership,
+    Job,
+)
+from awx.main.constants import ACTIVE_STATES
+from awx.main.dispatch.publish import task
+from awx.main.dispatch import get_local_queuename, reaper
+from awx.main.utils.common import (
+    ignore_inventory_computed_fields,
+    ignore_inventory_group_removal,
+    schedule_task_manager,
+)
+
+from awx.main.utils.external_logging import reconfigure_rsyslog
+from awx.main.utils.reload import stop_local_services
+from awx.main.utils.pglock import advisory_lock
+from awx.main.tasks.receptor import get_receptor_ctl, worker_info, worker_cleanup, administrative_workunit_reaper
+from awx.main.consumers import emit_channel_notification
+from awx.main import analytics
+from awx.conf import settings_registry
+from awx.main.analytics.subsystem_metrics import Metrics
+
+from rest_framework.exceptions import PermissionDenied
+
+logger = logging.getLogger('awx.main.tasks.system')
+
+OPENSSH_KEY_ERROR = u'''\
+It looks like you're trying to use a private key in OpenSSH format, which \
+isn't supported by the installed version of OpenSSH on this instance. \
+Try upgrading OpenSSH or providing your private key in an different format. \
+'''
+
+
+def dispatch_startup():
+    startup_logger = logging.getLogger('awx.main.tasks')
+    startup_logger.debug("Syncing Schedules")
+    for sch in Schedule.objects.all():
+        try:
+            sch.update_computed_fields()
+        except Exception:
+            logger.exception("Failed to rebuild schedule {}.".format(sch))
+
+    #
+    # When the dispatcher starts, if the instance cannot be found in the database,
+    # automatically register it.  This is mostly useful for openshift-based
+    # deployments where:
+    #
+    # 2 Instances come online
+    # Instance B encounters a network blip, Instance A notices, and
+    # deprovisions it
+    # Instance B's connectivity is restored, the dispatcher starts, and it
+    # re-registers itself
+    #
+    # In traditional container-less deployments, instances don't get
+    # deprovisioned when they miss their heartbeat, so this code is mostly a
+    # no-op.
+    #
+    apply_cluster_membership_policies()
+    cluster_node_heartbeat()
+    Metrics().clear_values()
+
+    # Update Tower's rsyslog.conf file based on loggins settings in the db
+    reconfigure_rsyslog()
+
+
+def inform_cluster_of_shutdown():
+    try:
+        this_inst = Instance.objects.get(hostname=settings.CLUSTER_HOST_ID)
+        this_inst.mark_offline(update_last_seen=True, errors=_('Instance received normal shutdown signal'))
+        try:
+            reaper.reap(this_inst)
+        except Exception:
+            logger.exception('failed to reap jobs for {}'.format(this_inst.hostname))
+        logger.warning('Normal shutdown signal for instance {}, ' 'removed self from capacity pool.'.format(this_inst.hostname))
+    except Exception:
+        logger.exception('Encountered problem with normal shutdown signal.')
+
+
+@task(queue=get_local_queuename)
+def apply_cluster_membership_policies():
+    from awx.main.signals import disable_activity_stream
+
+    started_waiting = time.time()
+    with advisory_lock('cluster_policy_lock', wait=True):
+        lock_time = time.time() - started_waiting
+        if lock_time > 1.0:
+            to_log = logger.info
+        else:
+            to_log = logger.debug
+        to_log('Waited {} seconds to obtain lock name: cluster_policy_lock'.format(lock_time))
+        started_compute = time.time()
+        # Hop nodes should never get assigned to an InstanceGroup.
+        all_instances = list(Instance.objects.exclude(node_type='hop').order_by('id'))
+        all_groups = list(InstanceGroup.objects.prefetch_related('instances'))
+
+        total_instances = len(all_instances)
+        actual_groups = []
+        actual_instances = []
+        Group = namedtuple('Group', ['obj', 'instances', 'prior_instances'])
+        Node = namedtuple('Instance', ['obj', 'groups'])
+
+        # Process policy instance list first, these will represent manually managed memberships
+        instance_hostnames_map = {inst.hostname: inst for inst in all_instances}
+        for ig in all_groups:
+            group_actual = Group(obj=ig, instances=[], prior_instances=[instance.pk for instance in ig.instances.all()])  # obtained in prefetch
+            for hostname in ig.policy_instance_list:
+                if hostname not in instance_hostnames_map:
+                    logger.info("Unknown instance {} in {} policy list".format(hostname, ig.name))
+                    continue
+                inst = instance_hostnames_map[hostname]
+                group_actual.instances.append(inst.id)
+                # NOTE: arguable behavior: policy-list-group is not added to
+                # instance's group count for consideration in minimum-policy rules
+            if group_actual.instances:
+                logger.debug("Policy List, adding Instances {} to Group {}".format(group_actual.instances, ig.name))
+
+            actual_groups.append(group_actual)
+
+        # Process Instance minimum policies next, since it represents a concrete lower bound to the
+        # number of instances to make available to instance groups
+        actual_instances = [Node(obj=i, groups=[]) for i in all_instances if i.managed_by_policy]
+        logger.debug("Total instances: {}, available for policy: {}".format(total_instances, len(actual_instances)))
+        for g in sorted(actual_groups, key=lambda x: len(x.instances)):
+            exclude_type = 'execution' if g.obj.name == settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME else 'control'
+            policy_min_added = []
+            for i in sorted(actual_instances, key=lambda x: len(x.groups)):
+                if i.obj.node_type == exclude_type:
+                    continue  # never place execution instances in controlplane group or control instances in other groups
+                if len(g.instances) >= g.obj.policy_instance_minimum:
+                    break
+                if i.obj.id in g.instances:
+                    # If the instance is already _in_ the group, it was
+                    # applied earlier via the policy list
+                    continue
+                g.instances.append(i.obj.id)
+                i.groups.append(g.obj.id)
+                policy_min_added.append(i.obj.id)
+            if policy_min_added:
+                logger.debug("Policy minimum, adding Instances {} to Group {}".format(policy_min_added, g.obj.name))
+
+        # Finally, process instance policy percentages
+        for g in sorted(actual_groups, key=lambda x: len(x.instances)):
+            exclude_type = 'execution' if g.obj.name == settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME else 'control'
+            candidate_pool_ct = sum(1 for i in actual_instances if i.obj.node_type != exclude_type)
+            if not candidate_pool_ct:
+                continue
+            policy_per_added = []
+            for i in sorted(actual_instances, key=lambda x: len(x.groups)):
+                if i.obj.node_type == exclude_type:
+                    continue
+                if i.obj.id in g.instances:
+                    # If the instance is already _in_ the group, it was
+                    # applied earlier via a minimum policy or policy list
+                    continue
+                if 100 * float(len(g.instances)) / candidate_pool_ct >= g.obj.policy_instance_percentage:
+                    break
+                g.instances.append(i.obj.id)
+                i.groups.append(g.obj.id)
+                policy_per_added.append(i.obj.id)
+            if policy_per_added:
+                logger.debug("Policy percentage, adding Instances {} to Group {}".format(policy_per_added, g.obj.name))
+
+        # Determine if any changes need to be made
+        needs_change = False
+        for g in actual_groups:
+            if set(g.instances) != set(g.prior_instances):
+                needs_change = True
+                break
+        if not needs_change:
+            logger.debug('Cluster policy no-op finished in {} seconds'.format(time.time() - started_compute))
+            return
+
+        # On a differential basis, apply instances to groups
+        with transaction.atomic():
+            with disable_activity_stream():
+                for g in actual_groups:
+                    if g.obj.is_container_group:
+                        logger.debug('Skipping containerized group {} for policy calculation'.format(g.obj.name))
+                        continue
+                    instances_to_add = set(g.instances) - set(g.prior_instances)
+                    instances_to_remove = set(g.prior_instances) - set(g.instances)
+                    if instances_to_add:
+                        logger.debug('Adding instances {} to group {}'.format(list(instances_to_add), g.obj.name))
+                        g.obj.instances.add(*instances_to_add)
+                    if instances_to_remove:
+                        logger.debug('Removing instances {} from group {}'.format(list(instances_to_remove), g.obj.name))
+                        g.obj.instances.remove(*instances_to_remove)
+        logger.debug('Cluster policy computation finished in {} seconds'.format(time.time() - started_compute))
+
+
+@task(queue='tower_broadcast_all')
+def handle_setting_changes(setting_keys):
+    orig_len = len(setting_keys)
+    for i in range(orig_len):
+        for dependent_key in settings_registry.get_dependent_settings(setting_keys[i]):
+            setting_keys.append(dependent_key)
+    cache_keys = set(setting_keys)
+    logger.debug('cache delete_many(%r)', cache_keys)
+    cache.delete_many(cache_keys)
+
+    if any([setting.startswith('LOG_AGGREGATOR') for setting in setting_keys]):
+        reconfigure_rsyslog()
+
+
+@task(queue='tower_broadcast_all')
+def delete_project_files(project_path):
+    # TODO: possibly implement some retry logic
+    lock_file = project_path + '.lock'
+    if os.path.exists(project_path):
+        try:
+            shutil.rmtree(project_path)
+            logger.debug('Success removing project files {}'.format(project_path))
+        except Exception:
+            logger.exception('Could not remove project directory {}'.format(project_path))
+    if os.path.exists(lock_file):
+        try:
+            os.remove(lock_file)
+            logger.debug('Success removing {}'.format(lock_file))
+        except Exception:
+            logger.exception('Could not remove lock file {}'.format(lock_file))
+
+
+@task(queue='tower_broadcast_all')
+def profile_sql(threshold=1, minutes=1):
+    if threshold <= 0:
+        cache.delete('awx-profile-sql-threshold')
+        logger.error('SQL PROFILING DISABLED')
+    else:
+        cache.set('awx-profile-sql-threshold', threshold, timeout=minutes * 60)
+        logger.error('SQL QUERIES >={}s ENABLED FOR {} MINUTE(S)'.format(threshold, minutes))
+
+
+@task(queue=get_local_queuename)
+def send_notifications(notification_list, job_id=None):
+    if not isinstance(notification_list, list):
+        raise TypeError("notification_list should be of type list")
+    if job_id is not None:
+        job_actual = UnifiedJob.objects.get(id=job_id)
+
+    notifications = Notification.objects.filter(id__in=notification_list)
+    if job_id is not None:
+        job_actual.notifications.add(*notifications)
+
+    for notification in notifications:
+        update_fields = ['status', 'notifications_sent']
+        try:
+            sent = notification.notification_template.send(notification.subject, notification.body)
+            notification.status = "successful"
+            notification.notifications_sent = sent
+            if job_id is not None:
+                job_actual.log_lifecycle("notifications_sent")
+        except Exception as e:
+            logger.exception("Send Notification Failed {}".format(e))
+            notification.status = "failed"
+            notification.error = smart_str(e)
+            update_fields.append('error')
+        finally:
+            try:
+                notification.save(update_fields=update_fields)
+            except Exception:
+                logger.exception('Error saving notification {} result.'.format(notification.id))
+
+
+@task(queue=get_local_queuename)
+def gather_analytics():
+    from awx.conf.models import Setting
+    from rest_framework.fields import DateTimeField
+
+    last_gather = Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_GATHER').first()
+    last_time = DateTimeField().to_internal_value(last_gather.value) if last_gather and last_gather.value else None
+    gather_time = now()
+
+    if not last_time or ((gather_time - last_time).total_seconds() > settings.AUTOMATION_ANALYTICS_GATHER_INTERVAL):
+        analytics.gather()
+
+
+@task(queue=get_local_queuename)
+def purge_old_stdout_files():
+    nowtime = time.time()
+    for f in os.listdir(settings.JOBOUTPUT_ROOT):
+        if os.path.getctime(os.path.join(settings.JOBOUTPUT_ROOT, f)) < nowtime - settings.LOCAL_STDOUT_EXPIRE_TIME:
+            os.unlink(os.path.join(settings.JOBOUTPUT_ROOT, f))
+            logger.debug("Removing {}".format(os.path.join(settings.JOBOUTPUT_ROOT, f)))
+
+
+def _cleanup_images_and_files(**kwargs):
+    if settings.IS_K8S:
+        return
+    this_inst = Instance.objects.me()
+    runner_cleanup_kwargs = this_inst.get_cleanup_task_kwargs(**kwargs)
+    if runner_cleanup_kwargs:
+        stdout = ''
+        with StringIO() as buffer:
+            with redirect_stdout(buffer):
+                ansible_runner.cleanup.run_cleanup(runner_cleanup_kwargs)
+                stdout = buffer.getvalue()
+        if '(changed: True)' in stdout:
+            logger.info(f'Performed local cleanup with kwargs {kwargs}, output:\n{stdout}')
+
+    # if we are the first instance alphabetically, then run cleanup on execution nodes
+    checker_instance = Instance.objects.filter(node_type__in=['hybrid', 'control'], enabled=True, capacity__gt=0).order_by('-hostname').first()
+    if checker_instance and this_inst.hostname == checker_instance.hostname:
+        for inst in Instance.objects.filter(node_type='execution', enabled=True, capacity__gt=0):
+            runner_cleanup_kwargs = inst.get_cleanup_task_kwargs(**kwargs)
+            if not runner_cleanup_kwargs:
+                continue
+            try:
+                stdout = worker_cleanup(inst.hostname, runner_cleanup_kwargs)
+                if '(changed: True)' in stdout:
+                    logger.info(f'Performed cleanup on execution node {inst.hostname} with output:\n{stdout}')
+            except RuntimeError:
+                logger.exception(f'Error running cleanup on execution node {inst.hostname}')
+
+
+@task(queue='tower_broadcast_all')
+def handle_removed_image(remove_images=None):
+    """Special broadcast invocation of this method to handle case of deleted EE"""
+    _cleanup_images_and_files(remove_images=remove_images, file_pattern='')
+
+
+@task(queue=get_local_queuename)
+def cleanup_images_and_files():
+    _cleanup_images_and_files()
+
+
+@task(queue=get_local_queuename)
+def cluster_node_health_check(node):
+    """
+    Used for the health check endpoint, refreshes the status of the instance, but must be ran on target node
+    """
+    if node == '':
+        logger.warn('Local health check incorrectly called with blank string')
+        return
+    elif node != settings.CLUSTER_HOST_ID:
+        logger.warn(f'Local health check for {node} incorrectly sent to {settings.CLUSTER_HOST_ID}')
+        return
+    try:
+        this_inst = Instance.objects.me()
+    except Instance.DoesNotExist:
+        logger.warn(f'Instance record for {node} missing, could not check capacity.')
+        return
+    this_inst.local_health_check()
+
+
+@task(queue=get_local_queuename)
+def execution_node_health_check(node):
+    if node == '':
+        logger.warn('Remote health check incorrectly called with blank string')
+        return
+    try:
+        instance = Instance.objects.get(hostname=node)
+    except Instance.DoesNotExist:
+        logger.warn(f'Instance record for {node} missing, could not check capacity.')
+        return
+
+    if instance.node_type != 'execution':
+        raise RuntimeError(f'Execution node health check ran against {instance.node_type} node {instance.hostname}')
+
+    data = worker_info(node)
+
+    prior_capacity = instance.capacity
+
+    instance.save_health_data(
+        version='ansible-runner-' + data.get('runner_version', '???'),
+        cpu=data.get('cpu_count', 0),
+        memory=data.get('mem_in_bytes', 0),
+        uuid=data.get('uuid'),
+        errors='\n'.join(data.get('errors', [])),
+    )
+
+    if data['errors']:
+        formatted_error = "\n".join(data["errors"])
+        if prior_capacity:
+            logger.warn(f'Health check marking execution node {node} as lost, errors:\n{formatted_error}')
+        else:
+            logger.info(f'Failed to find capacity of new or lost execution node {node}, errors:\n{formatted_error}')
+    else:
+        logger.info('Set capacity of execution node {} to {}, worker info data:\n{}'.format(node, instance.capacity, json.dumps(data, indent=2)))
+
+    return data
+
+
+def inspect_execution_nodes(instance_list):
+    with advisory_lock('inspect_execution_nodes_lock', wait=False):
+        node_lookup = {inst.hostname: inst for inst in instance_list}
+
+        ctl = get_receptor_ctl()
+        mesh_status = ctl.simple_command('status')
+
+        nowtime = now()
+        workers = mesh_status['Advertisements']
+        for ad in workers:
+            hostname = ad['NodeID']
+            if not any(cmd['WorkType'] == 'ansible-runner' for cmd in ad['WorkCommands'] or []):
+                continue
+
+            changed = False
+            if hostname in node_lookup:
+                instance = node_lookup[hostname]
+            else:
+                logger.warn(f"Unrecognized node on mesh advertising ansible-runner work type: {hostname}")
+                continue
+
+            was_lost = instance.is_lost(ref_time=nowtime)
+            last_seen = parse_date(ad['Time'])
+
+            if instance.last_seen and instance.last_seen >= last_seen:
+                continue
+            instance.last_seen = last_seen
+            instance.save(update_fields=['last_seen'])
+
+            if changed:
+                execution_node_health_check.apply_async([hostname])
+            elif was_lost:
+                # if the instance *was* lost, but has appeared again,
+                # attempt to re-establish the initial capacity and version
+                # check
+                logger.warn(f'Execution node attempting to rejoin as instance {hostname}.')
+                execution_node_health_check.apply_async([hostname])
+            elif instance.capacity == 0 and instance.enabled:
+                # nodes with proven connection but need remediation run health checks are reduced frequency
+                if not instance.last_health_check or (nowtime - instance.last_health_check).total_seconds() >= settings.EXECUTION_NODE_REMEDIATION_CHECKS:
+                    # Periodically re-run the health check of errored nodes, in case someone fixed it
+                    # TODO: perhaps decrease the frequency of these checks
+                    logger.debug(f'Restarting health check for execution node {hostname} with known errors.')
+                    execution_node_health_check.apply_async([hostname])
+
+
+@task(queue=get_local_queuename)
+def cluster_node_heartbeat():
+    logger.debug("Cluster node heartbeat task.")
+    nowtime = now()
+    instance_list = list(Instance.objects.all())
+    this_inst = None
+    lost_instances = []
+
+    for inst in instance_list:
+        if inst.hostname == settings.CLUSTER_HOST_ID:
+            this_inst = inst
+            instance_list.remove(inst)
+            break
+    else:
+        (changed, this_inst) = Instance.objects.get_or_register()
+        if changed:
+            logger.info("Registered tower control node '{}'".format(this_inst.hostname))
+
+    inspect_execution_nodes(instance_list)
+
+    for inst in list(instance_list):
+        if inst.is_lost(ref_time=nowtime):
+            lost_instances.append(inst)
+            instance_list.remove(inst)
+
+    if this_inst:
+        startup_event = this_inst.is_lost(ref_time=nowtime)
+        this_inst.local_health_check()
+        if startup_event and this_inst.capacity != 0:
+            logger.warning('Rejoining the cluster as instance {}.'.format(this_inst.hostname))
+            return
+    else:
+        raise RuntimeError("Cluster Host Not Found: {}".format(settings.CLUSTER_HOST_ID))
+    # IFF any node has a greater version than we do, then we'll shutdown services
+    for other_inst in instance_list:
+        if other_inst.version == "" or other_inst.version.startswith('ansible-runner') or other_inst.node_type == 'execution':
+            continue
+        if Version(other_inst.version.split('-', 1)[0]) > Version(awx_application_version.split('-', 1)[0]) and not settings.DEBUG:
+            logger.error(
+                "Host {} reports version {}, but this node {} is at {}, shutting down".format(
+                    other_inst.hostname, other_inst.version, this_inst.hostname, this_inst.version
+                )
+            )
+            # Shutdown signal will set the capacity to zero to ensure no Jobs get added to this instance.
+            # The heartbeat task will reset the capacity to the system capacity after upgrade.
+            stop_local_services(communicate=False)
+            raise RuntimeError("Shutting down.")
+
+    for other_inst in lost_instances:
+        try:
+            reaper.reap(other_inst)
+        except Exception:
+            logger.exception('failed to reap jobs for {}'.format(other_inst.hostname))
+        try:
+            # Capacity could already be 0 because:
+            #  * It's a new node and it never had a heartbeat
+            #  * It was set to 0 by another tower node running this method
+            #  * It was set to 0 by this node, but auto deprovisioning is off
+            #
+            # If auto deprovisining is on, don't bother setting the capacity to 0
+            # since we will delete the node anyway.
+            if other_inst.capacity != 0 and not settings.AWX_AUTO_DEPROVISION_INSTANCES:
+                other_inst.mark_offline(errors=_('Another cluster node has determined this instance to be unresponsive'))
+                logger.error("Host {} last checked in at {}, marked as lost.".format(other_inst.hostname, other_inst.last_seen))
+            elif settings.AWX_AUTO_DEPROVISION_INSTANCES:
+                deprovision_hostname = other_inst.hostname
+                other_inst.delete()
+                logger.info("Host {} Automatically Deprovisioned.".format(deprovision_hostname))
+        except DatabaseError as e:
+            if 'did not affect any rows' in str(e):
+                logger.debug('Another instance has marked {} as lost'.format(other_inst.hostname))
+            else:
+                logger.exception('Error marking {} as lost'.format(other_inst.hostname))
+
+
+@task(queue=get_local_queuename)
+def awx_receptor_workunit_reaper():
+    """
+    When an AWX job is launched via receptor, files such as status, stdin, and stdout are created
+    in a specific receptor directory. This directory on disk is a random 8 character string, e.g. qLL2JFNT
+    This is also called the work Unit ID in receptor, and is used in various receptor commands,
+    e.g. "work results qLL2JFNT"
+    After an AWX job executes, the receptor work unit directory is cleaned up by
+    issuing the work release command. In some cases the release process might fail, or
+    if AWX crashes during a job's execution, the work release command is never issued to begin with.
+    As such, this periodic task will obtain a list of all receptor work units, and find which ones
+    belong to AWX jobs that are in a completed state (status is canceled, error, or succeeded).
+    This task will call "work release" on each of these work units to clean up the files on disk.
+
+    Note that when we call "work release" on a work unit that actually represents remote work
+    both the local and remote work units are cleaned up.
+
+    Since we are cleaning up jobs that controller considers to be inactive, we take the added
+    precaution of calling "work cancel" in case the work unit is still active.
+    """
+    if not settings.RECEPTOR_RELEASE_WORK:
+        return
+    logger.debug("Checking for unreleased receptor work units")
+    receptor_ctl = get_receptor_ctl()
+    receptor_work_list = receptor_ctl.simple_command("work list")
+
+    unit_ids = [id for id in receptor_work_list]
+    jobs_with_unreleased_receptor_units = UnifiedJob.objects.filter(work_unit_id__in=unit_ids).exclude(status__in=ACTIVE_STATES)
+    for job in jobs_with_unreleased_receptor_units:
+        logger.debug(f"{job.log_format} is not active, reaping receptor work unit {job.work_unit_id}")
+        receptor_ctl.simple_command(f"work cancel {job.work_unit_id}")
+        receptor_ctl.simple_command(f"work release {job.work_unit_id}")
+
+    administrative_workunit_reaper(receptor_work_list)
+
+
+@task(queue=get_local_queuename)
+def awx_k8s_reaper():
+    if not settings.RECEPTOR_RELEASE_WORK:
+        return
+
+    from awx.main.scheduler.kubernetes import PodManager  # prevent circular import
+
+    for group in InstanceGroup.objects.filter(is_container_group=True).iterator():
+        logger.debug("Checking for orphaned k8s pods for {}.".format(group))
+        pods = PodManager.list_active_jobs(group)
+        for job in UnifiedJob.objects.filter(pk__in=pods.keys()).exclude(status__in=ACTIVE_STATES):
+            logger.debug('{} is no longer active, reaping orphaned k8s pod'.format(job.log_format))
+            try:
+                pm = PodManager(job)
+                pm.kube_api.delete_namespaced_pod(name=pods[job.id], namespace=pm.namespace, _request_timeout=settings.AWX_CONTAINER_GROUP_K8S_API_TIMEOUT)
+            except Exception:
+                logger.exception("Failed to delete orphaned pod {} from {}".format(job.log_format, group))
+
+
+@task(queue=get_local_queuename)
+def awx_periodic_scheduler():
+    with advisory_lock('awx_periodic_scheduler_lock', wait=False) as acquired:
+        if acquired is False:
+            logger.debug("Not running periodic scheduler, another task holds lock")
+            return
+        logger.debug("Starting periodic scheduler")
+
+        run_now = now()
+        state = TowerScheduleState.get_solo()
+        last_run = state.schedule_last_run
+        logger.debug("Last scheduler run was: %s", last_run)
+        state.schedule_last_run = run_now
+        state.save()
+
+        old_schedules = Schedule.objects.enabled().before(last_run)
+        for schedule in old_schedules:
+            schedule.update_computed_fields()
+        schedules = Schedule.objects.enabled().between(last_run, run_now)
+
+        invalid_license = False
+        try:
+            access_registry[Job](None).check_license(quiet=True)
+        except PermissionDenied as e:
+            invalid_license = e
+
+        for schedule in schedules:
+            template = schedule.unified_job_template
+            schedule.update_computed_fields()  # To update next_run timestamp.
+            if template.cache_timeout_blocked:
+                logger.warn("Cache timeout is in the future, bypassing schedule for template %s" % str(template.id))
+                continue
+            try:
+                job_kwargs = schedule.get_job_kwargs()
+                new_unified_job = schedule.unified_job_template.create_unified_job(**job_kwargs)
+                logger.debug('Spawned {} from schedule {}-{}.'.format(new_unified_job.log_format, schedule.name, schedule.pk))
+
+                if invalid_license:
+                    new_unified_job.status = 'failed'
+                    new_unified_job.job_explanation = str(invalid_license)
+                    new_unified_job.save(update_fields=['status', 'job_explanation'])
+                    new_unified_job.websocket_emit_status("failed")
+                    raise invalid_license
+                can_start = new_unified_job.signal_start()
+            except Exception:
+                logger.exception('Error spawning scheduled job.')
+                continue
+            if not can_start:
+                new_unified_job.status = 'failed'
+                new_unified_job.job_explanation = gettext_noop(
+                    "Scheduled job could not start because it \
+                    was not in the right state or required manual credentials"
+                )
+                new_unified_job.save(update_fields=['status', 'job_explanation'])
+                new_unified_job.websocket_emit_status("failed")
+            emit_channel_notification('schedules-changed', dict(id=schedule.id, group_name="schedules"))
+        state.save()
+
+
+@task(queue=get_local_queuename)
+def handle_work_success(task_actual):
+    try:
+        instance = UnifiedJob.get_instance_by_type(task_actual['type'], task_actual['id'])
+    except ObjectDoesNotExist:
+        logger.warning('Missing {} `{}` in success callback.'.format(task_actual['type'], task_actual['id']))
+        return
+    if not instance:
+        return
+
+    schedule_task_manager()
+
+
+@task(queue=get_local_queuename)
+def handle_work_error(task_id, *args, **kwargs):
+    subtasks = kwargs.get('subtasks', None)
+    logger.debug('Executing error task id %s, subtasks: %s' % (task_id, str(subtasks)))
+    first_instance = None
+    first_instance_type = ''
+    if subtasks is not None:
+        for each_task in subtasks:
+            try:
+                instance = UnifiedJob.get_instance_by_type(each_task['type'], each_task['id'])
+                if not instance:
+                    # Unknown task type
+                    logger.warn("Unknown task type: {}".format(each_task['type']))
+                    continue
+            except ObjectDoesNotExist:
+                logger.warning('Missing {} `{}` in error callback.'.format(each_task['type'], each_task['id']))
+                continue
+
+            if first_instance is None:
+                first_instance = instance
+                first_instance_type = each_task['type']
+
+            if instance.celery_task_id != task_id and not instance.cancel_flag and not instance.status == 'successful':
+                instance.status = 'failed'
+                instance.failed = True
+                if not instance.job_explanation:
+                    instance.job_explanation = 'Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (
+                        first_instance_type,
+                        first_instance.name,
+                        first_instance.id,
+                    )
+                instance.save()
+                instance.websocket_emit_status("failed")
+
+    # We only send 1 job complete message since all the job completion message
+    # handling does is trigger the scheduler. If we extend the functionality of
+    # what the job complete message handler does then we may want to send a
+    # completion event for each job here.
+    if first_instance:
+        schedule_task_manager()
+        pass
+
+
+@task(queue=get_local_queuename)
+def handle_success_and_failure_notifications(job_id):
+    uj = UnifiedJob.objects.get(pk=job_id)
+    retries = 0
+    while retries < 5:
+        if uj.finished:
+            uj.send_notification_templates('succeeded' if uj.status == 'successful' else 'failed')
+            return
+        else:
+            # wait a few seconds to avoid a race where the
+            # events are persisted _before_ the UJ.status
+            # changes from running -> successful
+            retries += 1
+            time.sleep(1)
+            uj = UnifiedJob.objects.get(pk=job_id)
+
+    logger.warn(f"Failed to even try to send notifications for job '{uj}' due to job not being in finished state.")
+
+
+@task(queue=get_local_queuename)
+def update_inventory_computed_fields(inventory_id):
+    """
+    Signal handler and wrapper around inventory.update_computed_fields to
+    prevent unnecessary recursive calls.
+    """
+    i = Inventory.objects.filter(id=inventory_id)
+    if not i.exists():
+        logger.error("Update Inventory Computed Fields failed due to missing inventory: " + str(inventory_id))
+        return
+    i = i[0]
+    try:
+        i.update_computed_fields()
+    except DatabaseError as e:
+        if 'did not affect any rows' in str(e):
+            logger.debug('Exiting duplicate update_inventory_computed_fields task.')
+            return
+        raise
+
+
+def update_smart_memberships_for_inventory(smart_inventory):
+    current = set(SmartInventoryMembership.objects.filter(inventory=smart_inventory).values_list('host_id', flat=True))
+    new = set(smart_inventory.hosts.values_list('id', flat=True))
+    additions = new - current
+    removals = current - new
+    if additions or removals:
+        with transaction.atomic():
+            if removals:
+                SmartInventoryMembership.objects.filter(inventory=smart_inventory, host_id__in=removals).delete()
+            if additions:
+                add_for_inventory = [SmartInventoryMembership(inventory_id=smart_inventory.id, host_id=host_id) for host_id in additions]
+                SmartInventoryMembership.objects.bulk_create(add_for_inventory, ignore_conflicts=True)
+        logger.debug(
+            'Smart host membership cached for {}, {} additions, {} removals, {} total count.'.format(
+                smart_inventory.pk, len(additions), len(removals), len(new)
+            )
+        )
+        return True  # changed
+    return False
+
+
+@task(queue=get_local_queuename)
+def update_host_smart_inventory_memberships():
+    smart_inventories = Inventory.objects.filter(kind='smart', host_filter__isnull=False, pending_deletion=False)
+    changed_inventories = set([])
+    for smart_inventory in smart_inventories:
+        try:
+            changed = update_smart_memberships_for_inventory(smart_inventory)
+            if changed:
+                changed_inventories.add(smart_inventory)
+        except IntegrityError:
+            logger.exception('Failed to update smart inventory memberships for {}'.format(smart_inventory.pk))
+    # Update computed fields for changed inventories outside atomic action
+    for smart_inventory in changed_inventories:
+        smart_inventory.update_computed_fields()
+
+
+@task(queue=get_local_queuename)
+def delete_inventory(inventory_id, user_id, retries=5):
+    # Delete inventory as user
+    if user_id is None:
+        user = None
+    else:
+        try:
+            user = User.objects.get(id=user_id)
+        except Exception:
+            user = None
+    with ignore_inventory_computed_fields(), ignore_inventory_group_removal(), impersonate(user):
+        try:
+            i = Inventory.objects.get(id=inventory_id)
+            for host in i.hosts.iterator():
+                host.job_events_as_primary_host.update(host=None)
+            i.delete()
+            emit_channel_notification('inventories-status_changed', {'group_name': 'inventories', 'inventory_id': inventory_id, 'status': 'deleted'})
+            logger.debug('Deleted inventory {} as user {}.'.format(inventory_id, user_id))
+        except Inventory.DoesNotExist:
+            logger.exception("Delete Inventory failed due to missing inventory: " + str(inventory_id))
+            return
+        except DatabaseError:
+            logger.exception('Database error deleting inventory {}, but will retry.'.format(inventory_id))
+            if retries > 0:
+                time.sleep(10)
+                delete_inventory(inventory_id, user_id, retries=retries - 1)
+
+
+def with_path_cleanup(f):
+    @functools.wraps(f)
+    def _wrapped(self, *args, **kwargs):
+        try:
+            return f(self, *args, **kwargs)
+        finally:
+            for p in self.cleanup_paths:
+                try:
+                    if os.path.isdir(p):
+                        shutil.rmtree(p, ignore_errors=True)
+                    elif os.path.exists(p):
+                        os.remove(p)
+                except OSError:
+                    logger.exception("Failed to remove tmp file: {}".format(p))
+            self.cleanup_paths = []
+
+    return _wrapped
+
+
+def _reconstruct_relationships(copy_mapping):
+    for old_obj, new_obj in copy_mapping.items():
+        model = type(old_obj)
+        for field_name in getattr(model, 'FIELDS_TO_PRESERVE_AT_COPY', []):
+            field = model._meta.get_field(field_name)
+            if isinstance(field, ForeignKey):
+                if getattr(new_obj, field_name, None):
+                    continue
+                related_obj = getattr(old_obj, field_name)
+                related_obj = copy_mapping.get(related_obj, related_obj)
+                setattr(new_obj, field_name, related_obj)
+            elif field.many_to_many:
+                for related_obj in getattr(old_obj, field_name).all():
+                    logger.debug('Deep copy: Adding {} to {}({}).{} relationship'.format(related_obj, new_obj, model, field_name))
+                    getattr(new_obj, field_name).add(copy_mapping.get(related_obj, related_obj))
+        new_obj.save()
+
+
+@task(queue=get_local_queuename)
+def deep_copy_model_obj(model_module, model_name, obj_pk, new_obj_pk, user_pk, uuid, permission_check_func=None):
+    sub_obj_list = cache.get(uuid)
+    if sub_obj_list is None:
+        logger.error('Deep copy {} from {} to {} failed unexpectedly.'.format(model_name, obj_pk, new_obj_pk))
+        return
+
+    logger.debug('Deep copy {} from {} to {}.'.format(model_name, obj_pk, new_obj_pk))
+    from awx.api.generics import CopyAPIView
+    from awx.main.signals import disable_activity_stream
+
+    model = getattr(importlib.import_module(model_module), model_name, None)
+    if model is None:
+        return
+    try:
+        obj = model.objects.get(pk=obj_pk)
+        new_obj = model.objects.get(pk=new_obj_pk)
+        creater = User.objects.get(pk=user_pk)
+    except ObjectDoesNotExist:
+        logger.warning("Object or user no longer exists.")
+        return
+    with transaction.atomic(), ignore_inventory_computed_fields(), disable_activity_stream():
+        copy_mapping = {}
+        for sub_obj_setup in sub_obj_list:
+            sub_model = getattr(importlib.import_module(sub_obj_setup[0]), sub_obj_setup[1], None)
+            if sub_model is None:
+                continue
+            try:
+                sub_obj = sub_model.objects.get(pk=sub_obj_setup[2])
+            except ObjectDoesNotExist:
+                continue
+            copy_mapping.update(CopyAPIView.copy_model_obj(obj, new_obj, sub_model, sub_obj, creater))
+        _reconstruct_relationships(copy_mapping)
+        if permission_check_func:
+            permission_check_func = getattr(getattr(importlib.import_module(permission_check_func[0]), permission_check_func[1]), permission_check_func[2])
+            permission_check_func(creater, copy_mapping.values())
+    if isinstance(new_obj, Inventory):
+        update_inventory_computed_fields.delay(new_obj.id)

awx/main/tests/conftest.py

@@ -84,6 +84,11 @@ def default_instance_group(instance_factory, instance_group_factory):
     return create_instance_group("default", instances=[create_instance("hostA")])
 
 
+@pytest.fixture
+def controlplane_instance_group(instance_factory, instance_group_factory):
+    return create_instance_group("controlplane", instances=[create_instance("hostA")])
+
+
 @pytest.fixture
 def job_template_with_survey_passwords_factory(job_template_factory):
     def rf(persisted):

awx/main/tests/functional/api/test_instance.py

@@ -0,0 +1,82 @@
+import pytest
+
+from unittest import mock
+
+from awx.api.versioning import reverse
+from awx.main.models.activity_stream import ActivityStream
+from awx.main.models.ha import Instance
+
+import redis
+
+# Django
+from django.test.utils import override_settings
+
+
+INSTANCE_KWARGS = dict(hostname='example-host', cpu=6, memory=36000000000, cpu_capacity=6, mem_capacity=42)
+
+
+@pytest.mark.django_db
+def test_disabled_zeros_capacity(patch, admin_user):
+    instance = Instance.objects.create(**INSTANCE_KWARGS)
+    assert ActivityStream.objects.filter(instance=instance).count() == 1
+
+    url = reverse('api:instance_detail', kwargs={'pk': instance.pk})
+
+    r = patch(url=url, data={'enabled': False}, user=admin_user)
+    assert r.data['capacity'] == 0
+
+    instance.refresh_from_db()
+    assert instance.capacity == 0
+    assert ActivityStream.objects.filter(instance=instance).count() == 2
+
+
+@pytest.mark.django_db
+def test_enabled_sets_capacity(patch, admin_user):
+    instance = Instance.objects.create(enabled=False, capacity=0, **INSTANCE_KWARGS)
+    assert instance.capacity == 0
+    assert ActivityStream.objects.filter(instance=instance).count() == 1
+
+    url = reverse('api:instance_detail', kwargs={'pk': instance.pk})
+
+    r = patch(url=url, data={'enabled': True}, user=admin_user)
+    assert r.data['capacity'] > 0
+
+    instance.refresh_from_db()
+    assert instance.capacity > 0
+    assert ActivityStream.objects.filter(instance=instance).count() == 2
+
+
+@pytest.mark.django_db
+def test_auditor_user_health_check(get, post, system_auditor):
+    instance = Instance.objects.create(**INSTANCE_KWARGS)
+    url = reverse('api:instance_health_check', kwargs={'pk': instance.pk})
+    r = get(url=url, user=system_auditor, expect=200)
+    assert r.data['cpu_capacity'] == instance.cpu_capacity
+    post(url=url, user=system_auditor, expect=403)
+
+
+@pytest.mark.django_db
+def test_health_check_throws_error(post, admin_user):
+    instance = Instance.objects.create(node_type='execution', **INSTANCE_KWARGS)
+    url = reverse('api:instance_health_check', kwargs={'pk': instance.pk})
+    # we will simulate a receptor error, similar to this one
+    # https://github.com/ansible/receptor/blob/156e6e24a49fbf868734507f9943ac96208ed8f5/receptorctl/receptorctl/socket_interface.py#L204
+    # related to issue https://github.com/ansible/tower/issues/5315
+    with mock.patch('awx.main.tasks.receptor.run_until_complete', side_effect=RuntimeError('Remote error: foobar')):
+        post(url=url, user=admin_user, expect=200)
+    instance.refresh_from_db()
+    assert 'Remote error: foobar' in instance.errors
+    assert instance.capacity == 0
+
+
+@pytest.mark.django_db
+@mock.patch.object(redis.client.Redis, 'ping', lambda self: True)
+def test_health_check_usage(get, post, admin_user):
+    instance = Instance.objects.create(**INSTANCE_KWARGS)
+    url = reverse('api:instance_health_check', kwargs={'pk': instance.pk})
+    r = get(url=url, user=admin_user, expect=200)
+    assert r.data['cpu_capacity'] == instance.cpu_capacity
+    assert r.data['last_health_check'] is None
+    with override_settings(CLUSTER_HOST_ID=instance.hostname):  # force direct call of cluster_node_health_check
+        r = post(url=url, user=admin_user, expect=200)
+        assert r.data['last_health_check'] is not None

awx/main/tests/functional/api/test_instance_group.py

@@ -4,6 +4,7 @@ import pytest
 
 from awx.api.versioning import reverse
 from awx.main.models import (
+    ActivityStream,
     Instance,
     InstanceGroup,
     ProjectUpdate,
@@ -23,6 +24,14 @@ def instance():
     return Instance.objects.create(hostname='instance')
 
 
+@pytest.fixture
+def node_type_instance():
+    def fn(hostname, node_type):
+        return Instance.objects.create(hostname=hostname, node_type=node_type)
+
+    return fn
+
+
 @pytest.fixture
 def instance_group(job_factory):
     ig = InstanceGroup(name="east")
@@ -198,3 +207,97 @@ def test_containerized_group_default_fields(instance_group, kube_credential):
     assert ig.policy_instance_list == []
     assert ig.policy_instance_minimum == 0
     assert ig.policy_instance_percentage == 0
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize('node_type', ['control', 'hybrid', 'execution'])
+def test_instance_attach_to_instance_group(post, instance_group, node_type_instance, admin, node_type):
+    instance = node_type_instance(hostname=node_type, node_type=node_type)
+
+    count = ActivityStream.objects.count()
+
+    url = reverse(f'api:instance_group_instance_list', kwargs={'pk': instance_group.pk})
+    post(url, {'associate': True, 'id': instance.id}, admin, expect=204 if node_type != 'control' else 400)
+
+    new_activity = ActivityStream.objects.all()[count:]
+    if node_type != 'control':
+        assert len(new_activity) == 2  # the second is an update of the instance group policy
+        new_activity = new_activity[0]
+        assert new_activity.operation == 'associate'
+        assert new_activity.object1 == 'instance_group'
+        assert new_activity.object2 == 'instance'
+        assert new_activity.instance.first() == instance
+        assert new_activity.instance_group.first() == instance_group
+    else:
+        assert not new_activity
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize('node_type', ['control', 'hybrid', 'execution'])
+def test_instance_unattach_from_instance_group(post, instance_group, node_type_instance, admin, node_type):
+    instance = node_type_instance(hostname=node_type, node_type=node_type)
+    instance_group.instances.add(instance)
+
+    count = ActivityStream.objects.count()
+
+    url = reverse(f'api:instance_group_instance_list', kwargs={'pk': instance_group.pk})
+    post(url, {'disassociate': True, 'id': instance.id}, admin, expect=204 if node_type != 'control' else 400)
+
+    new_activity = ActivityStream.objects.all()[count:]
+    if node_type != 'control':
+        assert len(new_activity) == 1
+        new_activity = new_activity[0]
+        assert new_activity.operation == 'disassociate'
+        assert new_activity.object1 == 'instance_group'
+        assert new_activity.object2 == 'instance'
+        assert new_activity.instance.first() == instance
+        assert new_activity.instance_group.first() == instance_group
+    else:
+        assert not new_activity
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize('node_type', ['control', 'hybrid', 'execution'])
+def test_instance_group_attach_to_instance(post, instance_group, node_type_instance, admin, node_type):
+    instance = node_type_instance(hostname=node_type, node_type=node_type)
+
+    count = ActivityStream.objects.count()
+
+    url = reverse(f'api:instance_instance_groups_list', kwargs={'pk': instance.pk})
+    post(url, {'associate': True, 'id': instance_group.id}, admin, expect=204 if node_type != 'control' else 400)
+
+    new_activity = ActivityStream.objects.all()[count:]
+    if node_type != 'control':
+        assert len(new_activity) == 2  # the second is an update of the instance group policy
+        new_activity = new_activity[0]
+        assert new_activity.operation == 'associate'
+        assert new_activity.object1 == 'instance'
+        assert new_activity.object2 == 'instance_group'
+        assert new_activity.instance.first() == instance
+        assert new_activity.instance_group.first() == instance_group
+    else:
+        assert not new_activity
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize('node_type', ['control', 'hybrid', 'execution'])
+def test_instance_group_unattach_from_instance(post, instance_group, node_type_instance, admin, node_type):
+    instance = node_type_instance(hostname=node_type, node_type=node_type)
+    instance_group.instances.add(instance)
+
+    count = ActivityStream.objects.count()
+
+    url = reverse(f'api:instance_instance_groups_list', kwargs={'pk': instance.pk})
+    post(url, {'disassociate': True, 'id': instance_group.id}, admin, expect=204 if node_type != 'control' else 400)
+
+    new_activity = ActivityStream.objects.all()[count:]
+    if node_type != 'control':
+        assert len(new_activity) == 1
+        new_activity = new_activity[0]
+        assert new_activity.operation == 'disassociate'
+        assert new_activity.object1 == 'instance'
+        assert new_activity.object2 == 'instance_group'
+        assert new_activity.instance.first() == instance
+        assert new_activity.instance_group.first() == instance_group
+    else:
+        assert not new_activity

awx/main/tests/functional/commands/test_inventory_import.py

@@ -5,6 +5,7 @@
 import pytest
 from unittest import mock
 import os
+import yaml
 
 # Django
 from django.core.management.base import CommandError
@@ -52,6 +53,110 @@ def mock_logging(self, level):
     pass
 
 
+@pytest.mark.django_db
+@mock.patch.object(inventory_import.Command, 'set_logging_level', mock_logging)
+class TestMigrationCases:
+    """In the case that we have any bugs with the declared instance ID variables
+    then it is inevitable that we will, at some point, import a host with a blank ID
+    and then later import it with the correct id.
+    """
+
+    @pytest.mark.parametrize('id_var', ('', 'foo.id', 'foo.id,other', 'other,foo.id'), ids=['none', 'simple', 'complex', 'backward'])
+    @pytest.mark.parametrize('host_name', ('host-1', 'fooval'), ids=['arbitrary', 'id'])
+    @pytest.mark.parametrize('has_var', (True, False))
+    def test_single_host_not_recreated(self, inventory, id_var, host_name, has_var):
+        inv_src = InventorySource.objects.create(inventory=inventory, source='gce')
+
+        options = dict(overwrite=True, instance_id_var=id_var)
+
+        vars = {'foo': {'id': 'fooval'}}
+        data = {
+            '_meta': {'hostvars': {host_name: vars if has_var else {'unrelated': 'value'}}},
+            "ungrouped": {"hosts": [host_name]},
+        }
+        old_id = None
+
+        for i in range(3):
+            inventory_import.Command().perform_update(options.copy(), data.copy(), inv_src.create_unified_job())
+
+            assert inventory.hosts.count() == inv_src.hosts.count() == 1
+            host = inventory.hosts.first()
+            assert host.name == host_name
+            assert host.instance_id in ('fooval', '')
+            if has_var:
+                assert yaml.safe_load(host.variables) == vars
+            else:
+                assert yaml.safe_load(host.variables) == {'unrelated': 'value'}
+
+            if old_id is not None:
+                assert host.id == old_id
+            old_id = host.id
+
+    @pytest.mark.parametrize('id_var_seq', [('', 'foo.id,other'), ('foo.id,other', '')], ids=['gained', 'lost'])  # second is problem case
+    @pytest.mark.parametrize('host_name', ('host-1', 'fooval'), ids=['arbitrary', 'id'])
+    def test_host_gains_or_loses_instance_id(self, inventory, id_var_seq, host_name):
+        inv_src = InventorySource.objects.create(inventory=inventory, source='gce')
+
+        options = dict(overwrite=True)
+
+        vars = {'foo': {'id': 'fooval'}}
+        old_id = None
+
+        for id_var in id_var_seq:
+            options['instance_id_var'] = id_var
+            data = {
+                '_meta': {'hostvars': {host_name: vars}},
+                "ungrouped": {"hosts": [host_name]},
+            }
+            inventory_import.Command().perform_update(options.copy(), data.copy(), inv_src.create_unified_job())
+
+            assert inventory.hosts.count() == inv_src.hosts.count() == 1
+            host = inventory.hosts.first()
+            assert host.name == host_name
+            assert host.instance_id == ('fooval' if id_var else '')
+            assert yaml.safe_load(host.variables) == vars
+
+            if old_id is not None:
+                assert host.id == old_id
+            old_id = host.id
+
+    @pytest.mark.parametrize('second_list', [('host-1', 'fooval'), ('host-1',), ('fooval',)])
+    def test_name_and_id_confusion(self, inventory, second_list):
+        inv_src = InventorySource.objects.create(inventory=inventory, source='gce')
+
+        CASES = [('', ['host-1', 'fooval']), ('foo.id', second_list)]
+
+        options = dict(overwrite=True)
+
+        vars = {'foo': {'id': 'fooval'}}
+        data = {
+            '_meta': {'hostvars': {}},
+            "ungrouped": {"hosts": []},
+        }
+        id_set = None
+
+        for id_var, hosts in CASES:
+            options['instance_id_var'] = id_var
+
+            data['_meta']['hostvars'] = {}
+            for host_name in hosts:
+                data['_meta']['hostvars'][host_name] = vars if id_var else {}
+            data['ungrouped']['hosts'] = hosts
+
+            inventory_import.Command().perform_update(options.copy(), data.copy(), inv_src.create_unified_job())
+
+            new_ids = set(inventory.hosts.values_list('id', flat=True))
+            if id_set is not None:
+                assert not (new_ids - id_set)
+            id_set = new_ids
+
+            assert inventory.hosts.count() == len(hosts), [(host.name, host.instance_id) for host in inventory.hosts.all()]
+            assert inv_src.hosts.count() == len(hosts), [(host.name, host.instance_id) for host in inventory.hosts.all()]
+            for host_name in hosts:
+                host = inventory.hosts.get(name=host_name)
+                assert host.instance_id == ('fooval' if id_var else '')
+
+
 @pytest.mark.django_db
 @pytest.mark.inventory_import
 @mock.patch.object(inventory_import.Command, 'check_license', mock.MagicMock())
@@ -89,7 +194,7 @@ class TestINIImports:
     def test_inventory_single_ini_import(self, inventory, capsys):
         inventory_import.AnsibleInventoryLoader._data = TEST_INVENTORY_CONTENT
         cmd = inventory_import.Command()
-        r = cmd.handle(inventory_id=inventory.pk, source=__file__, method='backport')
+        r = cmd.handle(inventory_id=inventory.pk, source=__file__)
         out, err = capsys.readouterr()
         assert r is None
         assert out == ''

awx/main/tests/functional/commands/test_register_queue.py

@@ -0,0 +1,26 @@
+from io import StringIO
+from contextlib import redirect_stdout
+
+import pytest
+
+from awx.main.management.commands.register_queue import RegisterQueue
+from awx.main.models.ha import InstanceGroup
+
+
+@pytest.mark.django_db
+def test_openshift_idempotence():
+    def perform_register():
+        with StringIO() as buffer:
+            with redirect_stdout(buffer):
+                RegisterQueue('default', 100, 0, [], is_container_group=True).register()
+                return buffer.getvalue()
+
+    assert '(changed: True)' in perform_register()
+    assert '(changed: True)' not in perform_register()
+    assert '(changed: True)' not in perform_register()
+
+    ig = InstanceGroup.objects.get(name='default')
+    assert ig.policy_instance_percentage == 100
+    assert ig.policy_instance_minimum == 0
+    assert ig.policy_instance_list == []
+    assert ig.is_container_group is True

awx/main/tests/functional/conftest.py

@@ -121,7 +121,7 @@ def run_computed_fields_right_away(request):
 
 @pytest.fixture
 @mock.patch.object(Project, "update", lambda self, **kwargs: None)
-def project(instance, organization):
+def project(organization):
     prj = Project.objects.create(
         name="test-proj",
         description="test-proj-desc",
@@ -136,7 +136,7 @@ def project(instance, organization):
 
 @pytest.fixture
 @mock.patch.object(Project, "update", lambda self, **kwargs: None)
-def manual_project(instance, organization):
+def manual_project(organization):
     prj = Project.objects.create(
         name="test-manual-proj",
         description="manual-proj-desc",
@@ -196,7 +196,7 @@ def instance(settings):
 
 
 @pytest.fixture
-def organization(instance):
+def organization():
     return Organization.objects.create(name="test-org", description="test-org-desc")
 
 

awx/main/tests/functional/models/test_activity_stream.py

@@ -170,7 +170,7 @@ def test_activity_stream_actor(admin_user):
 
 
 @pytest.mark.django_db
-def test_annon_user_action():
+def test_anon_user_action():
     with mock.patch('awx.main.signals.get_current_user') as u_mock:
         u_mock.return_value = AnonymousUser()
         inv = Inventory.objects.create(name='ainventory')

awx/main/tests/functional/models/test_execution_environment.py

@@ -0,0 +1,46 @@
+import pytest
+
+from awx.main.models.execution_environments import ExecutionEnvironment
+
+
+@pytest.fixture
+def cleanup_patch(mocker):
+    return mocker.patch('awx.main.signals.handle_removed_image')
+
+
+@pytest.mark.django_db
+def test_image_unchanged_no_delete_task(cleanup_patch):
+    """When an irrelevant EE field is changed, we do not run the image cleanup task"""
+    execution_environment = ExecutionEnvironment.objects.create(name='test-ee', image='quay.io/foo/bar')
+    execution_environment.description = 'foobar'
+    execution_environment.save()
+
+    cleanup_patch.delay.assert_not_called()
+
+
+@pytest.mark.django_db
+def test_image_changed_creates_delete_task(cleanup_patch):
+    execution_environment = ExecutionEnvironment.objects.create(name='test-ee', image='quay.io/foo/bar')
+    execution_environment.image = 'quay.io/new/image'
+    execution_environment.save()
+
+    cleanup_patch.delay.assert_called_once_with(remove_images=['quay.io/foo/bar'])
+
+
+@pytest.mark.django_db
+def test_image_still_in_use(cleanup_patch):
+    """When an image is still in use by another EE, we do not clean it up"""
+    ExecutionEnvironment.objects.create(name='unrelated-ee', image='quay.io/foo/bar')
+    execution_environment = ExecutionEnvironment.objects.create(name='test-ee', image='quay.io/foo/bar')
+    execution_environment.image = 'quay.io/new/image'
+    execution_environment.save()
+
+    cleanup_patch.delay.assert_not_called()
+
+
+@pytest.mark.django_db
+def test_image_deletion_creates_delete_task(cleanup_patch):
+    execution_environment = ExecutionEnvironment.objects.create(name='test-ee', image='quay.io/foo/bar')
+    execution_environment.delete()
+
+    cleanup_patch.delay.assert_called_once_with(remove_images=['quay.io/foo/bar'])

awx/main/tests/functional/task_management/test_container_groups.py

@@ -5,7 +5,7 @@ from collections import namedtuple
 from unittest import mock  # noqa
 import pytest
 
-from awx.main.tasks import AWXReceptorJob
+from awx.main.tasks.receptor import AWXReceptorJob
 from awx.main.utils import (
     create_temporary_fifo,
 )

awx/main/tests/functional/task_management/test_rampart_groups.py

@@ -3,7 +3,7 @@ from unittest import mock
 from datetime import timedelta
 from awx.main.scheduler import TaskManager
 from awx.main.models import InstanceGroup, WorkflowJob
-from awx.main.tasks import apply_cluster_membership_policies
+from awx.main.tasks.system import apply_cluster_membership_policies
 
 
 @pytest.mark.django_db
@@ -30,7 +30,7 @@ def test_multi_group_basic_job_launch(instance_factory, default_instance_group,
 
 
 @pytest.mark.django_db
-def test_multi_group_with_shared_dependency(instance_factory, default_instance_group, mocker, instance_group_factory, job_template_factory):
+def test_multi_group_with_shared_dependency(instance_factory, controlplane_instance_group, mocker, instance_group_factory, job_template_factory):
     i1 = instance_factory("i1")
     i2 = instance_factory("i2")
     ig1 = instance_group_factory("ig1", instances=[i1])
@@ -54,7 +54,7 @@ def test_multi_group_with_shared_dependency(instance_factory, default_instance_g
     with mocker.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
         pu = p.project_updates.first()
-        TaskManager.start_task.assert_called_once_with(pu, default_instance_group, [j1, j2], default_instance_group.instances.all()[0])
+        TaskManager.start_task.assert_called_once_with(pu, controlplane_instance_group, [j1, j2], controlplane_instance_group.instances.all()[0])
         pu.finished = pu.created + timedelta(seconds=1)
         pu.status = "successful"
         pu.save()

awx/main/tests/functional/task_management/test_scheduler.py

@@ -7,6 +7,7 @@ from awx.main.scheduler import TaskManager
 from awx.main.scheduler.dependency_graph import DependencyGraph
 from awx.main.utils import encrypt_field
 from awx.main.models import WorkflowJobTemplate, JobTemplate, Job
+from awx.main.models.ha import Instance, InstanceGroup
 
 
 @pytest.mark.django_db
@@ -99,6 +100,48 @@ class TestJobLifeCycle:
                 self.run_tm(tm, expect_schedule=[mock.call()])
             wfjts[0].refresh_from_db()
 
+    @pytest.fixture
+    def control_instance(self):
+        '''Control instance in the controlplane automatic IG'''
+        ig = InstanceGroup.objects.create(name='controlplane')
+        inst = Instance.objects.create(hostname='control-1', node_type='control', capacity=500)
+        ig.instances.add(inst)
+        return inst
+
+    @pytest.fixture
+    def execution_instance(self):
+        '''Execution node in the automatic default IG'''
+        ig = InstanceGroup.objects.create(name='default')
+        inst = Instance.objects.create(hostname='receptor-1', node_type='execution', capacity=500)
+        ig.instances.add(inst)
+        return inst
+
+    def test_control_and_execution_instance(self, project, system_job_template, job_template, inventory_source, control_instance, execution_instance):
+        assert Instance.objects.count() == 2
+
+        pu = project.create_unified_job()
+        sj = system_job_template.create_unified_job()
+        job = job_template.create_unified_job()
+        inv_update = inventory_source.create_unified_job()
+
+        all_ujs = (pu, sj, job, inv_update)
+        for uj in all_ujs:
+            uj.signal_start()
+
+        tm = TaskManager()
+        self.run_tm(tm)
+
+        for uj in all_ujs:
+            uj.refresh_from_db()
+            assert uj.status == 'waiting'
+
+        for uj in (pu, sj):  # control plane jobs
+            assert uj.capacity_type == 'control'
+            assert [uj.execution_node, uj.controller_node] == [control_instance.hostname, control_instance.hostname], uj
+        for uj in (job, inv_update):  # user-space jobs
+            assert uj.capacity_type == 'execution'
+            assert [uj.execution_node, uj.controller_node] == [execution_instance.hostname, control_instance.hostname], uj
+
 
 @pytest.mark.django_db
 def test_single_jt_multi_job_launch_blocks_last(default_instance_group, job_template_factory, mocker):
@@ -169,9 +212,9 @@ def test_multi_jt_capacity_blocking(default_instance_group, job_template_factory
 
 
 @pytest.mark.django_db
-def test_single_job_dependencies_project_launch(default_instance_group, job_template_factory, mocker):
+def test_single_job_dependencies_project_launch(controlplane_instance_group, job_template_factory, mocker):
     objects = job_template_factory('jt', organization='org1', project='proj', inventory='inv', credential='cred', jobs=["job_should_start"])
-    instance = default_instance_group.instances.all()[0]
+    instance = controlplane_instance_group.instances.all()[0]
     j = objects.jobs["job_should_start"]
     j.status = 'pending'
     j.save()
@@ -188,12 +231,12 @@ def test_single_job_dependencies_project_launch(default_instance_group, job_temp
             mock_pu.assert_called_once_with(j)
             pu = [x for x in p.project_updates.all()]
             assert len(pu) == 1
-            TaskManager.start_task.assert_called_once_with(pu[0], default_instance_group, [j], instance)
+            TaskManager.start_task.assert_called_once_with(pu[0], controlplane_instance_group, [j], instance)
             pu[0].status = "successful"
             pu[0].save()
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
-        TaskManager.start_task.assert_called_once_with(j, default_instance_group, [], instance)
+        TaskManager.start_task.assert_called_once_with(j, controlplane_instance_group, [], instance)
 
 
 @pytest.mark.django_db
@@ -254,8 +297,8 @@ def test_job_dependency_with_already_updated(default_instance_group, job_templat
 
 
 @pytest.mark.django_db
-def test_shared_dependencies_launch(default_instance_group, job_template_factory, mocker, inventory_source_factory):
-    instance = default_instance_group.instances.all()[0]
+def test_shared_dependencies_launch(controlplane_instance_group, job_template_factory, mocker, inventory_source_factory):
+    instance = controlplane_instance_group.instances.all()[0]
     objects = job_template_factory('jt', organization='org1', project='proj', inventory='inv', credential='cred', jobs=["first_job", "second_job"])
     j1 = objects.jobs["first_job"]
     j1.status = 'pending'
@@ -283,7 +326,7 @@ def test_shared_dependencies_launch(default_instance_group, job_template_factory
         pu = p.project_updates.first()
         iu = ii.inventory_updates.first()
         TaskManager.start_task.assert_has_calls(
-            [mock.call(iu, default_instance_group, [j1, j2, pu], instance), mock.call(pu, default_instance_group, [j1, j2, iu], instance)]
+            [mock.call(iu, controlplane_instance_group, [j1, j2, pu], instance), mock.call(pu, controlplane_instance_group, [j1, j2, iu], instance)]
         )
         pu.status = "successful"
         pu.finished = pu.created + timedelta(seconds=1)
@@ -293,12 +336,12 @@ def test_shared_dependencies_launch(default_instance_group, job_template_factory
         iu.save()
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
-        TaskManager.start_task.assert_called_once_with(j1, default_instance_group, [], instance)
+        TaskManager.start_task.assert_called_once_with(j1, controlplane_instance_group, [], instance)
         j1.status = "successful"
         j1.save()
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
-        TaskManager.start_task.assert_called_once_with(j2, default_instance_group, [], instance)
+        TaskManager.start_task.assert_called_once_with(j2, controlplane_instance_group, [], instance)
     pu = [x for x in p.project_updates.all()]
     iu = [x for x in ii.inventory_updates.all()]
     assert len(pu) == 1

awx/main/tests/functional/test_api_generics.py

@@ -0,0 +1,26 @@
+import pytest
+
+from django.test.utils import override_settings
+from awx.api.versioning import reverse
+
+
+@pytest.mark.django_db
+def test_change_400_error_log(caplog, post, admin_user):
+    with override_settings(API_400_ERROR_LOG_FORMAT='Test'):
+        post(url=reverse('api:setting_logging_test'), data={}, user=admin_user, expect=409)
+        assert 'Test' in caplog.text
+
+
+@pytest.mark.django_db
+def test_bad_400_error_log(caplog, post, admin_user):
+    with override_settings(API_400_ERROR_LOG_FORMAT="Not good {junk}"):
+        post(url=reverse('api:setting_logging_test'), data={}, user=admin_user, expect=409)
+        assert "Unable to format API_400_ERROR_LOG_FORMAT setting, defaulting log message: 'junk'" in caplog.text
+        assert 'status 409 received by user admin attempting to access /api/v2/settings/logging/test/ from 127.0.0.1' in caplog.text
+
+
+@pytest.mark.django_db
+def test_custom_400_error_log(caplog, post, admin_user):
+    with override_settings(API_400_ERROR_LOG_FORMAT="{status_code} {error}"):
+        post(url=reverse('api:setting_logging_test'), data={}, user=admin_user, expect=409)
+        assert '409 Logging not enabled' in caplog.text

awx/main/tests/functional/test_copy.py

@@ -5,7 +5,7 @@ from awx.api.versioning import reverse
 from awx.main.utils import decrypt_field
 from awx.main.models.workflow import WorkflowJobTemplate, WorkflowJobTemplateNode, WorkflowApprovalTemplate
 from awx.main.models.jobs import JobTemplate
-from awx.main.tasks import deep_copy_model_obj
+from awx.main.tasks.system import deep_copy_model_obj
 
 
 @pytest.mark.django_db

awx/main/tests/functional/test_ha.py

@@ -0,0 +1,19 @@
+import pytest
+
+# AWX
+from awx.main.ha import is_ha_environment
+from awx.main.models.ha import Instance
+
+
+@pytest.mark.django_db
+def test_multiple_instances():
+    for i in range(2):
+        Instance.objects.create(hostname=f'foo{i}', node_type='hybrid')
+    assert is_ha_environment()
+
+
+@pytest.mark.django_db
+def test_db_localhost():
+    Instance.objects.create(hostname='foo', node_type='hybrid')
+    Instance.objects.create(hostname='bar', node_type='execution')
+    assert is_ha_environment() is False

awx/main/tests/functional/test_instances.py

@@ -2,8 +2,9 @@ import pytest
 from unittest import mock
 
 from awx.main.models import AdHocCommand, InventoryUpdate, JobTemplate, ProjectUpdate
+from awx.main.models.activity_stream import ActivityStream
 from awx.main.models.ha import Instance, InstanceGroup
-from awx.main.tasks import apply_cluster_membership_policies
+from awx.main.tasks.system import apply_cluster_membership_policies
 from awx.api.versioning import reverse
 
 from django.utils.timezone import now
@@ -68,22 +69,24 @@ class TestPolicyTaskScheduling:
 
 
 @pytest.mark.django_db
-def test_instance_dup(org_admin, organization, project, instance_factory, instance_group_factory, get, system_auditor):
+def test_instance_dup(org_admin, organization, project, instance_factory, instance_group_factory, get, system_auditor, instance):
     i1 = instance_factory("i1")
     i2 = instance_factory("i2")
     i3 = instance_factory("i3")
+
     ig_all = instance_group_factory("all", instances=[i1, i2, i3])
     ig_dup = instance_group_factory("duplicates", instances=[i1])
     project.organization.instance_groups.add(ig_all, ig_dup)
-    actual_num_instances = Instance.objects.active_count()
+    actual_num_instances = Instance.objects.count()
     list_response = get(reverse('api:instance_list'), user=system_auditor)
     api_num_instances_auditor = list(list_response.data.items())[0][1]
 
     list_response2 = get(reverse('api:instance_list'), user=org_admin)
     api_num_instances_oa = list(list_response2.data.items())[0][1]
 
-    assert actual_num_instances == api_num_instances_auditor
-    # Note: The org_admin will not see the default 'tower' node because it is not in it's group, as expected
+    assert api_num_instances_auditor == actual_num_instances
+    # Note: The org_admin will not see the default 'tower' node
+    # (instance fixture) because it is not in its group, as expected
     assert api_num_instances_oa == (actual_num_instances - 1)
 
 
@@ -94,7 +97,13 @@ def test_policy_instance_few_instances(instance_factory, instance_group_factory)
     ig_2 = instance_group_factory("ig2", percentage=25)
     ig_3 = instance_group_factory("ig3", percentage=25)
     ig_4 = instance_group_factory("ig4", percentage=25)
+
+    count = ActivityStream.objects.count()
+
     apply_cluster_membership_policies()
+    # running apply_cluster_membership_policies shouldn't spam the activity stream
+    assert ActivityStream.objects.count() == count
+
     assert len(ig_1.instances.all()) == 1
     assert i1 in ig_1.instances.all()
     assert len(ig_2.instances.all()) == 1
@@ -103,8 +112,12 @@ def test_policy_instance_few_instances(instance_factory, instance_group_factory)
     assert i1 in ig_3.instances.all()
     assert len(ig_4.instances.all()) == 1
     assert i1 in ig_4.instances.all()
+
     i2 = instance_factory("i2")
+    count += 1
     apply_cluster_membership_policies()
+    assert ActivityStream.objects.count() == count
+
     assert len(ig_1.instances.all()) == 1
     assert i1 in ig_1.instances.all()
     assert len(ig_2.instances.all()) == 1
@@ -243,6 +256,41 @@ def test_policy_instance_list_explicitly_pinned(instance_factory, instance_group
     assert set(ig_2.instances.all()) == set([i2])
 
 
+@pytest.mark.django_db
+def test_control_plane_policy_exception(controlplane_instance_group):
+    controlplane_instance_group.policy_instance_percentage = 100
+    controlplane_instance_group.policy_instance_minimum = 2
+    controlplane_instance_group.save()
+    Instance.objects.create(hostname='foo-1', node_type='execution')
+    apply_cluster_membership_policies()
+    assert 'foo-1' not in [inst.hostname for inst in controlplane_instance_group.instances.all()]
+
+
+@pytest.mark.django_db
+def test_normal_instance_group_policy_exception():
+    ig = InstanceGroup.objects.create(name='bar', policy_instance_percentage=100, policy_instance_minimum=2)
+    Instance.objects.create(hostname='foo-1', node_type='control')
+    apply_cluster_membership_policies()
+    assert 'foo-1' not in [inst.hostname for inst in ig.instances.all()]
+
+
+@pytest.mark.django_db
+def test_percentage_as_fraction_of_execution_nodes():
+    """
+    If an instance requests 50 percent of instances, then those should be 50 percent
+    of available execution nodes (1 out of 2), as opposed to 50 percent
+    of all available nodes (2 out of 4) which include unusable control nodes
+    """
+    ig = InstanceGroup.objects.create(name='bar', policy_instance_percentage=50)
+    for i in range(2):
+        Instance.objects.create(hostname=f'foo-{i}', node_type='control')
+    for i in range(2):
+        Instance.objects.create(hostname=f'bar-{i}', node_type='execution')
+    apply_cluster_membership_policies()
+    assert ig.instances.count() == 1
+    assert ig.instances.first().hostname.startswith('bar-')
+
+
 @pytest.mark.django_db
 def test_basic_instance_group_membership(instance_group_factory, default_instance_group, job_factory):
     j = job_factory()
@@ -266,6 +314,12 @@ def test_inherited_instance_group_membership(instance_group_factory, default_ins
     assert default_instance_group not in j.preferred_instance_groups
 
 
+@pytest.mark.django_db
+def test_global_instance_groups_as_defaults(controlplane_instance_group, default_instance_group, job_factory):
+    j = job_factory()
+    assert j.preferred_instance_groups == [default_instance_group, controlplane_instance_group]
+
+
 @pytest.mark.django_db
 def test_mixed_group_membership(instance_factory, instance_group_factory):
     for i in range(5):
@@ -289,6 +343,23 @@ def test_instance_group_capacity(instance_factory, instance_group_factory):
     assert ig_single.capacity == 100
 
 
+@pytest.mark.django_db
+def test_health_check_clears_errors():
+    instance = Instance.objects.create(hostname='foo-1', enabled=True, capacity=0, errors='something went wrong')
+    data = dict(version='ansible-runner-4.2', cpu=782, memory=int(39e9), uuid='asdfasdfasdfasdfasdf', errors='')
+    instance.save_health_data(**data)
+    for k, v in data.items():
+        assert getattr(instance, k) == v
+
+
+@pytest.mark.django_db
+def test_health_check_oh_no():
+    instance = Instance.objects.create(hostname='foo-2', enabled=True, capacity=52, cpu=8, memory=int(40e9))
+    instance.save_health_data('', 0, 0, errors='This it not a real instance!')
+    assert instance.capacity == instance.cpu_capacity == 0
+    assert instance.errors == 'This it not a real instance!'
+
+
 @pytest.mark.django_db
 class TestInstanceGroupOrdering:
     def test_ad_hoc_instance_groups(self, instance_group_factory, inventory, default_instance_group):
@@ -314,15 +385,15 @@ class TestInstanceGroupOrdering:
         # API does not allow setting IGs on inventory source, so ignore those
         assert iu.preferred_instance_groups == [ig_inv, ig_org]
 
-    def test_project_update_instance_groups(self, instance_group_factory, project, default_instance_group):
+    def test_project_update_instance_groups(self, instance_group_factory, project, controlplane_instance_group):
         pu = ProjectUpdate.objects.create(project=project, organization=project.organization)
-        assert pu.preferred_instance_groups == [default_instance_group]
-        ig_org = instance_group_factory("OrgIstGrp", [default_instance_group.instances.first()])
-        ig_tmp = instance_group_factory("TmpIstGrp", [default_instance_group.instances.first()])
+        assert pu.preferred_instance_groups == [controlplane_instance_group]
+        ig_org = instance_group_factory("OrgIstGrp", [controlplane_instance_group.instances.first()])
+        ig_tmp = instance_group_factory("TmpIstGrp", [controlplane_instance_group.instances.first()])
         project.organization.instance_groups.add(ig_org)
-        assert pu.preferred_instance_groups == [ig_org]
+        assert pu.preferred_instance_groups == [ig_org, controlplane_instance_group]
         project.instance_groups.add(ig_tmp)
-        assert pu.preferred_instance_groups == [ig_tmp, ig_org]
+        assert pu.preferred_instance_groups == [ig_tmp, ig_org, controlplane_instance_group]
 
     def test_job_instance_groups(self, instance_group_factory, inventory, project, default_instance_group):
         jt = JobTemplate.objects.create(inventory=inventory, project=project)

awx/main/tests/functional/test_inventory_source_injectors.py

@@ -5,7 +5,7 @@ import json
 import re
 from collections import namedtuple
 
-from awx.main.tasks import RunInventoryUpdate
+from awx.main.tasks.jobs import RunInventoryUpdate
 from awx.main.models import InventorySource, Credential, CredentialType, UnifiedJob, ExecutionEnvironment
 from awx.main.constants import CLOUD_PROVIDERS, STANDARD_INVENTORY_UPDATE_ENV
 from awx.main.tests import data
@@ -257,6 +257,6 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential
         # Also do not send websocket status updates
         with mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()):
             # The point of this test is that we replace run with assertions
-            with mock.patch('awx.main.tasks.AWXReceptorJob.run', substitute_run):
+            with mock.patch('awx.main.tasks.receptor.AWXReceptorJob.run', substitute_run):
                 # so this sets up everything for a run and then yields control over to substitute_run
                 task.run(inventory_update.pk)

awx/main/tests/functional/test_jobs.py

@@ -4,7 +4,7 @@ from unittest import mock
 import json
 
 from awx.main.models import Job, Instance, JobHostSummary, InventoryUpdate, InventorySource, Project, ProjectUpdate, SystemJob, AdHocCommand
-from awx.main.tasks import cluster_node_heartbeat
+from awx.main.tasks.system import cluster_node_heartbeat
 from django.test.utils import override_settings
 
 
@@ -20,24 +20,27 @@ def test_orphan_unified_job_creation(instance, inventory):
 
 
 @pytest.mark.django_db
-@mock.patch('awx.main.utils.common.get_cpu_capacity', lambda: (2, 8))
-@mock.patch('awx.main.utils.common.get_mem_capacity', lambda: (8000, 62))
+@mock.patch('awx.main.tasks.system.inspect_execution_nodes', lambda *args, **kwargs: None)
+@mock.patch('awx.main.models.ha.get_cpu_effective_capacity', lambda cpu: 8)
+@mock.patch('awx.main.models.ha.get_mem_effective_capacity', lambda mem: 62)
 def test_job_capacity_and_with_inactive_node():
     i = Instance.objects.create(hostname='test-1')
-    with mock.patch.object(redis.client.Redis, 'ping', lambda self: True):
-        i.refresh_capacity()
+    i.save_health_data('18.0.1', 2, 8000)
+    assert i.enabled is True
+    assert i.capacity_adjustment == 1.0
     assert i.capacity == 62
     i.enabled = False
     i.save()
     with override_settings(CLUSTER_HOST_ID=i.hostname):
-        cluster_node_heartbeat()
+        with mock.patch.object(redis.client.Redis, 'ping', lambda self: True):
+            cluster_node_heartbeat()
         i = Instance.objects.get(id=i.id)
         assert i.capacity == 0
 
 
 @pytest.mark.django_db
-@mock.patch('awx.main.utils.common.get_cpu_capacity', lambda: (2, 8))
-@mock.patch('awx.main.utils.common.get_mem_capacity', lambda: (8000, 62))
+@mock.patch('awx.main.models.ha.get_cpu_effective_capacity', lambda cpu: 8)
+@mock.patch('awx.main.models.ha.get_mem_effective_capacity', lambda mem: 62)
 def test_job_capacity_with_redis_disabled():
     i = Instance.objects.create(hostname='test-1')
 
@@ -45,7 +48,7 @@ def test_job_capacity_with_redis_disabled():
         raise redis.ConnectionError()
 
     with mock.patch.object(redis.client.Redis, 'ping', _raise):
-        i.refresh_capacity()
+        i.local_health_check()
     assert i.capacity == 0
 
 

awx/main/tests/functional/test_licenses.py

@@ -9,6 +9,8 @@ try:
 except ImportError:
     from pip.req import parse_requirements
 
+from pip._internal.req.constructors import parse_req_from_line
+
 
 def test_python_and_js_licenses():
     def index_licenses(path):
@@ -48,21 +50,30 @@ def test_python_and_js_licenses():
 
     def read_api_requirements(path):
         ret = {}
+        skip_pbr_license_check = False
         for req_file in ['requirements.txt', 'requirements_git.txt']:
             fname = '%s/%s' % (path, req_file)
 
             for reqt in parse_requirements(fname, session=''):
-                name = reqt.name
-                version = str(reqt.specifier)
+                parsed_requirement = parse_req_from_line(reqt.requirement, None)
+                name = parsed_requirement.requirement.name
+                version = str(parsed_requirement.requirement.specifier)
                 if version.startswith('=='):
                     version = version[2:]
-                if reqt.link:
-                    (name, version) = reqt.link.filename.split('@', 1)
+                if parsed_requirement.link:
+                    if str(parsed_requirement.link).startswith(('http://', 'https://')):
+                        (name, version) = str(parsed_requirement.requirement).split('==', 1)
+                    else:
+                        (name, version) = parsed_requirement.link.filename.split('@', 1)
                     if name.endswith('.git'):
                         name = name[:-4]
                     if name == 'receptor':
                         name = 'receptorctl'
+                    if name == 'ansible-runner':
+                        skip_pbr_license_check = True
                 ret[name] = {'name': name, 'version': version}
+        if 'pbr' in ret and skip_pbr_license_check:
+            del ret['pbr']
         return ret
 
     def read_ui_requirements(path):

awx/main/tests/functional/test_tasks.py

@@ -2,8 +2,9 @@ import pytest
 from unittest import mock
 import os
 
-from awx.main.tasks import RunProjectUpdate, RunInventoryUpdate
-from awx.main.models import ProjectUpdate, InventoryUpdate, InventorySource
+from awx.main.tasks.jobs import RunProjectUpdate, RunInventoryUpdate
+from awx.main.tasks.system import execution_node_health_check
+from awx.main.models import ProjectUpdate, InventoryUpdate, InventorySource, Instance
 
 
 @pytest.fixture
@@ -15,6 +16,15 @@ def scm_revision_file(tmpdir_factory):
     return os.path.join(revision_file.dirname, 'revision.txt')
 
 
+@pytest.mark.django_db
+@pytest.mark.parametrize('node_type', ('control', 'hybrid'))
+def test_no_worker_info_on_AWX_nodes(node_type):
+    hostname = 'us-south-3-compute.invalid'
+    Instance.objects.create(hostname=hostname, node_type=node_type)
+    with pytest.raises(RuntimeError):
+        execution_node_health_check(hostname)
+
+
 @pytest.mark.django_db
 class TestDependentInventoryUpdate:
     def test_dependent_inventory_updates_is_called(self, scm_inventory_source, scm_revision_file):
@@ -40,7 +50,7 @@ class TestDependentInventoryUpdate:
         scm_inventory_source.scm_last_revision = ''
         proj_update = ProjectUpdate.objects.create(project=scm_inventory_source.source_project)
         with mock.patch.object(RunInventoryUpdate, 'run') as iu_run_mock:
-            with mock.patch('awx.main.tasks.create_partition'):
+            with mock.patch('awx.main.tasks.jobs.create_partition'):
                 task._update_dependent_inventories(proj_update, [scm_inventory_source])
                 assert InventoryUpdate.objects.count() == 1
                 inv_update = InventoryUpdate.objects.first()
@@ -64,7 +74,7 @@ class TestDependentInventoryUpdate:
             ProjectUpdate.objects.all().update(cancel_flag=True)
 
         with mock.patch.object(RunInventoryUpdate, 'run') as iu_run_mock:
-            with mock.patch('awx.main.tasks.create_partition'):
+            with mock.patch('awx.main.tasks.jobs.create_partition'):
                 iu_run_mock.side_effect = user_cancels_project
                 task._update_dependent_inventories(proj_update, [is1, is2])
                 # Verify that it bails after 1st update, detecting a cancel

awx/main/tests/unit/models/test_ha.py

@@ -1,15 +1,25 @@
 import pytest
 from unittest import mock
 from unittest.mock import Mock
+from decimal import Decimal
 
-from awx.main.models import (
-    InstanceGroup,
-)
+from awx.main.models import InstanceGroup, Instance
+
+
+@pytest.mark.parametrize('capacity_adjustment', [0.0, 0.25, 0.5, 0.75, 1, 1.5, 3])
+def test_capacity_adjustment_no_save(capacity_adjustment):
+    inst = Instance(hostname='test-host', capacity_adjustment=Decimal(capacity_adjustment), capacity=0, cpu_capacity=10, mem_capacity=1000)
+    assert inst.capacity == 0
+    assert inst.capacity_adjustment == capacity_adjustment  # sanity
+    inst.set_capacity_value()
+    assert inst.capacity > 0
+    assert inst.capacity == (float(inst.capacity_adjustment) * abs(inst.mem_capacity - inst.cpu_capacity) + min(inst.mem_capacity, inst.cpu_capacity))
 
 
 def T(impact):
-    j = mock.Mock(spec_set=['task_impact'])
+    j = mock.Mock(spec_set=['task_impact', 'capacity_type'])
     j.task_impact = impact
+    j.capacity_type = 'execution'
     return j
 
 
@@ -26,11 +36,13 @@ def Is(param):
             inst = Mock()
             inst.capacity = capacity
             inst.jobs_running = jobs_running
+            inst.node_type = 'execution'
             instances.append(inst)
     else:
         for i in param:
             inst = Mock()
             inst.remaining_capacity = i
+            inst.node_type = 'execution'
             instances.append(inst)
     return instances
 
@@ -77,3 +89,19 @@ class TestInstanceGroup(object):
             assert ig.find_largest_idle_instance(instances_online_only) is None, reason
         else:
             assert ig.find_largest_idle_instance(instances_online_only) == instances[instance_fit_index], reason
+
+
+def test_cleanup_params_defaults():
+    inst = Instance(hostname='foobar')
+    assert inst.get_cleanup_task_kwargs(exclude_strings=['awx_423_']) == {'exclude_strings': ['awx_423_'], 'file_pattern': '/tmp/awx_*_*'}
+
+
+def test_cleanup_params_for_image_cleanup():
+    inst = Instance(hostname='foobar')
+    # see CLI conversion in awx.main.tests.unit.utils.test_receptor
+    assert inst.get_cleanup_task_kwargs(file_pattern='', remove_images=['quay.invalid/foo/bar'], image_prune=True) == {
+        'file_pattern': '',
+        'process_isolation_executable': 'podman',
+        'remove_images': ['quay.invalid/foo/bar'],
+        'image_prune': True,
+    }

awx/main/tests/unit/notifications/test_rocketchat.py

@@ -7,8 +7,11 @@ import awx.main.notifications.rocketchat_backend as rocketchat_backend
 
 
 def test_send_messages():
-    with mock.patch('awx.main.notifications.rocketchat_backend.requests') as requests_mock:
+    with mock.patch('awx.main.notifications.rocketchat_backend.requests') as requests_mock, mock.patch(
+        'awx.main.notifications.rocketchat_backend.get_awx_http_client_headers'
+    ) as version_mock:
         requests_mock.post.return_value.status_code = 201
+        version_mock.return_value = {'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'}
         backend = rocketchat_backend.RocketChatBackend()
         message = EmailMessage(
             'test subject',
@@ -23,7 +26,12 @@ def test_send_messages():
                 message,
             ]
         )
-        requests_mock.post.assert_called_once_with('http://example.com', data='{"text": "test subject"}', verify=True)
+        requests_mock.post.assert_called_once_with(
+            'http://example.com',
+            data='{"text": "test subject"}',
+            headers={'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'},
+            verify=True,
+        )
         assert sent_messages == 1
 
 
@@ -84,8 +92,11 @@ def test_send_messages_with_icon_url():
 
 
 def test_send_messages_with_no_verify_ssl():
-    with mock.patch('awx.main.notifications.rocketchat_backend.requests') as requests_mock:
+    with mock.patch('awx.main.notifications.rocketchat_backend.requests') as requests_mock, mock.patch(
+        'awx.main.notifications.rocketchat_backend.get_awx_http_client_headers'
+    ) as version_mock:
         requests_mock.post.return_value.status_code = 201
+        version_mock.return_value = {'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'}
         backend = rocketchat_backend.RocketChatBackend(rocketchat_no_verify_ssl=True)
         message = EmailMessage(
             'test subject',
@@ -100,5 +111,10 @@ def test_send_messages_with_no_verify_ssl():
                 message,
             ]
         )
-        requests_mock.post.assert_called_once_with('http://example.com', data='{"text": "test subject"}', verify=False)
+        requests_mock.post.assert_called_once_with(
+            'http://example.com',
+            data='{"text": "test subject"}',
+            headers={'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'},
+            verify=False,
+        )
         assert sent_messages == 1

awx/main/tests/unit/notifications/test_slack.py

@@ -0,0 +1,75 @@
+import pytest
+
+from unittest import mock
+from django.core.mail.message import EmailMessage
+
+import awx.main.notifications.slack_backend as slack_backend
+
+
+def test_send_messages():
+    with mock.patch('awx.main.notifications.slack_backend.WebClient') as slack_sdk_mock:
+        WebClient_mock = slack_sdk_mock.return_value
+        WebClient_mock.chat_postMessage.return_value = {'ok': True}
+        backend = slack_backend.SlackBackend('slack_access_token')
+        message = EmailMessage(
+            'test subject',
+            'test body',
+            [],
+            [
+                '#random',
+            ],
+        )
+        sent_messages = backend.send_messages(
+            [
+                message,
+            ]
+        )
+        WebClient_mock.chat_postMessage.assert_called_once_with(channel='random', thread_ts=None, as_user=True, text='test subject')
+        assert sent_messages == 1
+
+
+def test_send_messages_with_color():
+    with mock.patch('awx.main.notifications.slack_backend.WebClient') as slack_sdk_mock:
+        WebClient_mock = slack_sdk_mock.return_value
+        WebClient_mock.chat_postMessage.return_value = {'ok': True}
+        backend = slack_backend.SlackBackend('slack_access_token', hex_color='#006699')
+        message = EmailMessage(
+            'test subject',
+            'test body',
+            [],
+            [
+                '#random',
+            ],
+        )
+        sent_messages = backend.send_messages(
+            [
+                message,
+            ]
+        )
+
+        WebClient_mock.chat_postMessage.assert_called_once_with(
+            channel='random', as_user=True, thread_ts=None, attachments=[{'color': '#006699', 'text': 'test subject'}]
+        )
+        assert sent_messages == 1
+
+
+def test_send_messages_fail():
+    with mock.patch('awx.main.notifications.slack_backend.WebClient') as slack_sdk_mock, pytest.raises(RuntimeError, match=r'.*not_in_channel.*'):
+        WebClient_mock = slack_sdk_mock.return_value
+        WebClient_mock.chat_postMessage.return_value = {'ok': False, 'error': 'not_in_channel'}
+        backend = slack_backend.SlackBackend('slack_access_token')
+        message = EmailMessage(
+            'test subject',
+            'test body',
+            [],
+            [
+                '#not_existing',
+            ],
+        )
+        sent_messages = backend.send_messages(
+            [
+                message,
+            ]
+        )
+        WebClient_mock.chat_postMessage.assert_called_once_with(channel='not_existing', as_user=True, text='test subject')
+        assert sent_messages == 0

awx/main/tests/unit/notifications/test_webhook.py

@@ -0,0 +1,221 @@
+import json
+
+from unittest import mock
+from django.core.mail.message import EmailMessage
+
+import awx.main.notifications.webhook_backend as webhook_backend
+
+
+def test_send_messages_as_POST():
+    with mock.patch('awx.main.notifications.webhook_backend.requests') as requests_mock, mock.patch(
+        'awx.main.notifications.webhook_backend.get_awx_http_client_headers'
+    ) as version_mock:
+        requests_mock.post.return_value.status_code = 200
+        version_mock.return_value = {'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'}
+        backend = webhook_backend.WebhookBackend('POST', None)
+        message = EmailMessage(
+            'test subject',
+            {'text': 'test body'},
+            [],
+            [
+                'http://example.com',
+            ],
+        )
+        sent_messages = backend.send_messages(
+            [
+                message,
+            ]
+        )
+        requests_mock.post.assert_called_once_with(
+            'http://example.com',
+            auth=None,
+            data=json.dumps({'text': 'test body'}, ensure_ascii=False).encode('utf-8'),
+            headers={'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'},
+            verify=True,
+        )
+        assert sent_messages == 1
+
+
+def test_send_messages_as_PUT():
+    with mock.patch('awx.main.notifications.webhook_backend.requests') as requests_mock, mock.patch(
+        'awx.main.notifications.webhook_backend.get_awx_http_client_headers'
+    ) as version_mock:
+        requests_mock.put.return_value.status_code = 200
+        version_mock.return_value = {'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'}
+        backend = webhook_backend.WebhookBackend('PUT', None)
+        message = EmailMessage(
+            'test subject 2',
+            {'text': 'test body 2'},
+            [],
+            [
+                'http://example.com',
+            ],
+        )
+        sent_messages = backend.send_messages(
+            [
+                message,
+            ]
+        )
+        requests_mock.put.assert_called_once_with(
+            'http://example.com',
+            auth=None,
+            data=json.dumps({'text': 'test body 2'}, ensure_ascii=False).encode('utf-8'),
+            headers={'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'},
+            verify=True,
+        )
+        assert sent_messages == 1
+
+
+def test_send_messages_with_username():
+    with mock.patch('awx.main.notifications.webhook_backend.requests') as requests_mock, mock.patch(
+        'awx.main.notifications.webhook_backend.get_awx_http_client_headers'
+    ) as version_mock:
+        requests_mock.post.return_value.status_code = 200
+        version_mock.return_value = {'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'}
+        backend = webhook_backend.WebhookBackend('POST', None, username='userstring')
+        message = EmailMessage(
+            'test subject',
+            {'text': 'test body'},
+            [],
+            [
+                'http://example.com',
+            ],
+        )
+        sent_messages = backend.send_messages(
+            [
+                message,
+            ]
+        )
+        requests_mock.post.assert_called_once_with(
+            'http://example.com',
+            auth=('userstring', None),
+            data=json.dumps({'text': 'test body'}, ensure_ascii=False).encode('utf-8'),
+            headers={'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'},
+            verify=True,
+        )
+        assert sent_messages == 1
+
+
+def test_send_messages_with_password():
+    with mock.patch('awx.main.notifications.webhook_backend.requests') as requests_mock, mock.patch(
+        'awx.main.notifications.webhook_backend.get_awx_http_client_headers'
+    ) as version_mock:
+        requests_mock.post.return_value.status_code = 200
+        version_mock.return_value = {'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'}
+        backend = webhook_backend.WebhookBackend('POST', None, password='passwordstring')
+        message = EmailMessage(
+            'test subject',
+            {'text': 'test body'},
+            [],
+            [
+                'http://example.com',
+            ],
+        )
+        sent_messages = backend.send_messages(
+            [
+                message,
+            ]
+        )
+        requests_mock.post.assert_called_once_with(
+            'http://example.com',
+            auth=(None, 'passwordstring'),
+            data=json.dumps({'text': 'test body'}, ensure_ascii=False).encode('utf-8'),
+            headers={'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'},
+            verify=True,
+        )
+        assert sent_messages == 1
+
+
+def test_send_messages_with_username_and_password():
+    with mock.patch('awx.main.notifications.webhook_backend.requests') as requests_mock, mock.patch(
+        'awx.main.notifications.webhook_backend.get_awx_http_client_headers'
+    ) as version_mock:
+        requests_mock.post.return_value.status_code = 200
+        version_mock.return_value = {'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'}
+        backend = webhook_backend.WebhookBackend('POST', None, username='userstring', password='passwordstring')
+        message = EmailMessage(
+            'test subject',
+            {'text': 'test body'},
+            [],
+            [
+                'http://example.com',
+            ],
+        )
+        sent_messages = backend.send_messages(
+            [
+                message,
+            ]
+        )
+        requests_mock.post.assert_called_once_with(
+            'http://example.com',
+            auth=('userstring', 'passwordstring'),
+            data=json.dumps({'text': 'test body'}, ensure_ascii=False).encode('utf-8'),
+            headers={'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'},
+            verify=True,
+        )
+        assert sent_messages == 1
+
+
+def test_send_messages_with_no_verify_ssl():
+    with mock.patch('awx.main.notifications.webhook_backend.requests') as requests_mock, mock.patch(
+        'awx.main.notifications.webhook_backend.get_awx_http_client_headers'
+    ) as version_mock:
+        requests_mock.post.return_value.status_code = 200
+        version_mock.return_value = {'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'}
+        backend = webhook_backend.WebhookBackend('POST', None, disable_ssl_verification=True)
+        message = EmailMessage(
+            'test subject',
+            {'text': 'test body'},
+            [],
+            [
+                'http://example.com',
+            ],
+        )
+        sent_messages = backend.send_messages(
+            [
+                message,
+            ]
+        )
+        requests_mock.post.assert_called_once_with(
+            'http://example.com',
+            auth=None,
+            data=json.dumps({'text': 'test body'}, ensure_ascii=False).encode('utf-8'),
+            headers={'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'},
+            verify=False,
+        )
+        assert sent_messages == 1
+
+
+def test_send_messages_with_additional_headers():
+    with mock.patch('awx.main.notifications.webhook_backend.requests') as requests_mock, mock.patch(
+        'awx.main.notifications.webhook_backend.get_awx_http_client_headers'
+    ) as version_mock:
+        requests_mock.post.return_value.status_code = 200
+        version_mock.return_value = {'Content-Type': 'application/json', 'User-Agent': 'AWX 0.0.1.dev (open)'}
+        backend = webhook_backend.WebhookBackend('POST', {'X-Test-Header1': 'test-content-1', 'X-Test-Header2': 'test-content-2'})
+        message = EmailMessage(
+            'test subject',
+            {'text': 'test body'},
+            [],
+            [
+                'http://example.com',
+            ],
+        )
+        sent_messages = backend.send_messages(
+            [
+                message,
+            ]
+        )
+        requests_mock.post.assert_called_once_with(
+            'http://example.com',
+            auth=None,
+            data=json.dumps({'text': 'test body'}, ensure_ascii=False).encode('utf-8'),
+            headers={
+                'Content-Type': 'application/json',
+                'User-Agent': 'AWX 0.0.1.dev (open)',
+                'X-Test-Header1': 'test-content-1',
+                'X-Test-Header2': 'test-content-2',
+            },
+            verify=True,
+        )
+        assert sent_messages == 1

awx/main/tests/unit/settings/test_defaults.py

@@ -7,7 +7,7 @@ from datetime import timedelta
 @pytest.mark.parametrize(
     "job_name,function_path",
     [
-        ('tower_scheduler', 'awx.main.tasks.awx_periodic_scheduler'),
+        ('tower_scheduler', 'awx.main.tasks.system.awx_periodic_scheduler'),
     ],
 )
 def test_CELERYBEAT_SCHEDULE(mocker, job_name, function_path):