Backports previously approved corrections. (#15121)

* Backports previously approved corrections.

* Deleted a blank line in inventories line 100

.github/actions/awx_devel_image/action.yml

@@ -11,6 +11,12 @@ runs:
       shell: bash
       run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
 
+    - name: Set lower case owner name
+      shell: bash
+      run: echo "OWNER_LC=${OWNER,,}" >> $GITHUB_ENV
+      env:
+        OWNER: '${{ github.repository_owner }}'
+
     - name: Log in to registry
       shell: bash
       run: |
@@ -18,11 +24,11 @@ runs:
 
     - name: Pre-pull latest devel image to warm cache
       shell: bash
-      run: docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }}
+      run: docker pull ghcr.io/${OWNER_LC}/awx_devel:${{ github.base_ref }}
 
     - name: Build image for current source checkout
       shell: bash
       run: |
-        DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} \
+        DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} \
         COMPOSE_TAG=${{ github.base_ref }} \
         make docker-compose-build

.github/actions/run_awx_devel/action.yml

@@ -35,7 +35,7 @@ runs:
     - name: Start AWX
       shell: bash
       run: |
-        DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} \
+        DEV_DOCKER_OWNER=${{ github.repository_owner }} \
         COMPOSE_TAG=${{ github.base_ref }} \
         COMPOSE_UP_OPTS="-d" \
         make docker-compose
@@ -71,7 +71,7 @@ runs:
       id: data
       shell: bash
       run: |
-        AWX_IP=$(docker inspect -f '{{.NetworkSettings.Networks._sources_awx.IPAddress}}' tools_awx_1)
+        AWX_IP=$(docker inspect -f '{{.NetworkSettings.Networks.awx.IPAddress}}' tools_awx_1)
         ADMIN_TOKEN=$(docker exec -i tools_awx_1 awx-manage create_oauth2_token --user admin)
         echo "ip=$AWX_IP" >> $GITHUB_OUTPUT
         echo "admin_token=$ADMIN_TOKEN" >> $GITHUB_OUTPUT

.github/pr_labeler.yml

@@ -15,5 +15,4 @@
 
 "dependencies":
   - any: ["awx/ui/package.json"]
-  - any: ["requirements/*.txt"]
-  - any: ["requirements/requirements.in"]
+  - any: ["requirements/*"]

.github/workflows/ci.yml

@@ -66,6 +66,8 @@ jobs:
   awx-operator:
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    env:
+      DEBUG_OUTPUT_DIR: /tmp/awx_operator_molecule_test
     steps:
       - name: Checkout awx
         uses: actions/checkout@v3
@@ -94,11 +96,11 @@ jobs:
       - name: Build AWX image
         working-directory: awx
         run: |
-          ansible-playbook -v tools/ansible/build.yml \
-            -e headless=yes \
-            -e awx_image=awx \
-            -e awx_image_tag=ci \
-            -e ansible_python_interpreter=$(which python3)
+          VERSION=`make version-for-buildyml` make awx-kube-build
+        env:
+          COMPOSE_TAG: ci
+          DEV_DOCKER_TAG_BASE: local
+          HEADLESS: yes
 
       - name: Run test deployment with awx-operator
         working-directory: awx-operator
@@ -107,10 +109,19 @@ jobs:
           ansible-galaxy collection install -r molecule/requirements.yml
           sudo rm -f $(which kustomize)
           make kustomize
-          KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule -v test -s kind
+          KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule -v test -s kind -- --skip-tags=replicas
         env:
-          AWX_TEST_IMAGE: awx
+          AWX_TEST_IMAGE: local/awx
           AWX_TEST_VERSION: ci
+          AWX_EE_TEST_IMAGE: quay.io/ansible/awx-ee:latest
+          STORE_DEBUG_OUTPUT: true
+
+      - name: Upload debug output
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: awx-operator-debug-output
+          path: ${{ env.DEBUG_OUTPUT_DIR }}
 
   collection-sanity:
     name: awx_collection sanity
@@ -127,10 +138,6 @@ jobs:
 
       - name: Run sanity tests
         run: make test_collection_sanity
-        env:
-          # needed due to cgroupsv2. This is fixed, but a stable release
-          # with the fix has not been made yet.
-          ANSIBLE_TEST_PREFER_PODMAN: 1
 
   collection-integration:
     name: awx_collection integration

.github/workflows/devel_images.yml

@@ -3,28 +3,50 @@ name: Build/Push Development Images
 env:
   LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
 on:
+  workflow_dispatch:
   push:
     branches:
       - devel
       - release_*
       - feature_*
 jobs:
-  push:
-    if: endsWith(github.repository, '/awx') || startsWith(github.ref, 'refs/heads/release_')
+  push-development-images:
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 120
     permissions:
       packages: write
       contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        build-targets:
+          - image-name: awx_devel
+            make-target: docker-compose-buildx
+          - image-name: awx_kube_devel
+            make-target: awx-kube-dev-buildx
+          - image-name: awx
+            make-target: awx-kube-buildx
     steps:
+
+      - name: Skipping build of awx image for non-awx repository
+        run: |
+          echo "Skipping build of awx image for non-awx repository"
+          exit 0
+        if: matrix.build-targets.image-name == 'awx' && !endsWith(github.repository, '/awx')
+
       - uses: actions/checkout@v3
 
-      - name: Get python version from Makefile
-        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
 
-      - name: Set lower case owner name
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set GITHUB_ENV variables
         run: |
-          echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV}
+          echo "DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER,,}" >> $GITHUB_ENV
+          echo "COMPOSE_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
         env:
           OWNER: '${{ github.repository_owner }}'
 
@@ -37,23 +59,19 @@ jobs:
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
-      - name: Pre-pull image to warm build cache
-        run: |
-          docker pull ghcr.io/${OWNER_LC}/awx_devel:${GITHUB_REF##*/} || :
-          docker pull ghcr.io/${OWNER_LC}/awx_kube_devel:${GITHUB_REF##*/} || :
-          docker pull ghcr.io/${OWNER_LC}/awx:${GITHUB_REF##*/} || :
+      - name: Setup node and npm
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16.13.1'
+        if: matrix.build-targets.image-name == 'awx'
 
-      - name: Build images
+      - name: Prebuild UI for awx image (to speed up build process)
         run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make docker-compose-build
-          DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-dev-build
-          DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-build
+          sudo apt-get install gettext
+          make ui-release
+          make ui-next
+        if: matrix.build-targets.image-name == 'awx'
 
-      - name: Push development images
+      - name: Build and push AWX devel images
         run: |
-          docker push ghcr.io/${OWNER_LC}/awx_devel:${GITHUB_REF##*/}
-          docker push ghcr.io/${OWNER_LC}/awx_kube_devel:${GITHUB_REF##*/}
-
-      - name: Push AWX k8s image, only for upstream and feature branches
-        run: docker push ghcr.io/${OWNER_LC}/awx:${GITHUB_REF##*/}
-        if: endsWith(github.repository, '/awx')
+          make ${{ matrix.build-targets.make-target }}

.github/workflows/promote.yml

@@ -7,7 +7,11 @@ env:
 on:
   release:
     types: [published]
-
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        description: 'Name for the tag of the release.'
+        required: true
 permissions:
   contents: read # to fetch code (actions/checkout)
 
@@ -17,6 +21,16 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 90
     steps:
+      - name: Set GitHub Env vars for workflow_dispatch event
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        run: |
+          echo "TAG_NAME=${{ github.event.inputs.tag_name }}" >> $GITHUB_ENV
+
+      - name: Set GitHub Env vars if release event
+        if: ${{ github.event_name == 'release' }}
+        run: |
+          echo "TAG_NAME=${{ env.TAG_NAME }}" >> $GITHUB_ENV
+
       - name: Checkout awx
         uses: actions/checkout@v3
 
@@ -43,16 +57,18 @@ jobs:
       - name: Build collection and publish to galaxy
         env:
           COLLECTION_NAMESPACE: ${{ env.collection_namespace }}
-          COLLECTION_VERSION: ${{ github.event.release.tag_name }}
+          COLLECTION_VERSION: ${{ env.TAG_NAME }}
           COLLECTION_TEMPLATE_VERSION: true
         run: |
           make build_collection
-          if [ "$(curl -L --head -sw '%{http_code}' https://galaxy.ansible.com/download/${{ env.collection_namespace }}-awx-${{ github.event.release.tag_name }}.tar.gz | tail -1)" == "302" ] ; then \
-              echo "Galaxy release already done"; \
-          else \
+          curl_with_redirects=$(curl --head -sLw '%{http_code}' https://galaxy.ansible.com/download/${{ env.collection_namespace }}-awx-${{ env.TAG_NAME }}.tar.gz | tail -1)
+          curl_without_redirects=$(curl --head -sw '%{http_code}' https://galaxy.ansible.com/download/${{ env.collection_namespace }}-awx-${{ env.TAG_NAME }}.tar.gz | tail -1)
+          if [[ "$curl_with_redirects" == "302" ]] || [[ "$curl_without_redirects" == "302" ]]; then
+              echo "Galaxy release already done";
+          else
               ansible-galaxy collection publish \
                 --token=${{ secrets.GALAXY_TOKEN }} \
-                awx_collection_build/${{ env.collection_namespace }}-awx-${{ github.event.release.tag_name }}.tar.gz; \
+                awx_collection_build/${{ env.collection_namespace }}-awx-${{ env.TAG_NAME }}.tar.gz;
           fi
 
       - name: Set official pypi info
@@ -64,6 +80,8 @@ jobs:
         if: ${{ github.repository_owner != 'ansible' }}
 
       - name: Build awxkit and upload to pypi
+        env:
+          SETUPTOOLS_SCM_PRETEND_VERSION: ${{ env.TAG_NAME }}
         run: |
           git reset --hard
           cd awxkit && python3 setup.py sdist bdist_wheel
@@ -83,11 +101,15 @@ jobs:
 
       - name: Re-tag and promote awx image
         run: |
-          docker pull ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name }}
-          docker tag ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name }} quay.io/${{ github.repository }}:${{ github.event.release.tag_name }}
-          docker tag ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name }} quay.io/${{ github.repository }}:latest
-          docker push quay.io/${{ github.repository }}:${{ github.event.release.tag_name }}
-          docker push quay.io/${{ github.repository }}:latest
-          docker pull ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.release.tag_name }}
-          docker tag ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.release.tag_name }} quay.io/${{ github.repository_owner }}/awx-ee:${{ github.event.release.tag_name }}
-          docker push quay.io/${{ github.repository_owner }}/awx-ee:${{ github.event.release.tag_name }}
+          docker buildx imagetools create \
+            ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }} \
+            --tag quay.io/${{ github.repository }}:${{ env.TAG_NAME }}
+          docker buildx imagetools create \
+            ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }} \
+            --tag quay.io/${{ github.repository }}:latest
+
+      - name: Re-tag and promote awx-ee image
+        run: |
+          docker buildx imagetools create \
+            ghcr.io/${{ github.repository_owner }}/awx-ee:${{ env.TAG_NAME }} \
+            --tag quay.io/${{ github.repository_owner }}/awx-ee:${{ env.TAG_NAME }}

.github/workflows/stage.yml

@@ -49,13 +49,11 @@ jobs:
         with:
           path: awx
 
-      - name: Get python version from Makefile
-        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
-
-      - name: Install python ${{ env.py_version }}
-        uses: actions/setup-python@v4
+      - name: Checkout awx-operator
+        uses: actions/checkout@v3
         with:
-          python-version: ${{ env.py_version }}
+          repository: ${{ github.repository_owner }}/awx-operator
+          path: awx-operator
 
       - name: Checkout awx-logos
         uses: actions/checkout@v3
@@ -63,57 +61,85 @@ jobs:
           repository: ansible/awx-logos
           path: awx-logos
 
-      - name: Checkout awx-operator
-        uses: actions/checkout@v3
+      - name: Get python version from Makefile
+        working-directory: awx
+        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
+
+      - name: Install python ${{ env.py_version }}
+        uses: actions/setup-python@v4
         with:
-          repository: ${{ github.repository_owner }}/awx-operator
-          path: awx-operator
+          python-version: ${{ env.py_version }}
 
       - name: Install playbook dependencies
         run: |
           python3 -m pip install docker
 
-      - name: Build and stage AWX
-        working-directory: awx
-        run: |
-          ansible-playbook -v tools/ansible/build.yml \
-            -e registry=ghcr.io \
-            -e registry_username=${{ github.actor }} \
-            -e registry_password=${{ secrets.GITHUB_TOKEN }} \
-            -e awx_image=${{ github.repository }} \
-            -e awx_version=${{ github.event.inputs.version }} \
-            -e ansible_python_interpreter=$(which python3) \
-            -e push=yes \
-            -e awx_official=yes
-
       - name: Log into registry ghcr.io
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d    # v3.0.0
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Log into registry quay.io
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d    # v3.0.0
+      - name: Copy logos for inclusion in sdist for official build
+        working-directory: awx
+        run: |
+          cp ../awx-logos/awx/ui/client/assets/* awx/ui/public/static/media/
+
+      - name: Setup node and npm
+        uses: actions/setup-node@v2
         with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_USER }}
-          password: ${{ secrets.QUAY_TOKEN }}
+          node-version: '16.13.1'
+
+      - name: Prebuild UI for awx image (to speed up build process)
+        working-directory: awx
+        run: |
+          sudo apt-get install gettext
+          make ui-release
+          make ui-next
+
+      - name: Set build env variables
+        run: |
+          echo "DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER,,}" >> $GITHUB_ENV
+          echo "COMPOSE_TAG=${{ github.event.inputs.version }}" >> $GITHUB_ENV
+          echo "VERSION=${{ github.event.inputs.version }}" >> $GITHUB_ENV
+          echo "AWX_TEST_VERSION=${{ github.event.inputs.version }}" >> $GITHUB_ENV
+          echo "AWX_TEST_IMAGE=ghcr.io/${OWNER,,}/awx" >> $GITHUB_ENV
+          echo "AWX_EE_TEST_IMAGE=ghcr.io/${OWNER,,}/awx-ee:${{ github.event.inputs.version }}" >> $GITHUB_ENV
+          echo "AWX_OPERATOR_TEST_IMAGE=ghcr.io/${OWNER,,}/awx-operator:${{ github.event.inputs.operator_version }}" >> $GITHUB_ENV
+        env:
+          OWNER: ${{ github.repository_owner }}
+
+      - name: Build and stage AWX
+        working-directory: awx
+        env:
+          DOCKER_BUILDX_PUSH: true
+          HEADLESS: false
+          PLATFORMS: linux/amd64,linux/arm64
+        run: |
+          make awx-kube-buildx
 
       - name: tag awx-ee:latest with version input
         run: |
-          docker pull quay.io/ansible/awx-ee:latest
-          docker tag quay.io/ansible/awx-ee:latest ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.inputs.version }}
-          docker push ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.inputs.version }}
+          docker buildx imagetools create \
+            quay.io/ansible/awx-ee:latest \
+            --tag ${AWX_EE_TEST_IMAGE}
 
       - name: Stage awx-operator image
         working-directory: awx-operator
         run: |
           BUILD_ARGS="--build-arg DEFAULT_AWX_VERSION=${{ github.event.inputs.version}} \
               --build-arg OPERATOR_VERSION=${{ github.event.inputs.operator_version }}" \
-          IMG=ghcr.io/${{ github.repository_owner }}/awx-operator:${{ github.event.inputs.operator_version }} \
+          IMG=${AWX_OPERATOR_TEST_IMAGE} \
           make docker-buildx
 
+      - name: Pulling images for test deployment with awx-operator
+        # awx operator molecue test expect to kind load image and buildx exports image to registry and not local
+        run: |
+          docker pull ${AWX_OPERATOR_TEST_IMAGE}
+          docker pull ${AWX_EE_TEST_IMAGE}
+          docker pull ${AWX_TEST_IMAGE}:${AWX_TEST_VERSION}
+
       - name: Run test deployment with awx-operator
         working-directory: awx-operator
         run: |
@@ -122,10 +148,6 @@ jobs:
           sudo rm -f $(which kustomize)
           make kustomize
           KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind
-        env:
-          AWX_TEST_IMAGE: ${{ github.repository }}
-          AWX_TEST_VERSION: ${{ github.event.inputs.version }}
-          AWX_EE_TEST_IMAGE: ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.inputs.version }}
 
       - name: Create draft release for AWX
         working-directory: awx

.gitignore

@@ -46,6 +46,11 @@ tools/docker-compose/overrides/
 tools/docker-compose-minikube/_sources
 tools/docker-compose/keycloak.awx.realm.json
 
+!tools/docker-compose/editable_dependencies
+tools/docker-compose/editable_dependencies/*
+!tools/docker-compose/editable_dependencies/README.md
+!tools/docker-compose/editable_dependencies/install.sh
+
 # Tower setup playbook testing
 setup/test/roles/postgresql
 **/provision_docker

.vscode/launch.json

@@ -0,0 +1,113 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "run_ws_heartbeat",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_ws_heartbeat"],
+      "django": true,
+      "preLaunchTask": "stop awx-ws-heartbeat",
+      "postDebugTask": "start awx-ws-heartbeat"
+    },
+    {
+      "name": "run_cache_clear",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_cache_clear"],
+      "django": true,
+      "preLaunchTask": "stop awx-cache-clear",
+      "postDebugTask": "start awx-cache-clear"
+    },
+    {
+      "name": "run_callback_receiver",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_callback_receiver"],
+      "django": true,
+      "preLaunchTask": "stop awx-receiver",
+      "postDebugTask": "start awx-receiver"
+    },
+    {
+      "name": "run_dispatcher",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_dispatcher"],
+      "django": true,
+      "preLaunchTask": "stop awx-dispatcher",
+      "postDebugTask": "start awx-dispatcher"
+    },
+    {
+      "name": "run_rsyslog_configurer",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_rsyslog_configurer"],
+      "django": true,
+      "preLaunchTask": "stop awx-rsyslog-configurer",
+      "postDebugTask": "start awx-rsyslog-configurer"
+    },
+    {
+      "name": "run_cache_clear",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_cache_clear"],
+      "django": true,
+      "preLaunchTask": "stop awx-cache-clear",
+      "postDebugTask": "start awx-cache-clear"
+    },
+    {
+      "name": "run_wsrelay",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_wsrelay"],
+      "django": true,
+      "preLaunchTask": "stop awx-wsrelay",
+      "postDebugTask": "start awx-wsrelay"
+    },
+    {
+      "name": "daphne",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "/var/lib/awx/venv/awx/bin/daphne",
+      "args": ["-b", "127.0.0.1", "-p", "8051", "awx.asgi:channel_layer"],
+      "django": true,
+      "preLaunchTask": "stop awx-daphne",
+      "postDebugTask": "start awx-daphne"
+    },
+    {
+      "name": "runserver(uwsgi alternative)",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["runserver", "127.0.0.1:8052"],
+      "django": true,
+      "preLaunchTask": "stop awx-uwsgi",
+      "postDebugTask": "start awx-uwsgi"
+    },
+    {
+      "name": "runserver_plus(uwsgi alternative)",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["runserver_plus", "127.0.0.1:8052"],
+      "django": true,
+      "preLaunchTask": "stop awx-uwsgi and install Werkzeug",
+      "postDebugTask": "start awx-uwsgi"
+    },
+    {
+      "name": "shell_plus",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["shell_plus"],
+      "django": true,
+    },
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,100 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "start awx-cache-clear",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-cache-clear"
+        },
+        {
+            "label": "stop awx-cache-clear",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-cache-clear"
+        },
+        {
+            "label": "start awx-daphne",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-daphne"
+        },
+        {
+            "label": "stop awx-daphne",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-daphne"
+        },
+        {
+            "label": "start awx-dispatcher",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-dispatcher"
+        },
+        {
+            "label": "stop awx-dispatcher",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-dispatcher"
+        },
+        {
+            "label": "start awx-receiver",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-receiver"
+        },
+        {
+            "label": "stop awx-receiver",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-receiver"
+        },
+        {
+            "label": "start awx-rsyslog-configurer",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-rsyslog-configurer"
+        },
+        {
+            "label": "stop awx-rsyslog-configurer",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-rsyslog-configurer"
+        },
+        {
+            "label": "start awx-rsyslogd",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-rsyslogd"
+        },
+        {
+            "label": "stop awx-rsyslogd",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-rsyslogd"
+        },
+        {
+            "label": "start awx-uwsgi",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-uwsgi"
+        },
+        {
+            "label": "stop awx-uwsgi",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-uwsgi"
+        },
+        {
+            "label": "stop awx-uwsgi and install Werkzeug",
+            "type": "shell",
+            "command": "pip install Werkzeug; supervisorctl stop tower-processes:awx-uwsgi"
+        },
+        {
+            "label": "start awx-ws-heartbeat",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-ws-heartbeat"
+        },
+        {
+            "label": "stop awx-ws-heartbeat",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-ws-heartbeat"
+        },
+        {
+            "label": "start awx-wsrelay",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-wsrelay"
+        },
+        {
+            "label": "stop awx-wsrelay",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-wsrelay"
+        }
+    ]
+}

ISSUES.md

@@ -80,7 +80,7 @@ If any of those items are missing your pull request will still get the `needs_tr
 Currently you can expect awxbot to add common labels such as `state:needs_triage`, `type:bug`, `component:docs`, etc...
 These labels are determined by the template data. Please use the template and fill it out as accurately as possible.
 
-The `state:needs_triage` label will will remain on your pull request until a person has looked at it.
+The `state:needs_triage` label will remain on your pull request until a person has looked at it.
 
 You can also expect the bot to CC maintainers of specific areas of the code, this will notify them that there is a pull request by placing a comment on the pull request.
 The comment will look something like `CC @matburt @wwitzel3 ...`.

Makefile

@@ -1,8 +1,8 @@
 -include awx/ui_next/Makefile
 
-PYTHON := $(notdir $(shell for i in python3.9 python3; do command -v $$i; done|sed 1q))
+PYTHON := $(notdir $(shell for i in python3.11 python3; do command -v $$i; done|sed 1q))
 SHELL := bash
-DOCKER_COMPOSE ?= docker-compose
+DOCKER_COMPOSE ?= docker compose
 OFFICIAL ?= no
 NODE ?= node
 NPM_BIN ?= npm
@@ -47,6 +47,8 @@ VAULT ?= false
 VAULT_TLS ?= false
 # If set to true docker-compose will also start a tacacs+ instance
 TACACS ?= false
+# If set to true docker-compose will install editable dependencies
+EDITABLE_DEPENDENCIES ?= false
 
 VENV_BASE ?= /var/lib/awx/venv
 
@@ -63,7 +65,7 @@ RECEPTOR_IMAGE ?= quay.io/ansible/receptor:devel
 SRC_ONLY_PKGS ?= cffi,pycparser,psycopg,twilio
 # These should be upgraded in the AWX and Ansible venv before attempting
 # to install the actual requirements
-VENV_BOOTSTRAP ?= pip==21.2.4 setuptools==65.6.3 setuptools_scm[toml]==8.0.4 wheel==0.38.4
+VENV_BOOTSTRAP ?= pip==21.2.4 setuptools==69.0.2 setuptools_scm[toml]==8.0.4 wheel==0.42.0
 
 NAME ?= awx
 
@@ -75,6 +77,9 @@ SDIST_TAR_FILE ?= $(SDIST_TAR_NAME).tar.gz
 
 I18N_FLAG_FILE = .i18n_built
 
+## PLATFORMS defines the target platforms for  the manager image be build to provide support to multiple
+PLATFORMS ?= linux/amd64,linux/arm64  # linux/ppc64le,linux/s390x
+
 .PHONY: awx-link clean clean-tmp clean-venv requirements requirements_dev \
 	develop refresh adduser migrate dbchange \
 	receiver test test_unit test_coverage coverage_html \
@@ -213,8 +218,6 @@ collectstatic:
 	fi; \
 	$(PYTHON) manage.py collectstatic --clear --noinput > /dev/null 2>&1
 
-DEV_RELOAD_COMMAND ?= supervisorctl restart tower-processes:*
-
 uwsgi: collectstatic
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
@@ -222,7 +225,7 @@ uwsgi: collectstatic
 	uwsgi /etc/tower/uwsgi.ini
 
 awx-autoreload:
-	@/awx_devel/tools/docker-compose/awx-autoreload /awx_devel/awx "$(DEV_RELOAD_COMMAND)"
+	@/awx_devel/tools/docker-compose/awx-autoreload /awx_devel/awx
 
 daphne:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -302,7 +305,7 @@ swagger: reports
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	(set -o pipefail && py.test $(PYTEST_ARGS) awx/conf/tests/functional awx/main/tests/functional/api awx/main/tests/docs --release=$(VERSION_TARGET) | tee reports/$@.report)
+	(set -o pipefail && py.test $(PYTEST_ARGS) awx/conf/tests/functional awx/main/tests/functional/api awx/main/tests/docs | tee reports/$@.report)
 
 check: black
 
@@ -532,16 +535,23 @@ docker-compose-sources: .git/hooks/pre-commit
 	    -e enable_vault=$(VAULT) \
 	    -e vault_tls=$(VAULT_TLS) \
 	    -e enable_tacacs=$(TACACS) \
-            $(EXTRA_SOURCES_ANSIBLE_OPTS)
+	    -e install_editable_dependencies=$(EDITABLE_DEPENDENCIES) \
+	    $(EXTRA_SOURCES_ANSIBLE_OPTS)
 
 docker-compose: awx/projects docker-compose-sources
 	ansible-galaxy install --ignore-certs -r tools/docker-compose/ansible/requirements.yml;
 	ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/initialize_containers.yml \
 	    -e enable_vault=$(VAULT) \
 	    -e vault_tls=$(VAULT_TLS) \
-	    -e enable_ldap=$(LDAP);
+	    -e enable_ldap=$(LDAP); \
+	$(MAKE) docker-compose-up
+
+docker-compose-up:
 	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans
 
+docker-compose-down:
+	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) down --remove-orphans
+
 docker-compose-credential-plugins: awx/projects docker-compose-sources
 	echo -e "\033[0;31mTo generate a CyberArk Conjur API key: docker exec -it tools_conjur_1 conjurctl account create quick-start\033[0m"
 	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml -f tools/docker-credential-plugins-override.yml up --no-recreate awx_1 --remove-orphans
@@ -586,35 +596,27 @@ docker-compose-build: Dockerfile.dev
 		--build-arg BUILDKIT_INLINE_CACHE=1 \
 		--cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) .
 
-# ## Build awx_devel image for docker compose development environment for multiple architectures
-# docker-compose-buildx: Dockerfile.dev
-# 	DOCKER_BUILDKIT=1 docker build \
-# 		-f Dockerfile.dev \
-# 		-t $(DEVEL_IMAGE_NAME) \
-# 		--build-arg BUILDKIT_INLINE_CACHE=1 \
-# 		--cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) .
 
-## Build awx_devel image for docker compose development environment for multiple architectures
-# PLATFORMS defines the target platforms for  the manager image be build to provide support to multiple
-# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
-# - able to use docker buildx . More info: https://docs.docker.com/build/buildx/
-# - have enable BuildKit, More info: https://docs.docker.com/develop/develop-images/build_enhancements/
-# - be able to push the image for your registry (i.e. if you do not inform a valid value via IMG=<myregistry/image:<tag>> than the export will fail)
-# To properly provided solutions that supports more than one platform you should use this option.
-PLATFORMS ?= linux/amd64,linux/arm64  # linux/ppc64le,linux/s390x
 .PHONY: docker-compose-buildx
-docker-compose-buildx: Dockerfile.dev ## Build and push docker image for the manager for cross-platform support
-	- docker buildx create --name project-v3-builder
-	docker buildx use project-v3-builder
-	- docker buildx build --push $(BUILD_ARGS) --platform=$(PLATFORMS) --tag $(DEVEL_IMAGE_NAME) -f Dockerfile.dev .
-	- docker buildx rm project-v3-builder
+## Build awx_devel image for docker compose development environment for multiple architectures
+docker-compose-buildx: Dockerfile.dev
+	- docker buildx create --name docker-compose-buildx
+	docker buildx use docker-compose-buildx
+	- docker buildx build \
+		--push \
+		--build-arg BUILDKIT_INLINE_CACHE=1 \
+		--cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) \
+		--platform=$(PLATFORMS) \
+		--tag $(DEVEL_IMAGE_NAME) \
+		-f Dockerfile.dev .
+	- docker buildx rm docker-compose-buildx
 
 docker-clean:
 	-$(foreach container_id,$(shell docker ps -f name=tools_awx -aq && docker ps -f name=tools_receptor -aq),docker stop $(container_id); docker rm -f $(container_id);)
 	-$(foreach image_id,$(shell docker images --filter=reference='*/*/*awx_devel*' --filter=reference='*/*awx_devel*' --filter=reference='*awx_devel*' -aq),docker rmi --force $(image_id);)
 
 docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean
-	docker volume rm -f tools_awx_db tools_vault_1 tools_ldap_1 tools_grafana_storage tools_prometheus_storage $(docker volume ls --filter name=tools_redis_socket_ -q)
+	docker volume rm -f tools_var_lib_awx tools_awx_db tools_awx_db_15 tools_vault_1 tools_ldap_1 tools_grafana_storage tools_prometheus_storage $(shell docker volume ls --filter name=tools_redis_socket_ -q)
 
 docker-refresh: docker-clean docker-compose
 
@@ -636,9 +638,6 @@ clean-elk:
 	docker rm tools_elasticsearch_1
 	docker rm tools_kibana_1
 
-psql-container:
-	docker run -it --net tools_default --rm postgres:12 sh -c 'exec psql -h "postgres" -p "5432" -U postgres'
-
 VERSION:
 	@echo "awx: $(VERSION)"
 
@@ -671,6 +670,21 @@ awx-kube-build: Dockerfile
 		--build-arg HEADLESS=$(HEADLESS) \
 		-t $(DEV_DOCKER_TAG_BASE)/awx:$(COMPOSE_TAG) .
 
+## Build multi-arch awx image for deployment on Kubernetes environment.
+awx-kube-buildx: Dockerfile
+	- docker buildx create --name awx-kube-buildx
+	docker buildx use awx-kube-buildx
+	- docker buildx build \
+		--push \
+		--build-arg VERSION=$(VERSION) \
+		--build-arg SETUPTOOLS_SCM_PRETEND_VERSION=$(VERSION) \
+		--build-arg HEADLESS=$(HEADLESS) \
+		--platform=$(PLATFORMS) \
+		--tag $(DEV_DOCKER_TAG_BASE)/awx:$(COMPOSE_TAG) \
+		-f Dockerfile .
+	- docker buildx rm awx-kube-buildx
+
+
 .PHONY: Dockerfile.kube-dev
 ## Generate Docker.kube-dev for awx_kube_devel image
 Dockerfile.kube-dev: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
@@ -687,6 +701,18 @@ awx-kube-dev-build: Dockerfile.kube-dev
 	    --cache-from=$(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG) \
 	    -t $(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG) .
 
+## Build and push multi-arch awx_kube_devel image for development on local Kubernetes environment.
+awx-kube-dev-buildx: Dockerfile.kube-dev
+	- docker buildx create --name awx-kube-dev-buildx
+	docker buildx use awx-kube-dev-buildx
+	- docker buildx build \
+		--push \
+		--build-arg BUILDKIT_INLINE_CACHE=1 \
+		--cache-from=$(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG) \
+		--platform=$(PLATFORMS) \
+		--tag $(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG) \
+		-f Dockerfile.kube-dev .
+	- docker buildx rm awx-kube-dev-buildx
 
 kind-dev-load: awx-kube-dev-build
 	$(KIND_BIN) load docker-image $(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG)

awx/__init__.py

@@ -154,10 +154,12 @@ def manage():
     from django.conf import settings
     from django.core.management import execute_from_command_line
 
-    # enforce the postgres version is equal to 12. if not, then terminate program with exit code of 1
+    # enforce the postgres version is a minimum of 12 (we need this for partitioning); if not, then terminate program with exit code of 1
+    # In the future if we require a feature of a version of postgres > 12 this should be updated to reflect that.
+    # The return of connection.pg_version is something like 12013
     if not os.getenv('SKIP_PG_VERSION_CHECK', False) and not MODE == 'development':
         if (connection.pg_version // 10000) < 12:
-            sys.stderr.write("Postgres version 12 is required\n")
+            sys.stderr.write("At a minimum, postgres version 12 is required\n")
             sys.exit(1)
 
     if len(sys.argv) >= 2 and sys.argv[1] in ('version', '--version'):  # pragma: no cover

awx/api/conf.py

@@ -93,6 +93,7 @@ register(
     default='',
     label=_('Login redirect override URL'),
     help_text=_('URL to which unauthorized users will be redirected to log in.  If blank, users will be sent to the login page.'),
+    warning_text=_('Changing the redirect URL could impact the ability to login if local authentication is also disabled.'),
     category=_('Authentication'),
     category_slug='authentication',
 )

awx/api/generics.py

@@ -30,11 +30,15 @@ from rest_framework.permissions import IsAuthenticated
 from rest_framework.renderers import StaticHTMLRenderer
 from rest_framework.negotiation import DefaultContentNegotiation
 
+# django-ansible-base
 from ansible_base.rest_filters.rest_framework.field_lookup_backend import FieldLookupBackend
 from ansible_base.lib.utils.models import get_all_field_names
+from ansible_base.rbac.models import RoleEvaluation, RoleDefinition
+from ansible_base.rbac.permission_registry import permission_registry
 
 # AWX
 from awx.main.models import UnifiedJob, UnifiedJobTemplate, User, Role, Credential, WorkflowJobTemplateNode, WorkflowApprovalTemplate
+from awx.main.models.rbac import give_creator_permissions
 from awx.main.access import optimize_queryset
 from awx.main.utils import camelcase_to_underscore, get_search_fields, getattrd, get_object_or_400, decrypt_field, get_awx_version
 from awx.main.utils.licensing import server_product_name
@@ -472,7 +476,11 @@ class ListAPIView(generics.ListAPIView, GenericAPIView):
 
 class ListCreateAPIView(ListAPIView, generics.ListCreateAPIView):
     # Base class for a list view that allows creating new objects.
-    pass
+    def perform_create(self, serializer):
+        super().perform_create(serializer)
+        if serializer.Meta.model in permission_registry.all_registered_models:
+            if self.request and self.request.user:
+                give_creator_permissions(self.request.user, serializer.instance)
 
 
 class ParentMixin(object):
@@ -792,6 +800,7 @@ class RetrieveUpdateDestroyAPIView(RetrieveUpdateAPIView, DestroyAPIView):
 
 
 class ResourceAccessList(ParentMixin, ListAPIView):
+    deprecated = True
     serializer_class = ResourceAccessListElementSerializer
     ordering = ('username',)
 
@@ -799,6 +808,15 @@ class ResourceAccessList(ParentMixin, ListAPIView):
         obj = self.get_parent_object()
 
         content_type = ContentType.objects.get_for_model(obj)
+
+        if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+            ancestors = set(RoleEvaluation.objects.filter(content_type_id=content_type.id, object_id=obj.id).values_list('role_id', flat=True))
+            qs = User.objects.filter(has_roles__in=ancestors) | User.objects.filter(is_superuser=True)
+            auditor_role = RoleDefinition.objects.filter(name="System Auditor").first()
+            if auditor_role:
+                qs |= User.objects.filter(role_assignments__role_definition=auditor_role)
+            return qs.distinct()
+
         roles = set(Role.objects.filter(content_type=content_type, object_id=obj.id))
 
         ancestors = set()
@@ -958,7 +976,7 @@ class CopyAPIView(GenericAPIView):
             None, None, self.model, obj, request.user, create_kwargs=create_kwargs, copy_name=serializer.validated_data.get('name', '')
         )
         if hasattr(new_obj, 'admin_role') and request.user not in new_obj.admin_role.members.all():
-            new_obj.admin_role.members.add(request.user)
+            give_creator_permissions(request.user, new_obj)
         if sub_objs:
             permission_check_func = None
             if hasattr(type(self), 'deep_copy_permission_check_func'):

awx/api/metadata.py

@@ -36,11 +36,13 @@ class Metadata(metadata.SimpleMetadata):
         field_info = OrderedDict()
         field_info['type'] = self.label_lookup[field]
         field_info['required'] = getattr(field, 'required', False)
+        field_info['hidden'] = getattr(field, 'hidden', False)
 
         text_attrs = [
             'read_only',
             'label',
             'help_text',
+            'warning_text',
             'min_length',
             'max_length',
             'min_value',

awx/api/serializers.py

@@ -43,11 +43,14 @@ from rest_framework.utils.serializer_helpers import ReturnList
 # Django-Polymorphic
 from polymorphic.models import PolymorphicModel
 
+# django-ansible-base
 from ansible_base.lib.utils.models import get_type_for_model
+from ansible_base.rbac.models import RoleEvaluation, ObjectRole
+from ansible_base.rbac import permission_registry
 
 # AWX
 from awx.main.access import get_user_capabilities
-from awx.main.constants import ACTIVE_STATES, CENSOR_VALUE
+from awx.main.constants import ACTIVE_STATES, CENSOR_VALUE, org_role_to_permission
 from awx.main.models import (
     ActivityStream,
     AdHocCommand,
@@ -102,7 +105,7 @@ from awx.main.models import (
     CLOUD_INVENTORY_SOURCES,
 )
 from awx.main.models.base import VERBOSITY_CHOICES, NEW_JOB_TYPE_CHOICES
-from awx.main.models.rbac import role_summary_fields_generator, RoleAncestorEntry
+from awx.main.models.rbac import role_summary_fields_generator, give_creator_permissions, get_role_codenames, to_permissions, get_role_from_object_role
 from awx.main.fields import ImplicitRoleField
 from awx.main.utils import (
     get_model_for_type,
@@ -191,6 +194,7 @@ SUMMARIZABLE_FK_FIELDS = {
     'webhook_credential': DEFAULT_SUMMARY_FIELDS + ('kind', 'cloud', 'credential_type_id'),
     'approved_or_denied_by': ('id', 'username', 'first_name', 'last_name'),
     'credential_type': DEFAULT_SUMMARY_FIELDS,
+    'resource': ('ansible_id', 'resource_type'),
 }
 
 
@@ -2762,13 +2766,26 @@ class ResourceAccessListElementSerializer(UserSerializer):
         team_content_type = ContentType.objects.get_for_model(Team)
         content_type = ContentType.objects.get_for_model(obj)
 
-        def get_roles_on_resource(parent_role):
-            "Returns a string list of the roles a parent_role has for current obj."
-            return list(
-                RoleAncestorEntry.objects.filter(ancestor=parent_role, content_type_id=content_type.id, object_id=obj.id)
-                .values_list('role_field', flat=True)
-                .distinct()
-            )
+        reversed_org_map = {}
+        for k, v in org_role_to_permission.items():
+            reversed_org_map[v] = k
+        reversed_role_map = {}
+        for k, v in to_permissions.items():
+            reversed_role_map[v] = k
+
+        def get_roles_from_perms(perm_list):
+            """given a list of permission codenames return a list of role names"""
+            role_names = set()
+            for codename in perm_list:
+                action = codename.split('_', 1)[0]
+                if action in reversed_role_map:
+                    role_names.add(reversed_role_map[action])
+                elif codename in reversed_org_map:
+                    if isinstance(obj, Organization):
+                        role_names.add(reversed_org_map[codename])
+                        if 'view_organization' not in role_names:
+                            role_names.add('read_role')
+            return list(role_names)
 
         def format_role_perm(role):
             role_dict = {'id': role.id, 'name': role.name, 'description': role.description}
@@ -2785,13 +2802,21 @@ class ResourceAccessListElementSerializer(UserSerializer):
             else:
                 # Singleton roles should not be managed from this view, as per copy/edit rework spec
                 role_dict['user_capabilities'] = {'unattach': False}
-            return {'role': role_dict, 'descendant_roles': get_roles_on_resource(role)}
+
+            model_name = content_type.model
+            if isinstance(obj, Organization):
+                descendant_perms = [codename for codename in get_role_codenames(role) if codename.endswith(model_name) or codename.startswith('add_')]
+            else:
+                descendant_perms = [codename for codename in get_role_codenames(role) if codename.endswith(model_name)]
+
+            return {'role': role_dict, 'descendant_roles': get_roles_from_perms(descendant_perms)}
 
         def format_team_role_perm(naive_team_role, permissive_role_ids):
             ret = []
+            team = naive_team_role.content_object
             team_role = naive_team_role
             if naive_team_role.role_field == 'admin_role':
-                team_role = naive_team_role.content_object.member_role
+                team_role = team.member_role
             for role in team_role.children.filter(id__in=permissive_role_ids).all():
                 role_dict = {
                     'id': role.id,
@@ -2811,10 +2836,87 @@ class ResourceAccessListElementSerializer(UserSerializer):
                 else:
                     # Singleton roles should not be managed from this view, as per copy/edit rework spec
                     role_dict['user_capabilities'] = {'unattach': False}
-                ret.append({'role': role_dict, 'descendant_roles': get_roles_on_resource(team_role)})
+
+                descendant_perms = list(
+                    RoleEvaluation.objects.filter(role__in=team.has_roles.all(), object_id=obj.id, content_type_id=content_type.id)
+                    .values_list('codename', flat=True)
+                    .distinct()
+                )
+
+                ret.append({'role': role_dict, 'descendant_roles': get_roles_from_perms(descendant_perms)})
+            return ret
+
+        gfk_kwargs = dict(content_type_id=content_type.id, object_id=obj.id)
+        direct_permissive_role_ids = Role.objects.filter(**gfk_kwargs).values_list('id', flat=True)
+
+        if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+            ret['summary_fields']['direct_access'] = []
+            ret['summary_fields']['indirect_access'] = []
+
+            new_roles_seen = set()
+            all_team_roles = set()
+            all_permissive_role_ids = set()
+            for evaluation in RoleEvaluation.objects.filter(role__in=user.has_roles.all(), **gfk_kwargs).prefetch_related('role'):
+                new_role = evaluation.role
+                if new_role.id in new_roles_seen:
+                    continue
+                new_roles_seen.add(new_role.id)
+                old_role = get_role_from_object_role(new_role)
+                all_permissive_role_ids.add(old_role.id)
+
+                if int(new_role.object_id) == obj.id and new_role.content_type_id == content_type.id:
+                    ret['summary_fields']['direct_access'].append(format_role_perm(old_role))
+                elif new_role.content_type_id == team_content_type.id:
+                    all_team_roles.add(old_role)
+                else:
+                    ret['summary_fields']['indirect_access'].append(format_role_perm(old_role))
+
+            # Lazy role creation gives us a big problem, where some intermediate roles are not easy to find
+            # like when a team has indirect permission, so here we get all roles the users teams have
+            # these contribute to all potential permission-granting roles of the object
+            user_teams_qs = permission_registry.team_model.objects.filter(member_roles__in=ObjectRole.objects.filter(users=user))
+            team_obj_roles = ObjectRole.objects.filter(teams__in=user_teams_qs)
+            for evaluation in RoleEvaluation.objects.filter(role__in=team_obj_roles, **gfk_kwargs).prefetch_related('role'):
+                new_role = evaluation.role
+                if new_role.id in new_roles_seen:
+                    continue
+                new_roles_seen.add(new_role.id)
+                old_role = get_role_from_object_role(new_role)
+                all_permissive_role_ids.add(old_role.id)
+
+            # In DAB RBAC, superuser is strictly a user flag, and global roles are not in the RoleEvaluation table
+            if user.is_superuser:
+                ret['summary_fields'].setdefault('indirect_access', [])
+                all_role_names = [field.name for field in obj._meta.get_fields() if isinstance(field, ImplicitRoleField)]
+                ret['summary_fields']['indirect_access'].append(
+                    {
+                        "role": {
+                            "id": None,
+                            "name": _("System Administrator"),
+                            "description": _("Can manage all aspects of the system"),
+                            "user_capabilities": {"unattach": False},
+                        },
+                        "descendant_roles": all_role_names,
+                    }
+                )
+            elif user.is_system_auditor:
+                ret['summary_fields'].setdefault('indirect_access', [])
+                ret['summary_fields']['indirect_access'].append(
+                    {
+                        "role": {
+                            "id": None,
+                            "name": _("System Auditor"),
+                            "description": _("Can view all aspects of the system"),
+                            "user_capabilities": {"unattach": False},
+                        },
+                        "descendant_roles": ["read_role"],
+                    }
+                )
+
+            ret['summary_fields']['direct_access'].extend([y for x in (format_team_role_perm(r, all_permissive_role_ids) for r in all_team_roles) for y in x])
+
             return ret
 
-        direct_permissive_role_ids = Role.objects.filter(content_type=content_type, object_id=obj.id).values_list('id', flat=True)
         all_permissive_role_ids = Role.objects.filter(content_type=content_type, object_id=obj.id).values_list('ancestors__id', flat=True)
 
         direct_access_roles = user.roles.filter(id__in=direct_permissive_role_ids).all()
@@ -3083,7 +3185,7 @@ class CredentialSerializerCreate(CredentialSerializer):
         credential = super(CredentialSerializerCreate, self).create(validated_data)
 
         if user:
-            credential.admin_role.members.add(user)
+            give_creator_permissions(user, credential)
         if team:
             if not credential.organization or team.organization.id != credential.organization.id:
                 raise serializers.ValidationError({"detail": _("Credential organization must be set and match before assigning to a team")})
@@ -5594,7 +5696,7 @@ class InstanceSerializer(BaseSerializer):
         res['jobs'] = self.reverse('api:instance_unified_jobs_list', kwargs={'pk': obj.pk})
         res['peers'] = self.reverse('api:instance_peers_list', kwargs={"pk": obj.pk})
         res['instance_groups'] = self.reverse('api:instance_instance_groups_list', kwargs={'pk': obj.pk})
-        if obj.node_type in [Instance.Types.EXECUTION, Instance.Types.HOP]:
+        if obj.node_type in [Instance.Types.EXECUTION, Instance.Types.HOP] and not obj.managed:
             res['install_bundle'] = self.reverse('api:instance_install_bundle', kwargs={'pk': obj.pk})
         if self.context['request'].user.is_superuser or self.context['request'].user.is_system_auditor:
             if obj.node_type == 'execution':

awx/api/versioning.py

@@ -2,28 +2,21 @@
 # All Rights Reserved.
 
 from django.conf import settings
-from django.urls import NoReverseMatch
 
-from rest_framework.reverse import _reverse
+from rest_framework.reverse import reverse as drf_reverse
 from rest_framework.versioning import URLPathVersioning as BaseVersioning
 
 
-def drf_reverse(viewname, args=None, kwargs=None, request=None, format=None, **extra):
-    """
-    Copy and monkey-patch `rest_framework.reverse.reverse` to prevent adding unwarranted
-    query string parameters.
-    """
-    scheme = getattr(request, 'versioning_scheme', None)
-    if scheme is not None:
-        try:
-            url = scheme.reverse(viewname, args, kwargs, request, format, **extra)
-        except NoReverseMatch:
-            # In case the versioning scheme reversal fails, fallback to the
-            # default implementation
-            url = _reverse(viewname, args, kwargs, request, format, **extra)
-    else:
-        url = _reverse(viewname, args, kwargs, request, format, **extra)
+def is_optional_api_urlpattern_prefix_request(request):
+    if settings.OPTIONAL_API_URLPATTERN_PREFIX and request:
+        if request.path.startswith(f"/api/{settings.OPTIONAL_API_URLPATTERN_PREFIX}"):
+            return True
+    return False
 
+
+def transform_optional_api_urlpattern_prefix_url(request, url):
+    if is_optional_api_urlpattern_prefix_request(request):
+        url = url.replace('/api', f"/api/{settings.OPTIONAL_API_URLPATTERN_PREFIX}")
     return url
 
 

awx/api/views/__init__.py

@@ -60,6 +60,9 @@ from oauth2_provider.models import get_access_token_model
 import pytz
 from wsgiref.util import FileWrapper
 
+# django-ansible-base
+from ansible_base.rbac.models import RoleEvaluation, ObjectRole
+
 # AWX
 from awx.main.tasks.system import send_notifications, update_inventory_computed_fields
 from awx.main.access import get_user_queryset
@@ -87,6 +90,7 @@ from awx.api.generics import (
 from awx.api.views.labels import LabelSubListCreateAttachDetachView
 from awx.api.versioning import reverse
 from awx.main import models
+from awx.main.models.rbac import get_role_definition
 from awx.main.utils import (
     camelcase_to_underscore,
     extract_ansible_vars,
@@ -272,16 +276,24 @@ class DashboardJobsGraphView(APIView):
 
         success_query = user_unified_jobs.filter(status='successful')
         failed_query = user_unified_jobs.filter(status='failed')
+        canceled_query = user_unified_jobs.filter(status='canceled')
+        error_query = user_unified_jobs.filter(status='error')
 
         if job_type == 'inv_sync':
             success_query = success_query.filter(instance_of=models.InventoryUpdate)
             failed_query = failed_query.filter(instance_of=models.InventoryUpdate)
+            canceled_query = canceled_query.filter(instance_of=models.InventoryUpdate)
+            error_query = error_query.filter(instance_of=models.InventoryUpdate)
         elif job_type == 'playbook_run':
             success_query = success_query.filter(instance_of=models.Job)
             failed_query = failed_query.filter(instance_of=models.Job)
+            canceled_query = canceled_query.filter(instance_of=models.Job)
+            error_query = error_query.filter(instance_of=models.Job)
         elif job_type == 'scm_update':
             success_query = success_query.filter(instance_of=models.ProjectUpdate)
             failed_query = failed_query.filter(instance_of=models.ProjectUpdate)
+            canceled_query = canceled_query.filter(instance_of=models.ProjectUpdate)
+            error_query = error_query.filter(instance_of=models.ProjectUpdate)
 
         end = now()
         interval = 'day'
@@ -297,10 +309,12 @@ class DashboardJobsGraphView(APIView):
         else:
             return Response({'error': _('Unknown period "%s"') % str(period)}, status=status.HTTP_400_BAD_REQUEST)
 
-        dashboard_data = {"jobs": {"successful": [], "failed": []}}
+        dashboard_data = {"jobs": {"successful": [], "failed": [], "canceled": [], "error": []}}
 
         succ_list = dashboard_data['jobs']['successful']
         fail_list = dashboard_data['jobs']['failed']
+        canceled_list = dashboard_data['jobs']['canceled']
+        error_list = dashboard_data['jobs']['error']
 
         qs_s = (
             success_query.filter(finished__range=(start, end))
@@ -318,6 +332,22 @@ class DashboardJobsGraphView(APIView):
             .annotate(agg=Count('id', distinct=True))
         )
         data_f = {item['d']: item['agg'] for item in qs_f}
+        qs_c = (
+            canceled_query.filter(finished__range=(start, end))
+            .annotate(d=Trunc('finished', interval, tzinfo=end.tzinfo))
+            .order_by()
+            .values('d')
+            .annotate(agg=Count('id', distinct=True))
+        )
+        data_c = {item['d']: item['agg'] for item in qs_c}
+        qs_e = (
+            error_query.filter(finished__range=(start, end))
+            .annotate(d=Trunc('finished', interval, tzinfo=end.tzinfo))
+            .order_by()
+            .values('d')
+            .annotate(agg=Count('id', distinct=True))
+        )
+        data_e = {item['d']: item['agg'] for item in qs_e}
 
         start_date = start.replace(hour=0, minute=0, second=0, microsecond=0)
         for d in itertools.count():
@@ -326,6 +356,8 @@ class DashboardJobsGraphView(APIView):
                 break
             succ_list.append([time.mktime(date.timetuple()), data_s.get(date, 0)])
             fail_list.append([time.mktime(date.timetuple()), data_f.get(date, 0)])
+            canceled_list.append([time.mktime(date.timetuple()), data_c.get(date, 0)])
+            error_list.append([time.mktime(date.timetuple()), data_e.get(date, 0)])
 
         return Response(dashboard_data)
 
@@ -508,6 +540,7 @@ class InstanceGroupAccessList(ResourceAccessList):
 
 
 class InstanceGroupObjectRolesList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.InstanceGroup
@@ -696,6 +729,7 @@ class TeamUsersList(BaseUsersList):
 
 
 class TeamRolesList(SubListAttachDetachAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializerWithParentAccess
     metadata_class = RoleMetadata
@@ -735,10 +769,12 @@ class TeamRolesList(SubListAttachDetachAPIView):
 
 
 class TeamObjectRolesList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.Team
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()
@@ -756,8 +792,15 @@ class TeamProjectsList(SubListAPIView):
         self.check_parent_access(team)
         model_ct = ContentType.objects.get_for_model(self.model)
         parent_ct = ContentType.objects.get_for_model(self.parent_model)
-        proj_roles = models.Role.objects.filter(Q(ancestors__content_type=parent_ct) & Q(ancestors__object_id=team.pk), content_type=model_ct)
-        return self.model.accessible_objects(self.request.user, 'read_role').filter(pk__in=[t.content_object.pk for t in proj_roles])
+
+        rd = get_role_definition(team.member_role)
+        role = ObjectRole.objects.filter(object_id=team.id, content_type=parent_ct, role_definition=rd).first()
+        if role is None:
+            # Team has no permissions, therefore team has no projects
+            return self.model.objects.none()
+        else:
+            project_qs = self.model.accessible_objects(self.request.user, 'read_role')
+            return project_qs.filter(id__in=RoleEvaluation.objects.filter(content_type_id=model_ct.id, role=role).values_list('object_id'))
 
 
 class TeamActivityStreamList(SubListAPIView):
@@ -772,10 +815,23 @@ class TeamActivityStreamList(SubListAPIView):
         self.check_parent_access(parent)
 
         qs = self.request.user.get_queryset(self.model)
+
         return qs.filter(
             Q(team=parent)
-            | Q(project__in=models.Project.accessible_objects(parent.member_role, 'read_role'))
-            | Q(credential__in=models.Credential.accessible_objects(parent.member_role, 'read_role'))
+            | Q(
+                project__in=RoleEvaluation.objects.filter(
+                    role__in=parent.has_roles.all(), content_type_id=ContentType.objects.get_for_model(models.Project).id, codename='view_project'
+                )
+                .values_list('object_id')
+                .distinct()
+            )
+            | Q(
+                credential__in=RoleEvaluation.objects.filter(
+                    role__in=parent.has_roles.all(), content_type_id=ContentType.objects.get_for_model(models.Credential).id, codename='view_credential'
+                )
+                .values_list('object_id')
+                .distinct()
+            )
         )
 
 
@@ -1027,10 +1083,12 @@ class ProjectAccessList(ResourceAccessList):
 
 
 class ProjectObjectRolesList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.Project
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()
@@ -1188,6 +1246,7 @@ class UserTeamsList(SubListAPIView):
 
 
 class UserRolesList(SubListAttachDetachAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializerWithParentAccess
     metadata_class = RoleMetadata
@@ -1462,10 +1521,12 @@ class CredentialAccessList(ResourceAccessList):
 
 
 class CredentialObjectRolesList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.Credential
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()
@@ -2252,13 +2313,6 @@ class JobTemplateList(ListCreateAPIView):
     serializer_class = serializers.JobTemplateSerializer
     always_allow_superuser = False
 
-    def post(self, request, *args, **kwargs):
-        ret = super(JobTemplateList, self).post(request, *args, **kwargs)
-        if ret.status_code == 201:
-            job_template = models.JobTemplate.objects.get(id=ret.data['id'])
-            job_template.admin_role.members.add(request.user)
-        return ret
-
 
 class JobTemplateDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIView):
     model = models.JobTemplate
@@ -2804,10 +2858,12 @@ class JobTemplateAccessList(ResourceAccessList):
 
 
 class JobTemplateObjectRolesList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.JobTemplate
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()
@@ -3190,10 +3246,12 @@ class WorkflowJobTemplateAccessList(ResourceAccessList):
 
 
 class WorkflowJobTemplateObjectRolesList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.WorkflowJobTemplate
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()
@@ -4202,6 +4260,7 @@ class ActivityStreamDetail(RetrieveAPIView):
 
 
 class RoleList(ListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     permission_classes = (IsAuthenticated,)
@@ -4209,11 +4268,13 @@ class RoleList(ListAPIView):
 
 
 class RoleDetail(RetrieveAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
 
 
 class RoleUsersList(SubListAttachDetachAPIView):
+    deprecated = True
     model = models.User
     serializer_class = serializers.UserSerializer
     parent_model = models.Role
@@ -4248,6 +4309,7 @@ class RoleUsersList(SubListAttachDetachAPIView):
 
 
 class RoleTeamsList(SubListAttachDetachAPIView):
+    deprecated = True
     model = models.Team
     serializer_class = serializers.TeamSerializer
     parent_model = models.Role
@@ -4292,10 +4354,12 @@ class RoleTeamsList(SubListAttachDetachAPIView):
             team.member_role.children.remove(role)
         else:
             team.member_role.children.add(role)
+
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
 class RoleParentsList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.Role
@@ -4309,6 +4373,7 @@ class RoleParentsList(SubListAPIView):
 
 
 class RoleChildrenList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.Role

awx/api/views/analytics.py

@@ -48,23 +48,23 @@ class AnalyticsRootView(APIView):
 
     def get(self, request, format=None):
         data = OrderedDict()
-        data['authorized'] = reverse('api:analytics_authorized')
-        data['reports'] = reverse('api:analytics_reports_list')
-        data['report_options'] = reverse('api:analytics_report_options_list')
-        data['adoption_rate'] = reverse('api:analytics_adoption_rate')
-        data['adoption_rate_options'] = reverse('api:analytics_adoption_rate_options')
-        data['event_explorer'] = reverse('api:analytics_event_explorer')
-        data['event_explorer_options'] = reverse('api:analytics_event_explorer_options')
-        data['host_explorer'] = reverse('api:analytics_host_explorer')
-        data['host_explorer_options'] = reverse('api:analytics_host_explorer_options')
-        data['job_explorer'] = reverse('api:analytics_job_explorer')
-        data['job_explorer_options'] = reverse('api:analytics_job_explorer_options')
-        data['probe_templates'] = reverse('api:analytics_probe_templates_explorer')
-        data['probe_templates_options'] = reverse('api:analytics_probe_templates_options')
-        data['probe_template_for_hosts'] = reverse('api:analytics_probe_template_for_hosts_explorer')
-        data['probe_template_for_hosts_options'] = reverse('api:analytics_probe_template_for_hosts_options')
-        data['roi_templates'] = reverse('api:analytics_roi_templates_explorer')
-        data['roi_templates_options'] = reverse('api:analytics_roi_templates_options')
+        data['authorized'] = reverse('api:analytics_authorized', request=request)
+        data['reports'] = reverse('api:analytics_reports_list', request=request)
+        data['report_options'] = reverse('api:analytics_report_options_list', request=request)
+        data['adoption_rate'] = reverse('api:analytics_adoption_rate', request=request)
+        data['adoption_rate_options'] = reverse('api:analytics_adoption_rate_options', request=request)
+        data['event_explorer'] = reverse('api:analytics_event_explorer', request=request)
+        data['event_explorer_options'] = reverse('api:analytics_event_explorer_options', request=request)
+        data['host_explorer'] = reverse('api:analytics_host_explorer', request=request)
+        data['host_explorer_options'] = reverse('api:analytics_host_explorer_options', request=request)
+        data['job_explorer'] = reverse('api:analytics_job_explorer', request=request)
+        data['job_explorer_options'] = reverse('api:analytics_job_explorer_options', request=request)
+        data['probe_templates'] = reverse('api:analytics_probe_templates_explorer', request=request)
+        data['probe_templates_options'] = reverse('api:analytics_probe_templates_options', request=request)
+        data['probe_template_for_hosts'] = reverse('api:analytics_probe_template_for_hosts_explorer', request=request)
+        data['probe_template_for_hosts_options'] = reverse('api:analytics_probe_template_for_hosts_options', request=request)
+        data['roi_templates'] = reverse('api:analytics_roi_templates_explorer', request=request)
+        data['roi_templates_options'] = reverse('api:analytics_roi_templates_options', request=request)
         return Response(data)
 
 

awx/api/views/inventory.py

@@ -152,6 +152,7 @@ class InventoryObjectRolesList(SubListAPIView):
     serializer_class = RoleSerializer
     parent_model = Inventory
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()

awx/api/views/organization.py

@@ -226,6 +226,7 @@ class OrganizationObjectRolesList(SubListAPIView):
     serializer_class = RoleSerializer
     parent_model = Organization
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()

awx/api/views/root.py

@@ -13,6 +13,7 @@ from django.utils.decorators import method_decorator
 from django.views.decorators.csrf import ensure_csrf_cookie
 from django.template.loader import render_to_string
 from django.utils.translation import gettext_lazy as _
+from django.urls import reverse as django_reverse
 
 from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.response import Response
@@ -27,7 +28,7 @@ from awx.main.analytics import all_collectors
 from awx.main.ha import is_ha_environment
 from awx.main.utils import get_awx_version, get_custom_venv_choices
 from awx.main.utils.licensing import validate_entitlement_manifest
-from awx.api.versioning import reverse, drf_reverse
+from awx.api.versioning import URLPathVersioning, is_optional_api_urlpattern_prefix_request, reverse, drf_reverse
 from awx.main.constants import PRIVILEGE_ESCALATION_METHODS
 from awx.main.models import Project, Organization, Instance, InstanceGroup, JobTemplate
 from awx.main.utils import set_environ
@@ -39,19 +40,19 @@ logger = logging.getLogger('awx.api.views.root')
 class ApiRootView(APIView):
     permission_classes = (AllowAny,)
     name = _('REST API')
-    versioning_class = None
+    versioning_class = URLPathVersioning
     swagger_topic = 'Versioning'
 
     @method_decorator(ensure_csrf_cookie)
     def get(self, request, format=None):
         '''List supported API versions'''
-
-        v2 = reverse('api:api_v2_root_view', kwargs={'version': 'v2'})
+        v2 = reverse('api:api_v2_root_view', request=request, kwargs={'version': 'v2'})
         data = OrderedDict()
         data['description'] = _('AWX REST API')
         data['current_version'] = v2
         data['available_versions'] = dict(v2=v2)
-        data['oauth2'] = drf_reverse('api:oauth_authorization_root_view')
+        if not is_optional_api_urlpattern_prefix_request(request):
+            data['oauth2'] = drf_reverse('api:oauth_authorization_root_view')
         data['custom_logo'] = settings.CUSTOM_LOGO
         data['custom_login_info'] = settings.CUSTOM_LOGIN_INFO
         data['login_redirect_override'] = settings.LOGIN_REDIRECT_OVERRIDE
@@ -130,6 +131,10 @@ class ApiVersionRootView(APIView):
         data['mesh_visualizer'] = reverse('api:mesh_visualizer_view', request=request)
         data['bulk'] = reverse('api:bulk', request=request)
         data['analytics'] = reverse('api:analytics_root_view', request=request)
+        data['service_index'] = django_reverse('service-index-root')
+        data['role_definitions'] = django_reverse('roledefinition-list')
+        data['role_user_assignments'] = django_reverse('roleuserassignment-list')
+        data['role_team_assignments'] = django_reverse('roleteamassignment-list')
         return Response(data)
 
 

awx/conf/conf.py

@@ -55,6 +55,7 @@ register(
     # Optional; category_slug will be slugified version of category if not
     # explicitly provided.
     category_slug='cows',
+    hidden=True,
 )
 
 

awx/conf/registry.py

@@ -127,6 +127,8 @@ class SettingsRegistry(object):
         encrypted = bool(field_kwargs.pop('encrypted', False))
         defined_in_file = bool(field_kwargs.pop('defined_in_file', False))
         unit = field_kwargs.pop('unit', None)
+        hidden = field_kwargs.pop('hidden', False)
+        warning_text = field_kwargs.pop('warning_text', None)
         if getattr(field_kwargs.get('child', None), 'source', None) is not None:
             field_kwargs['child'].source = None
         field_instance = field_class(**field_kwargs)
@@ -134,12 +136,14 @@ class SettingsRegistry(object):
         field_instance.category = category
         field_instance.depends_on = depends_on
         field_instance.unit = unit
+        field_instance.hidden = hidden
         if placeholder is not empty:
             field_instance.placeholder = placeholder
         field_instance.defined_in_file = defined_in_file
         if field_instance.defined_in_file:
             field_instance.help_text = str(_('This value has been set manually in a settings file.')) + '\n\n' + str(field_instance.help_text)
         field_instance.encrypted = encrypted
+        field_instance.warning_text = warning_text
         original_field_instance = field_instance
         if field_class != original_field_class:
             original_field_instance = original_field_class(**field_kwargs)

awx/conf/settings.py

@@ -1,6 +1,7 @@
 # Python
 import contextlib
 import logging
+import psycopg
 import threading
 import time
 import os
@@ -13,7 +14,7 @@ from django.conf import settings, UserSettingsHolder
 from django.core.cache import cache as django_cache
 from django.core.exceptions import ImproperlyConfigured, SynchronousOnlyOperation
 from django.db import transaction, connection
-from django.db.utils import Error as DBError, ProgrammingError
+from django.db.utils import DatabaseError, ProgrammingError
 from django.utils.functional import cached_property
 
 # Django REST Framework
@@ -80,18 +81,26 @@ def _ctit_db_wrapper(trans_safe=False):
                     logger.debug('Obtaining database settings in spite of broken transaction.')
                     transaction.set_rollback(False)
         yield
-    except DBError as exc:
+    except ProgrammingError as e:
+        # Exception raised for programming errors
+        # Examples may be table not found or already exists,
+        # this generally means we can't fetch Tower configuration
+        # because the database hasn't actually finished migrating yet;
+        # this is usually a sign that a service in a container (such as ws_broadcast)
+        # has come up *before* the database has finished migrating, and
+        # especially that the conf.settings table doesn't exist yet
+        # syntax error in the SQL statement, wrong number of parameters specified, etc.
         if trans_safe:
-            level = logger.warning
-            if isinstance(exc, ProgrammingError):
-                if 'relation' in str(exc) and 'does not exist' in str(exc):
-                    # this generally means we can't fetch Tower configuration
-                    # because the database hasn't actually finished migrating yet;
-                    # this is usually a sign that a service in a container (such as ws_broadcast)
-                    # has come up *before* the database has finished migrating, and
-                    # especially that the conf.settings table doesn't exist yet
-                    level = logger.debug
-            level(f'Database settings are not available, using defaults. error: {str(exc)}')
+            logger.debug(f'Database settings are not available, using defaults. error: {str(e)}')
+        else:
+            logger.exception('Error modifying something related to database settings.')
+    except DatabaseError as e:
+        if trans_safe:
+            cause = e.__cause__
+            if cause and hasattr(cause, 'sqlstate'):
+                sqlstate = cause.sqlstate
+                sqlstate_str = psycopg.errors.lookup(sqlstate)
+                logger.error('SQL Error state: {} - {}'.format(sqlstate, sqlstate_str))
         else:
             logger.exception('Error modifying something related to database settings.')
     finally:

awx/main/access.py

@@ -20,7 +20,10 @@ from rest_framework.exceptions import ParseError, PermissionDenied
 # Django OAuth Toolkit
 from awx.main.models.oauth import OAuth2Application, OAuth2AccessToken
 
+# django-ansible-base
 from ansible_base.lib.utils.validation import to_python_boolean
+from ansible_base.rbac.models import RoleEvaluation
+from ansible_base.rbac import permission_registry
 
 # AWX
 from awx.main.utils import (
@@ -72,8 +75,6 @@ from awx.main.models import (
     WorkflowJobTemplateNode,
     WorkflowApproval,
     WorkflowApprovalTemplate,
-    ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
-    ROLE_SINGLETON_SYSTEM_AUDITOR,
 )
 from awx.main.models.mixins import ResourceMixin
 
@@ -264,7 +265,11 @@ class BaseAccess(object):
         return self.can_change(obj, data)
 
     def can_delete(self, obj):
-        return self.user.is_superuser
+        if self.user.is_superuser:
+            return True
+        if obj._meta.model_name in [cls._meta.model_name for cls in permission_registry.all_registered_models]:
+            return self.user.has_obj_perm(obj, 'delete')
+        return False
 
     def can_copy(self, obj):
         return self.can_add({'reference_obj': obj})
@@ -639,7 +644,10 @@ class UserAccess(BaseAccess):
     """
 
     model = User
-    prefetch_related = ('profile',)
+    prefetch_related = (
+        'profile',
+        'resource',
+    )
 
     def filtered_queryset(self):
         if settings.ORG_ADMINS_CAN_SEE_ALL_USERS and (self.user.admin_of_organizations.exists() or self.user.auditor_of_organizations.exists()):
@@ -648,9 +656,7 @@ class UserAccess(BaseAccess):
             qs = (
                 User.objects.filter(pk__in=Organization.accessible_objects(self.user, 'read_role').values('member_role__members'))
                 | User.objects.filter(pk=self.user.id)
-                | User.objects.filter(
-                    pk__in=Role.objects.filter(singleton_name__in=[ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR]).values('members')
-                )
+                | User.objects.filter(is_superuser=True)
             ).distinct()
         return qs
 
@@ -708,6 +714,15 @@ class UserAccess(BaseAccess):
             if not allow_orphans:
                 # in these cases only superusers can modify orphan users
                 return False
+            if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+                # Permission granted if the user has all permissions that the target user has
+                target_perms = set(
+                    RoleEvaluation.objects.filter(role__in=obj.has_roles.all()).values_list('object_id', 'content_type_id', 'codename').distinct()
+                )
+                user_perms = set(
+                    RoleEvaluation.objects.filter(role__in=self.user.has_roles.all()).values_list('object_id', 'content_type_id', 'codename').distinct()
+                )
+                return not (target_perms - user_perms)
             return not obj.roles.all().exclude(ancestors__in=self.user.roles.all()).exists()
         else:
             return self.is_all_org_admin(obj)
@@ -835,6 +850,7 @@ class OrganizationAccess(NotificationAttachMixin, BaseAccess):
     prefetch_related = (
         'created_by',
         'modified_by',
+        'resource',  # dab_resource_registry
     )
     # organization admin_role is not a parent of organization auditor_role
     notification_attach_roles = ['admin_role', 'auditor_role']
@@ -945,9 +961,6 @@ class InventoryAccess(BaseAccess):
     def can_update(self, obj):
         return self.user in obj.update_role
 
-    def can_delete(self, obj):
-        return self.can_admin(obj, None)
-
     def can_run_ad_hoc_commands(self, obj):
         return self.user in obj.adhoc_role
 
@@ -1303,6 +1316,7 @@ class TeamAccess(BaseAccess):
         'created_by',
         'modified_by',
         'organization',
+        'resource',  # dab_resource_registry
     )
 
     def filtered_queryset(self):
@@ -1400,8 +1414,12 @@ class ExecutionEnvironmentAccess(BaseAccess):
     def can_change(self, obj, data):
         if obj and obj.organization_id is None:
             raise PermissionDenied
-        if self.user not in obj.organization.execution_environment_admin_role:
-            raise PermissionDenied
+        if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+            if not self.user.has_obj_perm(obj, 'change'):
+                raise PermissionDenied
+        else:
+            if self.user not in obj.organization.execution_environment_admin_role:
+                raise PermissionDenied
         if data and 'organization' in data:
             new_org = get_object_from_data('organization', Organization, data, obj=obj)
             if not new_org or self.user not in new_org.execution_environment_admin_role:
@@ -2587,6 +2605,8 @@ class ScheduleAccess(UnifiedCredentialsMixin, BaseAccess):
         if not JobLaunchConfigAccess(self.user).can_add(data):
             return False
         if not data:
+            if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+                return self.user.has_roles.filter(permission_partials__codename__in=['execute_jobtemplate', 'update_project', 'update_inventory']).exists()
             return Role.objects.filter(role_field__in=['update_role', 'execute_role'], ancestors__in=self.user.roles.all()).exists()
 
         return self.check_related('unified_job_template', UnifiedJobTemplate, data, role_field='execute_role', mandatory=True)
@@ -2615,6 +2635,8 @@ class NotificationTemplateAccess(BaseAccess):
     prefetch_related = ('created_by', 'modified_by', 'organization')
 
     def filtered_queryset(self):
+        if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+            return self.model.access_qs(self.user, 'view')
         return self.model.objects.filter(
             Q(organization__in=Organization.accessible_objects(self.user, 'notification_admin_role')) | Q(organization__in=self.user.auditor_of_organizations)
         ).distinct()
@@ -2783,7 +2805,7 @@ class ActivityStreamAccess(BaseAccess):
                 | Q(notification_template__organization__in=auditing_orgs)
                 | Q(notification__notification_template__organization__in=auditing_orgs)
                 | Q(label__organization__in=auditing_orgs)
-                | Q(role__in=Role.objects.filter(ancestors__in=self.user.roles.all()) if auditing_orgs else [])
+                | Q(role__in=Role.visible_roles(self.user) if auditing_orgs else [])
             )
 
         project_set = Project.accessible_pk_qs(self.user, 'read_role')
@@ -2840,13 +2862,10 @@ class RoleAccess(BaseAccess):
 
     def filtered_queryset(self):
         result = Role.visible_roles(self.user)
-        # Sanity check: is the requesting user an orphaned non-admin/auditor?
-        # if yes, make system admin/auditor mandatorily visible.
-        if not self.user.is_superuser and not self.user.is_system_auditor and not self.user.organizations.exists():
-            mandatories = ('system_administrator', 'system_auditor')
-            super_qs = Role.objects.filter(singleton_name__in=mandatories)
-            result = result | super_qs
-        return result
+        # Make system admin/auditor mandatorily visible.
+        mandatories = ('system_administrator', 'system_auditor')
+        super_qs = Role.objects.filter(singleton_name__in=mandatories)
+        return result | super_qs
 
     def can_add(self, obj, data):
         # Unsupported for now

awx/main/analytics/collectors.py

@@ -419,7 +419,7 @@ def _events_table(since, full_path, until, tbl, where_column, project_job_create
                           resolved_action,
                           resolved_role,
                           -- '-' operator listed here:
-                          -- https://www.postgresql.org/docs/12/functions-json.html
+                          -- https://www.postgresql.org/docs/15/functions-json.html
                           -- note that operator is only supported by jsonb objects
                           -- https://www.postgresql.org/docs/current/datatype-json.html
                           (CASE WHEN event = 'playbook_on_stats' THEN {event_data} - 'artifact_data' END) as playbook_on_stats,

awx/main/apps.py

@@ -1,7 +1,40 @@
 from django.apps import AppConfig
 from django.utils.translation import gettext_lazy as _
+from awx.main.utils.named_url_graph import _customize_graph, generate_graph
+from awx.conf import register, fields
 
 
 class MainConfig(AppConfig):
     name = 'awx.main'
     verbose_name = _('Main')
+
+    def load_named_url_feature(self):
+        models = [m for m in self.get_models() if hasattr(m, 'get_absolute_url')]
+        generate_graph(models)
+        _customize_graph()
+        register(
+            'NAMED_URL_FORMATS',
+            field_class=fields.DictField,
+            read_only=True,
+            label=_('Formats of all available named urls'),
+            help_text=_('Read-only list of key-value pairs that shows the standard format of all available named URLs.'),
+            category=_('Named URL'),
+            category_slug='named-url',
+        )
+        register(
+            'NAMED_URL_GRAPH_NODES',
+            field_class=fields.DictField,
+            read_only=True,
+            label=_('List of all named url graph nodes.'),
+            help_text=_(
+                'Read-only list of key-value pairs that exposes named URL graph topology.'
+                ' Use this list to programmatically generate named URLs for resources'
+            ),
+            category=_('Named URL'),
+            category_slug='named-url',
+        )
+
+    def ready(self):
+        super().ready()
+
+        self.load_named_url_feature()

awx/main/conf.py

@@ -92,6 +92,7 @@ register(
     ),
     category=_('System'),
     category_slug='system',
+    required=False,
 )
 
 register(
@@ -774,6 +775,7 @@ register(
     allow_null=True,
     category=_('System'),
     category_slug='system',
+    required=False,
 )
 register(
     'AUTOMATION_ANALYTICS_LAST_ENTRIES',
@@ -815,6 +817,7 @@ register(
     help_text=_('Max jobs to allow bulk jobs to launch'),
     category=_('Bulk Actions'),
     category_slug='bulk',
+    hidden=True,
 )
 
 register(
@@ -825,6 +828,7 @@ register(
     help_text=_('Max number of hosts to allow to be created in a single bulk action'),
     category=_('Bulk Actions'),
     category_slug='bulk',
+    hidden=True,
 )
 
 register(
@@ -835,6 +839,7 @@ register(
     help_text=_('Max number of hosts to allow to be deleted in a single bulk action'),
     category=_('Bulk Actions'),
     category_slug='bulk',
+    hidden=True,
 )
 
 register(
@@ -845,6 +850,7 @@ register(
     help_text=_('Enable preview of new user interface.'),
     category=_('System'),
     category_slug='system',
+    hidden=True,
 )
 
 register(

awx/main/constants.py

@@ -14,7 +14,7 @@ __all__ = [
     'STANDARD_INVENTORY_UPDATE_ENV',
 ]
 
-CLOUD_PROVIDERS = ('azure_rm', 'ec2', 'gce', 'vmware', 'openstack', 'rhv', 'satellite6', 'controller', 'insights')
+CLOUD_PROVIDERS = ('azure_rm', 'ec2', 'gce', 'vmware', 'openstack', 'rhv', 'satellite6', 'controller', 'insights', 'terraform')
 PRIVILEGE_ESCALATION_METHODS = [
     ('sudo', _('Sudo')),
     ('su', _('Su')),
@@ -114,3 +114,28 @@ SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS = 'unique_managed_hosts'
 
 # Shared prefetch to use for creating a queryset for the purpose of writing or saving facts
 HOST_FACTS_FIELDS = ('name', 'ansible_facts', 'ansible_facts_modified', 'modified', 'inventory_id')
+
+# Data for RBAC compatibility layer
+role_name_to_perm_mapping = {
+    'adhoc_role': ['adhoc_'],
+    'approval_role': ['approve_'],
+    'auditor_role': ['audit_'],
+    'admin_role': ['change_', 'add_', 'delete_'],
+    'execute_role': ['execute_'],
+    'read_role': ['view_'],
+    'update_role': ['update_'],
+    'member_role': ['member_'],
+    'use_role': ['use_'],
+}
+
+org_role_to_permission = {
+    'notification_admin_role': 'add_notificationtemplate',
+    'project_admin_role': 'add_project',
+    'execute_role': 'execute_jobtemplate',
+    'inventory_admin_role': 'add_inventory',
+    'credential_admin_role': 'add_credential',
+    'workflow_admin_role': 'add_workflowjobtemplate',
+    'job_template_admin_role': 'change_jobtemplate',  # TODO: this doesnt really work, solution not clear
+    'execution_environment_admin_role': 'add_executionenvironment',
+    'auditor_role': 'view_project',  # TODO: also doesnt really work
+}

awx/main/credential_plugins/azure_kv.py

@@ -1,9 +1,10 @@
+from azure.keyvault.secrets import SecretClient
+from azure.identity import ClientSecretCredential
+from msrestazure import azure_cloud
+
 from .plugin import CredentialPlugin
 
 from django.utils.translation import gettext_lazy as _
-from azure.keyvault import KeyVaultClient, KeyVaultAuthentication
-from azure.common.credentials import ServicePrincipalCredentials
-from msrestazure import azure_cloud
 
 
 # https://github.com/Azure/msrestazure-for-python/blob/master/msrestazure/azure_cloud.py
@@ -54,22 +55,9 @@ azure_keyvault_inputs = {
 
 
 def azure_keyvault_backend(**kwargs):
-    url = kwargs['url']
-    [cloud] = [c for c in clouds if c.name == kwargs.get('cloud_name', default_cloud.name)]
-
-    def auth_callback(server, resource, scope):
-        credentials = ServicePrincipalCredentials(
-            url=url,
-            client_id=kwargs['client'],
-            secret=kwargs['secret'],
-            tenant=kwargs['tenant'],
-            resource=f"https://{cloud.suffixes.keyvault_dns.split('.', 1).pop()}",
-        )
-        token = credentials.token
-        return token['token_type'], token['access_token']
-
-    kv = KeyVaultClient(KeyVaultAuthentication(auth_callback))
-    return kv.get_secret(url, kwargs['secret_field'], kwargs.get('secret_version', '')).value
+    csc = ClientSecretCredential(tenant_id=kwargs['tenant'], client_id=kwargs['client'], client_secret=kwargs['secret'])
+    kv = SecretClient(credential=csc, vault_url=kwargs['url'])
+    return kv.get_secret(name=kwargs['secret_field'], version=kwargs.get('secret_version', '')).value
 
 
 azure_keyvault_plugin = CredentialPlugin('Microsoft Azure Key Vault', inputs=azure_keyvault_inputs, backend=azure_keyvault_backend)

awx/main/dispatch/__init__.py

@@ -1,6 +1,7 @@
 import os
 import psycopg
 import select
+from copy import deepcopy
 
 from contextlib import contextmanager
 
@@ -94,8 +95,8 @@ class PubSub(object):
 
 
 def create_listener_connection():
-    conf = settings.DATABASES['default'].copy()
-    conf['OPTIONS'] = conf.get('OPTIONS', {}).copy()
+    conf = deepcopy(settings.DATABASES['default'])
+    conf['OPTIONS'] = deepcopy(conf.get('OPTIONS', {}))
     # Modify the application name to distinguish from other connections the process might use
     conf['OPTIONS']['application_name'] = get_application_name(settings.CLUSTER_HOST_ID, function='listener')
 

awx/main/dispatch/worker/base.py

@@ -259,6 +259,12 @@ class AWXConsumerPG(AWXConsumerBase):
                 current_downtime = time.time() - self.pg_down_time
                 if current_downtime > self.pg_max_wait:
                     logger.exception(f"Postgres event consumer has not recovered in {current_downtime} s, exiting")
+                    # Sending QUIT to multiprocess queue to signal workers to exit
+                    for worker in self.pool.workers:
+                        try:
+                            worker.quit()
+                        except Exception:
+                            logger.exception(f"Error sending QUIT to worker {worker}")
                     raise
                 # Wait for a second before next attempt, but still listen for any shutdown signals
                 for i in range(10):
@@ -270,6 +276,12 @@ class AWXConsumerPG(AWXConsumerBase):
             except Exception:
                 # Log unanticipated exception in addition to writing to stderr to get timestamps and other metadata
                 logger.exception('Encountered unhandled error in dispatcher main loop')
+                # Sending QUIT to multiprocess queue to signal workers to exit
+                for worker in self.pool.workers:
+                    try:
+                        worker.quit()
+                    except Exception:
+                        logger.exception(f"Error sending QUIT to worker {worker}")
                 raise
 
 

awx/main/management/commands/dump_auth_config.py

@@ -0,0 +1,195 @@
+import json
+import os
+import sys
+import re
+from typing import Any
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+from awx.conf import settings_registry
+
+
+class Command(BaseCommand):
+    help = 'Dump the current auth configuration in django_ansible_base.authenticator format, currently supports LDAP and SAML'
+
+    DAB_SAML_AUTHENTICATOR_KEYS = {
+        "SP_ENTITY_ID": True,
+        "SP_PUBLIC_CERT": True,
+        "SP_PRIVATE_KEY": True,
+        "ORG_INFO": True,
+        "TECHNICAL_CONTACT": True,
+        "SUPPORT_CONTACT": True,
+        "SP_EXTRA": False,
+        "SECURITY_CONFIG": False,
+        "EXTRA_DATA": False,
+        "ENABLED_IDPS": True,
+        "CALLBACK_URL": False,
+    }
+
+    DAB_LDAP_AUTHENTICATOR_KEYS = {
+        "SERVER_URI": True,
+        "BIND_DN": False,
+        "BIND_PASSWORD": False,
+        "CONNECTION_OPTIONS": False,
+        "GROUP_TYPE": True,
+        "GROUP_TYPE_PARAMS": True,
+        "GROUP_SEARCH": False,
+        "START_TLS": False,
+        "USER_DN_TEMPLATE": True,
+        "USER_ATTR_MAP": True,
+        "USER_SEARCH": False,
+    }
+
+    def is_enabled(self, settings, keys):
+        missing_fields = []
+        for key, required in keys.items():
+            if required and not settings.get(key):
+                missing_fields.append(key)
+        if missing_fields:
+            return False, missing_fields
+        return True, None
+
+    def get_awx_ldap_settings(self) -> dict[str, dict[str, Any]]:
+        awx_ldap_settings = {}
+
+        for awx_ldap_setting in settings_registry.get_registered_settings(category_slug='ldap'):
+            key = awx_ldap_setting.removeprefix("AUTH_LDAP_")
+            value = getattr(settings, awx_ldap_setting, None)
+            awx_ldap_settings[key] = value
+
+        grouped_settings = {}
+
+        for key, value in awx_ldap_settings.items():
+            match = re.search(r'(\d+)', key)
+            index = int(match.group()) if match else 0
+            new_key = re.sub(r'\d+_', '', key)
+
+            if index not in grouped_settings:
+                grouped_settings[index] = {}
+
+            grouped_settings[index][new_key] = value
+            if new_key == "GROUP_TYPE" and value:
+                grouped_settings[index][new_key] = type(value).__name__
+
+            if new_key == "SERVER_URI" and value:
+                value = value.split(", ")
+                grouped_settings[index][new_key] = value
+
+            if type(value).__name__ == "LDAPSearch":
+                data = []
+                data.append(value.base_dn)
+                data.append("SCOPE_SUBTREE")
+                data.append(value.filterstr)
+                grouped_settings[index][new_key] = data
+
+        return grouped_settings
+
+    def get_awx_saml_settings(self) -> dict[str, Any]:
+        awx_saml_settings = {}
+        for awx_saml_setting in settings_registry.get_registered_settings(category_slug='saml'):
+            awx_saml_settings[awx_saml_setting.removeprefix("SOCIAL_AUTH_SAML_")] = getattr(settings, awx_saml_setting, None)
+
+        return awx_saml_settings
+
+    def format_config_data(self, enabled, awx_settings, type, keys, name):
+        config = {
+            "type": f"ansible_base.authentication.authenticator_plugins.{type}",
+            "name": name,
+            "enabled": enabled,
+            "create_objects": True,
+            "users_unique": False,
+            "remove_users": True,
+            "configuration": {},
+        }
+        for k in keys:
+            v = awx_settings.get(k)
+            config["configuration"].update({k: v})
+
+        if type == "saml":
+            idp_to_key_mapping = {
+                "url": "IDP_URL",
+                "x509cert": "IDP_X509_CERT",
+                "entity_id": "IDP_ENTITY_ID",
+                "attr_email": "IDP_ATTR_EMAIL",
+                "attr_groups": "IDP_GROUPS",
+                "attr_username": "IDP_ATTR_USERNAME",
+                "attr_last_name": "IDP_ATTR_LAST_NAME",
+                "attr_first_name": "IDP_ATTR_FIRST_NAME",
+                "attr_user_permanent_id": "IDP_ATTR_USER_PERMANENT_ID",
+            }
+            for idp_name in awx_settings.get("ENABLED_IDPS", {}):
+                for key in idp_to_key_mapping:
+                    value = awx_settings["ENABLED_IDPS"][idp_name].get(key)
+                    if value is not None:
+                        config["name"] = idp_name
+                        config["configuration"].update({idp_to_key_mapping[key]: value})
+
+        return config
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "output_file",
+            nargs="?",
+            type=str,
+            default=None,
+            help="Output JSON file path",
+        )
+
+    def handle(self, *args, **options):
+        try:
+            data = []
+
+            # dump SAML settings
+            awx_saml_settings = self.get_awx_saml_settings()
+            awx_saml_enabled, saml_missing_fields = self.is_enabled(awx_saml_settings, self.DAB_SAML_AUTHENTICATOR_KEYS)
+            if awx_saml_enabled:
+                awx_saml_name = awx_saml_settings["ENABLED_IDPS"]
+                data.append(
+                    self.format_config_data(
+                        awx_saml_enabled,
+                        awx_saml_settings,
+                        "saml",
+                        self.DAB_SAML_AUTHENTICATOR_KEYS,
+                        awx_saml_name,
+                    )
+                )
+            else:
+                data.append({"SAML_missing_fields": saml_missing_fields})
+
+            # dump LDAP settings
+            awx_ldap_group_settings = self.get_awx_ldap_settings()
+            for awx_ldap_name, awx_ldap_settings in awx_ldap_group_settings.items():
+                awx_ldap_enabled, ldap_missing_fields = self.is_enabled(awx_ldap_settings, self.DAB_LDAP_AUTHENTICATOR_KEYS)
+                if awx_ldap_enabled:
+                    data.append(
+                        self.format_config_data(
+                            awx_ldap_enabled,
+                            awx_ldap_settings,
+                            "ldap",
+                            self.DAB_LDAP_AUTHENTICATOR_KEYS,
+                            f"LDAP_{awx_ldap_name}",
+                        )
+                    )
+                else:
+                    data.append({f"LDAP_{awx_ldap_name}_missing_fields": ldap_missing_fields})
+
+            # write to file if requested
+            if options["output_file"]:
+                # Define the path for the output JSON file
+                output_file = options["output_file"]
+
+                # Ensure the directory exists
+                os.makedirs(os.path.dirname(output_file), exist_ok=True)
+
+                # Write data to the JSON file
+                with open(output_file, "w") as f:
+                    json.dump(data, f, indent=4)
+
+                self.stdout.write(self.style.SUCCESS(f"Auth config data dumped to {output_file}"))
+            else:
+                self.stdout.write(json.dumps(data, indent=4))
+
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f"An error occurred: {str(e)}"))
+            sys.exit(1)

awx/main/management/commands/run_wsrelay.py

@@ -92,8 +92,6 @@ class Command(BaseCommand):
         return host_stats
 
     def handle(self, *arg, **options):
-        WebsocketsMetricsServer().start()
-
         # it's necessary to delay this import in case
         # database migrations are still running
         from awx.main.models.ha import Instance
@@ -166,8 +164,14 @@ class Command(BaseCommand):
 
             return
 
-        try:
-            websocket_relay_manager = WebSocketRelayManager()
-            asyncio.run(websocket_relay_manager.run())
-        except KeyboardInterrupt:
-            logger.info('Terminating Websocket Relayer')
+        WebsocketsMetricsServer().start()
+
+        while True:
+            try:
+                asyncio.run(WebSocketRelayManager().run())
+            except KeyboardInterrupt:
+                logger.info('Shutting down Websocket Relayer')
+                break
+            except Exception as e:
+                logger.exception('Error in Websocket Relayer, exception: {}. Restarting in 10 seconds'.format(e))
+                time.sleep(10)

awx/main/middleware.py

@@ -1,25 +1,25 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved.
 
+import functools
 import logging
 import threading
 import time
 import urllib.parse
+from pathlib import Path
 
 from django.conf import settings
 from django.contrib.auth import logout
-from django.contrib.auth.models import User
-from django.db.migrations.executor import MigrationExecutor
+from django.db.migrations.recorder import MigrationRecorder
 from django.db import connection
 from django.shortcuts import redirect
-from django.apps import apps
 from django.utils.deprecation import MiddlewareMixin
-from django.utils.translation import gettext_lazy as _
 from django.urls import reverse, resolve
 
-from awx.main.utils.named_url_graph import generate_graph, GraphNode
-from awx.conf import fields, register
+from awx.main import migrations
 from awx.main.utils.profiling import AWXProfiler
+from awx.main.utils.common import memoize
+from awx.urls import get_urlpatterns
 
 
 logger = logging.getLogger('awx.main.middleware')
@@ -97,49 +97,7 @@ class DisableLocalAuthMiddleware(MiddlewareMixin):
                 logout(request)
 
 
-def _customize_graph():
-    from awx.main.models import Instance, Schedule, UnifiedJobTemplate
-
-    for model in [Schedule, UnifiedJobTemplate]:
-        if model in settings.NAMED_URL_GRAPH:
-            settings.NAMED_URL_GRAPH[model].remove_bindings()
-            settings.NAMED_URL_GRAPH.pop(model)
-    if User not in settings.NAMED_URL_GRAPH:
-        settings.NAMED_URL_GRAPH[User] = GraphNode(User, ['username'], [])
-        settings.NAMED_URL_GRAPH[User].add_bindings()
-    if Instance not in settings.NAMED_URL_GRAPH:
-        settings.NAMED_URL_GRAPH[Instance] = GraphNode(Instance, ['hostname'], [])
-        settings.NAMED_URL_GRAPH[Instance].add_bindings()
-
-
 class URLModificationMiddleware(MiddlewareMixin):
-    def __init__(self, get_response):
-        models = [m for m in apps.get_app_config('main').get_models() if hasattr(m, 'get_absolute_url')]
-        generate_graph(models)
-        _customize_graph()
-        register(
-            'NAMED_URL_FORMATS',
-            field_class=fields.DictField,
-            read_only=True,
-            label=_('Formats of all available named urls'),
-            help_text=_('Read-only list of key-value pairs that shows the standard format of all available named URLs.'),
-            category=_('Named URL'),
-            category_slug='named-url',
-        )
-        register(
-            'NAMED_URL_GRAPH_NODES',
-            field_class=fields.DictField,
-            read_only=True,
-            label=_('List of all named url graph nodes.'),
-            help_text=_(
-                'Read-only list of key-value pairs that exposes named URL graph topology.'
-                ' Use this list to programmatically generate named URLs for resources'
-            ),
-            category=_('Named URL'),
-            category_slug='named-url',
-        )
-        super().__init__(get_response)
-
     @staticmethod
     def _hijack_for_old_jt_name(node, kwargs, named_url):
         try:
@@ -198,9 +156,46 @@ class URLModificationMiddleware(MiddlewareMixin):
             request.path_info = new_path
 
 
+@memoize(ttl=20)
+def is_migrating():
+    latest_number = 0
+    latest_name = ''
+    for migration_path in Path(migrations.__path__[0]).glob('[0-9]*.py'):
+        try:
+            migration_number = int(migration_path.name.split('_', 1)[0])
+        except ValueError:
+            continue
+        if migration_number > latest_number:
+            latest_number = migration_number
+            latest_name = migration_path.name[: -len('.py')]
+    return not MigrationRecorder(connection).migration_qs.filter(app='main', name=latest_name).exists()
+
+
 class MigrationRanCheckMiddleware(MiddlewareMixin):
     def process_request(self, request):
-        executor = MigrationExecutor(connection)
-        plan = executor.migration_plan(executor.loader.graph.leaf_nodes())
-        if bool(plan) and getattr(resolve(request.path), 'url_name', '') != 'migrations_notran':
+        if is_migrating() and getattr(resolve(request.path), 'url_name', '') != 'migrations_notran':
             return redirect(reverse("ui:migrations_notran"))
+
+
+class OptionalURLPrefixPath(MiddlewareMixin):
+    @functools.lru_cache
+    def _url_optional(self, prefix):
+        # Relavant Django code path https://github.com/django/django/blob/stable/4.2.x/django/core/handlers/base.py#L300
+        #
+        # resolve_request(request)
+        #   get_resolver(request.urlconf)
+        #     _get_cached_resolver(request.urlconf) <-- cached via @functools.cache
+        #
+        # Django will attempt to cache the value(s) of request.urlconf
+        # Being hashable is a prerequisit for being cachable.
+        # tuple() is hashable list() is not.
+        # Hence the tuple(list()) wrap.
+        return tuple(get_urlpatterns(prefix=prefix))
+
+    def process_request(self, request):
+        prefix = settings.OPTIONAL_API_URLPATTERN_PREFIX
+
+        if request.path.startswith(f"/api/{prefix}"):
+            request.urlconf = self._url_optional(prefix)
+        else:
+            request.urlconf = 'awx.urls'

awx/main/migrations/0190_alter_inventorysource_source_and_more.py

@@ -0,0 +1,59 @@
+# Generated by Django 4.2.6 on 2024-02-15 20:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0189_inbound_hop_nodes'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='inventorysource',
+            name='source',
+            field=models.CharField(
+                choices=[
+                    ('file', 'File, Directory or Script'),
+                    ('constructed', 'Template additional groups and hostvars at runtime'),
+                    ('scm', 'Sourced from a Project'),
+                    ('ec2', 'Amazon EC2'),
+                    ('gce', 'Google Compute Engine'),
+                    ('azure_rm', 'Microsoft Azure Resource Manager'),
+                    ('vmware', 'VMware vCenter'),
+                    ('satellite6', 'Red Hat Satellite 6'),
+                    ('openstack', 'OpenStack'),
+                    ('rhv', 'Red Hat Virtualization'),
+                    ('controller', 'Red Hat Ansible Automation Platform'),
+                    ('insights', 'Red Hat Insights'),
+                    ('terraform', 'Terraform State'),
+                ],
+                default=None,
+                max_length=32,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='inventoryupdate',
+            name='source',
+            field=models.CharField(
+                choices=[
+                    ('file', 'File, Directory or Script'),
+                    ('constructed', 'Template additional groups and hostvars at runtime'),
+                    ('scm', 'Sourced from a Project'),
+                    ('ec2', 'Amazon EC2'),
+                    ('gce', 'Google Compute Engine'),
+                    ('azure_rm', 'Microsoft Azure Resource Manager'),
+                    ('vmware', 'VMware vCenter'),
+                    ('satellite6', 'Red Hat Satellite 6'),
+                    ('openstack', 'OpenStack'),
+                    ('rhv', 'Red Hat Virtualization'),
+                    ('controller', 'Red Hat Ansible Automation Platform'),
+                    ('insights', 'Red Hat Insights'),
+                    ('terraform', 'Terraform State'),
+                ],
+                default=None,
+                max_length=32,
+            ),
+        ),
+    ]

awx/main/migrations/0191_add_django_permissions.py

@@ -0,0 +1,85 @@
+# Generated by Django 4.2.6 on 2023-11-13 20:10
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('main', '0190_alter_inventorysource_source_and_more'),
+        ('dab_rbac', '__first__'),
+    ]
+
+    operations = [
+        # Add custom permissions for all special actions, like update, use, adhoc, and so on
+        migrations.AlterModelOptions(
+            name='credential',
+            options={'ordering': ('name',), 'permissions': [('use_credential', 'Can use credential in a job or related resource')]},
+        ),
+        migrations.AlterModelOptions(
+            name='instancegroup',
+            options={'permissions': [('use_instancegroup', 'Can use instance group in a preference list of a resource')]},
+        ),
+        migrations.AlterModelOptions(
+            name='inventory',
+            options={
+                'ordering': ('name',),
+                'permissions': [
+                    ('use_inventory', 'Can use inventory in a job template'),
+                    ('adhoc_inventory', 'Can run ad hoc commands'),
+                    ('update_inventory', 'Can update inventory sources in inventory'),
+                ],
+                'verbose_name_plural': 'inventories',
+            },
+        ),
+        migrations.AlterModelOptions(
+            name='jobtemplate',
+            options={'ordering': ('name',), 'permissions': [('execute_jobtemplate', 'Can run this job template')]},
+        ),
+        migrations.AlterModelOptions(
+            name='project',
+            options={
+                'ordering': ('id',),
+                'permissions': [('update_project', 'Can run a project update'), ('use_project', 'Can use project in a job template')],
+            },
+        ),
+        migrations.AlterModelOptions(
+            name='workflowjobtemplate',
+            options={
+                'permissions': [
+                    ('execute_workflowjobtemplate', 'Can run this workflow job template'),
+                    ('approve_workflowjobtemplate', 'Can approve steps in this workflow job template'),
+                ]
+            },
+        ),
+        migrations.AlterModelOptions(
+            name='organization',
+            options={
+                'default_permissions': ('change', 'delete', 'view'),
+                'ordering': ('name',),
+                'permissions': [
+                    ('member_organization', 'Basic participation permissions for organization'),
+                    ('audit_organization', 'Audit everything inside the organization'),
+                ],
+            },
+        ),
+        migrations.AlterModelOptions(
+            name='team',
+            options={'ordering': ('organization__name', 'name'), 'permissions': [('member_team', 'Inherit all roles assigned to this team')]},
+        ),
+        # Remove add default permission for a few models
+        migrations.AlterModelOptions(
+            name='jobtemplate',
+            options={
+                'default_permissions': ('change', 'delete', 'view'),
+                'ordering': ('name',),
+                'permissions': [('execute_jobtemplate', 'Can run this job template')],
+            },
+        ),
+        migrations.AlterModelOptions(
+            name='instancegroup',
+            options={
+                'default_permissions': ('change', 'delete', 'view'),
+                'permissions': [('use_instancegroup', 'Can use instance group in a preference list of a resource')],
+            },
+        ),
+    ]

awx/main/migrations/0192_custom_roles.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.6 on 2023-11-21 02:06
+
+from django.db import migrations
+
+from awx.main.migrations._dab_rbac import migrate_to_new_rbac, create_permissions_as_operation, setup_managed_role_definitions
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('main', '0191_add_django_permissions'),
+        ('dab_rbac', '__first__'),
+    ]
+
+    operations = [
+        # make sure permissions and content types have been created by now
+        # these normally run in a post_migrate signal but we need them for our logic
+        migrations.RunPython(create_permissions_as_operation, migrations.RunPython.noop),
+        migrations.RunPython(setup_managed_role_definitions, migrations.RunPython.noop),
+        migrations.RunPython(migrate_to_new_rbac, migrations.RunPython.noop),
+    ]

awx/main/migrations/_dab_rbac.py

@@ -0,0 +1,343 @@
+import json
+import logging
+
+from django.apps import apps as global_apps
+from django.db.models import ForeignKey
+from django.conf import settings
+from ansible_base.rbac.migrations._utils import give_permissions
+from ansible_base.rbac.management import create_dab_permissions
+
+from awx.main.fields import ImplicitRoleField
+from awx.main.constants import role_name_to_perm_mapping
+
+from ansible_base.rbac.permission_registry import permission_registry
+
+
+logger = logging.getLogger('awx.main.migrations._dab_rbac')
+
+
+def create_permissions_as_operation(apps, schema_editor):
+    create_dab_permissions(global_apps.get_app_config("main"), apps=apps)
+
+
+"""
+Data structures and methods for the migration of old Role model to ObjectRole
+"""
+
+system_admin = ImplicitRoleField(name='system_administrator')
+system_auditor = ImplicitRoleField(name='system_auditor')
+system_admin.model = None
+system_auditor.model = None
+
+
+def resolve_parent_role(f, role_path):
+    """
+    Given a field and a path declared in parent_role from the field definition, like
+        execute_role = ImplicitRoleField(parent_role='admin_role')
+    This expects to be passed in (execute_role object, "admin_role")
+    It hould return the admin_role from that object
+    """
+    if role_path == 'singleton:system_administrator':
+        return system_admin
+    elif role_path == 'singleton:system_auditor':
+        return system_auditor
+    else:
+        related_field = f
+        current_model = f.model
+        for related_field_name in role_path.split('.'):
+            related_field = current_model._meta.get_field(related_field_name)
+            if isinstance(related_field, ForeignKey) and not isinstance(related_field, ImplicitRoleField):
+                current_model = related_field.related_model
+        return related_field
+
+
+def build_role_map(apps):
+    """
+    For the old Role model, this builds and returns dictionaries (children, parents)
+    which give a global mapping of the ImplicitRoleField instances according to the graph
+    """
+    models = set(apps.get_app_config('main').get_models())
+
+    all_fields = set()
+    parents = {}
+    children = {}
+
+    all_fields.add(system_admin)
+    all_fields.add(system_auditor)
+
+    for cls in models:
+        for f in cls._meta.get_fields():
+            if isinstance(f, ImplicitRoleField):
+                all_fields.add(f)
+
+    for f in all_fields:
+        if f.parent_role is not None:
+            if isinstance(f.parent_role, str):
+                parent_roles = [f.parent_role]
+            else:
+                parent_roles = f.parent_role
+
+            # SPECIAL CASE: organization auditor_role is not a child of admin_role
+            # this makes no practical sense and conflicts with expected managed role
+            # so we put it in as a hack here
+            if f.name == 'auditor_role' and f.model._meta.model_name == 'organization':
+                parent_roles.append('admin_role')
+
+            parent_list = []
+            for rel_name in parent_roles:
+                parent_list.append(resolve_parent_role(f, rel_name))
+
+            parents[f] = parent_list
+
+    # build children lookup from parents lookup
+    for child_field, parent_list in parents.items():
+        for parent_field in parent_list:
+            children.setdefault(parent_field, [])
+            children[parent_field].append(child_field)
+
+    return (parents, children)
+
+
+def get_descendents(f, children_map):
+    """
+    Given ImplicitRoleField F and the children mapping, returns all descendents
+    of that field, as a set of other fields, including itself
+    """
+    ret = {f}
+    if f in children_map:
+        for child_field in children_map[f]:
+            ret.update(get_descendents(child_field, children_map))
+    return ret
+
+
+def get_permissions_for_role(role_field, children_map, apps):
+    Permission = apps.get_model('dab_rbac', 'DABPermission')
+    ContentType = apps.get_model('contenttypes', 'ContentType')
+
+    perm_list = []
+    for child_field in get_descendents(role_field, children_map):
+        if child_field.name in role_name_to_perm_mapping:
+            for perm_name in role_name_to_perm_mapping[child_field.name]:
+                if perm_name == 'add_' and role_field.model._meta.model_name != 'organization':
+                    continue  # only organizations can contain add permissions
+                perm = Permission.objects.filter(content_type=ContentType.objects.get_for_model(child_field.model), codename__startswith=perm_name).first()
+                if perm is not None and perm not in perm_list:
+                    perm_list.append(perm)
+
+    # special case for two models that have object roles but no organization roles in old system
+    if role_field.name == 'notification_admin_role' or (role_field.name == 'admin_role' and role_field.model._meta.model_name == 'organization'):
+        ct = ContentType.objects.get_for_model(apps.get_model('main', 'NotificationTemplate'))
+        perm_list.extend(list(Permission.objects.filter(content_type=ct)))
+    if role_field.name == 'execution_environment_admin_role' or (role_field.name == 'admin_role' and role_field.model._meta.model_name == 'organization'):
+        ct = ContentType.objects.get_for_model(apps.get_model('main', 'ExecutionEnvironment'))
+        perm_list.extend(list(Permission.objects.filter(content_type=ct)))
+
+    # more special cases for those same above special org-level roles
+    if role_field.name == 'auditor_role':
+        for codename in ('view_notificationtemplate', 'view_executionenvironment'):
+            perm_list.append(Permission.objects.get(codename=codename))
+
+    return perm_list
+
+
+def migrate_to_new_rbac(apps, schema_editor):
+    """
+    This method moves the assigned permissions from the old rbac.py models
+    to the new RoleDefinition and ObjectRole models
+    """
+    Role = apps.get_model('main', 'Role')
+    RoleDefinition = apps.get_model('dab_rbac', 'RoleDefinition')
+    RoleUserAssignment = apps.get_model('dab_rbac', 'RoleUserAssignment')
+    Permission = apps.get_model('dab_rbac', 'DABPermission')
+
+    # remove add premissions that are not valid for migrations from old versions
+    for perm_str in ('add_organization', 'add_jobtemplate'):
+        perm = Permission.objects.filter(codename=perm_str).first()
+        if perm:
+            perm.delete()
+
+    managed_definitions = dict()
+    for role_definition in RoleDefinition.objects.filter(managed=True):
+        permissions = frozenset(role_definition.permissions.values_list('id', flat=True))
+        managed_definitions[permissions] = role_definition
+
+    # Build map of old role model
+    parents, children = build_role_map(apps)
+
+    # NOTE: this import is expected to break at some point, and then just move the data here
+    from awx.main.models.rbac import role_descriptions
+
+    for role in Role.objects.prefetch_related('members', 'parents').iterator():
+        if role.singleton_name:
+            continue  # only bothering to migrate object roles
+
+        team_roles = []
+        for parent in role.parents.all():
+            if parent.id not in json.loads(role.implicit_parents):
+                team_roles.append(parent)
+
+        # we will not create any roles that do not have any users or teams
+        if not (role.members.all() or team_roles):
+            logger.debug(f'Skipping role {role.role_field} for {role.content_type.model}-{role.object_id} due to no members')
+            continue
+
+        # get a list of permissions that the old role would grant
+        object_cls = apps.get_model(f'main.{role.content_type.model}')
+        object = object_cls.objects.get(pk=role.object_id)  # WORKAROUND, role.content_object does not work in migrations
+        f = object._meta.get_field(role.role_field)  # should be ImplicitRoleField
+        perm_list = get_permissions_for_role(f, children, apps)
+
+        permissions = frozenset(perm.id for perm in perm_list)
+
+        # With the needed permissions established, obtain the RoleDefinition this will need, priorities:
+        # 1. If it exists as a managed RoleDefinition then obviously use that
+        # 2. If we already created this for a prior role, use that
+        # 3. Create a new RoleDefinition that lists those permissions
+        if permissions in managed_definitions:
+            role_definition = managed_definitions[permissions]
+        else:
+            action = role.role_field.rsplit('_', 1)[0]  # remove the _field ending of the name
+            role_definition_name = f'{role.content_type.model_class().__name__} {action.title()}'
+
+            description = role_descriptions[role.role_field]
+            if type(description) == dict:
+                if role.content_type.model in description:
+                    description = description.get(role.content_type.model)
+                else:
+                    description = description.get('default')
+            if '%s' in description:
+                description = description % role.content_type.model
+
+            role_definition, created = RoleDefinition.objects.get_or_create(
+                name=role_definition_name,
+                defaults={'description': description, 'content_type_id': role.content_type_id},
+            )
+
+            if created:
+                logger.info(f'Created custom Role Definition {role_definition_name}, pk={role_definition.pk}')
+                role_definition.permissions.set(perm_list)
+
+        # Create the object role and add users to it
+        give_permissions(
+            apps,
+            role_definition,
+            users=role.members.all(),
+            teams=[tr.object_id for tr in team_roles],
+            object_id=role.object_id,
+            content_type_id=role.content_type_id,
+        )
+
+    # Create new replacement system auditor role
+    new_system_auditor, created = RoleDefinition.objects.get_or_create(
+        name='System Auditor',
+        defaults={'description': 'Migrated singleton role giving read permission to everything', 'managed': True},
+    )
+    new_system_auditor.permissions.add(*list(Permission.objects.filter(codename__startswith='view')))
+
+    # migrate is_system_auditor flag, because it is no longer handled by a system role
+    old_system_auditor = Role.objects.filter(singleton_name='system_auditor').first()
+    if old_system_auditor:
+        # if the system auditor role is not present, this is a new install and no users should exist
+        ct = 0
+        for user in role.members.all():
+            RoleUserAssignment.objects.create(user=user, role_definition=new_system_auditor)
+            ct += 1
+        if ct:
+            logger.info(f'Migrated {ct} users to new system auditor flag')
+
+
+def get_or_create_managed(name, description, ct, permissions, RoleDefinition):
+    role_definition, created = RoleDefinition.objects.get_or_create(name=name, defaults={'managed': True, 'description': description, 'content_type': ct})
+    role_definition.permissions.set(list(permissions))
+
+    if not role_definition.managed:
+        role_definition.managed = True
+        role_definition.save(update_fields=['managed'])
+
+    if created:
+        logger.info(f'Created RoleDefinition {role_definition.name} pk={role_definition} with {len(permissions)} permissions')
+
+    return role_definition
+
+
+def setup_managed_role_definitions(apps, schema_editor):
+    """
+    Idepotent method to create or sync the managed role definitions
+    """
+    to_create = settings.ANSIBLE_BASE_ROLE_PRECREATE
+
+    ContentType = apps.get_model('contenttypes', 'ContentType')
+    Permission = apps.get_model('dab_rbac', 'DABPermission')
+    RoleDefinition = apps.get_model('dab_rbac', 'RoleDefinition')
+    Organization = apps.get_model(settings.ANSIBLE_BASE_ORGANIZATION_MODEL)
+    org_ct = ContentType.objects.get_for_model(Organization)
+    managed_role_definitions = []
+
+    org_perms = set()
+    for cls in permission_registry._registry:
+        ct = ContentType.objects.get_for_model(cls)
+        object_perms = set(Permission.objects.filter(content_type=ct))
+        # Special case for InstanceGroup which has an organiation field, but is not an organization child object
+        if cls._meta.model_name != 'instancegroup':
+            org_perms.update(object_perms)
+
+        if 'object_admin' in to_create and cls != Organization:
+            indiv_perms = object_perms.copy()
+            add_perms = [perm for perm in indiv_perms if perm.codename.startswith('add_')]
+            if add_perms:
+                for perm in add_perms:
+                    indiv_perms.remove(perm)
+
+            managed_role_definitions.append(
+                get_or_create_managed(
+                    to_create['object_admin'].format(cls=cls), f'Has all permissions to a single {cls._meta.verbose_name}', ct, indiv_perms, RoleDefinition
+                )
+            )
+
+        if 'org_children' in to_create and cls != Organization:
+            org_child_perms = object_perms.copy()
+            org_child_perms.add(Permission.objects.get(codename='view_organization'))
+
+            managed_role_definitions.append(
+                get_or_create_managed(
+                    to_create['org_children'].format(cls=cls),
+                    f'Has all permissions to {cls._meta.verbose_name_plural} within an organization',
+                    org_ct,
+                    org_child_perms,
+                    RoleDefinition,
+                )
+            )
+
+        if 'special' in to_create:
+            special_perms = []
+            for perm in object_perms:
+                if perm.codename.split('_')[0] not in ('add', 'change', 'update', 'delete', 'view'):
+                    special_perms.append(perm)
+            for perm in special_perms:
+                action = perm.codename.split('_')[0]
+                view_perm = Permission.objects.get(content_type=ct, codename__startswith='view_')
+                managed_role_definitions.append(
+                    get_or_create_managed(
+                        to_create['special'].format(cls=cls, action=action.title()),
+                        f'Has {action} permissions to a single {cls._meta.verbose_name}',
+                        ct,
+                        [perm, view_perm],
+                        RoleDefinition,
+                    )
+                )
+
+    if 'org_admin' in to_create:
+        managed_role_definitions.append(
+            get_or_create_managed(
+                to_create['org_admin'].format(cls=Organization),
+                'Has all permissions to a single organization and all objects inside of it',
+                org_ct,
+                org_perms,
+                RoleDefinition,
+            )
+        )
+
+    unexpected_role_definitions = RoleDefinition.objects.filter(managed=True).exclude(pk__in=[rd.pk for rd in managed_role_definitions])
+    for role_definition in unexpected_role_definitions:
+        logger.info(f'Deleting old managed role definition {role_definition.name}, pk={role_definition.pk}')
+        role_definition.delete()

awx/main/models/__init__.py

@@ -1,12 +1,19 @@
 # Copyright (c) 2015 Ansible, Inc.
 # All Rights Reserved.
 
+import json
+
 # Django
 from django.conf import settings  # noqa
 from django.db import connection
 from django.db.models.signals import pre_delete  # noqa
 
+# django-ansible-base
+from ansible_base.resource_registry.fields import AnsibleResourceField
+from ansible_base.rbac import permission_registry
+from ansible_base.rbac.models import RoleDefinition, RoleUserAssignment
 from ansible_base.lib.utils.models import prevent_search
+from ansible_base.lib.utils.models import user_summary_fields
 
 # AWX
 from awx.main.models.base import BaseModel, PrimordialModel, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES  # noqa
@@ -99,6 +106,8 @@ from awx.main.access import get_user_queryset, check_user_access, check_user_acc
 User.add_to_class('get_queryset', get_user_queryset)
 User.add_to_class('can_access', check_user_access)
 User.add_to_class('can_access_with_errors', check_user_access_with_errors)
+User.add_to_class('resource', AnsibleResourceField(primary_key_field="id"))
+User.add_to_class('summary_fields', user_summary_fields)
 
 
 def convert_jsonfields():
@@ -191,11 +200,21 @@ User.add_to_class('auditor_of_organizations', user_get_auditor_of_organizations)
 User.add_to_class('created', created)
 
 
+def get_system_auditor_role():
+    rd, created = RoleDefinition.objects.get_or_create(
+        name='System Auditor', defaults={'description': 'Migrated singleton role giving read permission to everything'}
+    )
+    if created:
+        rd.permissions.add(*list(permission_registry.permission_qs.filter(codename__startswith='view')))
+    return rd
+
+
 @property
 def user_is_system_auditor(user):
     if not hasattr(user, '_is_system_auditor'):
         if user.pk:
-            user._is_system_auditor = user.roles.filter(singleton_name='system_auditor', role_field='system_auditor').exists()
+            rd = get_system_auditor_role()
+            user._is_system_auditor = RoleUserAssignment.objects.filter(user=user, role_definition=rd).exists()
         else:
             # Odd case where user is unsaved, this should never be relied on
             return False
@@ -209,17 +228,17 @@ def user_is_system_auditor(user, tf):
         # time they've logged in, and we've just created the new User in this
         # request), we need one to set up the system auditor role
         user.save()
-    if tf:
-        role = Role.singleton('system_auditor')
-        # must check if member to not duplicate activity stream
-        if user not in role.members.all():
-            role.members.add(user)
-        user._is_system_auditor = True
-    else:
-        role = Role.singleton('system_auditor')
-        if user in role.members.all():
-            role.members.remove(user)
-        user._is_system_auditor = False
+    rd = get_system_auditor_role()
+    assignment = RoleUserAssignment.objects.filter(user=user, role_definition=rd).first()
+    prior_value = bool(assignment)
+    if prior_value != bool(tf):
+        if assignment:
+            assignment.delete()
+        else:
+            rd.give_global_permission(user)
+        user._is_system_auditor = bool(tf)
+        entry = ActivityStream.objects.create(changes=json.dumps({"is_system_auditor": [prior_value, bool(tf)]}), object1='user', operation='update')
+        entry.user.add(user)
 
 
 User.add_to_class('is_system_auditor', user_is_system_auditor)
@@ -287,6 +306,10 @@ activity_stream_registrar.connect(WorkflowApprovalTemplate)
 activity_stream_registrar.connect(OAuth2Application)
 activity_stream_registrar.connect(OAuth2AccessToken)
 
+# Register models
+permission_registry.register(Project, Team, WorkflowJobTemplate, JobTemplate, Inventory, Organization, Credential, NotificationTemplate, ExecutionEnvironment)
+permission_registry.register(InstanceGroup, parent_field_name=None)  # Not part of an organization
+
 # prevent API filtering on certain Django-supplied sensitive fields
 prevent_search(User._meta.get_field('password'))
 prevent_search(OAuth2AccessToken._meta.get_field('token'))

awx/main/models/base.py

@@ -7,6 +7,9 @@ from django.core.exceptions import ValidationError, ObjectDoesNotExist
 from django.utils.translation import gettext_lazy as _
 from django.utils.timezone import now
 
+# django-ansible-base
+from ansible_base.lib.utils.models import get_type_for_model
+
 # Django-CRUM
 from crum import get_current_user
 
@@ -139,6 +142,23 @@ class BaseModel(models.Model):
             self.save(update_fields=update_fields)
         return update_fields
 
+    def summary_fields(self):
+        """
+        This exists for use by django-ansible-base,
+        which has standard patterns that differ from AWX, but we enable views from DAB
+        for those views to list summary_fields for AWX models, those models need to provide this
+        """
+        from awx.api.serializers import SUMMARIZABLE_FK_FIELDS
+
+        model_name = get_type_for_model(self)
+        related_fields = SUMMARIZABLE_FK_FIELDS.get(model_name, {})
+        summary_data = {}
+        for field_name in related_fields:
+            fval = getattr(self, field_name, None)
+            if fval is not None:
+                summary_data[field_name] = fval
+        return summary_data
+
 
 class CreatedModifiedModel(BaseModel):
     """

awx/main/models/credential/__init__.py

@@ -83,6 +83,7 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
         app_label = 'main'
         ordering = ('name',)
         unique_together = ('organization', 'name', 'credential_type')
+        permissions = [('use_credential', 'Can use credential in a job or related resource')]
 
     PASSWORD_FIELDS = ['inputs']
     FIELDS_TO_PRESERVE_AT_COPY = ['input_sources']
@@ -1231,6 +1232,14 @@ ManagedCredentialType(
                 'multiline': True,
                 'help_text': gettext_noop('Terraform backend config as Hashicorp configuration language.'),
             },
+            {
+                'id': 'gce_credentials',
+                'label': gettext_noop('Google Cloud Platform account credentials'),
+                'type': 'string',
+                'secret': True,
+                'multiline': True,
+                'help_text': gettext_noop('Google Cloud Platform account credentials in JSON format.'),
+            },
         ],
         'required': ['configuration'],
     },

awx/main/models/credential/injectors.py

@@ -130,3 +130,10 @@ def terraform(cred, env, private_data_dir):
         os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
         f.write(cred.get_input('configuration'))
     env['TF_BACKEND_CONFIG_FILE'] = to_container_path(path, private_data_dir)
+    # Handle env variables for GCP account credentials
+    if 'gce_credentials' in cred.inputs:
+        handle, path = tempfile.mkstemp(dir=os.path.join(private_data_dir, 'env'))
+        with os.fdopen(handle, 'w') as f:
+            os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
+            f.write(cred.get_input('gce_credentials'))
+        env['GOOGLE_BACKEND_CREDENTIALS'] = to_container_path(path, private_data_dir)

awx/main/models/ha.py

@@ -485,6 +485,9 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin, ResourceMi
 
     class Meta:
         app_label = 'main'
+        permissions = [('use_instancegroup', 'Can use instance group in a preference list of a resource')]
+        # Since this has no direct organization field only superuser can add, so remove add permission
+        default_permissions = ('change', 'delete', 'view')
 
     def set_default_policy_fields(self):
         self.policy_instance_list = []

awx/main/models/inventory.py

@@ -11,6 +11,8 @@ import os.path
 from urllib.parse import urljoin
 
 import yaml
+import tempfile
+import stat
 
 # Django
 from django.conf import settings
@@ -89,6 +91,11 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
         verbose_name_plural = _('inventories')
         unique_together = [('name', 'organization')]
         ordering = ('name',)
+        permissions = [
+            ('use_inventory', 'Can use inventory in a job template'),
+            ('adhoc_inventory', 'Can run ad hoc commands'),
+            ('update_inventory', 'Can update inventory sources in inventory'),
+        ]
 
     organization = models.ForeignKey(
         'Organization',
@@ -925,6 +932,7 @@ class InventorySourceOptions(BaseModel):
         ('rhv', _('Red Hat Virtualization')),
         ('controller', _('Red Hat Ansible Automation Platform')),
         ('insights', _('Red Hat Insights')),
+        ('terraform', _('Terraform State')),
     ]
 
     # From the options of the Django management base command
@@ -1399,7 +1407,7 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin,
         return selected_groups
 
 
-class CustomInventoryScript(CommonModelNameNotUnique, ResourceMixin):
+class CustomInventoryScript(CommonModelNameNotUnique):
     class Meta:
         app_label = 'main'
         ordering = ('name',)
@@ -1630,6 +1638,42 @@ class satellite6(PluginFileInjector):
         return ret
 
 
+class terraform(PluginFileInjector):
+    plugin_name = 'terraform_state'
+    namespace = 'cloud'
+    collection = 'terraform'
+    use_fqcn = True
+
+    def inventory_as_dict(self, inventory_update, private_data_dir):
+        ret = super().inventory_as_dict(inventory_update, private_data_dir)
+        credential = inventory_update.get_cloud_credential()
+        config_cred = credential.get_input('configuration')
+        if config_cred:
+            handle, path = tempfile.mkstemp(dir=os.path.join(private_data_dir, 'env'))
+            with os.fdopen(handle, 'w') as f:
+                os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
+                f.write(config_cred)
+            ret['backend_config_files'] = to_container_path(path, private_data_dir)
+        return ret
+
+    def build_plugin_private_data(self, inventory_update, private_data_dir):
+        credential = inventory_update.get_cloud_credential()
+
+        private_data = {'credentials': {}}
+        gce_cred = credential.get_input('gce_credentials')
+        if gce_cred:
+            private_data['credentials'][credential] = gce_cred
+        return private_data
+
+    def get_plugin_env(self, inventory_update, private_data_dir, private_data_files):
+        env = super(terraform, self).get_plugin_env(inventory_update, private_data_dir, private_data_files)
+        credential = inventory_update.get_cloud_credential()
+        cred_data = private_data_files['credentials']
+        if cred_data[credential]:
+            env['GOOGLE_BACKEND_CREDENTIALS'] = to_container_path(cred_data[credential], private_data_dir)
+        return env
+
+
 class controller(PluginFileInjector):
     plugin_name = 'tower'  # TODO: relying on routing for now, update after EEs pick up revised collection
     base_injector = 'template'

awx/main/models/jobs.py

@@ -205,6 +205,9 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
     class Meta:
         app_label = 'main'
         ordering = ('name',)
+        permissions = [('execute_jobtemplate', 'Can run this job template')]
+        # Remove add permission, ability to add comes from use permission for inventory, project, credentials
+        default_permissions = ('change', 'delete', 'view')
 
     job_type = models.CharField(
         max_length=64,

awx/main/models/mixins.py

@@ -19,13 +19,14 @@ from django.utils.translation import gettext_lazy as _
 from ansible_base.lib.utils.models import prevent_search
 
 # AWX
-from awx.main.models.rbac import Role, RoleAncestorEntry
+
+from awx.main.models.rbac import Role, RoleAncestorEntry, to_permissions
 from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser, polymorphic
 from awx.main.utils.execution_environments import get_default_execution_environment
 from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted
 from awx.main.utils.polymorphic import build_polymorphic_ctypes_map
 from awx.main.fields import AskForField
-from awx.main.constants import ACTIVE_STATES
+from awx.main.constants import ACTIVE_STATES, org_role_to_permission
 
 
 logger = logging.getLogger('awx.main.models.mixins')
@@ -64,6 +65,18 @@ class ResourceMixin(models.Model):
 
     @staticmethod
     def _accessible_pk_qs(cls, accessor, role_field, content_types=None):
+        if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+            if cls._meta.model_name == 'organization' and role_field in org_role_to_permission:
+                # Organization roles can not use the DAB RBAC shortcuts
+                # like Organization.access_qs(user, 'change_jobtemplate') is needed
+                # not just Organization.access_qs(user, 'change') is needed
+                if accessor.is_superuser:
+                    return cls.objects.values_list('id')
+
+                codename = org_role_to_permission[role_field]
+
+                return cls.access_ids_qs(accessor, codename, content_types=content_types)
+            return cls.access_ids_qs(accessor, to_permissions[role_field], content_types=content_types)
         if accessor._meta.model_name == 'user':
             ancestor_roles = accessor.roles.all()
         elif type(accessor) == Role:

awx/main/models/notifications.py

@@ -498,7 +498,7 @@ class JobNotificationMixin(object):
         # Body should have at least 2 CRLF, some clients will interpret
         # the email incorrectly with blank body.  So we will check that
 
-        if len(body.strip().splitlines()) <= 2:
+        if len(body.strip().splitlines()) < 1:
             # blank body
             body = '\r\n'.join(
                 [

awx/main/models/organization.py

@@ -10,6 +10,8 @@ from django.contrib.sessions.models import Session
 from django.utils.timezone import now as tz_now
 from django.utils.translation import gettext_lazy as _
 
+# django-ansible-base
+from ansible_base.resource_registry.fields import AnsibleResourceField
 
 # AWX
 from awx.api.versioning import reverse
@@ -33,6 +35,12 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi
     class Meta:
         app_label = 'main'
         ordering = ('name',)
+        permissions = [
+            ('member_organization', 'Basic participation permissions for organization'),
+            ('audit_organization', 'Audit everything inside the organization'),
+        ]
+        # Remove add permission, only superuser can add
+        default_permissions = ('change', 'delete', 'view')
 
     instance_groups = OrderedManyToManyField('InstanceGroup', blank=True, through='OrganizationInstanceGroupMembership')
     galaxy_credentials = OrderedManyToManyField(
@@ -103,6 +111,7 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi
     approval_role = ImplicitRoleField(
         parent_role='admin_role',
     )
+    resource = AnsibleResourceField(primary_key_field="id")
 
     def get_absolute_url(self, request=None):
         return reverse('api:organization_detail', kwargs={'pk': self.pk}, request=request)
@@ -134,6 +143,7 @@ class Team(CommonModelNameNotUnique, ResourceMixin):
         app_label = 'main'
         unique_together = [('organization', 'name')]
         ordering = ('organization__name', 'name')
+        permissions = [('member_team', 'Inherit all roles assigned to this team')]
 
     organization = models.ForeignKey(
         'Organization',
@@ -151,6 +161,7 @@ class Team(CommonModelNameNotUnique, ResourceMixin):
     read_role = ImplicitRoleField(
         parent_role=['organization.auditor_role', 'member_role'],
     )
+    resource = AnsibleResourceField(primary_key_field="id")
 
     def get_absolute_url(self, request=None):
         return reverse('api:team_detail', kwargs={'pk': self.pk}, request=request)

awx/main/models/projects.py

@@ -259,6 +259,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
     class Meta:
         app_label = 'main'
         ordering = ('id',)
+        permissions = [('update_project', 'Can run a project update'), ('use_project', 'Can use project in a job template')]
 
     default_environment = models.ForeignKey(
         'ExecutionEnvironment',

awx/main/models/rbac.py

@@ -7,14 +7,27 @@ import threading
 import contextlib
 import re
 
+# django-rest-framework
+from rest_framework.serializers import ValidationError
+
 # Django
 from django.db import models, transaction, connection
+from django.db.models.signals import m2m_changed
+from django.contrib.auth import get_user_model
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.utils.translation import gettext_lazy as _
+from django.apps import apps
+from django.conf import settings
+
+# Ansible_base app
+from ansible_base.rbac.models import RoleDefinition
+from ansible_base.lib.utils.models import get_type_for_model
 
 # AWX
 from awx.api.versioning import reverse
+from awx.main.migrations._dab_rbac import build_role_map, get_permissions_for_role
+from awx.main.constants import role_name_to_perm_mapping, org_role_to_permission
 
 __all__ = [
     'Role',
@@ -75,6 +88,11 @@ role_descriptions = {
 }
 
 
+to_permissions = {}
+for k, v in role_name_to_perm_mapping.items():
+    to_permissions[k] = v[0].strip('_')
+
+
 tls = threading.local()  # thread local storage
 
 
@@ -86,10 +104,8 @@ def check_singleton(func):
     """
 
     def wrapper(*args, **kwargs):
-        sys_admin = Role.singleton(ROLE_SINGLETON_SYSTEM_ADMINISTRATOR)
-        sys_audit = Role.singleton(ROLE_SINGLETON_SYSTEM_AUDITOR)
         user = args[0]
-        if user in sys_admin or user in sys_audit:
+        if user.is_superuser or user.is_system_auditor:
             if len(args) == 2:
                 return args[1]
             return Role.objects.all()
@@ -169,6 +185,24 @@ class Role(models.Model):
 
     def __contains__(self, accessor):
         if accessor._meta.model_name == 'user':
+            if accessor.is_superuser:
+                return True
+            if self.role_field == 'system_administrator':
+                return accessor.is_superuser
+            elif self.role_field == 'system_auditor':
+                return accessor.is_system_auditor
+            elif self.role_field in ('read_role', 'auditor_role') and accessor.is_system_auditor:
+                return True
+
+            if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+                if self.content_object and self.content_object._meta.model_name == 'organization' and self.role_field in org_role_to_permission:
+                    codename = org_role_to_permission[self.role_field]
+
+                    return accessor.has_obj_perm(self.content_object, codename)
+
+                if self.role_field not in to_permissions:
+                    raise Exception(f'{self.role_field} evaluated but not a translatable permission')
+                return accessor.has_obj_perm(self.content_object, to_permissions[self.role_field])
             return self.ancestors.filter(members=accessor).exists()
         else:
             raise RuntimeError(f'Role evaluations only valid for users, received {accessor}')
@@ -280,6 +314,9 @@ class Role(models.Model):
         #
         #
 
+        if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+            return
+
         if len(additions) == 0 and len(removals) == 0:
             return
 
@@ -412,6 +449,12 @@ class Role(models.Model):
         in their organization, but some of those roles descend from
         organization admin_role, but not auditor_role.
         """
+        if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+            from ansible_base.rbac.models import RoleEvaluation
+
+            q = RoleEvaluation.objects.filter(role__in=user.has_roles.all()).values_list('object_id', 'content_type_id').query
+            return roles_qs.extra(where=[f'(object_id,content_type_id) in ({q})'])
+
         return roles_qs.filter(
             id__in=RoleAncestorEntry.objects.filter(
                 descendent__in=RoleAncestorEntry.objects.filter(ancestor_id__in=list(user.roles.values_list('id', flat=True))).values_list(
@@ -434,6 +477,13 @@ class Role(models.Model):
         return self.singleton_name in [ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR]
 
 
+class AncestorManager(models.Manager):
+    def get_queryset(self):
+        if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+            raise RuntimeError('The old RBAC system has been disabled, this should never be called')
+        return super(AncestorManager, self).get_queryset()
+
+
 class RoleAncestorEntry(models.Model):
     class Meta:
         app_label = 'main'
@@ -451,6 +501,8 @@ class RoleAncestorEntry(models.Model):
     content_type_id = models.PositiveIntegerField(null=False)
     object_id = models.PositiveIntegerField(null=False)
 
+    objects = AncestorManager()
+
 
 def role_summary_fields_generator(content_object, role_field):
     global role_descriptions
@@ -479,3 +531,168 @@ def role_summary_fields_generator(content_object, role_field):
     summary['name'] = role_names[role_field]
     summary['id'] = getattr(content_object, '{}_id'.format(role_field))
     return summary
+
+
+# ----------------- Custom Role Compatibility -------------------------
+# The following are methods to connect this (old) RBAC system to the new
+# system which allows custom roles
+# this follows the ORM interface layer documented in docs/rbac.md
+def get_role_codenames(role):
+    obj = role.content_object
+    if obj is None:
+        return
+    f = obj._meta.get_field(role.role_field)
+    parents, children = build_role_map(apps)
+    return [perm.codename for perm in get_permissions_for_role(f, children, apps)]
+
+
+def get_role_definition(role):
+    """Given a old-style role, this gives a role definition in the new RBAC system for it"""
+    obj = role.content_object
+    if obj is None:
+        return
+    f = obj._meta.get_field(role.role_field)
+    action_name = f.name.rsplit("_", 1)[0]
+    rd_name = f'{type(obj).__name__} {action_name.title()} Compat'
+    perm_list = get_role_codenames(role)
+    defaults = {'content_type_id': role.content_type_id}
+    try:
+        rd, created = RoleDefinition.objects.get_or_create(name=rd_name, permissions=perm_list, defaults=defaults)
+    except ValidationError:
+        # This is a tricky case - practically speaking, users should not be allowed to create team roles
+        # or roles that include the team member permission.
+        # If we need to create this for compatibility purposes then we will create it as a managed non-editable role
+        defaults['managed'] = True
+        rd, created = RoleDefinition.objects.get_or_create(name=rd_name, permissions=perm_list, defaults=defaults)
+    return rd
+
+
+def get_role_from_object_role(object_role):
+    """
+    Given an object role from the new system, return the corresponding role from the old system
+    reverses naming from get_role_definition, and the ANSIBLE_BASE_ROLE_PRECREATE setting.
+    """
+    rd = object_role.role_definition
+    if rd.name.endswith(' Compat'):
+        model_name, role_name, _ = rd.name.split()
+        role_name = role_name.lower()
+        role_name += '_role'
+    elif rd.name.endswith(' Admin') and rd.name.count(' ') == 2:
+        # cases like "Organization Project Admin"
+        model_name, target_model_name, role_name = rd.name.split()
+        role_name = role_name.lower()
+        model_cls = apps.get_model('main', target_model_name)
+        target_model_name = get_type_for_model(model_cls)
+        if target_model_name == 'notification_template':
+            target_model_name = 'notification'  # total exception
+        role_name = f'{target_model_name}_admin_role'
+    elif rd.name.endswith(' Admin'):
+        # cases like "project-admin"
+        role_name = 'admin_role'
+    else:
+        print(rd.name)
+        model_name, role_name = rd.name.split()
+        role_name = role_name.lower()
+        role_name += '_role'
+    return getattr(object_role.content_object, role_name)
+
+
+def give_or_remove_permission(role, actor, giving=True):
+    obj = role.content_object
+    if obj is None:
+        return
+    rd = get_role_definition(role)
+    rd.give_or_remove_permission(actor, obj, giving=giving)
+
+
+class SyncEnabled(threading.local):
+    def __init__(self):
+        self.enabled = True
+
+
+rbac_sync_enabled = SyncEnabled()
+
+
+@contextlib.contextmanager
+def disable_rbac_sync():
+    try:
+        previous_value = rbac_sync_enabled.enabled
+        rbac_sync_enabled.enabled = False
+        yield
+    finally:
+        rbac_sync_enabled.enabled = previous_value
+
+
+def give_creator_permissions(user, obj):
+    assignment = RoleDefinition.objects.give_creator_permissions(user, obj)
+    if assignment:
+        with disable_rbac_sync():
+            old_role = get_role_from_object_role(assignment.object_role)
+            old_role.members.add(user)
+
+
+def sync_members_to_new_rbac(instance, action, model, pk_set, reverse, **kwargs):
+    if action.startswith('pre_'):
+        return
+    if not rbac_sync_enabled.enabled:
+        return
+
+    if action == 'post_add':
+        is_giving = True
+    elif action == 'post_remove':
+        is_giving = False
+    elif action == 'post_clear':
+        raise RuntimeError('Clearing of role members not supported')
+
+    if reverse:
+        user = instance
+    else:
+        role = instance
+
+    for user_or_role_id in pk_set:
+        if reverse:
+            role = Role.objects.get(pk=user_or_role_id)
+        else:
+            user = get_user_model().objects.get(pk=user_or_role_id)
+        give_or_remove_permission(role, user, giving=is_giving)
+
+
+def sync_parents_to_new_rbac(instance, action, model, pk_set, reverse, **kwargs):
+    if action.startswith('pre_'):
+        return
+
+    if action == 'post_add':
+        is_giving = True
+    elif action == 'post_remove':
+        is_giving = False
+    elif action == 'post_clear':
+        raise RuntimeError('Clearing of role members not supported')
+
+    if reverse:
+        parent_role = instance
+    else:
+        child_role = instance
+
+    for role_id in pk_set:
+        if reverse:
+            child_role = Role.objects.get(id=role_id)
+        else:
+            parent_role = Role.objects.get(id=role_id)
+
+        # To a fault, we want to avoid running this if triggered from implicit_parents management
+        # we only want to do anything if we know for sure this is a non-implicit team role
+        if parent_role.role_field == 'member_role' and parent_role.content_type.model == 'team':
+            # Team internal parents are member_role->read_role and admin_role->member_role
+            # for the same object, this parenting will also be implicit_parents management
+            # do nothing for internal parents, but OTHER teams may still be assigned permissions to a team
+            if (child_role.content_type_id == parent_role.content_type_id) and (child_role.object_id == parent_role.object_id):
+                return
+
+            from awx.main.models.organization import Team
+
+            team = Team.objects.get(pk=parent_role.object_id)
+            give_or_remove_permission(child_role, team, giving=is_giving)
+
+
+m2m_changed.connect(sync_members_to_new_rbac, Role.members.through)
+m2m_changed.connect(sync_parents_to_new_rbac, Role.parents.through)

awx/main/models/unified_jobs.py

@@ -37,7 +37,8 @@ from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel,
 from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.control import Control as ControlDispatcher
 from awx.main.registrar import activity_stream_registrar
-from awx.main.models.mixins import ResourceMixin, TaskManagerUnifiedJobMixin, ExecutionEnvironmentMixin
+from awx.main.models.mixins import TaskManagerUnifiedJobMixin, ExecutionEnvironmentMixin
+from awx.main.models.rbac import to_permissions
 from awx.main.utils.common import (
     camelcase_to_underscore,
     get_model_for_type,
@@ -210,7 +211,15 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, ExecutionEn
         # do not use this if in a subclass
         if cls != UnifiedJobTemplate:
             return super(UnifiedJobTemplate, cls).accessible_pk_qs(accessor, role_field)
-        return ResourceMixin._accessible_pk_qs(cls, accessor, role_field, content_types=cls._submodels_with_roles())
+        from ansible_base.rbac.models import RoleEvaluation
+
+        action = to_permissions[role_field]
+
+        return (
+            RoleEvaluation.objects.filter(role__in=accessor.has_roles.all(), codename__startswith=action, content_type_id__in=cls._submodels_with_roles())
+            .values_list('object_id')
+            .distinct()
+        )
 
     def _perform_unique_checks(self, unique_checks):
         # Handle the list of unique fields returned above. Replace with an
@@ -1599,7 +1608,8 @@ class UnifiedJob(
             extra["controller_node"] = self.controller_node or "NOT_SET"
         elif state == "execution_node_chosen":
             extra["execution_node"] = self.execution_node or "NOT_SET"
-        logger_job_lifecycle.info(msg, extra=extra)
+
+        logger_job_lifecycle.info(f"{msg} {json.dumps(extra)}")
 
     @property
     def launched_by(self):

awx/main/models/workflow.py

@@ -467,6 +467,10 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
 
     class Meta:
         app_label = 'main'
+        permissions = [
+            ('execute_workflowjobtemplate', 'Can run this workflow job template'),
+            ('approve_workflowjobtemplate', 'Can approve steps in this workflow job template'),
+        ]
 
     notification_templates_approvals = models.ManyToManyField(
         "NotificationTemplate",

awx/main/notifications/custom_notification_base.py

@@ -1,5 +1,6 @@
 # Copyright (c) 2019 Ansible, Inc.
 # All Rights Reserved.
+# -*-coding:utf-8-*-
 
 
 class CustomNotificationBase(object):

awx/main/routing.py

@@ -12,6 +12,7 @@ from . import consumers
 
 
 logger = logging.getLogger('awx.main.routing')
+_application = None
 
 
 class AWXProtocolTypeRouter(ProtocolTypeRouter):
@@ -66,11 +67,52 @@ websocket_relay_urlpatterns = [
     re_path(r'websocket/relay/$', consumers.RelayConsumer.as_asgi()),
 ]
 
-application = AWXProtocolTypeRouter(
-    {
-        'websocket': MultipleURLRouterAdapter(
-            URLRouter(websocket_relay_urlpatterns),
-            DrfAuthMiddlewareStack(URLRouter(websocket_urlpatterns)),
-        )
-    }
-)
+
+def application_func(cls=AWXProtocolTypeRouter) -> ProtocolTypeRouter:
+    return cls(
+        {
+            'websocket': MultipleURLRouterAdapter(
+                URLRouter(websocket_relay_urlpatterns),
+                DrfAuthMiddlewareStack(URLRouter(websocket_urlpatterns)),
+            )
+        }
+    )
+
+
+def __getattr__(name: str) -> ProtocolTypeRouter:
+    """
+    Defer instantiating application.
+    For testing, we just need it to NOT run on import.
+
+    https://peps.python.org/pep-0562/#specification
+
+    Normally, someone would get application from this module via:
+        from awx.main.routing import application
+
+    and do something with the application:
+        application.do_something()
+
+    What does the callstack look like when the import runs?
+    ...
+        awx.main.routing.__getattribute__(...)                              # <-- we don't define this so NOOP as far as we are concerned
+        if '__getattr__' in awx.main.routing.__dict__:                      # <-- this triggers the function we are in
+            return awx.main.routing.__dict__.__getattr__("application")
+
+    Why isn't this function simply implemented as:
+        def __getattr__(name):
+            if not _application:
+              _application = application_func()
+            return _application
+
+    It could. I manually tested it and it passes test_routing.py.
+
+    But my understanding after reading the PEP-0562 specification link above is that
+    performance would be a bit worse due to the extra __getattribute__ calls when
+    we reference non-global variables.
+    """
+    if name == "application":
+        globs = globals()
+        if not globs['_application']:
+            globs['_application'] = application_func()
+        return globs['_application']
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

awx/main/signals.py

@@ -126,6 +126,8 @@ def rebuild_role_ancestor_list(reverse, model, instance, pk_set, action, **kwarg
 
 def sync_superuser_status_to_rbac(instance, **kwargs):
     'When the is_superuser flag is changed on a user, reflect that in the membership of the System Admnistrator role'
+    if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+        return
     update_fields = kwargs.get('update_fields', None)
     if update_fields and 'is_superuser' not in update_fields:
         return
@@ -137,6 +139,8 @@ def sync_superuser_status_to_rbac(instance, **kwargs):
 
 def sync_rbac_to_superuser_status(instance, sender, **kwargs):
     'When the is_superuser flag is false but a user has the System Admin role, update the database to reflect that'
+    if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+        return
     if kwargs['action'] in ['post_add', 'post_remove', 'post_clear']:
         new_status_value = bool(kwargs['action'] == 'post_add')
         if hasattr(instance, 'singleton_name'):  # duck typing, role.members.add() vs user.roles.add()

awx/main/tasks/receptor.py

@@ -49,6 +49,70 @@ class ReceptorConnectionType(Enum):
     STREAMTLS = 2
 
 
+"""
+Translate receptorctl messages that come in over stdout into
+structured messages. Currently, these are error messages.
+"""
+
+
+class ReceptorErrorBase:
+    _MESSAGE = 'Receptor Error'
+
+    def __init__(self, node: str = 'N/A', state_name: str = 'N/A'):
+        self.node = node
+        self.state_name = state_name
+
+    def __str__(self):
+        return f"{self.__class__.__name__} '{self._MESSAGE}' on node '{self.node}' with state '{self.state_name}'"
+
+
+class WorkUnitError(ReceptorErrorBase):
+    _MESSAGE = 'unknown work unit '
+
+    def __init__(self, work_unit_id: str, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.work_unit_id = work_unit_id
+
+    def __str__(self):
+        return f"{super().__str__()} work unit id '{self.work_unit_id}'"
+
+
+class WorkUnitCancelError(WorkUnitError):
+    _MESSAGE = 'error cancelling remote unit:  unknown work unit '
+
+
+class WorkUnitResultsError(WorkUnitError):
+    _MESSAGE = 'Failed to get results: unknown work unit '
+
+
+class UnknownError(ReceptorErrorBase):
+    _MESSAGE = 'Unknown receptor ctl error'
+
+    def __init__(self, msg, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._MESSAGE = msg
+
+
+class FuzzyError:
+    def __new__(self, e: RuntimeError, node: str, state_name: str):
+        """
+        At the time of writing this comment all of the sub-classes detection
+        is centralized in this parent class. It's like a Router().
+        Someone may find it better to push down the error detection logic into
+        each sub-class.
+        """
+        msg = e.args[0]
+
+        common_startswith = (WorkUnitCancelError, WorkUnitResultsError, WorkUnitError)
+
+        for klass in common_startswith:
+            if msg.startswith(klass._MESSAGE):
+                work_unit_id = msg[len(klass._MESSAGE) :]
+                return klass(work_unit_id, node=node, state_name=state_name)
+
+        return UnknownError(msg, node=node, state_name=state_name)
+
+
 def read_receptor_config():
     # for K8S deployments, getting a lock is necessary as another process
     # may be re-writing the config at this time
@@ -185,6 +249,7 @@ def run_until_complete(node, timing_data=None, **kwargs):
         timing_data['transmit_timing'] = run_start - transmit_start
     run_timing = 0.0
     stdout = ''
+    state_name = 'local var never set'
 
     try:
         resultfile = receptor_ctl.get_work_results(unit_id)
@@ -205,13 +270,33 @@ def run_until_complete(node, timing_data=None, **kwargs):
         stdout = resultfile.read()
         stdout = str(stdout, encoding='utf-8')
 
+    except RuntimeError as e:
+        receptor_e = FuzzyError(e, node, state_name)
+        if type(receptor_e) in (
+            WorkUnitError,
+            WorkUnitResultsError,
+        ):
+            logger.warning(f'While consuming job results: {receptor_e}')
+        else:
+            raise
     finally:
         if settings.RECEPTOR_RELEASE_WORK:
-            res = receptor_ctl.simple_command(f"work release {unit_id}")
-            if res != {'released': unit_id}:
-                logger.warning(f'Could not confirm release of receptor work unit id {unit_id} from {node}, data: {res}')
+            try:
+                res = receptor_ctl.simple_command(f"work release {unit_id}")
 
-        receptor_ctl.close()
+                if res != {'released': unit_id}:
+                    logger.warning(f'Could not confirm release of receptor work unit id {unit_id} from {node}, data: {res}')
+
+                receptor_ctl.close()
+            except RuntimeError as e:
+                receptor_e = FuzzyError(e, node, state_name)
+                if type(receptor_e) in (
+                    WorkUnitError,
+                    WorkUnitCancelError,
+                ):
+                    logger.warning(f"While releasing work: {receptor_e}")
+                else:
+                    logger.error(f"While releasing work: {receptor_e}")
 
     if state_name.lower() == 'failed':
         work_detail = status.get('Detail', '')
@@ -275,7 +360,7 @@ def _convert_args_to_cli(vargs):
     args = ['cleanup']
     for option in ('exclude_strings', 'remove_images'):
         if vargs.get(option):
-            args.append('--{}={}'.format(option.replace('_', '-'), ' '.join(vargs.get(option))))
+            args.append('--{}="{}"'.format(option.replace('_', '-'), ' '.join(vargs.get(option))))
     for option in ('file_pattern', 'image_prune', 'process_isolation_executable', 'grace_period'):
         if vargs.get(option) is True:
             args.append('--{}'.format(option.replace('_', '-')))

awx/main/tasks/system.py

@@ -6,6 +6,7 @@ import itertools
 import json
 import logging
 import os
+import psycopg
 from io import StringIO
 from contextlib import redirect_stdout
 import shutil
@@ -416,7 +417,7 @@ def handle_removed_image(remove_images=None):
 
 @task(queue=get_task_queuename)
 def cleanup_images_and_files():
-    _cleanup_images_and_files()
+    _cleanup_images_and_files(image_prune=True)
 
 
 @task(queue=get_task_queuename)
@@ -630,10 +631,18 @@ def cluster_node_heartbeat(dispatch_time=None, worker_tasks=None):
                 logger.error("Host {} last checked in at {}, marked as lost.".format(other_inst.hostname, other_inst.last_seen))
 
         except DatabaseError as e:
-            if 'did not affect any rows' in str(e):
-                logger.debug('Another instance has marked {} as lost'.format(other_inst.hostname))
+            cause = e.__cause__
+            if cause and hasattr(cause, 'sqlstate'):
+                sqlstate = cause.sqlstate
+                sqlstate_str = psycopg.errors.lookup(sqlstate)
+                logger.debug('SQL Error state: {} - {}'.format(sqlstate, sqlstate_str))
+
+                if sqlstate == psycopg.errors.NoData:
+                    logger.debug('Another instance has marked {} as lost'.format(other_inst.hostname))
+                else:
+                    logger.exception("Error marking {} as lost.".format(other_inst.hostname))
             else:
-                logger.exception('Error marking {} as lost'.format(other_inst.hostname))
+                logger.exception('No SQL state available.  Error marking {} as lost'.format(other_inst.hostname))
 
     # Run local reaper
     if worker_tasks is not None:
@@ -788,10 +797,19 @@ def update_inventory_computed_fields(inventory_id):
     try:
         i.update_computed_fields()
     except DatabaseError as e:
-        if 'did not affect any rows' in str(e):
-            logger.debug('Exiting duplicate update_inventory_computed_fields task.')
-            return
-        raise
+        # https://github.com/django/django/blob/eff21d8e7a1cb297aedf1c702668b590a1b618f3/django/db/models/base.py#L1105
+        # django raises DatabaseError("Forced update did not affect any rows.")
+
+        # if sqlstate is set then there was a database error and otherwise will re-raise that error
+        cause = e.__cause__
+        if cause and hasattr(cause, 'sqlstate'):
+            sqlstate = cause.sqlstate
+            sqlstate_str = psycopg.errors.lookup(sqlstate)
+            logger.error('SQL Error state: {} - {}'.format(sqlstate, sqlstate_str))
+            raise
+
+        # otherwise
+        logger.debug('Exiting duplicate update_inventory_computed_fields task.')
 
 
 def update_smart_memberships_for_inventory(smart_inventory):

awx/main/tests/data/ansible_utils/playbooks/valid/hello_world.yml

@@ -3,5 +3,5 @@
   hosts: all
   tasks:
     - name: Hello Message
-      debug:
+      ansible.builtin.debug:
         msg: "Hello World!"

awx/main/tests/data/inventory/plugins/terraform/env.json

@@ -0,0 +1,3 @@
+{
+    "GOOGLE_BACKEND_CREDENTIALS": "{{ file_reference }}"
+}

awx/main/tests/docs/conftest.py

@@ -1,13 +1,8 @@
 from awx.main.tests.functional.conftest import *  # noqa
+import os
+import pytest
 
 
-def pytest_addoption(parser):
-    parser.addoption("--release", action="store", help="a release version number, e.g., 3.3.0")
-
-
-def pytest_generate_tests(metafunc):
-    # This is called for every test. Only get/set command line arguments
-    # if the argument is specified in the list of test "fixturenames".
-    option_value = metafunc.config.option.release
-    if 'release' in metafunc.fixturenames and option_value is not None:
-        metafunc.parametrize("release", [option_value])
+@pytest.fixture()
+def release():
+    return os.environ.get('VERSION_TARGET', '')

awx/main/tests/docs/test_swagger_generation.py

@@ -99,7 +99,7 @@ class TestSwaggerGeneration:
         # The number of API endpoints changes over time, but let's just check
         # for a reasonable number here; if this test starts failing, raise/lower the bounds
         paths = JSON['paths']
-        assert 250 < len(paths) < 375
+        assert 250 < len(paths) < 400
         assert set(list(paths['/api/'].keys())) == set(['get', 'parameters'])
         assert set(list(paths['/api/v2/'].keys())) == set(['get', 'parameters'])
         assert set(list(sorted(paths['/api/v2/credentials/'].keys()))) == set(['get', 'post', 'parameters'])

awx/main/tests/functional/analytics/test_metrics.py

@@ -4,7 +4,6 @@ from prometheus_client.parser import text_string_to_metric_families
 from awx.main import models
 from awx.main.analytics.metrics import metrics
 from awx.api.versioning import reverse
-from awx.main.models.rbac import Role
 
 EXPECTED_VALUES = {
     'awx_system_info': 1.0,
@@ -66,7 +65,6 @@ def test_metrics_permissions(get, admin, org_admin, alice, bob, organization):
     organization.auditor_role.members.add(bob)
     assert get(get_metrics_view_db_only(), user=bob).status_code == 403
 
-    Role.singleton('system_auditor').members.add(bob)
     bob.is_system_auditor = True
     assert get(get_metrics_view_db_only(), user=bob).status_code == 200
 

awx/main/tests/functional/api/test_auth.py

@@ -6,7 +6,7 @@ from django.test import Client
 from rest_framework.test import APIRequestFactory
 
 from awx.api.generics import LoggedLoginView
-from awx.api.versioning import drf_reverse
+from rest_framework.reverse import reverse as drf_reverse
 
 
 @pytest.mark.django_db

awx/main/tests/functional/api/test_credential.py

@@ -385,10 +385,9 @@ def test_list_created_org_credentials(post, get, organization, org_admin, org_me
 @pytest.mark.django_db
 def test_list_cannot_order_by_encrypted_field(post, get, organization, org_admin, credentialtype_ssh, order_by):
     for i, password in enumerate(('abc', 'def', 'xyz')):
-        response = post(reverse('api:credential_list'), {'organization': organization.id, 'name': 'C%d' % i, 'password': password}, org_admin)
+        post(reverse('api:credential_list'), {'organization': organization.id, 'name': 'C%d' % i, 'password': password}, org_admin, expect=400)
 
-    response = get(reverse('api:credential_list'), org_admin, QUERY_STRING='order_by=%s' % order_by, status=400)
-    assert response.status_code == 400
+    get(reverse('api:credential_list'), org_admin, QUERY_STRING='order_by=%s' % order_by, expect=400)
 
 
 @pytest.mark.django_db
@@ -399,8 +398,7 @@ def test_inputs_cannot_contain_extra_fields(get, post, organization, admin, cred
         'credential_type': credentialtype_ssh.pk,
         'inputs': {'invalid_field': 'foo'},
     }
-    response = post(reverse('api:credential_list'), params, admin)
-    assert response.status_code == 400
+    response = post(reverse('api:credential_list'), params, admin, expect=400)
     assert "'invalid_field' was unexpected" in response.data['inputs'][0]
 
 

awx/main/tests/functional/api/test_oauth.py

@@ -8,8 +8,10 @@ from django.db import connection
 from django.test.utils import override_settings
 from django.utils.encoding import smart_str, smart_bytes
 
+from rest_framework.reverse import reverse as drf_reverse
+
 from awx.main.utils.encryption import decrypt_value, get_encryption_key
-from awx.api.versioning import reverse, drf_reverse
+from awx.api.versioning import reverse
 from awx.main.models.oauth import OAuth2Application as Application, OAuth2AccessToken as AccessToken
 from awx.main.tests.functional import immediate_on_commit
 from awx.sso.models import UserEnterpriseAuth

awx/main/tests/functional/api/test_role.py

@@ -3,17 +3,6 @@ import pytest
 from awx.api.versioning import reverse
 
 
-@pytest.mark.django_db
-def test_admin_visible_to_orphaned_users(get, alice):
-    names = set()
-
-    response = get(reverse('api:role_list'), user=alice)
-    for item in response.data['results']:
-        names.add(item['name'])
-    assert 'System Auditor' in names
-    assert 'System Administrator' in names
-
-
 @pytest.mark.django_db
 @pytest.mark.parametrize('role,code', [('member_role', 400), ('admin_role', 400), ('inventory_admin_role', 204)])
 @pytest.mark.parametrize('reversed', [True, False])

awx/main/tests/functional/conftest.py

@@ -3,15 +3,19 @@ import pytest
 from unittest import mock
 import urllib.parse
 from unittest.mock import PropertyMock
+import importlib
 
 # Django
 from django.urls import resolve
 from django.http import Http404
+from django.apps import apps
 from django.core.handlers.exception import response_for_exception
 from django.contrib.auth.models import User
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db.backends.sqlite3.base import SQLiteCursorWrapper
 
+from django.db.models.signals import post_migrate
+
 # AWX
 from awx.main.models.projects import Project
 from awx.main.models.ha import Instance
@@ -28,7 +32,6 @@ from awx.main.models.organization import (
     Organization,
     Team,
 )
-from awx.main.models.rbac import Role
 from awx.main.models.notifications import NotificationTemplate, Notification
 from awx.main.models.events import (
     JobEvent,
@@ -41,10 +44,19 @@ from awx.main.models.workflow import WorkflowJobTemplate
 from awx.main.models.ad_hoc_commands import AdHocCommand
 from awx.main.models.oauth import OAuth2Application as Application
 from awx.main.models.execution_environments import ExecutionEnvironment
+from awx.main.utils import is_testing
 
 __SWAGGER_REQUESTS__ = {}
 
 
+# HACK: the dab_resource_registry app required ServiceID in migrations which checks do not run
+dab_rr_initial = importlib.import_module('ansible_base.resource_registry.migrations.0001_initial')
+
+
+if is_testing():
+    post_migrate.connect(lambda **kwargs: dab_rr_initial.create_service_id(apps, None))
+
+
 @pytest.fixture(scope="session")
 def swagger_autogen(requests=__SWAGGER_REQUESTS__):
     return requests
@@ -421,7 +433,7 @@ def admin(user):
 @pytest.fixture
 def system_auditor(user):
     u = user('an-auditor', False)
-    Role.singleton('system_auditor').members.add(u)
+    u.is_system_auditor = True
     return u
 
 

awx/main/tests/functional/dab_rbac/conftest.py

@@ -0,0 +1,10 @@
+import pytest
+from django.apps import apps
+
+from awx.main.migrations._dab_rbac import setup_managed_role_definitions
+
+
+@pytest.fixture
+def managed_roles():
+    "Run the migration script to pre-create managed role definitions"
+    setup_managed_role_definitions(apps, None)

awx/main/tests/functional/dab_rbac/test_access_list.py

@@ -0,0 +1,111 @@
+import pytest
+
+from awx.main.models import User
+from awx.api.versioning import reverse
+
+
+@pytest.mark.django_db
+def test_access_list_superuser(get, admin_user, inventory):
+    url = reverse('api:inventory_access_list', kwargs={'pk': inventory.id})
+
+    response = get(url, user=admin_user, expect=200)
+    by_username = {}
+    for entry in response.data['results']:
+        by_username[entry['username']] = entry
+    assert 'admin' in by_username
+
+    assert len(by_username['admin']['summary_fields']['indirect_access']) == 1
+    assert len(by_username['admin']['summary_fields']['direct_access']) == 0
+    access_entry = by_username['admin']['summary_fields']['indirect_access'][0]
+    assert sorted(access_entry['descendant_roles']) == sorted(['adhoc_role', 'use_role', 'update_role', 'read_role', 'admin_role'])
+
+
+@pytest.mark.django_db
+def test_access_list_system_auditor(get, admin_user, inventory):
+    sys_auditor = User.objects.create(username='sys-aud')
+    sys_auditor.is_system_auditor = True
+    assert sys_auditor.is_system_auditor
+    url = reverse('api:inventory_access_list', kwargs={'pk': inventory.id})
+
+    response = get(url, user=admin_user, expect=200)
+    by_username = {}
+    for entry in response.data['results']:
+        by_username[entry['username']] = entry
+    assert 'sys-aud' in by_username
+
+    assert len(by_username['sys-aud']['summary_fields']['indirect_access']) == 1
+    assert len(by_username['sys-aud']['summary_fields']['direct_access']) == 0
+    access_entry = by_username['sys-aud']['summary_fields']['indirect_access'][0]
+    assert access_entry['descendant_roles'] == ['read_role']
+
+
+@pytest.mark.django_db
+def test_access_list_direct_access(get, admin_user, inventory):
+    u1 = User.objects.create(username='u1')
+
+    inventory.admin_role.members.add(u1)
+
+    url = reverse('api:inventory_access_list', kwargs={'pk': inventory.id})
+    response = get(url, user=admin_user, expect=200)
+    by_username = {}
+    for entry in response.data['results']:
+        by_username[entry['username']] = entry
+    assert 'u1' in by_username
+
+    assert len(by_username['u1']['summary_fields']['direct_access']) == 1
+    assert len(by_username['u1']['summary_fields']['indirect_access']) == 0
+    access_entry = by_username['u1']['summary_fields']['direct_access'][0]
+    assert sorted(access_entry['descendant_roles']) == sorted(['adhoc_role', 'use_role', 'update_role', 'read_role', 'admin_role'])
+
+
+@pytest.mark.django_db
+def test_access_list_organization_access(get, admin_user, inventory):
+    u2 = User.objects.create(username='u2')
+
+    inventory.organization.inventory_admin_role.members.add(u2)
+
+    # User has indirect access to the inventory
+    url = reverse('api:inventory_access_list', kwargs={'pk': inventory.id})
+    response = get(url, user=admin_user, expect=200)
+    by_username = {}
+    for entry in response.data['results']:
+        by_username[entry['username']] = entry
+    assert 'u2' in by_username
+
+    assert len(by_username['u2']['summary_fields']['indirect_access']) == 1
+    assert len(by_username['u2']['summary_fields']['direct_access']) == 0
+    access_entry = by_username['u2']['summary_fields']['indirect_access'][0]
+    assert sorted(access_entry['descendant_roles']) == sorted(['adhoc_role', 'use_role', 'update_role', 'read_role', 'admin_role'])
+
+    # Test that user shows up in the organization access list with direct access of expected roles
+    url = reverse('api:organization_access_list', kwargs={'pk': inventory.organization_id})
+    response = get(url, user=admin_user, expect=200)
+    by_username = {}
+    for entry in response.data['results']:
+        by_username[entry['username']] = entry
+    assert 'u2' in by_username
+
+    assert len(by_username['u2']['summary_fields']['direct_access']) == 1
+    assert len(by_username['u2']['summary_fields']['indirect_access']) == 0
+    access_entry = by_username['u2']['summary_fields']['direct_access'][0]
+    assert sorted(access_entry['descendant_roles']) == sorted(['inventory_admin_role', 'read_role'])
+
+
+@pytest.mark.django_db
+def test_team_indirect_access(get, team, admin_user, inventory):
+    u1 = User.objects.create(username='u1')
+    team.member_role.members.add(u1)
+
+    inventory.organization.inventory_admin_role.parents.add(team.member_role)
+
+    url = reverse('api:inventory_access_list', kwargs={'pk': inventory.id})
+    response = get(url, user=admin_user, expect=200)
+    by_username = {}
+    for entry in response.data['results']:
+        by_username[entry['username']] = entry
+    assert 'u1' in by_username
+
+    assert len(by_username['u1']['summary_fields']['direct_access']) == 1
+    assert len(by_username['u1']['summary_fields']['indirect_access']) == 0
+    access_entry = by_username['u1']['summary_fields']['direct_access'][0]
+    assert sorted(access_entry['descendant_roles']) == sorted(['adhoc_role', 'use_role', 'update_role', 'read_role', 'admin_role'])

awx/main/tests/functional/dab_rbac/test_dab_migration.py

@@ -0,0 +1,45 @@
+import pytest
+from django.apps import apps
+from django.test.utils import override_settings
+
+from awx.main.migrations._dab_rbac import setup_managed_role_definitions
+
+from ansible_base.rbac.models import RoleDefinition
+
+INVENTORY_OBJ_PERMISSIONS = ['view_inventory', 'adhoc_inventory', 'use_inventory', 'change_inventory', 'delete_inventory', 'update_inventory']
+
+
+@pytest.mark.django_db
+def test_managed_definitions_precreate():
+    with override_settings(
+        ANSIBLE_BASE_ROLE_PRECREATE={
+            'object_admin': '{cls._meta.model_name}-admin',
+            'org_admin': 'organization-admin',
+            'org_children': 'organization-{cls._meta.model_name}-admin',
+            'special': '{cls._meta.model_name}-{action}',
+        }
+    ):
+        setup_managed_role_definitions(apps, None)
+    rd = RoleDefinition.objects.get(name='inventory-admin')
+    assert rd.managed is True
+    # add permissions do not go in the object-level admin
+    assert set(rd.permissions.values_list('codename', flat=True)) == set(INVENTORY_OBJ_PERMISSIONS)
+
+    # test org-level object admin permissions
+    rd = RoleDefinition.objects.get(name='organization-inventory-admin')
+    assert rd.managed is True
+    assert set(rd.permissions.values_list('codename', flat=True)) == set(['add_inventory', 'view_organization'] + INVENTORY_OBJ_PERMISSIONS)
+
+
+@pytest.mark.django_db
+def test_managed_definitions_custom_obj_admin_name():
+    with override_settings(
+        ANSIBLE_BASE_ROLE_PRECREATE={
+            'object_admin': 'foo-{cls._meta.model_name}-foo',
+        }
+    ):
+        setup_managed_role_definitions(apps, None)
+    rd = RoleDefinition.objects.get(name='foo-inventory-foo')
+    assert rd.managed is True
+    # add permissions do not go in the object-level admin
+    assert set(rd.permissions.values_list('codename', flat=True)) == set(INVENTORY_OBJ_PERMISSIONS)

awx/main/tests/functional/dab_rbac/test_dab_rbac_api.py

@@ -0,0 +1,75 @@
+import pytest
+
+from django.contrib.contenttypes.models import ContentType
+from django.urls import reverse as django_reverse
+
+from awx.api.versioning import reverse
+from awx.main.models import JobTemplate, Inventory, Organization
+
+from ansible_base.rbac.models import RoleDefinition
+
+
+@pytest.mark.django_db
+def test_managed_roles_created(managed_roles):
+    "Managed RoleDefinitions are created in post_migration signal, we expect to see them here"
+    for cls in (JobTemplate, Inventory):
+        ct = ContentType.objects.get_for_model(cls)
+        rds = list(RoleDefinition.objects.filter(content_type=ct))
+        assert len(rds) > 1
+        assert f'{cls.__name__} Admin' in [rd.name for rd in rds]
+        for rd in rds:
+            assert rd.managed is True
+
+
+@pytest.mark.django_db
+def test_custom_read_role(admin_user, post, managed_roles):
+    rd_url = django_reverse('roledefinition-list')
+    resp = post(
+        url=rd_url, data={"name": "read role made for test", "content_type": "awx.inventory", "permissions": ['view_inventory']}, user=admin_user, expect=201
+    )
+    rd_id = resp.data['id']
+    rd = RoleDefinition.objects.get(id=rd_id)
+    assert rd.content_type == ContentType.objects.get_for_model(Inventory)
+
+
+@pytest.mark.django_db
+def test_custom_system_roles_prohibited(admin_user, post):
+    rd_url = django_reverse('roledefinition-list')
+    resp = post(url=rd_url, data={"name": "read role made for test", "content_type": None, "permissions": ['view_inventory']}, user=admin_user, expect=400)
+    assert 'System-wide roles are not enabled' in str(resp.data)
+
+
+@pytest.mark.django_db
+def test_assign_managed_role(admin_user, alice, rando, inventory, post, managed_roles):
+    rd = RoleDefinition.objects.get(name='Inventory Admin')
+    rd.give_permission(alice, inventory)
+    # Now that alice has full permissions to the inventory, she will give rando permission
+    url = django_reverse('roleuserassignment-list')
+    post(url=url, data={"user": rando.id, "role_definition": rd.id, "object_id": inventory.id}, user=alice, expect=201)
+    assert rando.has_obj_perm(inventory, 'change') is True
+
+
+@pytest.mark.django_db
+def test_assign_custom_delete_role(admin_user, rando, inventory, delete, patch):
+    rd, _ = RoleDefinition.objects.get_or_create(
+        name='inventory-delete', permissions=['delete_inventory', 'view_inventory'], content_type=ContentType.objects.get_for_model(Inventory)
+    )
+    rd.give_permission(rando, inventory)
+    inv_id = inventory.pk
+    inv_url = reverse('api:inventory_detail', kwargs={'pk': inv_id})
+    patch(url=inv_url, data={"description": "new"}, user=rando, expect=403)
+    delete(url=inv_url, user=rando, expect=202)
+    assert Inventory.objects.get(id=inv_id).pending_deletion
+
+
+@pytest.mark.django_db
+def test_assign_custom_add_role(admin_user, rando, organization, post, managed_roles):
+    rd, _ = RoleDefinition.objects.get_or_create(
+        name='inventory-add', permissions=['add_inventory', 'view_organization'], content_type=ContentType.objects.get_for_model(Organization)
+    )
+    rd.give_permission(rando, organization)
+    url = reverse('api:inventory_list')
+    r = post(url=url, data={'name': 'abc', 'organization': organization.id}, user=rando, expect=201)
+    inv_id = r.data['id']
+    inventory = Inventory.objects.get(id=inv_id)
+    assert rando.has_obj_perm(inventory, 'change')

awx/main/tests/functional/dab_rbac/test_translation_layer.py

@@ -0,0 +1,107 @@
+from unittest import mock
+
+import pytest
+
+from awx.main.models.rbac import get_role_from_object_role, give_creator_permissions
+from awx.main.models import User, Organization, WorkflowJobTemplate, WorkflowJobTemplateNode, Team
+from awx.api.versioning import reverse
+
+from ansible_base.rbac.models import RoleUserAssignment
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    'role_name',
+    ['execution_environment_admin_role', 'project_admin_role', 'admin_role', 'auditor_role', 'read_role', 'execute_role', 'notification_admin_role'],
+)
+def test_round_trip_roles(organization, rando, role_name, managed_roles):
+    """
+    Make an assignment with the old-style role,
+    get the equivelent new role
+    get the old role again
+    """
+    getattr(organization, role_name).members.add(rando)
+    assignment = RoleUserAssignment.objects.get(user=rando)
+    print(assignment.role_definition.name)
+    old_role = get_role_from_object_role(assignment.object_role)
+    assert old_role.id == getattr(organization, role_name).id
+
+
+@pytest.mark.django_db
+def test_organization_level_permissions(organization, inventory, managed_roles):
+    u1 = User.objects.create(username='alice')
+    u2 = User.objects.create(username='bob')
+
+    organization.inventory_admin_role.members.add(u1)
+    organization.workflow_admin_role.members.add(u2)
+
+    assert u1 in inventory.admin_role
+    assert u1 in organization.inventory_admin_role
+    assert u2 in organization.workflow_admin_role
+
+    assert u2 not in organization.inventory_admin_role
+    assert u1 not in organization.workflow_admin_role
+    assert not (set(u1.has_roles.all()) & set(u2.has_roles.all()))  # user have no roles in common
+
+    # Old style
+    assert set(Organization.accessible_objects(u1, 'inventory_admin_role')) == set([organization])
+    assert set(Organization.accessible_objects(u2, 'inventory_admin_role')) == set()
+    assert set(Organization.accessible_objects(u1, 'workflow_admin_role')) == set()
+    assert set(Organization.accessible_objects(u2, 'workflow_admin_role')) == set([organization])
+
+    # New style
+    assert set(Organization.access_qs(u1, 'add_inventory')) == set([organization])
+    assert set(Organization.access_qs(u1, 'change_inventory')) == set([organization])
+    assert set(Organization.access_qs(u2, 'add_inventory')) == set()
+    assert set(Organization.access_qs(u1, 'add_workflowjobtemplate')) == set()
+    assert set(Organization.access_qs(u2, 'add_workflowjobtemplate')) == set([organization])
+
+
+@pytest.mark.django_db
+def test_organization_execute_role(organization, rando, managed_roles):
+    organization.execute_role.members.add(rando)
+    assert rando in organization.execute_role
+    assert set(Organization.accessible_objects(rando, 'execute_role')) == set([organization])
+
+
+@pytest.mark.django_db
+def test_workflow_approval_list(get, post, admin_user, managed_roles):
+    workflow_job_template = WorkflowJobTemplate.objects.create()
+    approval_node = WorkflowJobTemplateNode.objects.create(workflow_job_template=workflow_job_template)
+    url = reverse('api:workflow_job_template_node_create_approval', kwargs={'pk': approval_node.pk, 'version': 'v2'})
+    post(url, {'name': 'URL Test', 'description': 'An approval', 'timeout': 0}, user=admin_user)
+    approval_node.refresh_from_db()
+    approval_jt = approval_node.unified_job_template
+    approval_jt.create_unified_job()
+
+    r = get(url=reverse('api:workflow_approval_list'), user=admin_user, expect=200)
+    assert r.data['count'] >= 1
+
+
+@pytest.mark.django_db
+def test_creator_permission(rando, admin_user, inventory, managed_roles):
+    give_creator_permissions(rando, inventory)
+    assert rando in inventory.admin_role
+    assert rando in inventory.admin_role.members.all()
+
+
+@pytest.mark.django_db
+def test_team_team_read_role(rando, team, admin_user, post, managed_roles):
+    orgs = [Organization.objects.create(name=f'foo-{i}') for i in range(2)]
+    teams = [Team.objects.create(name=f'foo-{i}', organization=orgs[i]) for i in range(2)]
+    teams[1].member_role.members.add(rando)
+
+    # give second team read permission to first team through the API for regression testing
+    url = reverse('api:role_teams_list', kwargs={'pk': teams[0].read_role.pk, 'version': 'v2'})
+    post(url, {'id': teams[1].id}, user=admin_user)
+
+    # user should be able to view the first team
+    assert rando in teams[0].read_role
+
+
+@pytest.mark.django_db
+def test_implicit_parents_no_assignments(organization):
+    """Through the normal course of creating models, we should not be changing DAB RBAC permissions"""
+    with mock.patch('awx.main.models.rbac.give_or_remove_permission') as mck:
+        Team.objects.create(name='random team', organization=organization)
+    mck.assert_not_called()

awx/main/tests/functional/dab_resource_registry/test_ansible_id_display.py

@@ -0,0 +1,39 @@
+import pytest
+
+from ansible_base.resource_registry.models import Resource
+
+from awx.api.versioning import reverse
+
+
+def assert_has_resource(list_response, obj=None):
+    data = list_response.data
+    assert 'resource' in data['results'][0]['summary_fields']
+    resource_data = data['results'][0]['summary_fields']['resource']
+    assert resource_data['ansible_id']
+    resource = Resource.objects.filter(ansible_id=resource_data['ansible_id']).first()
+    assert resource
+    assert resource.content_object
+    if obj:
+        objects = [Resource.objects.get(ansible_id=entry['summary_fields']['resource']['ansible_id']).content_object for entry in data['results']]
+        assert obj in objects
+
+
+@pytest.mark.django_db
+def test_organization_ansible_id(organization, admin_user, get):
+    url = reverse('api:organization_list')
+    response = get(url=url, user=admin_user, expect=200)
+    assert_has_resource(response, obj=organization)
+
+
+@pytest.mark.django_db
+def test_team_ansible_id(team, admin_user, get):
+    url = reverse('api:team_list')
+    response = get(url=url, user=admin_user, expect=200)
+    assert_has_resource(response, obj=team)
+
+
+@pytest.mark.django_db
+def test_user_ansible_id(rando, admin_user, get):
+    url = reverse('api:user_list')
+    response = get(url=url, user=admin_user, expect=200)
+    assert_has_resource(response, obj=rando)

awx/main/tests/functional/models/test_activity_stream.py

@@ -104,11 +104,13 @@ class TestRolesAssociationEntries:
         else:
             assert len(entry_qs) == 1
         # unfortunate, the original creation does _not_ set a real is_auditor field
-        assert 'is_system_auditor' not in json.loads(entry_qs[0].changes)
+        assert 'is_system_auditor' not in json.loads(entry_qs[0].changes)  # NOTE: if this fails, see special note
+        # special note - if system auditor flag is moved to user model then we expect this assertion to be changed
+        # make sure that an extra entry is not created, expectation for count would change to 1
         if value:
-            auditor_changes = json.loads(entry_qs[1].changes)
-            assert auditor_changes['object2'] == 'user'
-            assert auditor_changes['object2_pk'] == u.pk
+            entry = entry_qs[1]
+            assert json.loads(entry.changes) == {'is_system_auditor': [False, True]}
+            assert entry.object1 == 'user'
 
     def test_user_no_op_api(self, system_auditor):
         as_ct = ActivityStream.objects.count()

awx/main/tests/functional/models/test_context_managers.py

@@ -1,7 +1,6 @@
 import pytest
 
 # AWX context managers for testing
-from awx.main.models.rbac import batch_role_ancestor_rebuilding
 from awx.main.signals import disable_activity_stream, disable_computed_fields, update_inventory_computed_fields
 
 # AWX models
@@ -10,15 +9,6 @@ from awx.main.models import ActivityStream, Job
 from awx.main.tests.functional import immediate_on_commit
 
 
-@pytest.mark.django_db
-def test_rbac_batch_rebuilding(rando, organization):
-    with batch_role_ancestor_rebuilding():
-        organization.admin_role.members.add(rando)
-        inventory = organization.inventories.create(name='test-inventory')
-        assert rando not in inventory.admin_role
-    assert rando in inventory.admin_role
-
-
 @pytest.mark.django_db
 def test_disable_activity_stream():
     with disable_activity_stream():

awx/main/tests/functional/models/test_inventory.py

@@ -193,6 +193,7 @@ class TestInventorySourceInjectors:
             ('satellite6', 'theforeman.foreman.foreman'),
             ('insights', 'redhatinsights.insights.insights'),
             ('controller', 'awx.awx.tower'),
+            ('terraform', 'cloud.terraform.terraform_state'),
         ],
     )
     def test_plugin_proper_names(self, source, proper_name):

awx/main/tests/functional/test_fixture_factories.py

@@ -50,13 +50,13 @@ def test_org_factory_roles(organization_factory):
         teams=['team1', 'team2'],
         users=['team1:foo', 'bar'],
         projects=['baz', 'bang'],
-        roles=['team2.member_role:foo', 'team1.admin_role:bar', 'team1.admin_role:team2.admin_role', 'baz.admin_role:foo'],
+        roles=['team2.member_role:foo', 'team1.admin_role:bar', 'team1.member_role:team2.admin_role', 'baz.admin_role:foo'],
     )
 
     assert objects.users.bar in objects.teams.team2.admin_role
     assert objects.users.foo in objects.projects.baz.admin_role
     assert objects.users.foo in objects.teams.team1.member_role
-    assert objects.teams.team2.admin_role in objects.teams.team1.admin_role.children.all()
+    assert objects.teams.team2.admin_role in objects.teams.team1.member_role.children.all()
 
 
 @pytest.mark.django_db

awx/main/tests/functional/test_inventory_source_injectors.py

@@ -107,6 +107,7 @@ def read_content(private_data_dir, raw_env, inventory_update):
             for filename in os.listdir(os.path.join(private_data_dir, subdir)):
                 filename_list.append(os.path.join(subdir, filename))
     filename_list = sorted(filename_list, key=lambda fn: inverse_env.get(os.path.join(private_data_dir, fn), [fn])[0])
+    inventory_content = ""
     for filename in filename_list:
         if filename in ('args', 'project'):
             continue  # Ansible runner
@@ -130,6 +131,7 @@ def read_content(private_data_dir, raw_env, inventory_update):
                 dir_contents[abs_file_path] = f.read()
             # Declare a reference to inventory plugin file if it exists
             if abs_file_path.endswith('.yml') and 'plugin: ' in dir_contents[abs_file_path]:
+                inventory_content = dir_contents[abs_file_path]
                 referenced_paths.add(abs_file_path)  # used as inventory file
             elif cache_file_regex.match(abs_file_path):
                 file_aliases[abs_file_path] = 'cache_file'
@@ -157,7 +159,11 @@ def read_content(private_data_dir, raw_env, inventory_update):
     content = {}
     for abs_file_path, file_content in dir_contents.items():
         # assert that all files laid down are used
-        if abs_file_path not in referenced_paths and abs_file_path not in ignore_files:
+        if (
+            abs_file_path not in referenced_paths
+            and to_container_path(abs_file_path, private_data_dir) not in inventory_content
+            and abs_file_path not in ignore_files
+        ):
             raise AssertionError(
                 "File {} is not referenced. References and files:\n{}\n{}".format(abs_file_path, json.dumps(env, indent=4), json.dumps(dir_contents, indent=4))
             )

awx/main/tests/functional/test_named_url.py

@@ -1,9 +1,6 @@
 # -*- coding: utf-8 -*-
-from unittest import mock
-
 import pytest
 
-from django.core.exceptions import ImproperlyConfigured
 from django.conf import settings
 
 from awx.api.versioning import reverse
@@ -23,25 +20,6 @@ from awx.main.models import (  # noqa
     User,
     WorkflowJobTemplate,
 )
-from awx.conf import settings_registry
-
-
-def setup_module(module):
-    # In real-world scenario, named url graph structure is populated by __init__
-    # of URLModificationMiddleware. The way Django bootstraps ensures the initialization
-    # will happen *once and only once*, while the number of initialization is uncontrollable
-    # in unit test environment. So it is wrapped by try-except block to mute any
-    # unwanted exceptions.
-    try:
-        URLModificationMiddleware(mock.Mock())
-    except ImproperlyConfigured:
-        pass
-
-
-def teardown_module(module):
-    # settings_registry will be persistent states unless we explicitly clean them up.
-    settings_registry.unregister('NAMED_URL_FORMATS')
-    settings_registry.unregister('NAMED_URL_GRAPH_NODES')
 
 
 @pytest.mark.django_db

awx/main/tests/functional/test_projects.py

@@ -411,14 +411,14 @@ def test_project_delete(delete, organization, admin_user):
 
 
 @pytest.mark.parametrize(
-    'order_by, expected_names, expected_ids',
+    'order_by, expected_names',
     [
-        ('name', ['alice project', 'bob project', 'shared project'], [1, 2, 3]),
-        ('-name', ['shared project', 'bob project', 'alice project'], [3, 2, 1]),
+        ('name', ['alice project', 'bob project', 'shared project']),
+        ('-name', ['shared project', 'bob project', 'alice project']),
     ],
 )
 @pytest.mark.django_db
-def test_project_list_ordering_by_name(get, order_by, expected_names, expected_ids, organization_factory):
+def test_project_list_ordering_by_name(get, order_by, expected_names, organization_factory):
     'ensure sorted order of project list is maintained correctly when the requested order is invalid or not applicable'
     objects = organization_factory(
         'org1',
@@ -426,13 +426,11 @@ def test_project_list_ordering_by_name(get, order_by, expected_names, expected_i
         superusers=['admin'],
     )
     project_names = []
-    project_ids = []
     # TODO: ask for an order by here that doesn't apply
     results = get(reverse('api:project_list'), objects.superusers.admin, QUERY_STRING='order_by=%s' % order_by).data['results']
     for x in range(len(results)):
         project_names.append(results[x]['name'])
-        project_ids.append(results[x]['id'])
-    assert project_names == expected_names and project_ids == expected_ids
+    assert project_names == expected_names
 
 
 @pytest.mark.parametrize('order_by', ('name', '-name'))
@@ -450,7 +448,8 @@ def test_project_list_ordering_with_duplicate_names(get, order_by, organization_
     for x in range(3):
         results = get(reverse('api:project_list'), objects.superusers.admin, QUERY_STRING='order_by=%s' % order_by).data['results']
         project_ids[x] = [proj['id'] for proj in results]
-    assert project_ids[0] == project_ids[1] == project_ids[2] == [1, 2, 3, 4, 5]
+    assert project_ids[0] == project_ids[1] == project_ids[2]
+    assert project_ids[0] == sorted(project_ids[0])
 
 
 @pytest.mark.django_db

awx/main/tests/functional/test_rbac_api.py

@@ -3,7 +3,9 @@ import pytest
 
 from django.db import transaction
 from awx.api.versioning import reverse
-from awx.main.models.rbac import Role, ROLE_SINGLETON_SYSTEM_ADMINISTRATOR
+from awx.main.models.rbac import Role
+
+from django.test.utils import override_settings
 
 
 @pytest.fixture
@@ -31,8 +33,6 @@ def test_get_roles_list_user(organization, inventory, team, get, user):
     'Users can see all roles they have access to, but not all roles'
     this_user = user('user-test_get_roles_list_user')
     organization.member_role.members.add(this_user)
-    custom_role = Role.objects.create(role_field='custom_role-test_get_roles_list_user')
-    organization.member_role.children.add(custom_role)
 
     url = reverse('api:role_list')
     response = get(url, this_user)
@@ -46,10 +46,8 @@ def test_get_roles_list_user(organization, inventory, team, get, user):
     for r in roles['results']:
         role_hash[r['id']] = r
 
-    assert Role.singleton(ROLE_SINGLETON_SYSTEM_ADMINISTRATOR).id in role_hash
     assert organization.admin_role.id in role_hash
     assert organization.member_role.id in role_hash
-    assert custom_role.id in role_hash
 
     assert inventory.admin_role.id not in role_hash
     assert team.member_role.id not in role_hash
@@ -57,7 +55,8 @@ def test_get_roles_list_user(organization, inventory, team, get, user):
 
 @pytest.mark.django_db
 def test_roles_visibility(get, organization, project, admin, alice, bob):
-    Role.singleton('system_auditor').members.add(alice)
+    alice.is_system_auditor = True
+    alice.save()
     assert get(reverse('api:role_list') + '?id=%d' % project.update_role.id, user=admin).data['count'] == 1
     assert get(reverse('api:role_list') + '?id=%d' % project.update_role.id, user=alice).data['count'] == 1
     assert get(reverse('api:role_list') + '?id=%d' % project.update_role.id, user=bob).data['count'] == 0
@@ -67,7 +66,8 @@ def test_roles_visibility(get, organization, project, admin, alice, bob):
 
 @pytest.mark.django_db
 def test_roles_filter_visibility(get, organization, project, admin, alice, bob):
-    Role.singleton('system_auditor').members.add(alice)
+    alice.is_system_auditor = True
+    alice.save()
     project.update_role.members.add(admin)
 
     assert get(reverse('api:user_roles_list', kwargs={'pk': admin.id}) + '?id=%d' % project.update_role.id, user=admin).data['count'] == 1
@@ -105,15 +105,6 @@ def test_cant_delete_role(delete, admin, inventory):
 #
 
 
-@pytest.mark.django_db
-def test_get_user_roles_list(get, admin):
-    url = reverse('api:user_roles_list', kwargs={'pk': admin.id})
-    response = get(url, admin)
-    assert response.status_code == 200
-    roles = response.data
-    assert roles['count'] > 0  # 'system_administrator' role if nothing else
-
-
 @pytest.mark.django_db
 def test_user_view_other_user_roles(organization, inventory, team, get, alice, bob):
     'Users can see roles for other users, but only the roles that that user has access to see as well'
@@ -141,7 +132,6 @@ def test_user_view_other_user_roles(organization, inventory, team, get, alice, b
 
     assert organization.admin_role.id in role_hash
     assert custom_role.id not in role_hash  # doesn't show up in the user roles list, not an explicit grant
-    assert Role.singleton(ROLE_SINGLETON_SYSTEM_ADMINISTRATOR).id not in role_hash
     assert inventory.admin_role.id not in role_hash
     assert team.member_role.id not in role_hash  # alice can't see this
 
@@ -197,6 +187,7 @@ def test_remove_role_from_user(role, post, admin):
 
 
 @pytest.mark.django_db
+@override_settings(ANSIBLE_BASE_ALLOW_TEAM_ORG_ADMIN=True)
 def test_get_teams_roles_list(get, team, organization, admin):
     team.member_role.children.add(organization.admin_role)
     url = reverse('api:team_roles_list', kwargs={'pk': team.id})

awx/main/tests/functional/test_rbac_core.py

@@ -1,213 +0,0 @@
-import pytest
-
-from awx.main.models import (
-    Role,
-    Organization,
-    Project,
-)
-from awx.main.fields import update_role_parentage_for_instance
-
-
-@pytest.mark.django_db
-def test_auto_inheritance_by_children(organization, alice):
-    A = Role.objects.create()
-    B = Role.objects.create()
-    A.members.add(alice)
-
-    assert alice not in organization.admin_role
-    assert Organization.accessible_objects(alice, 'admin_role').count() == 0
-    A.children.add(B)
-    assert alice not in organization.admin_role
-    assert Organization.accessible_objects(alice, 'admin_role').count() == 0
-    A.children.add(organization.admin_role)
-    assert alice in organization.admin_role
-    assert Organization.accessible_objects(alice, 'admin_role').count() == 1
-    A.children.remove(organization.admin_role)
-    assert alice not in organization.admin_role
-    B.children.add(organization.admin_role)
-    assert alice in organization.admin_role
-    B.children.remove(organization.admin_role)
-    assert alice not in organization.admin_role
-    assert Organization.accessible_objects(alice, 'admin_role').count() == 0
-
-    # We've had the case where our pre/post save init handlers in our field descriptors
-    # end up creating a ton of role objects because of various not-so-obvious issues
-    assert Role.objects.count() < 50
-
-
-@pytest.mark.django_db
-def test_auto_inheritance_by_parents(organization, alice):
-    A = Role.objects.create()
-    B = Role.objects.create()
-    A.members.add(alice)
-
-    assert alice not in organization.admin_role
-    B.parents.add(A)
-    assert alice not in organization.admin_role
-    organization.admin_role.parents.add(A)
-    assert alice in organization.admin_role
-    organization.admin_role.parents.remove(A)
-    assert alice not in organization.admin_role
-    organization.admin_role.parents.add(B)
-    assert alice in organization.admin_role
-    organization.admin_role.parents.remove(B)
-    assert alice not in organization.admin_role
-
-
-@pytest.mark.django_db
-def test_accessible_objects(organization, alice, bob):
-    A = Role.objects.create()
-    A.members.add(alice)
-    B = Role.objects.create()
-    B.members.add(alice)
-    B.members.add(bob)
-
-    assert Organization.accessible_objects(alice, 'admin_role').count() == 0
-    assert Organization.accessible_objects(bob, 'admin_role').count() == 0
-    A.children.add(organization.admin_role)
-    assert Organization.accessible_objects(alice, 'admin_role').count() == 1
-    assert Organization.accessible_objects(bob, 'admin_role').count() == 0
-
-
-@pytest.mark.django_db
-def test_team_symantics(organization, team, alice):
-    assert alice not in organization.auditor_role
-    team.member_role.children.add(organization.auditor_role)
-    assert alice not in organization.auditor_role
-    team.member_role.members.add(alice)
-    assert alice in organization.auditor_role
-    team.member_role.members.remove(alice)
-    assert alice not in organization.auditor_role
-
-
-@pytest.mark.django_db
-def test_auto_field_adjustments(organization, inventory, team, alice):
-    'Ensures the auto role reparenting is working correctly through non m2m fields'
-    org2 = Organization.objects.create(name='Org 2', description='org 2')
-    org2.admin_role.members.add(alice)
-    assert alice not in inventory.admin_role
-    inventory.organization = org2
-    inventory.save()
-    assert alice in inventory.admin_role
-    inventory.organization = organization
-    inventory.save()
-    assert alice not in inventory.admin_role
-    # assert False
-
-
-@pytest.mark.django_db
-def test_implicit_deletes(alice):
-    'Ensures implicit resources and roles delete themselves'
-    delorg = Organization.objects.create(name='test-org')
-    child = Role.objects.create()
-    child.parents.add(delorg.admin_role)
-    delorg.admin_role.members.add(alice)
-
-    admin_role_id = delorg.admin_role.id
-    auditor_role_id = delorg.auditor_role.id
-
-    assert child.ancestors.count() > 1
-    assert Role.objects.filter(id=admin_role_id).count() == 1
-    assert Role.objects.filter(id=auditor_role_id).count() == 1
-    n_alice_roles = alice.roles.count()
-    n_system_admin_children = Role.singleton('system_administrator').children.count()
-
-    delorg.delete()
-
-    assert Role.objects.filter(id=admin_role_id).count() == 0
-    assert Role.objects.filter(id=auditor_role_id).count() == 0
-    assert alice.roles.count() == (n_alice_roles - 1)
-    assert Role.singleton('system_administrator').children.count() == (n_system_admin_children - 1)
-    assert child.ancestors.count() == 1
-    assert child.ancestors.all()[0] == child
-
-
-@pytest.mark.django_db
-def test_content_object(user):
-    'Ensure our content_object stuf seems to be working'
-
-    org = Organization.objects.create(name='test-org')
-    assert org.admin_role.content_object.id == org.id
-
-
-@pytest.mark.django_db
-def test_hierarchy_rebuilding_multi_path():
-    'Tests a subdtle cases around role hierarchy rebuilding when you have multiple paths to the same role of different length'
-
-    X = Role.objects.create()
-    A = Role.objects.create()
-    B = Role.objects.create()
-    C = Role.objects.create()
-    D = Role.objects.create()
-
-    A.children.add(B)
-    A.children.add(D)
-    B.children.add(C)
-    C.children.add(D)
-
-    assert A.is_ancestor_of(D)
-    assert X.is_ancestor_of(D) is False
-
-    X.children.add(A)
-
-    assert X.is_ancestor_of(D) is True
-
-    X.children.remove(A)
-
-    # This can be the stickler, the rebuilder needs to ensure that D's role
-    # hierarchy is built after both A and C are updated.
-    assert X.is_ancestor_of(D) is False
-
-
-@pytest.mark.django_db
-def test_auto_parenting():
-    org1 = Organization.objects.create(name='org1')
-    org2 = Organization.objects.create(name='org2')
-
-    prj1 = Project.objects.create(name='prj1')
-    prj2 = Project.objects.create(name='prj2')
-
-    assert org1.admin_role.is_ancestor_of(prj1.admin_role) is False
-    assert org1.admin_role.is_ancestor_of(prj2.admin_role) is False
-    assert org2.admin_role.is_ancestor_of(prj1.admin_role) is False
-    assert org2.admin_role.is_ancestor_of(prj2.admin_role) is False
-
-    prj1.organization = org1
-    prj1.save()
-
-    assert org1.admin_role.is_ancestor_of(prj1.admin_role)
-    assert org1.admin_role.is_ancestor_of(prj2.admin_role) is False
-    assert org2.admin_role.is_ancestor_of(prj1.admin_role) is False
-    assert org2.admin_role.is_ancestor_of(prj2.admin_role) is False
-
-    prj2.organization = org1
-    prj2.save()
-
-    assert org1.admin_role.is_ancestor_of(prj1.admin_role)
-    assert org1.admin_role.is_ancestor_of(prj2.admin_role)
-    assert org2.admin_role.is_ancestor_of(prj1.admin_role) is False
-    assert org2.admin_role.is_ancestor_of(prj2.admin_role) is False
-
-    prj1.organization = org2
-    prj1.save()
-
-    assert org1.admin_role.is_ancestor_of(prj1.admin_role) is False
-    assert org1.admin_role.is_ancestor_of(prj2.admin_role)
-    assert org2.admin_role.is_ancestor_of(prj1.admin_role)
-    assert org2.admin_role.is_ancestor_of(prj2.admin_role) is False
-
-    prj2.organization = org2
-    prj2.save()
-
-    assert org1.admin_role.is_ancestor_of(prj1.admin_role) is False
-    assert org1.admin_role.is_ancestor_of(prj2.admin_role) is False
-    assert org2.admin_role.is_ancestor_of(prj1.admin_role)
-    assert org2.admin_role.is_ancestor_of(prj2.admin_role)
-
-
-@pytest.mark.django_db
-def test_update_parents_keeps_teams(team, project):
-    project.update_role.parents.add(team.member_role)
-    assert list(Project.accessible_objects(team.member_role, 'update_role')) == [project]  # test prep sanity check
-    update_role_parentage_for_instance(project)
-    assert list(Project.accessible_objects(team.member_role, 'update_role')) == [project]  # actual assertion

awx/main/tests/functional/test_rbac_job_templates.py

@@ -4,7 +4,7 @@ import pytest
 from awx.api.versioning import reverse
 from awx.main.access import BaseAccess, JobTemplateAccess, ScheduleAccess
 from awx.main.models.jobs import JobTemplate
-from awx.main.models import Project, Organization, Inventory, Schedule, User
+from awx.main.models import Project, Organization, Schedule
 
 
 @mock.patch.object(BaseAccess, 'check_license', return_value=None)
@@ -177,7 +177,7 @@ def test_job_template_creator_access(project, organization, rando, post):
     jt_pk = response.data['id']
     jt_obj = JobTemplate.objects.get(pk=jt_pk)
     # Creating a JT should place the creator in the admin role
-    assert rando in jt_obj.admin_role.members.all()
+    assert rando in jt_obj.admin_role
 
 
 @pytest.mark.django_db
@@ -283,48 +283,3 @@ class TestProjectOrganization:
         assert org_admin not in jt.admin_role
         patch(url=jt.get_absolute_url(), data={'project': project.id}, user=admin_user, expect=200)
         assert org_admin in jt.admin_role
-
-    def test_inventory_read_transfer_direct(self, patch):
-        orgs = []
-        invs = []
-        admins = []
-        for i in range(2):
-            org = Organization.objects.create(name='org{}'.format(i))
-            org_admin = User.objects.create(username='user{}'.format(i))
-            inv = Inventory.objects.create(organization=org, name='inv{}'.format(i))
-            org.auditor_role.members.add(org_admin)
-
-            orgs.append(org)
-            admins.append(org_admin)
-            invs.append(inv)
-
-        jt = JobTemplate.objects.create(name='foo', inventory=invs[0])
-        assert admins[0] in jt.read_role
-        assert admins[1] not in jt.read_role
-
-        jt.inventory = invs[1]
-        jt.save(update_fields=['inventory'])
-        assert admins[0] not in jt.read_role
-        assert admins[1] in jt.read_role
-
-    def test_inventory_read_transfer_indirect(self, patch):
-        orgs = []
-        admins = []
-        for i in range(2):
-            org = Organization.objects.create(name='org{}'.format(i))
-            org_admin = User.objects.create(username='user{}'.format(i))
-            org.auditor_role.members.add(org_admin)
-
-            orgs.append(org)
-            admins.append(org_admin)
-
-        inv = Inventory.objects.create(organization=orgs[0], name='inv{}'.format(i))
-
-        jt = JobTemplate.objects.create(name='foo', inventory=inv)
-        assert admins[0] in jt.read_role
-        assert admins[1] not in jt.read_role
-
-        inv.organization = orgs[1]
-        inv.save(update_fields=['organization'])
-        assert admins[0] not in jt.read_role
-        assert admins[1] in jt.read_role

awx/main/tests/functional/test_rbac_migration.py

@@ -1,9 +1,7 @@
 import pytest
 
-from django.apps import apps
-
 from awx.main.migrations import _rbac as rbac
-from awx.main.models import UnifiedJobTemplate, InventorySource, Inventory, JobTemplate, Project, Organization, User
+from awx.main.models import UnifiedJobTemplate, InventorySource, Inventory, JobTemplate, Project, Organization
 
 
 @pytest.mark.django_db
@@ -49,27 +47,3 @@ def test_implied_organization_subquery_job_template():
                 assert jt.test_field is None
             else:
                 assert jt.test_field == jt.project.organization_id
-
-
-@pytest.mark.django_db
-def test_give_explicit_inventory_permission():
-    dual_admin = User.objects.create(username='alice')
-    inv_admin = User.objects.create(username='bob')
-    inv_org = Organization.objects.create(name='inv-org')
-    proj_org = Organization.objects.create(name='proj-org')
-
-    inv_org.admin_role.members.add(inv_admin, dual_admin)
-    proj_org.admin_role.members.add(dual_admin)
-
-    proj = Project.objects.create(name="test-proj", organization=proj_org)
-    inv = Inventory.objects.create(name='test-inv', organization=inv_org)
-
-    jt = JobTemplate.objects.create(name='foo', project=proj, inventory=inv)
-
-    assert dual_admin in jt.admin_role
-
-    rbac.restore_inventory_admins(apps, None)
-
-    assert inv_admin in jt.admin_role.members.all()
-    assert dual_admin not in jt.admin_role.members.all()
-    assert dual_admin in jt.admin_role

awx/main/tests/functional/test_rbac_team.py

@@ -92,7 +92,7 @@ def test_team_accessible_by(team, user, project):
     u = user('team_member', False)
 
     team.member_role.children.add(project.use_role)
-    assert list(Project.accessible_objects(team.member_role, 'read_role')) == [project]
+    assert list(Project.accessible_objects(team, 'read_role')) == [project]
     assert u not in project.read_role
 
     team.member_role.members.add(u)
@@ -104,7 +104,7 @@ def test_team_accessible_objects(team, user, project):
     u = user('team_member', False)
 
     team.member_role.children.add(project.use_role)
-    assert len(Project.accessible_objects(team.member_role, 'read_role')) == 1
+    assert len(Project.accessible_objects(team, 'read_role')) == 1
     assert not Project.accessible_objects(u, 'read_role')
 
     team.member_role.members.add(u)

awx/main/tests/functional/test_rbac_user.py

@@ -4,7 +4,7 @@ from unittest import mock
 from django.test import TransactionTestCase
 
 from awx.main.access import UserAccess, RoleAccess, TeamAccess
-from awx.main.models import User, Organization, Inventory, Role
+from awx.main.models import User, Organization, Inventory, get_system_auditor_role
 
 
 class TestSysAuditorTransactional(TransactionTestCase):
@@ -18,7 +18,8 @@ class TestSysAuditorTransactional(TransactionTestCase):
 
     def test_auditor_caching(self):
         rando = self.rando()
-        with self.assertNumQueries(1):
+        get_system_auditor_role()  # pre-create role, normally done by migrations
+        with self.assertNumQueries(2):
             v = rando.is_system_auditor
         assert not v
         with self.assertNumQueries(0):
@@ -153,34 +154,3 @@ def test_org_admin_cannot_delete_member_attached_to_other_group(org_admin, org_m
     access = UserAccess(org_admin)
     other_org.member_role.members.add(org_member)
     assert not access.can_delete(org_member)
-
-
-@pytest.mark.parametrize('reverse', (True, False))
-@pytest.mark.django_db
-def test_consistency_of_is_superuser_flag(reverse):
-    users = [User.objects.create(username='rando_{}'.format(i)) for i in range(2)]
-    for u in users:
-        assert u.is_superuser is False
-
-    system_admin = Role.singleton('system_administrator')
-    if reverse:
-        for u in users:
-            u.roles.add(system_admin)
-    else:
-        system_admin.members.add(*[u.id for u in users])  # like .add(42, 54)
-
-    for u in users:
-        u.refresh_from_db()
-        assert u.is_superuser is True
-
-    users[0].roles.clear()
-    for u in users:
-        u.refresh_from_db()
-    assert users[0].is_superuser is False
-    assert users[1].is_superuser is True
-
-    system_admin.members.clear()
-
-    for u in users:
-        u.refresh_from_db()
-        assert u.is_superuser is False

awx/main/tests/functional/test_routing.py

@@ -0,0 +1,90 @@
+import pytest
+
+from django.contrib.auth.models import AnonymousUser
+
+from channels.routing import ProtocolTypeRouter
+from channels.testing.websocket import WebsocketCommunicator
+
+
+from awx.main.consumers import WebsocketSecretAuthHelper
+
+
+@pytest.fixture
+def application():
+    # code in routing hits the db on import because .. settings cache
+    from awx.main.routing import application_func
+
+    yield application_func(ProtocolTypeRouter)
+
+
+@pytest.fixture
+def websocket_server_generator(application):
+    def fn(endpoint):
+        return WebsocketCommunicator(application, endpoint)
+
+    return fn
+
+
+@pytest.mark.asyncio
+@pytest.mark.django_db
+class TestWebsocketRelay:
+    @pytest.fixture
+    def websocket_relay_secret_generator(self, settings):
+        def fn(secret, set_broadcast_websocket_secret=False):
+            secret_backup = settings.BROADCAST_WEBSOCKET_SECRET
+            settings.BROADCAST_WEBSOCKET_SECRET = 'foobar'
+            res = ('secret'.encode('utf-8'), WebsocketSecretAuthHelper.construct_secret().encode('utf-8'))
+            if set_broadcast_websocket_secret is False:
+                settings.BROADCAST_WEBSOCKET_SECRET = secret_backup
+            return res
+
+        return fn
+
+    @pytest.fixture
+    def websocket_relay_secret(self, settings, websocket_relay_secret_generator):
+        return websocket_relay_secret_generator('foobar', set_broadcast_websocket_secret=True)
+
+    async def test_authorized(self, websocket_server_generator, websocket_relay_secret):
+        server = websocket_server_generator('/websocket/relay/')
+
+        server.scope['headers'] = (websocket_relay_secret,)
+        connected, _ = await server.connect()
+        assert connected is True
+
+    async def test_not_authorized(self, websocket_server_generator):
+        server = websocket_server_generator('/websocket/relay/')
+        connected, _ = await server.connect()
+        assert connected is False, "Connection to the relay websocket without auth. We expected the client to be denied."
+
+    async def test_wrong_secret(self, websocket_server_generator, websocket_relay_secret_generator):
+        server = websocket_server_generator('/websocket/relay/')
+
+        server.scope['headers'] = (websocket_relay_secret_generator('foobar', set_broadcast_websocket_secret=False),)
+        connected, _ = await server.connect()
+        assert connected is False
+
+
+@pytest.mark.asyncio
+@pytest.mark.django_db
+class TestWebsocketEventConsumer:
+    async def test_unauthorized_anonymous(self, websocket_server_generator):
+        server = websocket_server_generator('/websocket/')
+
+        server.scope['user'] = AnonymousUser()
+        connected, _ = await server.connect()
+        assert connected is False, "Anonymous user should NOT be allowed to login."
+
+    @pytest.mark.skip(reason="Ran out of coding time.")
+    async def test_authorized(self, websocket_server_generator, application, admin):
+        server = websocket_server_generator('/websocket/')
+
+        """
+        I ran out of time. Here is what I was thinking ...
+        Inject a valid session into the cookies in the header
+
+        server.scope['headers'] = (
+            (b'cookie', ...),
+        )
+        """
+        connected, _ = await server.connect()
+        assert connected is True, "User should be allowed in via cookies auth via a session key in the cookies"

awx/main/tests/functional/test_teams.py

@@ -1,14 +0,0 @@
-import pytest
-
-
-@pytest.mark.django_db()
-def test_admin_not_member(team):
-    """Test to ensure we don't add admin_role as a parent to team.member_role, as
-    this creates a cycle with organization administration, which we've decided
-    to remove support for
-
-    (2016-06-16) I think this might have been resolved. I'm asserting
-    this to be true in the mean time.
-    """
-
-    assert team.admin_role.is_ancestor_of(team.member_role) is True

awx/main/tests/settings_for_test.py

@@ -1,11 +1,6 @@
 # Python
-from unittest import mock
 import uuid
 
-# patch python-ldap
-with mock.patch('__main__.__builtins__.dir', return_value=[]):
-    import ldap  # NOQA
-
 # Load development settings for base variables.
 from awx.settings.development import *  # NOQA
 

awx/main/tests/unit/commands/test_dump_auth_config.py

@@ -0,0 +1,132 @@
+from io import StringIO
+import json
+from django.core.management import call_command
+from django.test import TestCase, override_settings
+
+
+settings_dict = {
+    "SOCIAL_AUTH_SAML_SP_ENTITY_ID": "SP_ENTITY_ID",
+    "SOCIAL_AUTH_SAML_SP_PUBLIC_CERT": "SP_PUBLIC_CERT",
+    "SOCIAL_AUTH_SAML_SP_PRIVATE_KEY": "SP_PRIVATE_KEY",
+    "SOCIAL_AUTH_SAML_ORG_INFO": "ORG_INFO",
+    "SOCIAL_AUTH_SAML_TECHNICAL_CONTACT": "TECHNICAL_CONTACT",
+    "SOCIAL_AUTH_SAML_SUPPORT_CONTACT": "SUPPORT_CONTACT",
+    "SOCIAL_AUTH_SAML_SP_EXTRA": "SP_EXTRA",
+    "SOCIAL_AUTH_SAML_SECURITY_CONFIG": "SECURITY_CONFIG",
+    "SOCIAL_AUTH_SAML_EXTRA_DATA": "EXTRA_DATA",
+    "SOCIAL_AUTH_SAML_ENABLED_IDPS": {
+        "Keycloak": {
+            "attr_last_name": "last_name",
+            "attr_groups": "groups",
+            "attr_email": "email",
+            "attr_user_permanent_id": "name_id",
+            "attr_username": "username",
+            "entity_id": "https://example.com/auth/realms/awx",
+            "url": "https://example.com/auth/realms/awx/protocol/saml",
+            "x509cert": "-----BEGIN CERTIFICATE-----\nMIIDDjCCAfYCCQCPBeVvpo8+VzANBgkqhkiG9w0BAQsFADBJMQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCTkMxDzANBgNVBAcMBkR1cmhhbTEMMAoGA1UECgwDYXd4MQ4w\nDAYDVQQDDAVsb2NhbDAeFw0yNDAxMTgxNDA4MzFaFw0yNTAxMTcxNDA4MzFaMEkx\nCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJOQzEPMA0GA1UEBwwGRHVyaGFtMQwwCgYD\nVQQKDANhd3gxDjAMBgNVBAMMBWxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\nMIIBCgKCAQEAzouj93oyFXsHEABdPESh3CYpp5QJJBM4TLYIIolk6PFOFIVwBuFY\nfExi5w7Hh4A42lPM6RkrT+u3h7LV39H9MRUfqygOSmaxICTOI0sU9ROHc44fWWzN\n756OP4B5zSiqG82q8X7nYVkcID+2F/3ekPLMOlWn53OrcdfKKDIcqavoTkQJefc2\nggXU3WgVCxGki/qCm+e5cZ1Cpl/ykSLOT8dWMEzDd12kin66zJ3KYz9F2Q5kQTh4\nKRAChnBBoEqzOfENHEAaHALiXOlVSy61VcLbtvskRMMwBtsydlnd9n/HGnktgrid\n3Ca0z5wBTHWjAOBvCKxKJuDa+jmyHEnpcQIDAQABMA0GCSqGSIb3DQEBCwUAA4IB\nAQBXvmyPWgXhC26cHYJBgQqj57dZ+n7p00kM1J+27oDMjGmbmX+XIKXLWazw/rG3\ngDjw9MXI2tVCrQMX0ohjphaULXhb/VBUPDOiW+k7C6AB3nZySFRflcR3cM4f83zF\nMoBd0549h5Red4p72FeOKNJRTN8YO4ooH9YNh5g0FQkgqn7fV9w2CNlomeKIW9zP\nm8tjFw0cJUk2wEYBVl8O7ko5rgNlzhkLoZkMvJhKa99AQJA6MAdyoLl1lv56Kq4X\njk+mMEiz9SaInp+ILQ1uQxZEwuC7DoGRW76rV4Fnie6+DLft4WKZfX1497mx8NV3\noR0abutJaKnCj07dwRu4/EsK\n-----END CERTIFICATE-----",
+            "attr_first_name": "first_name",
+        }
+    },
+    "SOCIAL_AUTH_SAML_CALLBACK_URL": "CALLBACK_URL",
+    "AUTH_LDAP_1_SERVER_URI": "SERVER_URI",
+    "AUTH_LDAP_1_BIND_DN": "BIND_DN",
+    "AUTH_LDAP_1_BIND_PASSWORD": "BIND_PASSWORD",
+    "AUTH_LDAP_1_GROUP_SEARCH": ["GROUP_SEARCH"],
+    "AUTH_LDAP_1_GROUP_TYPE": "string object",
+    "AUTH_LDAP_1_GROUP_TYPE_PARAMS": {"member_attr": "member", "name_attr": "cn"},
+    "AUTH_LDAP_1_USER_DN_TEMPLATE": "USER_DN_TEMPLATE",
+    "AUTH_LDAP_1_USER_SEARCH": ["USER_SEARCH"],
+    "AUTH_LDAP_1_USER_ATTR_MAP": {
+        "email": "email",
+        "last_name": "last_name",
+        "first_name": "first_name",
+    },
+    "AUTH_LDAP_1_CONNECTION_OPTIONS": {},
+    "AUTH_LDAP_1_START_TLS": None,
+}
+
+
+@override_settings(**settings_dict)
+class TestDumpAuthConfigCommand(TestCase):
+    def setUp(self):
+        super().setUp()
+        self.expected_config = [
+            {
+                "type": "ansible_base.authentication.authenticator_plugins.saml",
+                "name": "Keycloak",
+                "enabled": True,
+                "create_objects": True,
+                "users_unique": False,
+                "remove_users": True,
+                "configuration": {
+                    "SP_ENTITY_ID": "SP_ENTITY_ID",
+                    "SP_PUBLIC_CERT": "SP_PUBLIC_CERT",
+                    "SP_PRIVATE_KEY": "SP_PRIVATE_KEY",
+                    "ORG_INFO": "ORG_INFO",
+                    "TECHNICAL_CONTACT": "TECHNICAL_CONTACT",
+                    "SUPPORT_CONTACT": "SUPPORT_CONTACT",
+                    "SP_EXTRA": "SP_EXTRA",
+                    "SECURITY_CONFIG": "SECURITY_CONFIG",
+                    "EXTRA_DATA": "EXTRA_DATA",
+                    "ENABLED_IDPS": {
+                        "Keycloak": {
+                            "attr_last_name": "last_name",
+                            "attr_groups": "groups",
+                            "attr_email": "email",
+                            "attr_user_permanent_id": "name_id",
+                            "attr_username": "username",
+                            "entity_id": "https://example.com/auth/realms/awx",
+                            "url": "https://example.com/auth/realms/awx/protocol/saml",
+                            "x509cert": "-----BEGIN CERTIFICATE-----\nMIIDDjCCAfYCCQCPBeVvpo8+VzANBgkqhkiG9w0BAQsFADBJMQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCTkMxDzANBgNVBAcMBkR1cmhhbTEMMAoGA1UECgwDYXd4MQ4w\nDAYDVQQDDAVsb2NhbDAeFw0yNDAxMTgxNDA4MzFaFw0yNTAxMTcxNDA4MzFaMEkx\nCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJOQzEPMA0GA1UEBwwGRHVyaGFtMQwwCgYD\nVQQKDANhd3gxDjAMBgNVBAMMBWxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\nMIIBCgKCAQEAzouj93oyFXsHEABdPESh3CYpp5QJJBM4TLYIIolk6PFOFIVwBuFY\nfExi5w7Hh4A42lPM6RkrT+u3h7LV39H9MRUfqygOSmaxICTOI0sU9ROHc44fWWzN\n756OP4B5zSiqG82q8X7nYVkcID+2F/3ekPLMOlWn53OrcdfKKDIcqavoTkQJefc2\nggXU3WgVCxGki/qCm+e5cZ1Cpl/ykSLOT8dWMEzDd12kin66zJ3KYz9F2Q5kQTh4\nKRAChnBBoEqzOfENHEAaHALiXOlVSy61VcLbtvskRMMwBtsydlnd9n/HGnktgrid\n3Ca0z5wBTHWjAOBvCKxKJuDa+jmyHEnpcQIDAQABMA0GCSqGSIb3DQEBCwUAA4IB\nAQBXvmyPWgXhC26cHYJBgQqj57dZ+n7p00kM1J+27oDMjGmbmX+XIKXLWazw/rG3\ngDjw9MXI2tVCrQMX0ohjphaULXhb/VBUPDOiW+k7C6AB3nZySFRflcR3cM4f83zF\nMoBd0549h5Red4p72FeOKNJRTN8YO4ooH9YNh5g0FQkgqn7fV9w2CNlomeKIW9zP\nm8tjFw0cJUk2wEYBVl8O7ko5rgNlzhkLoZkMvJhKa99AQJA6MAdyoLl1lv56Kq4X\njk+mMEiz9SaInp+ILQ1uQxZEwuC7DoGRW76rV4Fnie6+DLft4WKZfX1497mx8NV3\noR0abutJaKnCj07dwRu4/EsK\n-----END CERTIFICATE-----",
+                            "attr_first_name": "first_name",
+                        }
+                    },
+                    "CALLBACK_URL": "CALLBACK_URL",
+                    "IDP_URL": "https://example.com/auth/realms/awx/protocol/saml",
+                    "IDP_X509_CERT": "-----BEGIN CERTIFICATE-----\nMIIDDjCCAfYCCQCPBeVvpo8+VzANBgkqhkiG9w0BAQsFADBJMQswCQYDVQQGEwJV\nUzELMAkGA1UECAwCTkMxDzANBgNVBAcMBkR1cmhhbTEMMAoGA1UECgwDYXd4MQ4w\nDAYDVQQDDAVsb2NhbDAeFw0yNDAxMTgxNDA4MzFaFw0yNTAxMTcxNDA4MzFaMEkx\nCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJOQzEPMA0GA1UEBwwGRHVyaGFtMQwwCgYD\nVQQKDANhd3gxDjAMBgNVBAMMBWxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A\nMIIBCgKCAQEAzouj93oyFXsHEABdPESh3CYpp5QJJBM4TLYIIolk6PFOFIVwBuFY\nfExi5w7Hh4A42lPM6RkrT+u3h7LV39H9MRUfqygOSmaxICTOI0sU9ROHc44fWWzN\n756OP4B5zSiqG82q8X7nYVkcID+2F/3ekPLMOlWn53OrcdfKKDIcqavoTkQJefc2\nggXU3WgVCxGki/qCm+e5cZ1Cpl/ykSLOT8dWMEzDd12kin66zJ3KYz9F2Q5kQTh4\nKRAChnBBoEqzOfENHEAaHALiXOlVSy61VcLbtvskRMMwBtsydlnd9n/HGnktgrid\n3Ca0z5wBTHWjAOBvCKxKJuDa+jmyHEnpcQIDAQABMA0GCSqGSIb3DQEBCwUAA4IB\nAQBXvmyPWgXhC26cHYJBgQqj57dZ+n7p00kM1J+27oDMjGmbmX+XIKXLWazw/rG3\ngDjw9MXI2tVCrQMX0ohjphaULXhb/VBUPDOiW+k7C6AB3nZySFRflcR3cM4f83zF\nMoBd0549h5Red4p72FeOKNJRTN8YO4ooH9YNh5g0FQkgqn7fV9w2CNlomeKIW9zP\nm8tjFw0cJUk2wEYBVl8O7ko5rgNlzhkLoZkMvJhKa99AQJA6MAdyoLl1lv56Kq4X\njk+mMEiz9SaInp+ILQ1uQxZEwuC7DoGRW76rV4Fnie6+DLft4WKZfX1497mx8NV3\noR0abutJaKnCj07dwRu4/EsK\n-----END CERTIFICATE-----",
+                    "IDP_ENTITY_ID": "https://example.com/auth/realms/awx",
+                    "IDP_ATTR_EMAIL": "email",
+                    "IDP_GROUPS": "groups",
+                    "IDP_ATTR_USERNAME": "username",
+                    "IDP_ATTR_LAST_NAME": "last_name",
+                    "IDP_ATTR_FIRST_NAME": "first_name",
+                    "IDP_ATTR_USER_PERMANENT_ID": "name_id",
+                },
+            },
+            {
+                "type": "ansible_base.authentication.authenticator_plugins.ldap",
+                "name": "LDAP_1",
+                "enabled": True,
+                "create_objects": True,
+                "users_unique": False,
+                "remove_users": True,
+                "configuration": {
+                    "SERVER_URI": ["SERVER_URI"],
+                    "BIND_DN": "BIND_DN",
+                    "BIND_PASSWORD": "BIND_PASSWORD",
+                    "CONNECTION_OPTIONS": {},
+                    "GROUP_TYPE": "str",
+                    "GROUP_TYPE_PARAMS": {"member_attr": "member", "name_attr": "cn"},
+                    "GROUP_SEARCH": ["GROUP_SEARCH"],
+                    "START_TLS": None,
+                    "USER_DN_TEMPLATE": "USER_DN_TEMPLATE",
+                    "USER_ATTR_MAP": {"email": "email", "last_name": "last_name", "first_name": "first_name"},
+                    "USER_SEARCH": ["USER_SEARCH"],
+                },
+            },
+        ]
+
+    def test_json_returned_from_cmd(self):
+        output = StringIO()
+        call_command("dump_auth_config", stdout=output)
+        cmmd_output = json.loads(output.getvalue())
+
+        # check configured SAML return
+        assert cmmd_output[0] == self.expected_config[0]
+
+        # check configured LDAP return
+        assert cmmd_output[2] == self.expected_config[1]
+
+        # check unconfigured LDAP return
+        assert "LDAP_0_missing_fields" in cmmd_output[1]
+        assert cmmd_output[1]["LDAP_0_missing_fields"] == ['SERVER_URI', 'GROUP_TYPE', 'GROUP_TYPE_PARAMS', 'USER_DN_TEMPLATE', 'USER_ATTR_MAP']

awx/main/tests/unit/tasks/test_system.py

@@ -0,0 +1,64 @@
+import pytest
+from unittest.mock import MagicMock, patch
+from awx.main.tasks.system import update_inventory_computed_fields
+from awx.main.models import Inventory
+from django.db import DatabaseError
+
+
+@pytest.fixture
+def mock_logger():
+    with patch("awx.main.tasks.system.logger") as logger:
+        yield logger
+
+
+@pytest.fixture
+def mock_inventory():
+    return MagicMock(spec=Inventory)
+
+
+def test_update_inventory_computed_fields_existing_inventory(mock_logger, mock_inventory):
+    # Mocking the Inventory.objects.filter method to return a non-empty queryset
+    with patch("awx.main.tasks.system.Inventory.objects.filter") as mock_filter:
+        mock_filter.return_value.exists.return_value = True
+        mock_filter.return_value.__getitem__.return_value = mock_inventory
+
+        # Mocking the update_computed_fields method
+        with patch.object(mock_inventory, "update_computed_fields") as mock_update_computed_fields:
+            update_inventory_computed_fields(1)
+
+            # Assertions
+            mock_filter.assert_called_once_with(id=1)
+            mock_update_computed_fields.assert_called_once()
+
+            # You can add more assertions based on your specific requirements
+
+
+def test_update_inventory_computed_fields_missing_inventory(mock_logger):
+    # Mocking the Inventory.objects.filter method to return an empty queryset
+    with patch("awx.main.tasks.system.Inventory.objects.filter") as mock_filter:
+        mock_filter.return_value.exists.return_value = False
+
+        update_inventory_computed_fields(1)
+
+        # Assertions
+        mock_filter.assert_called_once_with(id=1)
+        mock_logger.error.assert_called_once_with("Update Inventory Computed Fields failed due to missing inventory: 1")
+
+
+def test_update_inventory_computed_fields_database_error_nosqlstate(mock_logger, mock_inventory):
+    # Mocking the Inventory.objects.filter method to return a non-empty queryset
+    with patch("awx.main.tasks.system.Inventory.objects.filter") as mock_filter:
+        mock_filter.return_value.exists.return_value = True
+        mock_filter.return_value.__getitem__.return_value = mock_inventory
+
+        # Mocking the update_computed_fields method
+        with patch.object(mock_inventory, "update_computed_fields") as mock_update_computed_fields:
+            # Simulating the update_computed_fields method to explicitly raise a DatabaseError
+            mock_update_computed_fields.side_effect = DatabaseError("Some error")
+
+            update_inventory_computed_fields(1)
+
+            # Assertions
+            mock_filter.assert_called_once_with(id=1)
+            mock_update_computed_fields.assert_called_once()
+            mock_inventory.update_computed_fields.assert_called_once()

awx/main/tests/unit/test_tasks.py

@@ -1106,6 +1106,44 @@ class TestJobCredentials(TestJobExecution):
         config = open(local_path, 'r').read()
         assert config == hcl_config
 
+    def test_terraform_gcs_backend_credentials(self, job, private_data_dir, mock_me):
+        terraform = CredentialType.defaults['terraform']()
+        hcl_config = '''
+        backend "gcs" {
+            bucket = "gce_storage"
+        }
+        '''
+        gce_backend_credentials = '''
+        {
+            "type": "service_account",
+            "project_id": "sample",
+            "private_key_id": "eeeeeeeeeeeeeeeeeeeeeeeeeee",
+            "private_key": "-----BEGIN PRIVATE KEY-----\naaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n-----END PRIVATE KEY-----\n",
+            "client_email": "sample@sample.iam.gserviceaccount.com",
+            "client_id": "0123456789",
+            "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+            "token_uri": "https://oauth2.googleapis.com/token",
+            "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+            "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/cloud-content-robot%40sample.iam.gserviceaccount.com",
+        }
+        '''
+        credential = Credential(pk=1, credential_type=terraform, inputs={'configuration': hcl_config, 'gce_credentials': gce_backend_credentials})
+        credential.inputs['configuration'] = encrypt_field(credential, 'configuration')
+        credential.inputs['gce_credentials'] = encrypt_field(credential, 'gce_credentials')
+        job.credentials.add(credential)
+
+        env = {}
+        safe_env = {}
+        credential.credential_type.inject_credential(credential, env, safe_env, [], private_data_dir)
+
+        local_path = to_host_path(env['TF_BACKEND_CONFIG_FILE'], private_data_dir)
+        config = open(local_path, 'r').read()
+        assert config == hcl_config
+
+        credentials_path = to_host_path(env['GOOGLE_BACKEND_CREDENTIALS'], private_data_dir)
+        credentials = open(credentials_path, 'r').read()
+        assert credentials == gce_backend_credentials
+
     def test_custom_environment_injectors_with_jinja_syntax_error(self, private_data_dir, mock_me):
         some_cloud = CredentialType(
             kind='cloud',

awx/main/tests/unit/utils/test_common.py

@@ -121,6 +121,10 @@ def test_get_model_for_valid_type(model_type, model_class):
     assert common.get_model_for_type(model_type) == model_class
 
 
+def test_is_testing():
+    assert common.is_testing() is True
+
+
 @pytest.mark.parametrize("model_type,model_class", [(name, cls) for cls, name in TEST_MODELS])
 def test_get_capacity_type(model_type, model_class):
     if model_type in ('job', 'ad_hoc_command', 'inventory_update', 'job_template'):

awx/main/tests/unit/utils/test_filters.py

@@ -69,7 +69,7 @@ class mockHost:
 @mock.patch('awx.main.utils.filters.get_model', return_value=mockHost())
 class TestSmartFilterQueryFromString:
     @mock.patch(
-        'ansible_base.rest_filters.rest_framework.field_lookup_backend.get_fields_from_path', lambda model, path: ([model], path)
+        'ansible_base.rest_filters.rest_framework.field_lookup_backend.get_fields_from_path', lambda model, path, **kwargs: ([model], path)
     )  # disable field filtering, because a__b isn't a real Host field
     @pytest.mark.parametrize(
         "filter_string,q_expected",

awx/main/tests/unit/utils/test_receptor.py

@@ -3,7 +3,7 @@ from awx.main.tasks.receptor import _convert_args_to_cli
 
 def test_file_cleanup_scenario():
     args = _convert_args_to_cli({'exclude_strings': ['awx_423_', 'awx_582_'], 'file_pattern': '/tmp/awx_*_*'})
-    assert ' '.join(args) == 'cleanup --exclude-strings=awx_423_ awx_582_ --file-pattern=/tmp/awx_*_*'
+    assert ' '.join(args) == 'cleanup --exclude-strings="awx_423_ awx_582_" --file-pattern=/tmp/awx_*_*'
 
 
 def test_image_cleanup_scenario():
@@ -17,5 +17,5 @@ def test_image_cleanup_scenario():
         }
     )
     assert (
-        ' '.join(args) == 'cleanup --remove-images=quay.invalid/foo/bar:latest quay.invalid/foo/bar:devel --image-prune --process-isolation-executable=podman'
+        ' '.join(args) == 'cleanup --remove-images="quay.invalid/foo/bar:latest quay.invalid/foo/bar:devel" --image-prune --process-isolation-executable=podman'
     )

awx/main/utils/common.py

@@ -7,6 +7,7 @@ import json
 import yaml
 import logging
 import time
+import psycopg
 import os
 import subprocess
 import re
@@ -23,7 +24,7 @@ from django.core.exceptions import ObjectDoesNotExist, FieldDoesNotExist
 from django.utils.dateparse import parse_datetime
 from django.utils.translation import gettext_lazy as _
 from django.utils.functional import cached_property
-from django.db import connection, transaction, ProgrammingError, IntegrityError
+from django.db import connection, DatabaseError, transaction, ProgrammingError, IntegrityError
 from django.db.models.fields.related import ForeignObjectRel, ManyToManyField
 from django.db.models.fields.related_descriptors import ForwardManyToOneDescriptor, ManyToManyDescriptor
 from django.db.models.query import QuerySet
@@ -136,7 +137,7 @@ def underscore_to_camelcase(s):
 @functools.cache
 def is_testing(argv=None):
     '''Return True if running django or py.test unit tests.'''
-    if 'PYTEST_CURRENT_TEST' in os.environ.keys():
+    if os.environ.get('DJANGO_SETTINGS_MODULE') == 'awx.main.tests.settings_for_test':
         return True
     argv = sys.argv if argv is None else argv
     if len(argv) >= 1 and ('py.test' in argv[0] or 'py/test.py' in argv[0]):
@@ -1155,11 +1156,25 @@ def create_partition(tblname, start=None):
                     f'ALTER TABLE {tblname} ATTACH PARTITION {tblname}_{partition_label} '
                     f'FOR VALUES FROM (\'{start_timestamp}\') TO (\'{end_timestamp}\');'
                 )
+
     except (ProgrammingError, IntegrityError) as e:
-        if 'already exists' in str(e):
-            logger.info(f'Caught known error due to partition creation race: {e}')
-        else:
-            raise
+        cause = e.__cause__
+        if cause and hasattr(cause, 'sqlstate'):
+            sqlstate = cause.sqlstate
+            sqlstate_cls = psycopg.errors.lookup(sqlstate)
+
+            if psycopg.errors.DuplicateTable == sqlstate_cls or psycopg.errors.UniqueViolation == sqlstate_cls:
+                logger.info(f'Caught known error due to partition creation race: {e}')
+            else:
+                logger.error('SQL Error state: {} - {}'.format(sqlstate, sqlstate_cls))
+                raise
+    except DatabaseError as e:
+        cause = e.__cause__
+        if cause and hasattr(cause, 'sqlstate'):
+            sqlstate = cause.sqlstate
+            sqlstate_str = psycopg.errors.lookup(sqlstate)
+            logger.error('SQL Error state: {} - {}'.format(sqlstate, sqlstate_str))
+        raise
 
 
 def cleanup_new_process(func):

awx/main/utils/formatters.py

@@ -3,7 +3,6 @@
 
 from copy import copy
 import json
-import json_log_formatter
 import logging
 import traceback
 import socket
@@ -15,15 +14,6 @@ from django.core.serializers.json import DjangoJSONEncoder
 from django.conf import settings
 
 
-class JobLifeCycleFormatter(json_log_formatter.JSONFormatter):
-    def json_record(self, message: str, extra: dict, record: logging.LogRecord):
-        if 'time' not in extra:
-            extra['time'] = now()
-        if record.exc_info:
-            extra['exc_info'] = self.formatException(record.exc_info)
-        return extra
-
-
 class TimeFormatter(logging.Formatter):
     """
     Custom log formatter used for inventory imports