Fix Redis broken pipe error in queue.py with reconnection logic and enhanced diagnostics for long-running jobs

Update code to pull subscriptions from
console.redhat.com instead of
subscription.rhsm.redhat.com

Uses service account client ID and client secret
instead of username/password, which is being
deprecated in July 2025.

Additional changes:

- In awx.awx.subscriptions module, use new service
account params rather than old basic auth params

- Update awx.awx.license module to use subscription_id
instead of pool_id. This is due to using a different API,
which identifies unique subscriptions by subscriptionID
instead of pool ID.

Signed-off-by: Seth Foster <fosterbseth@gmail.com>
Co-authored-by: Chris Meyers <chris.meyers.fsu@gmail.com>
Co-authored-by: Peter Braun <pbraun@redhat.com>

.codecov.yml

@@ -0,0 +1,57 @@
+---
+
+codecov:
+  notify:
+    after_n_builds: 6  # Number of test matrix+lint jobs uploading coverage
+    wait_for_ci: false
+
+  require_ci_to_pass: false
+
+  token: >-  # repo-scoped, upload-only, needed for stability in PRs from forks
+    2b8c7a7a-7293-4a00-bf02-19bd55a1389b
+
+comment:
+  require_changes: true
+
+coverage:
+  range: 100..100
+  status:
+    patch:
+      default:
+        target: 100%
+      pytest:
+        target: 100%
+        flags:
+          - pytest
+      typing:
+        flags:
+          - MyPy
+    project:
+      default:
+        target: 75%
+      lib:
+        flags:
+          - pytest
+        paths:
+          - awx/
+        target: 75%
+      tests:
+        flags:
+          - pytest
+        paths:
+          - tests/
+          - >-
+            **/test/
+          - >-
+            **/tests/
+          - >-
+            **/test/**
+          - >-
+            **/tests/**
+        target: 95%
+      typing:
+        flags:
+          - MyPy
+        target: 100%
+
+...

.coveragerc

@@ -1,16 +1,6 @@
-[run]
-source = awx
-branch = True
-omit =
-    awx/main/migrations/*
-    awx/lib/site-packages/*
-
 [report]
 # Regexes for lines to exclude from consideration
-exclude_lines =
-    # Have to re-enable the standard pragma
-    pragma: no cover
-
+exclude_also =
     # Don't complain about missing debug-only code:
     def __repr__
     if self\.debug
@@ -23,7 +13,18 @@ exclude_lines =
     if 0:
     if __name__ == .__main__.:
 
-ignore_errors = True
+    ^\s*@pytest\.mark\.xfail
+
+[run]
+branch = True
+omit =
+    awx/main/migrations/*
+    awx/settings/defaults.py
+    awx/settings/*_defaults.py
+source =
+    .
+source_pkgs =
+    awx
 
 [xml]
 output = ./reports/coverage.xml

.github/actions/awx_devel_image/action.yml

@@ -0,0 +1,44 @@
+name: Setup images for AWX
+description: Builds new awx_devel image
+inputs:
+  github-token:
+    description: GitHub Token for registry access
+    required: true
+  private-github-key:
+    description: GitHub private key for private repositories
+    required: false
+    default: ''
+runs:
+  using: composite
+  steps:
+    - uses: ./.github/actions/setup-python
+
+    - name: Set lower case owner name
+      shell: bash
+      run: echo "OWNER_LC=${OWNER,,}" >> $GITHUB_ENV
+      env:
+        OWNER: '${{ github.repository_owner }}'
+
+    - name: Log in to registry
+      shell: bash
+      run: |
+        echo "${{ inputs.github-token }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+    - uses: ./.github/actions/setup-ssh-agent
+      with:
+        ssh-private-key: ${{ inputs.private-github-key }}
+
+    - name: Pre-pull latest devel image to warm cache
+      shell: bash
+      run: |
+        DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} \
+        COMPOSE_TAG=${{ github.base_ref || github.ref_name }} \
+        docker pull -q `make print-DEVEL_IMAGE_NAME`
+      continue-on-error: true
+
+    - name: Build image for current source checkout
+      shell: bash
+      run: |
+        DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} \
+        COMPOSE_TAG=${{ github.base_ref || github.ref_name }} \
+        make docker-compose-build

.github/actions/run_awx_devel/action.yml

@@ -0,0 +1,77 @@
+name: Run AWX docker-compose
+description: Runs AWX with `make docker-compose`
+inputs:
+  github-token:
+    description: GitHub Token to pass to awx_devel_image
+    required: true
+  build-ui:
+    description: Should the UI be built?
+    required: false
+    default: false
+    type: boolean
+  private-github-key:
+    description: GitHub private key for private repositories
+    required: false
+    default: ''
+outputs:
+  ip:
+    description: The IP of the tools_awx_1 container
+    value: ${{ steps.data.outputs.ip }}
+runs:
+  using: composite
+  steps:
+    - name: Disable apparmor for rsyslogd, first step
+      shell: bash
+      run: sudo ln -s /etc/apparmor.d/usr.sbin.rsyslogd /etc/apparmor.d/disable/
+
+    - name: Disable apparmor for rsyslogd, second step
+      shell: bash
+      run: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.rsyslogd
+
+    - name: Build awx_devel image for running checks
+      uses: ./.github/actions/awx_devel_image
+      with:
+        github-token: ${{ inputs.github-token }}
+        private-github-key: ${{ inputs.private-github-key }}
+
+    - name: Upgrade ansible-core
+      shell: bash
+      run: python3 -m pip install --upgrade ansible-core
+
+    - name: Install system deps
+      shell: bash
+      run: sudo apt-get install -y gettext
+
+    - name: Start AWX
+      shell: bash
+      run: |
+        DEV_DOCKER_OWNER=${{ github.repository_owner }} \
+        COMPOSE_TAG=${{ github.base_ref || github.ref_name }} \
+        DJANGO_COLORS=nocolor \
+        SUPERVISOR_ARGS="-n -t" \
+        COMPOSE_UP_OPTS="-d --no-color" \
+        make docker-compose
+
+    - name: Update default AWX password
+      shell: bash
+      run: |
+        SECONDS=0
+        while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' -k https://localhost:8043/api/v2/ping/)" != "200" ]]; do
+          if [[ $SECONDS -gt 600 ]]; then
+            echo "Timing out, AWX never came up"
+            exit 1
+          fi
+          echo "Waiting for AWX..."
+          sleep 5
+        done
+        echo "AWX is up, updating the password..."
+        docker exec -i tools_awx_1 sh <<-EOSH
+          awx-manage update_password --username=admin --password=password
+        EOSH
+
+    - name: Get instance data
+      id: data
+      shell: bash
+      run: |
+        AWX_IP=$(docker inspect -f '{{.NetworkSettings.Networks.awx.IPAddress}}' tools_awx_1)
+        echo "ip=$AWX_IP" >> $GITHUB_OUTPUT

.github/actions/setup-python/action.yml

@@ -0,0 +1,27 @@
+name: 'Setup Python from Makefile'
+description: 'Extract and set up Python version from Makefile'
+inputs:
+  python-version:
+    description: 'Override Python version (optional)'
+    required: false
+    default: ''
+  working-directory:
+    description: 'Directory containing the Makefile'
+    required: false
+    default: '.'
+runs:
+  using: composite
+  steps:
+    - name: Get python version from Makefile
+      shell: bash
+      run: |
+        if [ -n "${{ inputs.python-version }}" ]; then
+          echo "py_version=${{ inputs.python-version }}" >> $GITHUB_ENV
+        else
+          cd ${{ inputs.working-directory }}
+          echo "py_version=`make PYTHON_VERSION`" >> $GITHUB_ENV
+        fi
+    - name: Install python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ env.py_version }}

.github/actions/setup-ssh-agent/action.yml

@@ -0,0 +1,29 @@
+name: 'Setup SSH for GitHub'
+description: 'Configure SSH for private repository access'
+inputs:
+  ssh-private-key:
+    description: 'SSH private key for repository access'
+    required: false
+    default: ''
+runs:
+  using: composite
+  steps:
+    - name: Generate placeholder SSH private key if SSH auth for private repos is not needed
+      id: generate_key
+      shell: bash
+      run: |
+        if [[ -z "${{ inputs.ssh-private-key }}" ]]; then
+          ssh-keygen -t ed25519 -C "github-actions" -N "" -f ~/.ssh/id_ed25519
+          echo "SSH_PRIVATE_KEY<<EOF" >> $GITHUB_OUTPUT
+          cat ~/.ssh/id_ed25519 >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+        else
+          echo "SSH_PRIVATE_KEY<<EOF" >> $GITHUB_OUTPUT
+          echo "${{ inputs.ssh-private-key }}" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Add private GitHub key to SSH agent
+      uses: webfactory/ssh-agent@v0.9.0
+      with:
+        ssh-private-key: ${{ steps.generate_key.outputs.SSH_PRIVATE_KEY }}

.github/actions/upload_awx_devel_logs/action.yml

@@ -0,0 +1,19 @@
+name: Upload logs
+description: Upload logs from `make docker-compose` devel environment to GitHub as an artifact
+inputs:
+  log-filename:
+    description: "*Unique* name of the log file"
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Get AWX logs
+      shell: bash
+      run: |
+        docker logs tools_awx_1 > ${{ inputs.log-filename }}
+
+    - name: Upload AWX logs as artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: docker-compose-logs-${{ inputs.log-filename }}
+        path: ${{ inputs.log-filename }}

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "docs/docsite/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 2
+    labels:
+      - "docs"
+      - "dependencies"

.github/issue_labeler.yml

@@ -6,8 +6,6 @@ needs_triage:
   - "Feature Summary"
 "component:ui":
   - "\\[X\\] UI"
-"component:ui_next":
-  - "\\[X\\] UI \\(tech preview\\)"
 "component:api":
   - "\\[X\\] API"
 "component:docs":

.github/pr_labeler.yml

@@ -1,8 +1,5 @@
 "component:api":
-  - any: ["awx/**/*", "!awx/ui/**"]
-
-"component:ui":
-  - any: ["awx/ui/**/*"]
+  - any: ["awx/**/*"]
 
 "component:docs":
   - any: ["docs/**/*"]
@@ -14,6 +11,4 @@
   - any: ["awx_collection/**/*"]
 
 "dependencies":
-  - any: ["awx/ui/package.json"]
-  - any: ["requirements/*.txt"]
-  - any: ["requirements/requirements.in"]
+  - any: ["requirements/*"]

.github/triage_replies.md

@@ -1,14 +1,13 @@
 ## General
-- For the roundup of all the different mailing lists available from AWX, Ansible, and beyond visit: https://docs.ansible.com/ansible/latest/community/communication.html
 - Hello, we think your question is answered in our FAQ. Does this: https://www.ansible.com/products/awx-project/faq cover your question?
-- You can find the latest documentation here: https://docs.ansible.com/automation-controller/latest/html/userguide/index.html
+- You can find the latest documentation here: https://ansible.readthedocs.io/projects/awx/en/latest/userguide/index.html
 
 
 
 ## PRs/Issues
 
-### Visit our mailing list
-- Hello, this appears to be less of a bug report or feature request and more of a question. Could you please ask this on our mailing list? See https://github.com/ansible/awx/#get-involved for information for ways to connect with us.
+### Visit the Forum or Matrix
+- Hello, this appears to be less of a bug report or feature request and more of a question. Could you please ask this on either the [Ansible AWX channel on Matrix](https://matrix.to/#/#awx:ansible.com) or the [Ansible Community Forum](https://forum.ansible.com/tag/awx)?
 
 ### Denied Submission
 
@@ -83,7 +82,7 @@ The Ansible Community is looking at building an EE that corresponds to all of th
 ## Mailing List Triage
 
 ### Create an issue
-- Hello, thanks for reaching out on list. We think this merits an issue on our Github, https://github.com/ansible/awx/issues. If you could open an issue up on Github it will get tagged and integrated into our planning and workflow. All future work will be tracked there. Issues should include as much information as possible, including screenshots, log outputs, or any reproducers.
+- Hello, thanks for reaching out on list. We think this merits an issue on our GitHub, https://github.com/ansible/awx/issues. If you could open an issue up on GitHub it will get tagged and integrated into our planning and workflow. All future work will be tracked there. Issues should include as much information as possible, including screenshots, log outputs, or any reproducers.
 
 ### Create a Pull Request
 - Hello, we think your idea is good! Please consider contributing a PR for this following our contributing guidelines: https://github.com/ansible/awx/blob/devel/CONTRIBUTING.md
@@ -93,8 +92,8 @@ The Ansible Community is looking at building an EE that corresponds to all of th
 - Hello, your issue seems related to receptor. Could you please open an issue in the receptor repository? https://github.com/ansible/receptor. Thanks!
 
 ### Ansible Engine not AWX
-- Hello, your question seems to be about Ansible development, not about AWX. Try asking on the Ansible-devel specific mailing list: https://groups.google.com/g/ansible-devel
-- Hello, your question seems to be about using Ansible, not about AWX. https://groups.google.com/g/ansible-project is the best place to visit for user questions about Ansible. Thanks!
+- Hello, your question seems to be about Ansible development, not about AWX. Try asking on in the Forum https://forum.ansible.com/tag/development
+- Hello, your question seems to be about using Ansible Core, not about AWX. https://forum.ansible.com/tag/ansible-core is the best place to visit for user questions about Ansible. Thanks!
 
 ### Ansible Galaxy not AWX
 - Hey there. That sounds like an FAQ question. Did this: https://www.ansible.com/products/awx-project/faq cover your question?
@@ -104,7 +103,7 @@ The Ansible Community is looking at building an EE that corresponds to all of th
 - AWX-Operator: https://github.com/ansible/awx-operator/blob/devel/CONTRIBUTING.md
 
 ### Oracle AWX
-We'd be happy to help if you can reproduce this with AWX since we do not have Oracle's Linux Automation Manager. If you need help with this specific version of Oracles Linux Automation Manager you will need to contact your Oracle for support. 
+We'd be happy to help if you can reproduce this with AWX since we do not have Oracle's Linux Automation Manager. If you need help with this specific version of Oracles Linux Automation Manager you will need to contact your Oracle for support.
 
 ### Community Resolved
 Hi,

.github/workflows/ci.yml

@@ -5,12 +5,17 @@ env:
   CI_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   DEV_DOCKER_OWNER: ${{ github.repository_owner }}
   COMPOSE_TAG: ${{ github.base_ref || 'devel' }}
+  UPSTREAM_REPOSITORY_ID: 91594105
 on:
   pull_request:
+  push:
+    branches:
+      - devel  # needed to publish code coverage post-merge
 jobs:
   common-tests:
     name: ${{ matrix.tests.name }}
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     permissions:
       packages: write
       contents: read
@@ -19,57 +24,157 @@ jobs:
       matrix:
         tests:
           - name: api-test
-            command: /start_tests.sh
+            command: /start_tests.sh test_coverage
+            coverage-upload-name: ""
+          - name: api-migrations
+            command: /start_tests.sh test_migrations
+            coverage-upload-name: ""
           - name: api-lint
             command: /var/lib/awx/venv/awx/bin/tox -e linters
+            coverage-upload-name: ""
           - name: api-swagger
             command: /start_tests.sh swagger
+            coverage-upload-name: ""
           - name: awx-collection
             command: /start_tests.sh test_collection_all
+            coverage-upload-name: "awx-collection"
           - name: api-schema
-            command: /start_tests.sh detect-schema-change SCHEMA_DIFF_BASE_BRANCH=${{ github.event.pull_request.base.ref }}
-          - name: ui-lint
-            command: make ui-lint
-          - name: ui-test-screens
-            command: make ui-test-screens
-          - name: ui-test-general
-            command: make ui-test-general
+            command: >-
+              /start_tests.sh detect-schema-change SCHEMA_DIFF_BASE_BRANCH=${{
+                github.event.pull_request.base.ref || github.ref_name
+              }}
+            coverage-upload-name: ""
+
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+
+      - name: Build awx_devel image for running checks
+        uses: ./.github/actions/awx_devel_image
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          private-github-key: ${{ secrets.PRIVATE_GITHUB_KEY }}
 
       - name: Run check ${{ matrix.tests.name }}
-        run: AWX_DOCKER_CMD='${{ matrix.tests.command }}' make github_ci_runner
+        id: make-run
+        run: >-
+          AWX_DOCKER_ARGS='-e GITHUB_ACTIONS -e GITHUB_OUTPUT -v "${GITHUB_OUTPUT}:${GITHUB_OUTPUT}:rw,Z"'
+          AWX_DOCKER_CMD='${{ matrix.tests.command }}'
+          make docker-runner
+
+      - name: Upload test coverage to Codecov
+        if: >-
+          !cancelled()
+          && steps.make-run.outputs.cov-report-files != ''
+        uses: codecov/codecov-action@v4
+        with:
+          fail_ci_if_error: >-
+            ${{
+              toJSON(env.UPSTREAM_REPOSITORY_ID == github.repository_id)
+            }}
+          files: >-
+            ${{ steps.make-run.outputs.cov-report-files }}
+          flags: >-
+            CI-GHA,
+            pytest,
+            OS-${{
+              runner.os
+            }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+      - name: Upload test results to Codecov
+        if: >-
+          !cancelled()
+          && steps.make-run.outputs.test-result-files != ''
+        uses: codecov/test-results-action@v1
+        with:
+          fail_ci_if_error: >-
+            ${{
+              toJSON(env.UPSTREAM_REPOSITORY_ID == github.repository_id)
+            }}
+          files: >-
+            ${{ steps.make-run.outputs.test-result-files }}
+          flags: >-
+            CI-GHA,
+            pytest,
+            OS-${{
+              runner.os
+            }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Upload awx jUnit test reports
+        if: >-
+          !cancelled()
+          && steps.make-run.outputs.test-result-files != ''
+          && github.event_name == 'push'
+          && env.UPSTREAM_REPOSITORY_ID == github.repository_id
+          && github.ref_name == github.event.repository.default_branch
+        run: |
+          for junit_file in $(echo '${{ steps.make-run.outputs.test-result-files }}' | sed 's/,/ /')
+          do
+              curl \
+                -v \
+                --user "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_USER }}:${{ secrets.PDE_ORG_RESULTS_UPLOAD_PASSWORD }}" \
+                --form "xunit_xml=@${junit_file}" \
+                --form "component_name=${{ matrix.tests.coverage-upload-name || 'awx' }}" \
+                --form "git_commit_sha=${{ github.sha }}" \
+                --form "git_repository_url=https://github.com/${{ github.repository }}" \
+                "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}/api/results/upload/"
+          done
 
   dev-env:
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
-      - name: Run smoke test
-        run: make github_ci_setup && ansible-playbook tools/docker-compose/ansible/smoke-test.yml -v
+      - uses: ./.github/actions/setup-python
+        with:
+          python-version: '3.x'
+
+      - uses: ./.github/actions/run_awx_devel
+        id: awx
+        with:
+          build-ui: false
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          private-github-key: ${{ secrets.PRIVATE_GITHUB_KEY }}
+
+      - name: Run live dev env tests
+        run: docker exec tools_awx_1 /bin/bash -c "make live_test"
+
+      - uses: ./.github/actions/upload_awx_devel_logs
+        if: always()
+        with:
+          log-filename: live-tests.log
 
   awx-operator:
     runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      DEBUG_OUTPUT_DIR: /tmp/awx_operator_molecule_test
     steps:
       - name: Checkout awx
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
+          show-progress: false
           path: awx
 
-      - name: Checkout awx-operator
-        uses: actions/checkout@v2
+      - uses: ./awx/.github/actions/setup-ssh-agent
         with:
+          ssh-private-key: ${{ secrets.PRIVATE_GITHUB_KEY }}
+
+      - name: Checkout awx-operator
+        uses: actions/checkout@v4
+        with:
+          show-progress: false\
           repository: ansible/awx-operator
           path: awx-operator
 
-      - name: Get python version from Makefile
-        working-directory: awx
-        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
-
-      - name: Install python ${{ env.py_version }}
-        uses: actions/setup-python@v2
+      - uses: ./awx/.github/actions/setup-python
         with:
-          python-version: ${{ env.py_version }}
+          working-directory: awx
 
       - name: Install playbook dependencies
         run: |
@@ -78,39 +183,260 @@ jobs:
       - name: Build AWX image
         working-directory: awx
         run: |
-          ansible-playbook -v tools/ansible/build.yml \
-            -e headless=yes \
-            -e awx_image=awx \
-            -e awx_image_tag=ci \
-            -e ansible_python_interpreter=$(which python3)
+          VERSION=`make version-for-buildyml` make awx-kube-build
+        env:
+          COMPOSE_TAG: ci
+          DEV_DOCKER_TAG_BASE: local
+          HEADLESS: yes
 
       - name: Run test deployment with awx-operator
         working-directory: awx-operator
         run: |
           python3 -m pip install -r molecule/requirements.txt
+          python3 -m pip install PyYAML # for awx/tools/scripts/rewrite-awx-operator-requirements.py
+          $(realpath ../awx/tools/scripts/rewrite-awx-operator-requirements.py) molecule/requirements.yml $(realpath ../awx)
           ansible-galaxy collection install -r molecule/requirements.yml
           sudo rm -f $(which kustomize)
           make kustomize
-          KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule -v test -s kind
+          KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule -v test -s kind -- --skip-tags=replicas
         env:
-          AWX_TEST_IMAGE: awx
+          AWX_TEST_IMAGE: local/awx
           AWX_TEST_VERSION: ci
+          AWX_EE_TEST_IMAGE: quay.io/ansible/awx-ee:latest
+          STORE_DEBUG_OUTPUT: true
+
+      - name: Upload debug output
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: awx-operator-debug-output
+          path: ${{ env.DEBUG_OUTPUT_DIR }}
 
   collection-sanity:
     name: awx_collection sanity
     runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        ansible:
+          - stable-2.17
+          # - devel
+    steps:
+      - name: Perform sanity testing
+        uses: ansible-community/ansible-test-gh-action@release/v1
+        with:
+          ansible-core-version: ${{ matrix.ansible }}
+          codecov-token: ${{ secrets.CODECOV_TOKEN }}
+          collection-root: awx_collection
+          pre-test-cmd: >-
+            ansible-playbook
+            -i localhost,
+            tools/template_galaxy.yml
+            -e collection_package=awx
+            -e collection_namespace=awx
+            -e collection_version=1.0.0
+            -e '{"awx_template_version": false}'
+          testing-type: sanity
+
+      - name: Upload awx jUnit test reports to the unified dashboard
+        if: >-
+          !cancelled()
+          && steps.make-run.outputs.test-result-files != ''
+          && github.event_name == 'push'
+          && env.UPSTREAM_REPOSITORY_ID == github.repository_id
+          && github.ref_name == github.event.repository.default_branch
+        run: |
+          for junit_file in $(echo '${{ steps.make-run.outputs.test-result-files }}' | sed 's/,/ /')
+          do
+              curl \
+                -v \
+                --user "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_USER }}:${{ secrets.PDE_ORG_RESULTS_UPLOAD_PASSWORD }}" \
+                --form "xunit_xml=@${junit_file}" \
+                --form "component_name=awx" \
+                --form "git_commit_sha=${{ github.sha }}" \
+                --form "git_repository_url=https://github.com/${{ github.repository }}" \
+                "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}/api/results/upload/"
+          done
+
+  collection-integration:
+    name: awx_collection integration
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        target-regex:
+          - name: a-h
+            regex: ^[a-h]
+          - name: i-p
+            regex: ^[i-p]
+          - name: r-z0-9
+            regex: ^[r-z0-9]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+
+      - uses: ./.github/actions/setup-python
+        with:
+          python-version: '3.x'
+
+      - uses: ./.github/actions/run_awx_devel
+        id: awx
+        with:
+          build-ui: false
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          private-github-key: ${{ secrets.PRIVATE_GITHUB_KEY }}
+
+      - name: Install dependencies for running tests
+        run: |
+          python3 -m pip install -e ./awxkit/
+          python3 -m pip install -r awx_collection/requirements.txt
+
+      - name: Run integration tests
+        id: make-run
+        run: |
+          echo "::remove-matcher owner=python::"  # Disable annoying annotations from setup-python
+          echo '[general]' > ~/.tower_cli.cfg
+          echo 'host = https://${{ steps.awx.outputs.ip }}:8043' >> ~/.tower_cli.cfg
+          echo 'username = admin' >> ~/.tower_cli.cfg
+          echo 'password = password' >> ~/.tower_cli.cfg
+          echo 'verify_ssl = false' >> ~/.tower_cli.cfg
+          TARGETS="$(ls awx_collection/tests/integration/targets | grep '${{ matrix.target-regex.regex }}' | tr '\n' ' ')"
+          make COLLECTION_VERSION=100.100.100-git COLLECTION_TEST_TARGET="--requirements $TARGETS" test_collection_integration
+        env:
+          ANSIBLE_TEST_PREFER_PODMAN: 1
+
+      - name: Upload test coverage to Codecov
+        if: >-
+          !cancelled()
+          && steps.make-run.outputs.cov-report-files != ''
+        uses: codecov/codecov-action@v4
+        with:
+          fail_ci_if_error: >-
+            ${{
+              toJSON(env.UPSTREAM_REPOSITORY_ID == github.repository_id)
+            }}
+          files: >-
+            ${{ steps.make-run.outputs.cov-report-files }}
+          flags: >-
+            CI-GHA,
+            ansible-test,
+            integration,
+            OS-${{
+              runner.os
+            }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+      # Upload coverage report as artifact
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: coverage-${{ matrix.target-regex.name }}
+          path: ~/.ansible/collections/ansible_collections/awx/awx/tests/output/coverage/
+
+      - uses: ./.github/actions/upload_awx_devel_logs
+        if: always()
+        with:
+          log-filename: collection-integration-${{ matrix.target-regex.name }}.log
+
+  collection-integration-coverage-combine:
+    name: combine awx_collection integration coverage
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    needs:
+      - collection-integration
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+
+      - uses: ./.github/actions/setup-python
+        with:
+          python-version: '3.x'
 
-      # The containers that GitHub Actions use have Ansible installed, so upgrade to make sure we have the latest version.
       - name: Upgrade ansible-core
         run: python3 -m pip install --upgrade ansible-core
 
-      - name: Run sanity tests
-        run: make test_collection_sanity
+      - name: Download coverage artifacts A to H
+        uses: actions/download-artifact@v4
+        with:
+          name: coverage-a-h
+          path: coverage
+
+      - name: Download coverage artifacts I to P
+        uses: actions/download-artifact@v4
+        with:
+          name: coverage-i-p
+          path: coverage
+
+      - name: Download coverage artifacts Z to Z
+        uses: actions/download-artifact@v4
+        with:
+          name: coverage-r-z0-9
+          path: coverage
+
+      - name: Combine coverage
+        run: |
+          make COLLECTION_VERSION=100.100.100-git install_collection
+          mkdir -p ~/.ansible/collections/ansible_collections/awx/awx/tests/output/coverage
+          cp -rv coverage/* ~/.ansible/collections/ansible_collections/awx/awx/tests/output/coverage/
+          cd ~/.ansible/collections/ansible_collections/awx/awx
+          ansible-test coverage combine --requirements
+          ansible-test coverage html
+          echo '## AWX Collection Integration Coverage' >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          ansible-test coverage report >> $GITHUB_STEP_SUMMARY
+          echo '```' >> $GITHUB_STEP_SUMMARY
+          echo >> $GITHUB_STEP_SUMMARY
+          echo '## AWX Collection Integration Coverage HTML' >> $GITHUB_STEP_SUMMARY
+          echo 'Download the HTML artifacts to view the coverage report.' >> $GITHUB_STEP_SUMMARY
+
+      # This is a huge hack, there's no official action for removing artifacts currently.
+      # Also ACTIONS_RUNTIME_URL and ACTIONS_RUNTIME_TOKEN aren't available in normal run
+      # steps, so we have to use github-script to get them.
+      #
+      # The advantage of doing this, though, is that we save on artifact storage space.
+
+      - name: Get secret artifact runtime URL
+        uses: actions/github-script@v6
+        id: get-runtime-url
+        with:
+          result-encoding: string
+          script: |
+            const { ACTIONS_RUNTIME_URL } = process.env;
+            return ACTIONS_RUNTIME_URL;
+
+      - name: Get secret artifact runtime token
+        uses: actions/github-script@v6
+        id: get-runtime-token
+        with:
+          result-encoding: string
+          script: |
+            const { ACTIONS_RUNTIME_TOKEN } = process.env;
+            return ACTIONS_RUNTIME_TOKEN;
+
+      - name: Remove intermediary artifacts
         env:
-          # needed due to cgroupsv2. This is fixed, but a stable release
-          # with the fix has not been made yet.
-          ANSIBLE_TEST_PREFER_PODMAN: 1
+          ACTIONS_RUNTIME_URL: ${{ steps.get-runtime-url.outputs.result }}
+          ACTIONS_RUNTIME_TOKEN: ${{ steps.get-runtime-token.outputs.result }}
+        run: |
+          echo "::add-mask::${ACTIONS_RUNTIME_TOKEN}"
+          artifacts=$(
+            curl -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" \
+              ${ACTIONS_RUNTIME_URL}_apis/pipelines/workflows/${{ github.run_id }}/artifacts?api-version=6.0-preview \
+            | jq -r '.value | .[] | select(.name | startswith("coverage-")) | .url'
+          )
+
+          for artifact in $artifacts; do
+            curl -i -X DELETE -H "Accept: application/json;api-version=6.0-preview" -H "Authorization: Bearer $ACTIONS_RUNTIME_TOKEN" "$artifact"
+          done
+
+      - name: Upload coverage report as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: awx-collection-integration-coverage-html
+          path: ~/.ansible/collections/ansible_collections/awx/awx/tests/output/reports/coverage

.github/workflows/dab-release.yml

@@ -0,0 +1,57 @@
+---
+name: django-ansible-base requirements update
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 6 * * *' # once an day @ 6 AM
+permissions:
+  pull-requests: write
+  contents: write
+jobs:
+  dab-pin-newest:
+    if: (github.repository_owner == 'ansible' && endsWith(github.repository, 'awx')) || github.event_name != 'schedule'
+    runs-on: ubuntu-latest
+    steps:
+      - id: dab-release
+        name: Get current django-ansible-base release version
+        uses: pozetroninc/github-action-get-latest-release@2a61c339ea7ef0a336d1daa35ef0cb1418e7676c # v0.8.0
+        with:
+          owner: ansible
+          repo: django-ansible-base
+          excludes: prerelease, draft
+
+      - name: Check out respository code
+        uses: actions/checkout@v4
+
+      - id: dab-pinned
+        name: Get current django-ansible-base pinned version
+        run:
+          echo "version=$(requirements/django-ansible-base-pinned-version.sh)" >> "$GITHUB_OUTPUT"
+
+      - name: Update django-ansible-base pinned version to upstream release
+        run:
+          requirements/django-ansible-base-pinned-version.sh -s ${{ steps.dab-release.outputs.release }}
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6
+        with:
+          base: devel
+          branch: bump-django-ansible-base
+          title: Bump django-ansible-base to ${{ steps.dab-release.outputs.release }}
+          body: |
+            ##### SUMMARY
+            Automated .github/workflows/dab-release.yml
+
+            django-ansible-base upstream released version == ${{ steps.dab-release.outputs.release }}
+            requirements_git.txt django-ansible-base pinned version ==  ${{ steps.dab-pinned.outputs.version }}
+
+            ##### ISSUE TYPE
+            - Bug, Docs Fix or other nominal change
+
+            ##### COMPONENT NAME
+            - API
+
+          commit-message: |
+            Update django-ansible-base version to ${{ steps.dab-pinned.outputs.version }}
+          add-paths:
+            requirements/requirements_git.txt

.github/workflows/devel_images.yml

@@ -2,57 +2,77 @@
 name: Build/Push Development Images
 env:
   LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
+  DOCKER_CACHE: "--no-cache" # using the cache will not rebuild git requirements and other things
 on:
+  workflow_dispatch:
   push:
     branches:
       - devel
       - release_*
       - feature_*
 jobs:
-  push:
-    if: endsWith(github.repository, '/awx') || startsWith(github.ref, 'refs/heads/release_')
+  push-development-images:
     runs-on: ubuntu-latest
+    timeout-minutes: 120
     permissions:
       packages: write
       contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        build-targets:
+          - image-name: awx_devel
+            make-target: docker-compose-buildx
+          - image-name: awx_kube_devel
+            make-target: awx-kube-dev-buildx
+          - image-name: awx
+            make-target: awx-kube-buildx
     steps:
-      - uses: actions/checkout@v2
 
-      - name: Get python version from Makefile
-        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
-
-      - name: Set lower case owner name
+      - name: Skipping build of awx image for non-awx repository
         run: |
-          echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV}
+          echo "Skipping build of awx image for non-awx repository"
+          exit 0
+        if: matrix.build-targets.image-name == 'awx' && !endsWith(github.repository, '/awx')
+
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set GITHUB_ENV variables
+        run: |
+          echo "DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER,,}" >> $GITHUB_ENV
+          echo "COMPOSE_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV
         env:
           OWNER: '${{ github.repository_owner }}'
 
-      - name: Install python ${{ env.py_version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ env.py_version }}
+      - uses: ./.github/actions/setup-python
 
       - name: Log in to registry
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
-      - name: Pre-pull image to warm build cache
-        run: |
-          docker pull ghcr.io/${OWNER_LC}/awx_devel:${GITHUB_REF##*/} || :
-          docker pull ghcr.io/${OWNER_LC}/awx_kube_devel:${GITHUB_REF##*/} || :
-          docker pull ghcr.io/${OWNER_LC}/awx:${GITHUB_REF##*/} || :
+      - name: Setup node and npm for the new UI build
+        uses: actions/setup-node@v2
+        with:
+          node-version: '18'
+        if: matrix.build-targets.image-name == 'awx'
 
-      - name: Build images
+      - name: Prebuild new UI for awx image (to speed up build process)
         run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make docker-compose-build
-          DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-dev-build
-          DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-build
+          make ui
+        if: matrix.build-targets.image-name == 'awx'
 
-      - name: Push development images
+      - uses: ./.github/actions/setup-ssh-agent
+        with:
+          ssh-private-key: ${{ secrets.PRIVATE_GITHUB_KEY }}
+
+      - name: Build and push AWX devel images
         run: |
-          docker push ghcr.io/${OWNER_LC}/awx_devel:${GITHUB_REF##*/}
-          docker push ghcr.io/${OWNER_LC}/awx_kube_devel:${GITHUB_REF##*/}
-
-      - name: Push AWX k8s image, only for upstream and feature branches
-        run: docker push ghcr.io/${OWNER_LC}/awx:${GITHUB_REF##*/}
-        if: endsWith(github.repository, '/awx')
+          make ${{ matrix.build-targets.make-target }}

.github/workflows/docs.yml

@@ -0,0 +1,23 @@
+---
+name: Docsite CI
+on:
+  pull_request:
+jobs:
+  docsite-build:
+    name: docsite test build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+
+      - uses: ./.github/actions/setup-python
+        with:
+          python-version: '3.x'
+
+      - name: install tox
+        run: pip install tox
+
+      - name: Assure docs can be built
+        run: tox -e docs

.github/workflows/e2e_test.yml

@@ -1,109 +0,0 @@
----
-name: E2E Tests
-env:
-  LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
-
-on:
-  pull_request_target:
-    types: [labeled]
-jobs:
-  e2e-test:
-    if: contains(github.event.pull_request.labels.*.name, 'qe:e2e')
-    runs-on: ubuntu-latest
-    timeout-minutes: 40
-    permissions:
-      packages: write
-      contents: read
-    strategy:
-      matrix:
-        job: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Get python version from Makefile
-        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
-
-      - name: Install python ${{ env.py_version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ env.py_version }}
-
-      - name: Install system deps
-        run: sudo apt-get install -y gettext
-
-      - name: Log in to registry
-        run: |
-          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Pre-pull image to warm build cache
-        run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }}
-
-      - name: Build UI
-        run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make ui-devel
-
-      - name: Start AWX
-        run: |
-          DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} COMPOSE_TAG=${{ github.base_ref }} make docker-compose &> make-docker-compose-output.log &
-
-      - name: Pull awx_cypress_base image
-        run: |
-          docker pull quay.io/awx/awx_cypress_base:latest
-
-      - name: Checkout test project
-        uses: actions/checkout@v2
-        with:
-          repository: ${{ github.repository_owner }}/tower-qa
-          ssh-key: ${{ secrets.QA_REPO_KEY }}
-          path: tower-qa
-          ref: devel
-
-      - name: Build cypress
-        run: |
-          cd ${{ secrets.E2E_PROJECT }}/ui-tests/awx-pf-tests
-          docker build -t awx-pf-tests .
-
-      - name: Update default AWX password
-        run: |
-          while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' -k https://localhost:8043/api/v2/ping/)" != "200" ]]
-          do
-          echo "Waiting for AWX..."
-          sleep 5;
-          done
-          echo "AWX is up, updating the password..."
-          docker exec -i tools_awx_1 sh <<-EOSH
-            awx-manage update_password --username=admin --password=password
-          EOSH
-
-      - name: Run E2E tests
-        env:
-          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-        run: |
-          export COMMIT_INFO_BRANCH=$GITHUB_HEAD_REF
-          export COMMIT_INFO_AUTHOR=$GITHUB_ACTOR
-          export COMMIT_INFO_SHA=$GITHUB_SHA
-          export COMMIT_INFO_REMOTE=$GITHUB_REPOSITORY_OWNER
-          cd ${{ secrets.E2E_PROJECT }}/ui-tests/awx-pf-tests
-          AWX_IP=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' tools_awx_1)
-          printenv > .env
-          echo "Executing tests:"
-          docker run \
-          --network '_sources_default' \
-          --ipc=host \
-          --env-file=.env \
-          -e CYPRESS_baseUrl="https://$AWX_IP:8043" \
-          -e CYPRESS_AWX_E2E_USERNAME=admin \
-          -e CYPRESS_AWX_E2E_PASSWORD='password' \
-          -e COMMAND="npm run cypress-concurrently-gha" \
-          -v /dev/shm:/dev/shm \
-          -v $PWD:/e2e \
-          -w /e2e \
-          awx-pf-tests run --project .
-
-      - name: Save AWX logs
-        uses: actions/upload-artifact@v2
-        with:
-          name: AWX-logs-${{ matrix.job }}
-          path: make-docker-compose-output.log

.github/workflows/feature_branch_deletion.yml

@@ -2,13 +2,12 @@
 name: Feature branch deletion cleanup
 env:
   LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
-on:
-  delete:
-    branches:
-      - feature_**
+on: delete
 jobs:
-  push:
+  branch_delete:
+    if: ${{ github.event.ref_type == 'branch' && startsWith(github.event.ref, 'feature_') }}
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     permissions:
       packages: write
       contents: read
@@ -21,6 +20,4 @@ jobs:
         run: |
           ansible localhost -c local, -m command -a "{{ ansible_python_interpreter + ' -m pip install boto3'}}"
           ansible localhost -c local -m aws_s3 \
-            -a "bucket=awx-public-ci-files object=${GITHUB_REF##*/}/schema.json mode=delete permission=public-read"
-
-
+            -a "bucket=awx-public-ci-files object=${GITHUB_REF##*/}/schema.json mode=delobj permission=public-read"

.github/workflows/label_issue.yml

@@ -13,6 +13,7 @@ permissions:
 jobs:
   triage:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     name: Label Issue
 
     steps:
@@ -26,12 +27,18 @@ jobs:
 
   community:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     name: Label Issue - Community
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+
+      - uses: ./.github/actions/setup-python
+
       - name: Install python requests
         run: pip install requests
+
       - name: Check if user is a member of Ansible org
         uses: jannekem/run-python-script-action@v1
         id: check_user

.github/workflows/label_pr.yml

@@ -14,6 +14,7 @@ permissions:
 jobs:
   triage:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     name: Label PR
 
     steps:
@@ -25,10 +26,17 @@ jobs:
 
   community:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     name: Label PR - Community
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+
+      - uses: ./.github/actions/setup-python
+        with:
+          python-version: '3.x'
+
       - name: Install python requests
         run: pip install requests
       - name: Check if user is a member of Ansible org

.github/workflows/pr_body_check.yml

@@ -10,6 +10,7 @@ jobs:
     if: github.repository_owner == 'ansible' && endsWith(github.repository, 'awx')
     name: Scan PR description for semantic versioning keywords
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     permissions:
       packages: write
       contents: read

.github/workflows/promote.yml

@@ -7,25 +7,36 @@ env:
 on:
   release:
     types: [published]
-
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        description: 'Name for the tag of the release.'
+        required: true
 permissions:
   contents: read # to fetch code (actions/checkout)
 
 jobs:
   promote:
-    if: endsWith(github.repository, '/awx') 
+    if: endsWith(github.repository, '/awx')
     runs-on: ubuntu-latest
+    timeout-minutes: 90
     steps:
+      - name: Set GitHub Env vars for workflow_dispatch event
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        run: |
+          echo "TAG_NAME=${{ github.event.inputs.tag_name }}" >> $GITHUB_ENV
+
+      - name: Set GitHub Env vars if release event
+        if: ${{ github.event_name == 'release' }}
+        run: |
+          echo "TAG_NAME=${{ github.event.release.tag_name }}" >> $GITHUB_ENV
+
       - name: Checkout awx
-        uses: actions/checkout@v2
-
-      - name: Get python version from Makefile
-        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
-
-      - name: Install python ${{ env.py_version }}
-        uses: actions/setup-python@v2
+        uses: actions/checkout@v4
         with:
-          python-version: ${{ env.py_version }}
+          show-progress: false
+
+      - uses: ./.github/actions/setup-python
 
       - name: Install dependencies
         run: |
@@ -40,14 +51,23 @@ jobs:
         if: ${{ github.repository_owner != 'ansible' }}
 
       - name: Build collection and publish to galaxy
+        env:
+          COLLECTION_NAMESPACE: ${{ env.collection_namespace }}
+          COLLECTION_VERSION: ${{ env.TAG_NAME }}
+          COLLECTION_TEMPLATE_VERSION: true
         run: |
-          COLLECTION_TEMPLATE_VERSION=true COLLECTION_NAMESPACE=${{ env.collection_namespace }} make build_collection
-          if [ "$(curl --head -sw '%{http_code}' https://galaxy.ansible.com/download/${{ env.collection_namespace }}-awx-${{ github.event.release.tag_name }}.tar.gz | tail -1)" == "302" ] ; then \
-              echo "Galaxy release already done"; \
-          else \
+          sudo apt-get install jq
+          make build_collection
+          count=$(curl -s https://galaxy.ansible.com/api/v3/plugin/ansible/search/collection-versions/\?namespace\=${COLLECTION_NAMESPACE}\&name\=awx\&version\=${COLLECTION_VERSION} | jq .meta.count)
+          if [[ "$count" == "1" ]]; then
+              echo "Galaxy release already done";
+          elif [[ "$count" == "0" ]]; then
               ansible-galaxy collection publish \
                 --token=${{ secrets.GALAXY_TOKEN }} \
-                awx_collection_build/${{ env.collection_namespace }}-awx-${{ github.event.release.tag_name }}.tar.gz; \
+                awx_collection_build/${COLLECTION_NAMESPACE}-awx-${COLLECTION_VERSION}.tar.gz;
+          else
+              echo "Unexpected count from galaxy search: $count";
+              exit 1;
           fi
 
       - name: Set official pypi info
@@ -59,9 +79,11 @@ jobs:
         if: ${{ github.repository_owner != 'ansible' }}
 
       - name: Build awxkit and upload to pypi
+        env:
+          SETUPTOOLS_SCM_PRETEND_VERSION: ${{ env.TAG_NAME }}
         run: |
           git reset --hard
-          cd awxkit && python3 setup.py bdist_wheel
+          cd awxkit && python3 setup.py sdist bdist_wheel
           twine upload \
             -r ${{ env.pypi_repo }} \
             -u ${{ secrets.PYPI_USERNAME }} \
@@ -78,11 +100,15 @@ jobs:
 
       - name: Re-tag and promote awx image
         run: |
-          docker pull ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name }}
-          docker tag ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name }} quay.io/${{ github.repository }}:${{ github.event.release.tag_name }}
-          docker tag ghcr.io/${{ github.repository }}:${{ github.event.release.tag_name }} quay.io/${{ github.repository }}:latest
-          docker push quay.io/${{ github.repository }}:${{ github.event.release.tag_name }}
-          docker push quay.io/${{ github.repository }}:latest
-          docker pull ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.release.tag_name }}
-          docker tag ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.release.tag_name }} quay.io/${{ github.repository_owner }}/awx-ee:${{ github.event.release.tag_name }}
-          docker push quay.io/${{ github.repository_owner }}/awx-ee:${{ github.event.release.tag_name }}
+          docker buildx imagetools create \
+            ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }} \
+            --tag quay.io/${{ github.repository }}:${{ env.TAG_NAME }}
+          docker buildx imagetools create \
+            ghcr.io/${{ github.repository }}:${{ env.TAG_NAME }} \
+            --tag quay.io/${{ github.repository }}:latest
+
+      - name: Re-tag and promote awx-ee image
+        run: |
+          docker buildx imagetools create \
+            ghcr.io/${{ github.repository_owner }}/awx-ee:${{ env.TAG_NAME }} \
+            --tag quay.io/${{ github.repository_owner }}/awx-ee:${{ env.TAG_NAME }}

.github/workflows/stage.yml

@@ -23,6 +23,7 @@ jobs:
   stage:
     if: endsWith(github.repository, '/awx')
     runs-on: ubuntu-latest
+    timeout-minutes: 90
     permissions:
       packages: write
       contents: write
@@ -44,68 +45,95 @@ jobs:
           exit 0
 
       - name: Checkout awx
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
+          show-progress: false
           path: awx
 
-      - name: Get python version from Makefile
-        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
-
-      - name: Install python ${{ env.py_version }}
-        uses: actions/setup-python@v2
+      - name: Checkout awx-operator
+        uses: actions/checkout@v4
         with:
-          python-version: ${{ env.py_version }}
+          show-progress: false
+          repository: ${{ github.repository_owner }}/awx-operator
+          path: awx-operator
 
       - name: Checkout awx-logos
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
+          show-progress: false
           repository: ansible/awx-logos
           path: awx-logos
 
-      - name: Checkout awx-operator
-        uses: actions/checkout@v2
+      - uses: ./awx/.github/actions/setup-python
         with:
-          repository: ${{ github.repository_owner }}/awx-operator
-          path: awx-operator
+          working-directory: awx
 
       - name: Install playbook dependencies
         run: |
           python3 -m pip install docker
 
-      - name: Build and stage AWX
+      - name: Log into registry ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Copy logos for inclusion in sdist for official build
         working-directory: awx
         run: |
-          ansible-playbook -v tools/ansible/build.yml \
-            -e registry=ghcr.io \
-            -e registry_username=${{ github.actor }} \
-            -e registry_password=${{ secrets.GITHUB_TOKEN }} \
-            -e awx_image=${{ github.repository }} \
-            -e awx_version=${{ github.event.inputs.version }} \
-            -e ansible_python_interpreter=$(which python3) \
-            -e push=yes \
-            -e awx_official=yes
+          cp ../awx-logos/awx/ui/client/assets/* awx/ui/public/static/media/
 
-      - name: Log in to GHCR
-        run: |
-          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: Setup node and npm for new UI build
+        uses: actions/setup-node@v2
+        with:
+          node-version: '18'
 
-      - name: Log in to Quay
+      - name: Prebuild new UI for awx image (to speed up build process)
+        working-directory: awx
+        run: make ui
+
+      - name: Set build env variables
         run: |
-          echo ${{ secrets.QUAY_TOKEN }} | docker login quay.io -u ${{ secrets.QUAY_USER }} --password-stdin
+          echo "DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER,,}" >> $GITHUB_ENV
+          echo "COMPOSE_TAG=${{ github.event.inputs.version }}" >> $GITHUB_ENV
+          echo "VERSION=${{ github.event.inputs.version }}" >> $GITHUB_ENV
+          echo "AWX_TEST_VERSION=${{ github.event.inputs.version }}" >> $GITHUB_ENV
+          echo "AWX_TEST_IMAGE=ghcr.io/${OWNER,,}/awx" >> $GITHUB_ENV
+          echo "AWX_EE_TEST_IMAGE=ghcr.io/${OWNER,,}/awx-ee:${{ github.event.inputs.version }}" >> $GITHUB_ENV
+          echo "AWX_OPERATOR_TEST_IMAGE=ghcr.io/${OWNER,,}/awx-operator:${{ github.event.inputs.operator_version }}" >> $GITHUB_ENV
+        env:
+          OWNER: ${{ github.repository_owner }}
+
+      - name: Build and stage AWX
+        working-directory: awx
+        env:
+          DOCKER_BUILDX_PUSH: true
+          HEADLESS: false
+          PLATFORMS: linux/amd64,linux/arm64
+        run: |
+          make awx-kube-buildx
 
       - name: tag awx-ee:latest with version input
         run: |
-          docker pull quay.io/ansible/awx-ee:latest
-          docker tag quay.io/ansible/awx-ee:latest ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.inputs.version }}
-          docker push ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.inputs.version }}
+          docker buildx imagetools create \
+            quay.io/ansible/awx-ee:latest \
+            --tag ${AWX_EE_TEST_IMAGE}
 
-      - name: Build and stage awx-operator
+      - name: Stage awx-operator image
         working-directory: awx-operator
         run: |
-          BUILD_ARGS="--build-arg DEFAULT_AWX_VERSION=${{ github.event.inputs.version }} \
-                      --build-arg OPERATOR_VERSION=${{ github.event.inputs.operator_version }}" \
-          IMAGE_TAG_BASE=ghcr.io/${{ github.repository_owner }}/awx-operator \
-          VERSION=${{ github.event.inputs.operator_version }} make docker-build docker-push
+          BUILD_ARGS="--build-arg DEFAULT_AWX_VERSION=${{ github.event.inputs.version}} \
+              --build-arg OPERATOR_VERSION=${{ github.event.inputs.operator_version }}" \
+          IMG=${AWX_OPERATOR_TEST_IMAGE} \
+          make docker-buildx
+
+      - name: Pulling images for test deployment with awx-operator
+        # awx operator molecue test expect to kind load image and buildx exports image to registry and not local
+        run: |
+          docker pull -q ${AWX_OPERATOR_TEST_IMAGE}
+          docker pull -q ${AWX_EE_TEST_IMAGE}
+          docker pull -q ${AWX_TEST_IMAGE}:${AWX_TEST_VERSION}
 
       - name: Run test deployment with awx-operator
         working-directory: awx-operator
@@ -115,10 +143,6 @@ jobs:
           sudo rm -f $(which kustomize)
           make kustomize
           KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind
-        env:
-          AWX_TEST_IMAGE: ${{ github.repository }}
-          AWX_TEST_VERSION: ${{ github.event.inputs.version }}
-          AWX_EE_TEST_IMAGE: ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.inputs.version }}
 
       - name: Create draft release for AWX
         working-directory: awx

.github/workflows/update_dependabot_prs.yml

@@ -9,10 +9,13 @@ jobs:
     name: Update Dependabot Prs
     if:  contains(github.event.pull_request.labels.*.name, 'dependencies') && contains(github.event.pull_request.labels.*.name, 'component:ui')
     runs-on: ubuntu-latest
+    timeout-minutes: 20
 
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Update PR Body
         env:

.github/workflows/upload_schema.yml

@@ -5,6 +5,7 @@ env:
   LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - devel
@@ -13,27 +14,28 @@ on:
 jobs:
   push:
     runs-on: ubuntu-latest
+    timeout-minutes: 60
     permissions:
       packages: write
       contents: read
     steps:
-      - uses: actions/checkout@v2
-
-      - name: Get python version from Makefile
-        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
-
-      - name: Install python ${{ env.py_version }}
-        uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
         with:
-          python-version: ${{ env.py_version }}       
+          show-progress: false
+
+      - uses: ./.github/actions/setup-python
 
       - name: Log in to registry
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
+      - uses: ./.github/actions/setup-ssh-agent
+        with:
+          ssh-private-key: ${{ secrets.PRIVATE_GITHUB_KEY }}
+
       - name: Pre-pull image to warm build cache
         run: |
-          docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${GITHUB_REF##*/} || :
+          docker pull -q ghcr.io/${{ github.repository_owner }}/awx_devel:${GITHUB_REF##*/} || :
 
       - name: Build image
         run: |
@@ -53,5 +55,3 @@ jobs:
           ansible localhost -c local, -m command -a "{{ ansible_python_interpreter + ' -m pip install boto3'}}"
           ansible localhost -c local -m aws_s3 \
             -a "src=${{ github.workspace }}/schema.json bucket=awx-public-ci-files object=${GITHUB_REF##*/}/schema.json mode=put permission=public-read"
-
-

.gitignore

@@ -20,23 +20,10 @@ awx/projects
 awx/job_output
 awx/public/media
 awx/public/static
-awx/ui/tests/test-results.xml
-awx/ui/client/src/local_settings.json
 awx/main/fixtures
 awx/*.log
 tower/tower_warnings.log
 celerybeat-schedule
-awx/ui/static
-awx/ui/build_test
-awx/ui/client/languages
-awx/ui/templates/ui/index.html
-awx/ui/templates/ui/installing.html
-awx/ui/node_modules/
-awx/ui/src/locales/*/messages.js
-awx/ui/coverage/
-awx/ui/build
-awx/ui/.env.local
-awx/ui/instrumented
 rsyslog.pid
 tools/docker-compose/ansible/awx_dump.sql
 tools/docker-compose/Dockerfile
@@ -44,7 +31,11 @@ tools/docker-compose/_build
 tools/docker-compose/_sources
 tools/docker-compose/overrides/
 tools/docker-compose-minikube/_sources
-tools/docker-compose/keycloak.awx.realm.json
+
+!tools/docker-compose/editable_dependencies
+tools/docker-compose/editable_dependencies/*
+!tools/docker-compose/editable_dependencies/README.md
+!tools/docker-compose/editable_dependencies/install.sh
 
 # Tower setup playbook testing
 setup/test/roles/postgresql
@@ -74,11 +65,6 @@ __pycache__
 /tmp
 **/npm-debug.log*
 
-# UI build flag files
-awx/ui/.deps_built
-awx/ui/.release_built
-awx/ui/.release_deps_built
-
 # Testing
 .cache
 .coverage
@@ -156,12 +142,20 @@ use_dev_supervisor.txt
 .idea/*
 *.unison.tmp
 *.#
-/awx/ui/.ui-built
 /_build/
 /_build_kube_dev/
 /Dockerfile
 /Dockerfile.dev
 /Dockerfile.kube-dev
 
-awx/ui_next/src
-awx/ui_next/build
+awx/ui/src
+awx/ui/build
+awx/ui/.ui-built
+awx/ui_next
+
+# Docs build stuff
+docs/docsite/build/
+_readthedocs/
+
+# Pyenv
+.python-version

.gitleaks.toml

@@ -0,0 +1,5 @@
+[allowlist]
+description = "Documentation contains example secrets and passwords"
+paths = [
+  "docs/docsite/rst/administration/oauth2_token_auth.rst",
+]

.pip-tools.toml

@@ -0,0 +1,5 @@
+[tool.pip-tools]
+resolver = "backtracking"
+allow-unsafe = true
+strip-extras = true
+quiet = true

.readthedocs.yaml

@@ -0,0 +1,16 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: >-
+      3.11
+  commands:
+    - pip install --user tox
+    - python3 -m tox -e docs --notest -v
+    - python3 -m tox -e docs --skip-pkg-install -q
+    - mkdir -p _readthedocs/html/
+    - mv docs/docsite/build/html/* _readthedocs/html/

.vscode/launch.json

@@ -0,0 +1,113 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "run_ws_heartbeat",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_ws_heartbeat"],
+      "django": true,
+      "preLaunchTask": "stop awx-ws-heartbeat",
+      "postDebugTask": "start awx-ws-heartbeat"
+    },
+    {
+      "name": "run_cache_clear",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_cache_clear"],
+      "django": true,
+      "preLaunchTask": "stop awx-cache-clear",
+      "postDebugTask": "start awx-cache-clear"
+    },
+    {
+      "name": "run_callback_receiver",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_callback_receiver"],
+      "django": true,
+      "preLaunchTask": "stop awx-receiver",
+      "postDebugTask": "start awx-receiver"
+    },
+    {
+      "name": "run_dispatcher",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_dispatcher"],
+      "django": true,
+      "preLaunchTask": "stop awx-dispatcher",
+      "postDebugTask": "start awx-dispatcher"
+    },
+    {
+      "name": "run_rsyslog_configurer",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_rsyslog_configurer"],
+      "django": true,
+      "preLaunchTask": "stop awx-rsyslog-configurer",
+      "postDebugTask": "start awx-rsyslog-configurer"
+    },
+    {
+      "name": "run_cache_clear",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_cache_clear"],
+      "django": true,
+      "preLaunchTask": "stop awx-cache-clear",
+      "postDebugTask": "start awx-cache-clear"
+    },
+    {
+      "name": "run_wsrelay",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["run_wsrelay"],
+      "django": true,
+      "preLaunchTask": "stop awx-wsrelay",
+      "postDebugTask": "start awx-wsrelay"
+    },
+    {
+      "name": "daphne",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "/var/lib/awx/venv/awx/bin/daphne",
+      "args": ["-b", "127.0.0.1", "-p", "8051", "awx.asgi:channel_layer"],
+      "django": true,
+      "preLaunchTask": "stop awx-daphne",
+      "postDebugTask": "start awx-daphne"
+    },
+    {
+      "name": "runserver(uwsgi alternative)",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["runserver", "127.0.0.1:8052"],
+      "django": true,
+      "preLaunchTask": "stop awx-uwsgi",
+      "postDebugTask": "start awx-uwsgi"
+    },
+    {
+      "name": "runserver_plus(uwsgi alternative)",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["runserver_plus", "127.0.0.1:8052"],
+      "django": true,
+      "preLaunchTask": "stop awx-uwsgi and install Werkzeug",
+      "postDebugTask": "start awx-uwsgi"
+    },
+    {
+      "name": "shell_plus",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "manage.py",
+      "args": ["shell_plus"],
+      "django": true,
+    },
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,100 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "start awx-cache-clear",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-cache-clear"
+        },
+        {
+            "label": "stop awx-cache-clear",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-cache-clear"
+        },
+        {
+            "label": "start awx-daphne",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-daphne"
+        },
+        {
+            "label": "stop awx-daphne",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-daphne"
+        },
+        {
+            "label": "start awx-dispatcher",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-dispatcher"
+        },
+        {
+            "label": "stop awx-dispatcher",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-dispatcher"
+        },
+        {
+            "label": "start awx-receiver",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-receiver"
+        },
+        {
+            "label": "stop awx-receiver",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-receiver"
+        },
+        {
+            "label": "start awx-rsyslog-configurer",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-rsyslog-configurer"
+        },
+        {
+            "label": "stop awx-rsyslog-configurer",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-rsyslog-configurer"
+        },
+        {
+            "label": "start awx-rsyslogd",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-rsyslogd"
+        },
+        {
+            "label": "stop awx-rsyslogd",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-rsyslogd"
+        },
+        {
+            "label": "start awx-uwsgi",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-uwsgi"
+        },
+        {
+            "label": "stop awx-uwsgi",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-uwsgi"
+        },
+        {
+            "label": "stop awx-uwsgi and install Werkzeug",
+            "type": "shell",
+            "command": "pip install Werkzeug; supervisorctl stop tower-processes:awx-uwsgi"
+        },
+        {
+            "label": "start awx-ws-heartbeat",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-ws-heartbeat"
+        },
+        {
+            "label": "stop awx-ws-heartbeat",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-ws-heartbeat"
+        },
+        {
+            "label": "start awx-wsrelay",
+            "type": "shell",
+            "command": "supervisorctl start tower-processes:awx-wsrelay"
+        },
+        {
+            "label": "stop awx-wsrelay",
+            "type": "shell",
+            "command": "supervisorctl stop tower-processes:awx-wsrelay"
+        }
+    ]
+}

.yamllint

@@ -5,11 +5,12 @@ ignore: |
   awx/main/tests/data/inventory/plugins/**
   # vault files
   awx/main/tests/data/ansible_utils/playbooks/valid/vault.yml
-  awx/ui/test/e2e/tests/smoke-vars.yml
-  awx/ui/node_modules
   tools/docker-compose/_sources
   # django template files
   awx/api/templates/instance_install_bundle/**
+  .readthedocs.yaml
+  tools/loki
+  tools/otel
 
 extends: default
 

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 Hi there! We're excited to have you as a contributor.
 
-Have questions about this document or anything not covered here? Come chat with us at `#ansible-awx` on irc.libera.chat, or submit your question to the [mailing list](https://groups.google.com/forum/#!forum/awx-project).
+Have questions about this document or anything not covered here? Create a topic using the [AWX tag on the Ansible Forum](https://forum.ansible.com/tag/awx).
 
 ## Table of contents
 
@@ -30,7 +30,7 @@ Have questions about this document or anything not covered here? Come chat with
 - You must use `git commit --signoff` for any commit to be merged, and agree that usage of --signoff constitutes agreement with the terms of [DCO 1.1](./DCO_1_1.md).
 - Take care to make sure no merge commits are in the submission, and use `git rebase` vs `git merge` for this reason.
   - If collaborating with someone else on the same branch, consider using `--force-with-lease` instead of `--force`. This will prevent you from accidentally overwriting commits pushed by someone else. For more information, see [git push docs](https://git-scm.com/docs/git-push#git-push---force-with-leaseltrefnamegt).
-- If submitting a large code change, it's a good idea to join the `#ansible-awx` channel on irc.libera.chat, and talk about what you would like to do or add first. This not only helps everyone know what's going on, it also helps save time and effort, if the community decides some changes are needed.
+- If submitting a large code change, it's a good idea to create a [forum topic tagged with 'awx'](https://forum.ansible.com/tag/awx), and talk about what you would like to do or add first. This not only helps everyone know what's going on, it also helps save time and effort, if the community decides some changes are needed.
 - We ask all of our community members and contributors to adhere to the [Ansible code of conduct](http://docs.ansible.com/ansible/latest/community/code_of_conduct.html). If you have questions, or need assistance, please reach out to our community team at [codeofconduct@ansible.com](mailto:codeofconduct@ansible.com)
 
 ## Setting up your development environment
@@ -67,7 +67,7 @@ If you're not using Docker for Mac, or Docker for Windows, you may need, or choo
 
 #### Frontend Development
 
-See [the ui development documentation](awx/ui/CONTRIBUTING.md).
+See [the ansible-ui development documentation](https://github.com/ansible/ansible-ui/blob/main/CONTRIBUTING.md).
 
 #### Fork and clone the AWX repo
 
@@ -121,18 +121,18 @@ If it has someone assigned to it then that person is the person responsible for
 
 **NOTES**
 
-> Issue assignment will only be done for maintainers of the project. If you decide to work on an issue, please feel free to add a comment in the issue to let others know that you are working on it; but know that we will accept the first pull request from whomever is able to fix an issue. Once your PR is accepted we can add you as an assignee to an issue upon request. 
+> Issue assignment will only be done for maintainers of the project. If you decide to work on an issue, please feel free to add a comment in the issue to let others know that you are working on it; but know that we will accept the first pull request from whomever is able to fix an issue. Once your PR is accepted we can add you as an assignee to an issue upon request.
 
 
-> If you work in a part of the codebase that is going through active development, your changes may be rejected, or you may be asked to `rebase`. A good idea before starting work is to have a discussion with us in the `#ansible-awx` channel on irc.libera.chat, or on the [mailing list](https://groups.google.com/forum/#!forum/awx-project).
+> If you work in a part of the codebase that is going through active development, your changes may be rejected, or you may be asked to `rebase`. A good idea before starting work is to have a discussion with us in the [Ansible Forum](https://forum.ansible.com/tag/awx).
 
-> If you're planning to develop features or fixes for the UI, please review the [UI Developer doc](./awx/ui/README.md).
+> If you're planning to develop features or fixes for the UI, please review the [UI Developer doc](https://github.com/ansible/ansible-ui/blob/main/CONTRIBUTING.md).
 
 ### Translations
 
 At this time we do not accept PRs for adding additional language translations as we have an automated process for generating our translations. This is because translations require constant care as new strings are added and changed in the code base. Because of this the .po files are overwritten during every translation release cycle. We also can't support a lot of translations on AWX as its an open source project and each language adds time and cost to maintain. If you would like to see AWX translated into a new language please create an issue and ask others you know to upvote the issue. Our translation team will review the needs of the community and see what they can do around supporting additional language.
 
-If you find an issue with an existing translation, please see the [Reporting Issues](#reporting-issues) section to open an issue and our translation team will work with you on a resolution. 
+If you find an issue with an existing translation, please see the [Reporting Issues](#reporting-issues) section to open an issue and our translation team will work with you on a resolution.
 
 
 ## Submitting Pull Requests
@@ -143,15 +143,13 @@ Here are a few things you can do to help the visibility of your change, and incr
 
 - No issues when running linters/code checkers
   - Python: black: `(container)/awx_devel$ make black`
-  - Javascript: `(container)/awx_devel$ make ui-lint`
 - No issues from unit tests
   - Python: py.test: `(container)/awx_devel$ make test`
-  - JavaScript: `(container)/awx_devel$ make ui-test`
 - Write tests for new functionality, update/add tests for bug fixes
 - Make the smallest change possible
 - Write good commit messages. See [How to write a Git commit message](https://chris.beams.io/posts/git-commit/).
 
-It's generally a good idea to discuss features with us first by engaging us in the `#ansible-awx` channel on irc.libera.chat, or on the [mailing list](https://groups.google.com/forum/#!forum/awx-project).
+It's generally a good idea to discuss features with us first by engaging on the [Ansible Forum](https://forum.ansible.com/tag/awx).
 
 We like to keep our commit history clean, and will require resubmission of pull requests that contain merge commits. Use `git pull --rebase`, rather than
 `git pull`, and `git rebase`, rather than `git merge`.
@@ -161,11 +159,11 @@ Sometimes it might take us a while to fully review your PR. We try to keep the `
 When your PR is initially submitted the checks will not be run until a maintainer allows them to be. Once a maintainer has done a quick review of your work the PR will have the linter and unit tests run against them via GitHub Actions, and the status reported in the PR.
 
 ## Reporting Issues
- 
+
 We welcome your feedback, and encourage you to file an issue when you run into a problem. But before opening a new issues, we ask that you please view our [Issues guide](./ISSUES.md).
 
 ## Getting Help
 
-If you require additional assistance, please reach out to us at `#ansible-awx` on irc.libera.chat, or submit your question to the [mailing list](https://groups.google.com/forum/#!forum/awx-project).
+If you require additional assistance, please submit your question to the [Ansible Forum](https://forum.ansible.com/tag/awx).
 
 For extra information on debugging tools, see [Debugging](./docs/debugging/).

ISSUES.md

@@ -1,11 +1,11 @@
 # Issues
 
-## Reporting 
+## Reporting
 
 Use the GitHub [issue tracker](https://github.com/ansible/awx/issues) for filing bugs. In order to save time, and help us respond to issues quickly, make sure to fill out as much of the issue template
 as possible. Version information, and an accurate reproducing scenario are critical to helping us identify the problem.
 
-Please don't use the issue tracker as a way to ask how to do something. Instead, use the [mailing list](https://groups.google.com/forum/#!forum/awx-project) , and the `#ansible-awx` channel on irc.libera.chat to get help.
+Please don't use the issue tracker as a way to ask how to do something. Instead, use the [Ansible Forum](https://forum.ansible.com/tag/awx).
 
 Before opening a new issue, please use the issue search feature to see if what you're experiencing has already been reported. If you have any extra detail to provide, please comment. Otherwise, rather than posting a "me too" comment, please consider giving it a ["thumbs up"](https://github.com/blog/2119-add-reactions-to-pull-requests-issues-and-comment) to give us an indication of the severity of the problem.
 
@@ -14,7 +14,7 @@ Before opening a new issue, please use the issue search feature to see if what y
 When reporting issues for the UI, we also appreciate having screen shots and any error messages from the web browser's console. It's not unusual for browser extensions
 and plugins to cause problems. Reporting those will also help speed up analyzing and resolving UI bugs.
 
-### API and backend issues 
+### API and backend issues
 
 For the API and backend services, please capture all of the logs that you can from the time the problem occurred.
 
@@ -80,7 +80,7 @@ If any of those items are missing your pull request will still get the `needs_tr
 Currently you can expect awxbot to add common labels such as `state:needs_triage`, `type:bug`, `component:docs`, etc...
 These labels are determined by the template data. Please use the template and fill it out as accurately as possible.
 
-The `state:needs_triage` label will will remain on your pull request until a person has looked at it.
+The `state:needs_triage` label will remain on your pull request until a person has looked at it.
 
 You can also expect the bot to CC maintainers of specific areas of the code, this will notify them that there is a pull request by placing a comment on the pull request.
 The comment will look something like `CC @matburt @wwitzel3 ...`.

MANIFEST.in

@@ -4,9 +4,7 @@ recursive-include awx *.mo
 recursive-include awx/static *
 recursive-include awx/templates *.html
 recursive-include awx/api/templates *.md *.html *.yml
-recursive-include awx/ui/build *.html
 recursive-include awx/ui/build *
-recursive-include awx/ui_next/build *
 recursive-include awx/playbooks *.yml
 recursive-include awx/lib/site-packages *
 recursive-include awx/plugins *.ps1
@@ -17,12 +15,11 @@ recursive-include licenses *
 recursive-exclude awx devonly.py*
 recursive-exclude awx/api/tests *
 recursive-exclude awx/main/tests *
-recursive-exclude awx/ui/client *
 recursive-exclude awx/settings local_settings.py*
 include tools/scripts/request_tower_configuration.sh
 include tools/scripts/request_tower_configuration.ps1
 include tools/scripts/automation-controller-service
-include tools/scripts/failure-event-handler
+include tools/scripts/rsyslog-4xx-recovery
 include tools/scripts/awx-python
 include awx/playbooks/library/mkfifo.py
 include tools/sosreport/*

Makefile

@@ -1,15 +1,17 @@
--include awx/ui_next/Makefile
+-include awx/ui/Makefile
 
-PYTHON := $(notdir $(shell for i in python3.9 python3; do command -v $$i; done|sed 1q))
+PYTHON := $(notdir $(shell for i in python3.11 python3; do command -v $$i; done|sed 1q))
 SHELL := bash
-DOCKER_COMPOSE ?= docker-compose
+DOCKER_COMPOSE ?= docker compose
 OFFICIAL ?= no
 NODE ?= node
 NPM_BIN ?= npm
+KIND_BIN ?= $(shell which kind)
 CHROMIUM_BIN=/tmp/chrome-linux/chrome
+GIT_REPO_NAME ?= $(shell basename `git rev-parse --show-toplevel`)
 GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 MANAGEMENT_COMMAND ?= awx-manage
-VERSION ?= $(shell $(PYTHON) tools/scripts/scm_version.py)
+VERSION ?= $(shell $(PYTHON) tools/scripts/scm_version.py 2> /dev/null)
 
 # ansible-test requires semver compatable version, so we allow overrides to hack it
 COLLECTION_VERSION ?= $(shell $(PYTHON) tools/scripts/scm_version.py | cut -d . -f 1-3)
@@ -22,7 +24,7 @@ COLLECTION_TEST_TARGET ?=
 # args for collection install
 COLLECTION_PACKAGE ?= awx
 COLLECTION_NAMESPACE ?= awx
-COLLECTION_INSTALL = ~/.ansible/collections/ansible_collections/$(COLLECTION_NAMESPACE)/$(COLLECTION_PACKAGE)
+COLLECTION_INSTALL = $(HOME)/.ansible/collections/ansible_collections/$(COLLECTION_NAMESPACE)/$(COLLECTION_PACKAGE)
 COLLECTION_TEMPLATE_VERSION ?= false
 
 # NOTE: This defaults the container image version to the branch that's active
@@ -30,10 +32,6 @@ COMPOSE_TAG ?= $(GIT_BRANCH)
 MAIN_NODE_TYPE ?= hybrid
 # If set to true docker-compose will also start a pgbouncer instance and use it
 PGBOUNCER ?= false
-# If set to true docker-compose will also start a keycloak instance
-KEYCLOAK ?= false
-# If set to true docker-compose will also start an ldap instance
-LDAP ?= false
 # If set to true docker-compose will also start a splunk instance
 SPLUNK ?= false
 # If set to true docker-compose will also start a prometheus instance
@@ -42,8 +40,16 @@ PROMETHEUS ?= false
 GRAFANA ?= false
 # If set to true docker-compose will also start a hashicorp vault instance
 VAULT ?= false
-# If set to true docker-compose will also start a tacacs+ instance
-TACACS ?= false
+# If set to true docker-compose will also start a hashicorp vault instance with TLS enabled
+VAULT_TLS ?= false
+# If set to true docker-compose will also start an OpenTelemetry Collector instance
+OTEL ?= false
+# If set to true docker-compose will also start a Loki instance
+LOKI ?= false
+# If set to true docker-compose will install editable dependencies
+EDITABLE_DEPENDENCIES ?= false
+# If set to true, use tls for postgres connection
+PG_TLS ?= false
 
 VENV_BASE ?= /var/lib/awx/venv
 
@@ -51,7 +57,12 @@ DEV_DOCKER_OWNER ?= ansible
 # Docker will only accept lowercase, so github names like Paul need to be paul
 DEV_DOCKER_OWNER_LOWER = $(shell echo $(DEV_DOCKER_OWNER) | tr A-Z a-z)
 DEV_DOCKER_TAG_BASE ?= ghcr.io/$(DEV_DOCKER_OWNER_LOWER)
-DEVEL_IMAGE_NAME ?= $(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG)
+DEVEL_IMAGE_NAME ?= $(DEV_DOCKER_TAG_BASE)/$(GIT_REPO_NAME)_devel:$(COMPOSE_TAG)
+IMAGE_KUBE_DEV=$(DEV_DOCKER_TAG_BASE)/$(GIT_REPO_NAME)_kube_devel:$(COMPOSE_TAG)
+IMAGE_KUBE=$(DEV_DOCKER_TAG_BASE)/$(GIT_REPO_NAME):$(COMPOSE_TAG)
+
+# Common command to use for running ansible-playbook
+ANSIBLE_PLAYBOOK ?= ansible-playbook -e ansible_python_interpreter=$(PYTHON)
 
 RECEPTOR_IMAGE ?= quay.io/ansible/receptor:devel
 
@@ -60,7 +71,7 @@ RECEPTOR_IMAGE ?= quay.io/ansible/receptor:devel
 SRC_ONLY_PKGS ?= cffi,pycparser,psycopg,twilio
 # These should be upgraded in the AWX and Ansible venv before attempting
 # to install the actual requirements
-VENV_BOOTSTRAP ?= pip==21.2.4 setuptools==65.6.3 setuptools_scm[toml]==7.0.5 wheel==0.38.4
+VENV_BOOTSTRAP ?= pip==21.2.4 setuptools==70.3.0 setuptools_scm[toml]==8.1.0 wheel==0.45.1 cython==3.0.11
 
 NAME ?= awx
 
@@ -72,13 +83,27 @@ SDIST_TAR_FILE ?= $(SDIST_TAR_NAME).tar.gz
 
 I18N_FLAG_FILE = .i18n_built
 
+## PLATFORMS defines the target platforms for  the manager image be build to provide support to multiple
+PLATFORMS ?= linux/amd64,linux/arm64  # linux/ppc64le,linux/s390x
+
+# Set up cache variables for image builds, allowing to control whether cache is used or not, ex:
+# DOCKER_CACHE=--no-cache make docker-compose-build
+ifeq ($(DOCKER_CACHE),)
+ DOCKER_DEVEL_CACHE_FLAG=--cache-from=$(DEVEL_IMAGE_NAME)
+ DOCKER_KUBE_DEV_CACHE_FLAG=--cache-from=$(IMAGE_KUBE_DEV)
+ DOCKER_KUBE_CACHE_FLAG=--cache-from=$(IMAGE_KUBE)
+else
+ DOCKER_DEVEL_CACHE_FLAG=$(DOCKER_CACHE)
+ DOCKER_KUBE_DEV_CACHE_FLAG=$(DOCKER_CACHE)
+ DOCKER_KUBE_CACHE_FLAG=$(DOCKER_CACHE)
+endif
+
 .PHONY: awx-link clean clean-tmp clean-venv requirements requirements_dev \
 	develop refresh adduser migrate dbchange \
 	receiver test test_unit test_coverage coverage_html \
 	sdist \
-	ui-release ui-devel \
 	VERSION PYTHON_VERSION docker-compose-sources \
-	.git/hooks/pre-commit github_ci_setup github_ci_runner
+	.git/hooks/pre-commit
 
 clean-tmp:
 	rm -rf tmp/
@@ -99,7 +124,7 @@ clean-languages:
 	find ./awx/locale/ -type f -regex '.*\.mo$$' -delete
 
 ## Remove temporary build files, compiled Python files.
-clean: clean-ui clean-api clean-awxkit clean-dist
+clean: clean-api clean-awxkit clean-dist
 	rm -rf awx/public
 	rm -rf awx/lib/site-packages
 	rm -rf awx/job_status
@@ -198,20 +223,12 @@ migrate:
 dbchange:
 	$(MANAGEMENT_COMMAND) makemigrations
 
-supervisor:
-	@if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/awx/bin/activate; \
-	fi; \
-	supervisord --pidfile=/tmp/supervisor_pid -n
-
 collectstatic:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
 	$(PYTHON) manage.py collectstatic --clear --noinput > /dev/null 2>&1
 
-DEV_RELOAD_COMMAND ?= supervisorctl restart tower-processes:*
-
 uwsgi: collectstatic
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
@@ -219,7 +236,7 @@ uwsgi: collectstatic
 	uwsgi /etc/tower/uwsgi.ini
 
 awx-autoreload:
-	@/awx_devel/tools/docker-compose/awx-autoreload /awx_devel/awx "$(DEV_RELOAD_COMMAND)"
+	@/awx_devel/tools/docker-compose/awx-autoreload /awx_devel/awx
 
 daphne:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -299,7 +316,12 @@ swagger: reports
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	(set -o pipefail && py.test $(PYTEST_ARGS) awx/conf/tests/functional awx/main/tests/functional/api awx/main/tests/docs --release=$(VERSION_TARGET) | tee reports/$@.report)
+	(set -o pipefail && py.test --cov --cov-report=xml --junitxml=reports/junit.xml $(PYTEST_ARGS) awx/conf/tests/functional awx/main/tests/functional/api awx/main/tests/docs | tee reports/$@.report)
+	@if [ "${GITHUB_ACTIONS}" = "true" ]; \
+	then \
+	  echo 'cov-report-files=reports/coverage.xml' >> "${GITHUB_OUTPUT}"; \
+	  echo 'test-result-files=reports/junit.xml' >> "${GITHUB_OUTPUT}"; \
+	fi
 
 check: black
 
@@ -312,7 +334,7 @@ api-lint:
 awx-link:
 	[ -d "/awx_devel/awx.egg-info" ] || $(PYTHON) /awx_devel/tools/scripts/egg_info_dev
 
-TEST_DIRS ?= awx/main/tests/unit awx/main/tests/functional awx/conf/tests awx/sso/tests
+TEST_DIRS ?= awx/main/tests/unit awx/main/tests/functional awx/conf/tests
 PYTEST_ARGS ?= -n auto
 ## Run all API unit tests.
 test:
@@ -323,20 +345,29 @@ test:
 	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py3
 	awx-manage check_migrations --dry-run --check  -n 'missing_migration_file'
 
-## Login to Github container image registry, pull image, then build image.
-github_ci_setup:
-	# GITHUB_ACTOR is automatic github actions env var
-	# CI_GITHUB_TOKEN is defined in .github files
-	echo $(CI_GITHUB_TOKEN) | docker login ghcr.io -u $(GITHUB_ACTOR) --password-stdin
-	docker pull $(DEVEL_IMAGE_NAME) || :  # Pre-pull image to warm build cache
-	$(MAKE) docker-compose-build
+live_test:
+	cd awx/main/tests/live && py.test tests/
+
+## Run all API unit tests with coverage enabled.
+test_coverage:
+	$(MAKE) test PYTEST_ARGS="--create-db --cov --cov-report=xml --junitxml=reports/junit.xml"
+	@if [ "${GITHUB_ACTIONS}" = "true" ]; \
+	then \
+	  echo 'cov-report-files=awxkit/coverage.xml,reports/coverage.xml' >> "${GITHUB_OUTPUT}"; \
+	  echo 'test-result-files=awxkit/report.xml,reports/junit.xml' >> "${GITHUB_OUTPUT}"; \
+	fi
+
+test_migrations:
+	PYTHONDONTWRITEBYTECODE=1 py.test -p no:cacheprovider --migrations -m migration_test --create-db --cov=awx --cov-report=xml --junitxml=reports/junit.xml $(PYTEST_ARGS) $(TEST_DIRS)
+	@if [ "${GITHUB_ACTIONS}" = "true" ]; \
+	then \
+	  echo 'cov-report-files=reports/coverage.xml' >> "${GITHUB_OUTPUT}"; \
+	  echo 'test-result-files=reports/junit.xml' >> "${GITHUB_OUTPUT}"; \
+	fi
 
 ## Runs AWX_DOCKER_CMD inside a new docker container.
 docker-runner:
-	docker run -u $(shell id -u) --rm -v $(shell pwd):/awx_devel/:Z --workdir=/awx_devel $(DEVEL_IMAGE_NAME) $(AWX_DOCKER_CMD)
-
-## Builds image and runs AWX_DOCKER_CMD in it, mainly for .github checks.
-github_ci_runner: github_ci_setup docker-runner
+	docker run -u $(shell id -u) --rm -v $(shell pwd):/awx_devel/:Z $(AWX_DOCKER_ARGS) --workdir=/awx_devel $(DEVEL_IMAGE_NAME) $(AWX_DOCKER_CMD)
 
 test_collection:
 	rm -f $(shell ls -d $(VENV_BASE)/awx/lib/python* | head -n 1)/no-global-site-packages.txt
@@ -345,7 +376,12 @@ test_collection:
 	fi && \
 	if ! [ -x "$(shell command -v ansible-playbook)" ]; then pip install ansible-core; fi
 	ansible --version
-	py.test $(COLLECTION_TEST_DIRS) -v
+	py.test $(COLLECTION_TEST_DIRS) --cov --cov-report=xml --junitxml=reports/junit.xml -v
+	@if [ "${GITHUB_ACTIONS}" = "true" ]; \
+	then \
+	  echo 'cov-report-files=reports/coverage.xml' >> "${GITHUB_OUTPUT}"; \
+	  echo 'test-result-files=reports/junit.xml' >> "${GITHUB_OUTPUT}"; \
+	fi
 	# The python path needs to be modified so that the tests can find Ansible within the container
 	# First we will use anything expility set as PYTHONPATH
 	# Second we will load any libraries out of the virtualenv (if it's unspecified that should be ok because python should not load out of an empty directory)
@@ -361,7 +397,7 @@ symlink_collection:
 	ln -s $(shell pwd)/awx_collection $(COLLECTION_INSTALL)
 
 awx_collection_build: $(shell find awx_collection -type f)
-	ansible-playbook -i localhost, awx_collection/tools/template_galaxy.yml \
+	$(ANSIBLE_PLAYBOOK) -i localhost, awx_collection/tools/template_galaxy.yml \
 	  -e collection_package=$(COLLECTION_PACKAGE) \
 	  -e collection_namespace=$(COLLECTION_NAMESPACE) \
 	  -e collection_version=$(COLLECTION_VERSION) \
@@ -380,23 +416,29 @@ test_collection_sanity:
 	if ! [ -x "$(shell command -v ansible-test)" ]; then pip install ansible-core; fi
 	ansible --version
 	COLLECTION_VERSION=1.0.0 $(MAKE) install_collection
-	cd $(COLLECTION_INSTALL) && ansible-test sanity $(COLLECTION_SANITY_ARGS)
+	cd $(COLLECTION_INSTALL) && \
+		ansible-test sanity $(COLLECTION_SANITY_ARGS) --coverage --junit && \
+		ansible-test coverage xml --requirements --group-by command --group-by version
+	@if [ "${GITHUB_ACTIONS}" = "true" ]; \
+	then \
+	  echo cov-report-files="$$(find "$(COLLECTION_INSTALL)/tests/output/reports/" -type f -name 'coverage=sanity*.xml' -print0 | tr '\0' ',' | sed 's#,$$##')" >> "${GITHUB_OUTPUT}"; \
+	  echo test-result-files="$$(find "$(COLLECTION_INSTALL)/tests/output/junit/" -type f -name '*.xml' -print0 | tr '\0' ',' | sed 's#,$$##')" >> "${GITHUB_OUTPUT}"; \
+	fi
 
 test_collection_integration: install_collection
-	cd $(COLLECTION_INSTALL) && ansible-test integration $(COLLECTION_TEST_TARGET)
+	cd $(COLLECTION_INSTALL) && \
+		ansible-test integration --coverage -vvv $(COLLECTION_TEST_TARGET) && \
+		ansible-test coverage xml --requirements --group-by command --group-by version
+	@if [ "${GITHUB_ACTIONS}" = "true" ]; \
+	then \
+	  echo cov-report-files="$$(find "$(COLLECTION_INSTALL)/tests/output/reports/" -type f -name 'coverage=integration*.xml' -print0 | tr '\0' ',' | sed 's#,$$##')" >> "${GITHUB_OUTPUT}"; \
+	fi
 
 test_unit:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
-	py.test awx/main/tests/unit awx/conf/tests/unit awx/sso/tests/unit
-
-## Run all API unit tests with coverage enabled.
-test_coverage:
-	@if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/awx/bin/activate; \
-	fi; \
-	py.test --create-db --cov=awx --cov-report=xml --junitxml=./reports/junit.xml $(TEST_DIRS)
+	py.test awx/main/tests/unit awx/conf/tests/unit
 
 ## Output test coverage as HTML (into htmlcov directory).
 coverage_html:
@@ -415,76 +457,7 @@ bulk_data:
 	fi; \
 	$(PYTHON) tools/data_generators/rbac_dummy_data_generator.py --preset=$(DATA_GEN_PRESET)
 
-
-# UI TASKS
-# --------------------------------------
-
-UI_BUILD_FLAG_FILE = awx/ui/.ui-built
-
-clean-ui:
-	rm -rf node_modules
-	rm -rf awx/ui/node_modules
-	rm -rf awx/ui/build
-	rm -rf awx/ui/src/locales/_build
-	rm -rf $(UI_BUILD_FLAG_FILE)
-        # the collectstatic command doesn't like it if this dir doesn't exist.
-	mkdir -p awx/ui/build/static
-
-awx/ui/node_modules:
-	NODE_OPTIONS=--max-old-space-size=6144 $(NPM_BIN) --prefix awx/ui --loglevel warn --force ci
-
-$(UI_BUILD_FLAG_FILE):
-	$(MAKE) awx/ui/node_modules
-	$(PYTHON) tools/scripts/compilemessages.py
-	$(NPM_BIN) --prefix awx/ui --loglevel warn run compile-strings
-	$(NPM_BIN) --prefix awx/ui --loglevel warn run build
-	touch $@
-
-ui-release: $(UI_BUILD_FLAG_FILE)
-
-ui-devel: awx/ui/node_modules
-	@$(MAKE) -B $(UI_BUILD_FLAG_FILE)
-	@if [ -d "/var/lib/awx" ] ; then \
-		mkdir -p /var/lib/awx/public/static/css; \
-		mkdir -p /var/lib/awx/public/static/js; \
-		mkdir -p /var/lib/awx/public/static/media; \
-		cp -r awx/ui/build/static/css/* /var/lib/awx/public/static/css; \
-		cp -r awx/ui/build/static/js/* /var/lib/awx/public/static/js; \
-		cp -r awx/ui/build/static/media/* /var/lib/awx/public/static/media; \
-	fi
-
-ui-devel-instrumented: awx/ui/node_modules
-	$(NPM_BIN) --prefix awx/ui --loglevel warn run start-instrumented
-
-ui-devel-test: awx/ui/node_modules
-	$(NPM_BIN) --prefix awx/ui --loglevel warn run start
-
-ui-lint:
-	$(NPM_BIN) --prefix awx/ui install
-	$(NPM_BIN) run --prefix awx/ui lint
-	$(NPM_BIN) run --prefix awx/ui prettier-check
-
-ui-test:
-	$(NPM_BIN) --prefix awx/ui install
-	$(NPM_BIN) run --prefix awx/ui test
-
-ui-test-screens:
-	$(NPM_BIN) --prefix awx/ui install
-	$(NPM_BIN) run --prefix awx/ui pretest
-	$(NPM_BIN) run --prefix awx/ui test-screens --runInBand
-
-ui-test-general:
-	$(NPM_BIN) --prefix awx/ui install
-	$(NPM_BIN) run --prefix awx/ui pretest
-	$(NPM_BIN) run --prefix awx/ui/ test-general --runInBand
-
-# NOTE: The make target ui-next is imported from awx/ui_next/Makefile
-HEADLESS ?= no
-ifeq ($(HEADLESS), yes)
 dist/$(SDIST_TAR_FILE):
-else
-dist/$(SDIST_TAR_FILE): $(UI_BUILD_FLAG_FILE) ui-next
-endif
 	$(PYTHON) -m build -s
 	ln -sf $(SDIST_TAR_FILE) dist/awx.tar.gz
 
@@ -515,32 +488,41 @@ endif
 
 docker-compose-sources: .git/hooks/pre-commit
 	@if [ $(MINIKUBE_CONTAINER_GROUP) = true ]; then\
-	    ansible-playbook -i tools/docker-compose/inventory -e minikube_setup=$(MINIKUBE_SETUP) tools/docker-compose-minikube/deploy.yml; \
+	    $(ANSIBLE_PLAYBOOK) -i tools/docker-compose/inventory -e minikube_setup=$(MINIKUBE_SETUP) tools/docker-compose-minikube/deploy.yml; \
 	fi;
 
-	ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/sources.yml \
-	    -e awx_image=$(DEV_DOCKER_TAG_BASE)/awx_devel \
+	$(ANSIBLE_PLAYBOOK) -i tools/docker-compose/inventory tools/docker-compose/ansible/sources.yml \
+	    -e awx_image=$(DEV_DOCKER_TAG_BASE)/$(GIT_REPO_NAME)_devel \
 	    -e awx_image_tag=$(COMPOSE_TAG) \
 	    -e receptor_image=$(RECEPTOR_IMAGE) \
 	    -e control_plane_node_count=$(CONTROL_PLANE_NODE_COUNT) \
 	    -e execution_node_count=$(EXECUTION_NODE_COUNT) \
 	    -e minikube_container_group=$(MINIKUBE_CONTAINER_GROUP) \
 	    -e enable_pgbouncer=$(PGBOUNCER) \
-	    -e enable_keycloak=$(KEYCLOAK) \
-	    -e enable_ldap=$(LDAP) \
 	    -e enable_splunk=$(SPLUNK) \
 	    -e enable_prometheus=$(PROMETHEUS) \
 	    -e enable_grafana=$(GRAFANA) \
 	    -e enable_vault=$(VAULT) \
-	    -e enable_tacacs=$(TACACS) \
-            $(EXTRA_SOURCES_ANSIBLE_OPTS)
+	    -e vault_tls=$(VAULT_TLS) \
+	    -e enable_otel=$(OTEL) \
+	    -e enable_loki=$(LOKI) \
+	    -e install_editable_dependencies=$(EDITABLE_DEPENDENCIES) \
+	    -e pg_tls=$(PG_TLS) \
+	    $(EXTRA_SOURCES_ANSIBLE_OPTS)
 
 docker-compose: awx/projects docker-compose-sources
 	ansible-galaxy install --ignore-certs -r tools/docker-compose/ansible/requirements.yml;
-	ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/initialize_containers.yml \
-	  -e enable_vault=$(VAULT);
+	$(ANSIBLE_PLAYBOOK) -i tools/docker-compose/inventory tools/docker-compose/ansible/initialize_containers.yml \
+	    -e enable_vault=$(VAULT) \
+	    -e vault_tls=$(VAULT_TLS); \
+	$(MAKE) docker-compose-up
+
+docker-compose-up:
 	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans
 
+docker-compose-down:
+	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) down --remove-orphans
+
 docker-compose-credential-plugins: awx/projects docker-compose-sources
 	echo -e "\033[0;31mTo generate a CyberArk Conjur API key: docker exec -it tools_conjur_1 conjurctl account create quick-start\033[0m"
 	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml -f tools/docker-credential-plugins-override.yml up --no-recreate awx_1 --remove-orphans
@@ -572,7 +554,7 @@ docker-compose-container-group-clean:
 .PHONY: Dockerfile.dev
 ## Generate Dockerfile.dev for awx_devel image
 Dockerfile.dev: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
-	ansible-playbook tools/ansible/dockerfile.yml \
+	$(ANSIBLE_PLAYBOOK) tools/ansible/dockerfile.yml \
 		-e dockerfile_name=Dockerfile.dev \
 		-e build_dev=True \
 		-e receptor_image=$(RECEPTOR_IMAGE)
@@ -580,41 +562,39 @@ Dockerfile.dev: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
 ## Build awx_devel image for docker compose development environment
 docker-compose-build: Dockerfile.dev
 	DOCKER_BUILDKIT=1 docker build \
+		--ssh default=$(SSH_AUTH_SOCK) \
 		-f Dockerfile.dev \
 		-t $(DEVEL_IMAGE_NAME) \
 		--build-arg BUILDKIT_INLINE_CACHE=1 \
-		--cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) .
+		$(DOCKER_DEVEL_CACHE_FLAG) .
+
+.PHONY: docker-compose-buildx
+## Build awx_devel image for docker compose development environment for multiple architectures
+docker-compose-buildx: Dockerfile.dev
+	- docker buildx create --name docker-compose-buildx
+	docker buildx use docker-compose-buildx
+	- docker buildx build \
+		--ssh default=$(SSH_AUTH_SOCK) \
+		--push \
+		--build-arg BUILDKIT_INLINE_CACHE=1 \
+		$(DOCKER_DEVEL_CACHE_FLAG) \
+		--platform=$(PLATFORMS) \
+		--tag $(DEVEL_IMAGE_NAME) \
+		-f Dockerfile.dev .
+	- docker buildx rm docker-compose-buildx
 
 docker-clean:
 	-$(foreach container_id,$(shell docker ps -f name=tools_awx -aq && docker ps -f name=tools_receptor -aq),docker stop $(container_id); docker rm -f $(container_id);)
 	-$(foreach image_id,$(shell docker images --filter=reference='*/*/*awx_devel*' --filter=reference='*/*awx_devel*' --filter=reference='*awx_devel*' -aq),docker rmi --force $(image_id);)
 
 docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean
-	docker volume rm -f tools_awx_db tools_vault_1 tools_grafana_storage tools_prometheus_storage $(docker volume ls --filter name=tools_redis_socket_ -q)
+	docker volume rm -f tools_var_lib_awx tools_awx_db tools_awx_db_15 tools_vault_1 tools_grafana_storage tools_prometheus_storage $(shell docker volume ls --filter name=tools_redis_socket_ -q)
 
 docker-refresh: docker-clean docker-compose
 
-## Docker Development Environment with Elastic Stack Connected
-docker-compose-elk: awx/projects docker-compose-sources
-	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
-
-docker-compose-cluster-elk: awx/projects docker-compose-sources
-	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml -f tools/elastic/docker-compose.logstash-link-cluster.yml -f tools/elastic/docker-compose.elastic-override.yml up --no-recreate
-
 docker-compose-container-group:
 	MINIKUBE_CONTAINER_GROUP=true $(MAKE) docker-compose
 
-clean-elk:
-	docker stop tools_kibana_1
-	docker stop tools_logstash_1
-	docker stop tools_elasticsearch_1
-	docker rm tools_logstash_1
-	docker rm tools_elasticsearch_1
-	docker rm tools_kibana_1
-
-psql-container:
-	docker run -it --net tools_default --rm postgres:12 sh -c 'exec psql -h "postgres" -p "5432" -U postgres'
-
 VERSION:
 	@echo "awx: $(VERSION)"
 
@@ -635,22 +615,41 @@ version-for-buildyml:
 .PHONY: Dockerfile
 ## Generate Dockerfile for awx image
 Dockerfile: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
-	ansible-playbook tools/ansible/dockerfile.yml \
+	$(ANSIBLE_PLAYBOOK) tools/ansible/dockerfile.yml \
 		-e receptor_image=$(RECEPTOR_IMAGE) \
 		-e headless=$(HEADLESS)
 
 ## Build awx image for deployment on Kubernetes environment.
 awx-kube-build: Dockerfile
 	DOCKER_BUILDKIT=1 docker build -f Dockerfile \
+		--ssh default=$(SSH_AUTH_SOCK) \
 		--build-arg VERSION=$(VERSION) \
 		--build-arg SETUPTOOLS_SCM_PRETEND_VERSION=$(VERSION) \
 		--build-arg HEADLESS=$(HEADLESS) \
-		-t $(DEV_DOCKER_TAG_BASE)/awx:$(COMPOSE_TAG) .
+		$(DOCKER_KUBE_CACHE_FLAG) \
+		-t $(IMAGE_KUBE) .
+
+## Build multi-arch awx image for deployment on Kubernetes environment.
+awx-kube-buildx: Dockerfile
+	- docker buildx create --name awx-kube-buildx
+	docker buildx use awx-kube-buildx
+	- docker buildx build \
+		--ssh default=$(SSH_AUTH_SOCK) \
+		--push \
+		--build-arg VERSION=$(VERSION) \
+		--build-arg SETUPTOOLS_SCM_PRETEND_VERSION=$(VERSION) \
+		--build-arg HEADLESS=$(HEADLESS) \
+		--platform=$(PLATFORMS) \
+		$(DOCKER_KUBE_CACHE_FLAG) \
+		--tag $(IMAGE_KUBE) \
+		-f Dockerfile .
+	- docker buildx rm awx-kube-buildx
+
 
 .PHONY: Dockerfile.kube-dev
 ## Generate Docker.kube-dev for awx_kube_devel image
 Dockerfile.kube-dev: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
-	ansible-playbook tools/ansible/dockerfile.yml \
+	$(ANSIBLE_PLAYBOOK) tools/ansible/dockerfile.yml \
 	    -e dockerfile_name=Dockerfile.kube-dev \
 	    -e kube_dev=True \
 	    -e template_dest=_build_kube_dev \
@@ -659,24 +658,31 @@ Dockerfile.kube-dev: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
 ## Build awx_kube_devel image for development on local Kubernetes environment.
 awx-kube-dev-build: Dockerfile.kube-dev
 	DOCKER_BUILDKIT=1 docker build -f Dockerfile.kube-dev \
+		--ssh default=$(SSH_AUTH_SOCK) \
 	    --build-arg BUILDKIT_INLINE_CACHE=1 \
-	    --cache-from=$(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG) \
-	    -t $(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG) .
+	     $(DOCKER_KUBE_DEV_CACHE_FLAG) \
+	    -t $(IMAGE_KUBE_DEV) .
 
+## Build and push multi-arch awx_kube_devel image for development on local Kubernetes environment.
+awx-kube-dev-buildx: Dockerfile.kube-dev
+	- docker buildx create --name awx-kube-dev-buildx
+	docker buildx use awx-kube-dev-buildx
+	- docker buildx build \
+		--ssh default=$(SSH_AUTH_SOCK) \
+		--push \
+		--build-arg BUILDKIT_INLINE_CACHE=1 \
+		$(DOCKER_KUBE_DEV_CACHE_FLAG) \
+		--platform=$(PLATFORMS) \
+		--tag $(IMAGE_KUBE_DEV) \
+		-f Dockerfile.kube-dev .
+	- docker buildx rm awx-kube-dev-buildx
+
+kind-dev-load: awx-kube-dev-build
+	$(KIND_BIN) load docker-image $(IMAGE_KUBE_DEV)
 
 # Translation TASKS
 # --------------------------------------
 
-## generate UI .pot file, an empty template of strings yet to be translated
-pot: $(UI_BUILD_FLAG_FILE)
-	$(NPM_BIN) --prefix awx/ui --loglevel warn run extract-template --clean
-	$(NPM_BIN) --prefix awx/ui_next --loglevel warn run extract-template --clean
-
-## generate UI .po files for each locale (will update translated strings for `en`)
-po: $(UI_BUILD_FLAG_FILE)
-	$(NPM_BIN) --prefix awx/ui --loglevel warn run extract-strings -- --clean
-	$(NPM_BIN) --prefix awx/ui_next --loglevel warn run extract-strings -- --clean
-
 ## generate API django .pot .po
 messages:
 	@if [ "$(VENV_BASE)" ]; then \
@@ -723,6 +729,6 @@ help/generate:
 	{ lastLine = $$0 }' $(MAKEFILE_LIST) | sort -u
 	@printf "\n"
 
-## Display help for ui-next targets
-help/ui-next:
-	@$(MAKE) -s help MAKEFILE_LIST="awx/ui_next/Makefile"
+## Display help for ui targets
+help/ui:
+	@$(MAKE) -s help MAKEFILE_LIST="awx/ui/Makefile"

README.md

@@ -1,13 +1,24 @@
-[![CI](https://github.com/ansible/awx/actions/workflows/ci.yml/badge.svg?branch=devel)](https://github.com/ansible/awx/actions/workflows/ci.yml) [![Code of Conduct](https://img.shields.io/badge/code%20of%20conduct-Ansible-yellow.svg)](https://docs.ansible.com/ansible/latest/community/code_of_conduct.html) [![Apache v2 License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](https://github.com/ansible/awx/blob/devel/LICENSE.md) [![AWX Mailing List](https://img.shields.io/badge/mailing%20list-AWX-orange.svg)](https://groups.google.com/g/awx-project)
-[![IRC Chat - #ansible-awx](https://img.shields.io/badge/IRC-%23ansible--awx-blueviolet.svg)](https://libera.chat)
+[![CI](https://github.com/ansible/awx/actions/workflows/ci.yml/badge.svg?branch=devel)](https://github.com/ansible/awx/actions/workflows/ci.yml) [![codecov](https://codecov.io/github/ansible/awx/graph/badge.svg?token=4L4GSP9IAR)](https://codecov.io/github/ansible/awx) [![Code of Conduct](https://img.shields.io/badge/code%20of%20conduct-Ansible-yellow.svg)](https://docs.ansible.com/ansible/latest/community/code_of_conduct.html) [![Apache v2 License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](https://github.com/ansible/awx/blob/devel/LICENSE.md) [![AWX on the Ansible Forum](https://img.shields.io/badge/mailing%20list-AWX-orange.svg)](https://forum.ansible.com/tag/awx)
+[![Ansible Matrix](https://img.shields.io/badge/matrix-Ansible%20Community-blueviolet.svg?logo=matrix)](https://chat.ansible.im/#/welcome) [![Ansible Discourse](https://img.shields.io/badge/discourse-Ansible%20Community-yellowgreen.svg?logo=discourse)](https://forum.ansible.com)
 
 <img src="https://raw.githubusercontent.com/ansible/awx-logos/master/awx/ui/client/assets/logo-login.svg?sanitize=true" width=200 alt="AWX" />
 
+> [!CAUTION]
+> The last release of this repository was released on Jul 2, 2024.
+> **Releases of this project are now paused during a large scale refactoring.**
+> For more information, follow [the Forum](https://forum.ansible.com/) and - more specifically - see the various communications on the matter:
+>
+> * [Blog: Upcoming Changes to the AWX Project](https://www.ansible.com/blog/upcoming-changes-to-the-awx-project/)
+> * [Streamlining AWX Releases](https://forum.ansible.com/t/streamlining-awx-releases/6894) Primary update
+> * [Refactoring AWX into a Pluggable, Service-Oriented Architecture](https://forum.ansible.com/t/refactoring-awx-into-a-pluggable-service-oriented-architecture/7404)
+> * [Upcoming changes to AWX Operator installation methods](https://forum.ansible.com/t/upcoming-changes-to-awx-operator-installation-methods/7598)
+> * [AWX UI and credential types transitioning to the new pluggable architecture](https://forum.ansible.com/t/awx-ui-and-credential-types-transitioning-to-the-new-pluggable-architecture/8027)
+
 AWX provides a web-based user interface, REST API, and task engine built on top of [Ansible](https://github.com/ansible/ansible). It is one of the upstream projects for [Red Hat Ansible Automation Platform](https://www.ansible.com/products/automation-platform).
 
 To install AWX, please view the [Install guide](./INSTALL.md).
 
-To learn more about using AWX, and Tower, view the [Tower docs site](http://docs.ansible.com/ansible-tower/index.html).
+To learn more about using AWX, view the [AWX docs site](https://ansible.readthedocs.io/projects/awx/en/latest/).
 
 The AWX Project Frequently Asked Questions can be found [here](https://www.ansible.com/awx-project-faq).
 
@@ -18,9 +29,9 @@ Contributing
 
 - Refer to the [Contributing guide](./CONTRIBUTING.md) to get started developing, testing, and building AWX.
 - All code submissions are made through pull requests against the `devel` branch.
-- All contributors must use git commit --signoff for any commit to be merged and agree that usage of --signoff constitutes agreement with the terms of [DCO 1.1](./DCO_1_1.md)
+- All contributors must use `git commit --signoff` for any commit to be merged and agree that usage of `--signoff` constitutes agreement with the terms of [DCO 1.1](./DCO_1_1.md)
 - Take care to make sure no merge commits are in the submission, and use `git rebase` vs. `git merge` for this reason.
-- If submitting a large code change, it's a good idea to join the `#ansible-awx` channel on web.libera.chat and talk about what you would like to do or add first. This not only helps everyone know what's going on, but it also helps save time and effort if the community decides some changes are needed.
+- If submitting a large code change, it's a good idea to join discuss via the [Ansible Forum](https://forum.ansible.com/tag/awx). This helps everyone know what's going on, and it also helps save time and effort if the community decides some changes are needed.
 
 Reporting Issues
 ----------------
@@ -30,12 +41,11 @@ If you're experiencing a problem that you feel is a bug in AWX or have ideas for
 Code of Conduct
 ---------------
 
-We ask all of our community members and contributors to adhere to the [Ansible code of conduct](http://docs.ansible.com/ansible/latest/community/code_of_conduct.html). If you have questions or need assistance, please reach out to our community team at [codeofconduct@ansible.com](mailto:codeofconduct@ansible.com)   
+We require all of our community members and contributors to adhere to the [Ansible code of conduct](http://docs.ansible.com/ansible/latest/community/code_of_conduct.html). If you have questions or need assistance, please reach out to our community team at [codeofconduct@ansible.com](mailto:codeofconduct@ansible.com)
 
 Get Involved
 ------------
 
-We welcome your feedback and ideas. Here's how to reach us with feedback and questions:
+We welcome your feedback and ideas via the [Ansible Forum](https://forum.ansible.com/tag/awx).
 
-- Join the `#ansible-awx` channel on irc.libera.chat
-- Join the [mailing list](https://groups.google.com/forum/#!forum/awx-project) 
+For a full list of all the ways to talk with the Ansible Community, see the [AWX Communication guide](https://ansible.readthedocs.io/projects/awx/en/latest/contributor/communication.html).

awx/__init__.py

@@ -5,6 +5,7 @@ from __future__ import absolute_import, unicode_literals
 import os
 import sys
 import warnings
+from importlib.metadata import PackageNotFoundError, version as _get_version
 
 
 def get_version():
@@ -34,10 +35,8 @@ def version_file():
 
 
 try:
-    import pkg_resources
-
-    __version__ = pkg_resources.get_distribution('awx').version
-except pkg_resources.DistributionNotFound:
+    __version__ = _get_version('awx')
+except PackageNotFoundError:
     __version__ = get_version()
 
 __all__ = ['__version__']
@@ -52,124 +51,25 @@ try:
 except ImportError:  # pragma: no cover
     MODE = 'production'
 
-import hashlib
 
 try:
     import django  # noqa: F401
-
-    HAS_DJANGO = True
 except ImportError:
-    HAS_DJANGO = False
+    pass
 else:
-    from django.db.backends.base import schema
-    from django.db.models import indexes
-    from django.db.backends.utils import names_digest
     from django.db import connection
 
-if HAS_DJANGO is True:
-    # See upgrade blocker note in requirements/README.md
-    try:
-        names_digest('foo', 'bar', 'baz', length=8)
-    except ValueError:
-
-        def names_digest(*args, length):
-            """
-            Generate a 32-bit digest of a set of arguments that can be used to shorten
-            identifying names.  Support for use in FIPS environments.
-            """
-            h = hashlib.md5(usedforsecurity=False)
-            for arg in args:
-                h.update(arg.encode())
-            return h.hexdigest()[:length]
-
-        schema.names_digest = names_digest
-        indexes.names_digest = names_digest
-
-
-def find_commands(management_dir):
-    # Modified version of function from django/core/management/__init__.py.
-    command_dir = os.path.join(management_dir, 'commands')
-    commands = []
-    try:
-        for f in os.listdir(command_dir):
-            if f.startswith('_'):
-                continue
-            elif f.endswith('.py') and f[:-3] not in commands:
-                commands.append(f[:-3])
-            elif f.endswith('.pyc') and f[:-4] not in commands:  # pragma: no cover
-                commands.append(f[:-4])
-    except OSError:
-        pass
-    return commands
-
-
-def oauth2_getattribute(self, attr):
-    # Custom method to override
-    # oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__
-    from django.conf import settings
-    from oauth2_provider.settings import DEFAULTS
-
-    val = None
-    if (isinstance(attr, str)) and (attr in DEFAULTS) and (not attr.startswith('_')):
-        # certain Django OAuth Toolkit migrations actually reference
-        # setting lookups for references to model classes (e.g.,
-        # oauth2_settings.REFRESH_TOKEN_MODEL)
-        # If we're doing an OAuth2 setting lookup *while running* a migration,
-        # don't do our usual database settings lookup
-        val = settings.OAUTH2_PROVIDER.get(attr)
-    if val is None:
-        val = object.__getattribute__(self, attr)
-    return val
-
 
 def prepare_env():
     # Update the default settings environment variable based on current mode.
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE)
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings')
+    os.environ.setdefault('AWX_MODE', MODE)
     # Hide DeprecationWarnings when running in production.  Need to first load
     # settings to apply our filter after Django's own warnings filter.
     from django.conf import settings
 
     if not settings.DEBUG:  # pragma: no cover
         warnings.simplefilter('ignore', DeprecationWarning)
-    # Monkeypatch Django find_commands to also work with .pyc files.
-    import django.core.management
-
-    django.core.management.find_commands = find_commands
-
-    # Monkeypatch Oauth2 toolkit settings class to check for settings
-    # in django.conf settings each time, not just once during import
-    import oauth2_provider.settings
-
-    oauth2_provider.settings.OAuth2ProviderSettings.__getattribute__ = oauth2_getattribute
-
-    # Use the AWX_TEST_DATABASE_* environment variables to specify the test
-    # database settings to use when management command is run as an external
-    # program via unit tests.
-    for opt in ('ENGINE', 'NAME', 'USER', 'PASSWORD', 'HOST', 'PORT'):  # pragma: no cover
-        if os.environ.get('AWX_TEST_DATABASE_%s' % opt, None):
-            settings.DATABASES['default'][opt] = os.environ['AWX_TEST_DATABASE_%s' % opt]
-    # Disable capturing all SQL queries in memory when in DEBUG mode.
-    if settings.DEBUG and not getattr(settings, 'SQL_DEBUG', True):
-        from django.db.backends.base.base import BaseDatabaseWrapper
-        from django.db.backends.utils import CursorWrapper
-
-        BaseDatabaseWrapper.make_debug_cursor = lambda self, cursor: CursorWrapper(cursor, self)
-
-    # Use the default devserver addr/port defined in settings for runserver.
-    default_addr = getattr(settings, 'DEVSERVER_DEFAULT_ADDR', '127.0.0.1')
-    default_port = getattr(settings, 'DEVSERVER_DEFAULT_PORT', 8000)
-    from django.core.management.commands import runserver as core_runserver
-
-    original_handle = core_runserver.Command.handle
-
-    def handle(self, *args, **options):
-        if not options.get('addrport'):
-            options['addrport'] = '%s:%d' % (default_addr, int(default_port))
-        elif options.get('addrport').isdigit():
-            options['addrport'] = '%s:%d' % (default_addr, int(options['addrport']))
-        return original_handle(self, *args, **options)
-
-    core_runserver.Command.handle = handle
 
 
 def manage():
@@ -179,10 +79,12 @@ def manage():
     from django.conf import settings
     from django.core.management import execute_from_command_line
 
-    # enforce the postgres version is equal to 12. if not, then terminate program with exit code of 1
+    # enforce the postgres version is a minimum of 12 (we need this for partitioning); if not, then terminate program with exit code of 1
+    # In the future if we require a feature of a version of postgres > 12 this should be updated to reflect that.
+    # The return of connection.pg_version is something like 12013
     if not os.getenv('SKIP_PG_VERSION_CHECK', False) and not MODE == 'development':
         if (connection.pg_version // 10000) < 12:
-            sys.stderr.write("Postgres version 12 is required\n")
+            sys.stderr.write("At a minimum, postgres version 12 is required\n")
             sys.exit(1)
 
     if len(sys.argv) >= 2 and sys.argv[1] in ('version', '--version'):  # pragma: no cover

awx/api/authentication.py

@@ -11,9 +11,6 @@ from django.utils.encoding import smart_str
 # Django REST Framework
 from rest_framework import authentication
 
-# Django-OAuth-Toolkit
-from oauth2_provider.contrib.rest_framework import OAuth2Authentication
-
 logger = logging.getLogger('awx.api.authentication')
 
 
@@ -36,16 +33,3 @@ class LoggedBasicAuthentication(authentication.BasicAuthentication):
 class SessionAuthentication(authentication.SessionAuthentication):
     def authenticate_header(self, request):
         return 'Session'
-
-
-class LoggedOAuth2Authentication(OAuth2Authentication):
-    def authenticate(self, request):
-        ret = super(LoggedOAuth2Authentication, self).authenticate(request)
-        if ret:
-            user, token = ret
-            username = user.username if user else '<none>'
-            logger.info(
-                smart_str(u"User {} performed a {} to {} through the API using OAuth 2 token {}.".format(username, request.method, request.path, token.pk))
-            )
-            setattr(user, 'oauth_scopes', [x for x in token.scope.split() if x])
-        return ret

awx/api/conf.py

@@ -6,9 +6,6 @@ from rest_framework import serializers
 
 # AWX
 from awx.conf import fields, register, register_validate
-from awx.api.fields import OAuth2ProviderField
-from oauth2_provider.settings import oauth2_settings
-from awx.sso.common import is_remote_auth_enabled
 
 
 register(
@@ -35,10 +32,7 @@ register(
     'DISABLE_LOCAL_AUTH',
     field_class=fields.BooleanField,
     label=_('Disable the built-in authentication system'),
-    help_text=_(
-        "Controls whether users are prevented from using the built-in authentication system. "
-        "You probably want to do this if you are using an LDAP or SAML integration."
-    ),
+    help_text=_("Controls whether users are prevented from using the built-in authentication system. "),
     category=_('Authentication'),
     category_slug='authentication',
 )
@@ -50,41 +44,6 @@ register(
     category=_('Authentication'),
     category_slug='authentication',
 )
-register(
-    'OAUTH2_PROVIDER',
-    field_class=OAuth2ProviderField,
-    default={
-        'ACCESS_TOKEN_EXPIRE_SECONDS': oauth2_settings.ACCESS_TOKEN_EXPIRE_SECONDS,
-        'AUTHORIZATION_CODE_EXPIRE_SECONDS': oauth2_settings.AUTHORIZATION_CODE_EXPIRE_SECONDS,
-        'REFRESH_TOKEN_EXPIRE_SECONDS': oauth2_settings.REFRESH_TOKEN_EXPIRE_SECONDS,
-    },
-    label=_('OAuth 2 Timeout Settings'),
-    help_text=_(
-        'Dictionary for customizing OAuth 2 timeouts, available items are '
-        '`ACCESS_TOKEN_EXPIRE_SECONDS`, the duration of access tokens in the number '
-        'of seconds, `AUTHORIZATION_CODE_EXPIRE_SECONDS`, the duration of '
-        'authorization codes in the number of seconds, and `REFRESH_TOKEN_EXPIRE_SECONDS`, '
-        'the duration of refresh tokens, after expired access tokens, '
-        'in the number of seconds.'
-    ),
-    category=_('Authentication'),
-    category_slug='authentication',
-    unit=_('seconds'),
-)
-register(
-    'ALLOW_OAUTH2_FOR_EXTERNAL_USERS',
-    field_class=fields.BooleanField,
-    default=False,
-    label=_('Allow External Users to Create OAuth2 Tokens'),
-    help_text=_(
-        'For security reasons, users from external auth providers (LDAP, SAML, '
-        'SSO, Radius, and others) are not allowed to create OAuth2 tokens. '
-        'To change this behavior, enable this setting. Existing tokens will '
-        'not be deleted when this setting is toggled off.'
-    ),
-    category=_('Authentication'),
-    category_slug='authentication',
-)
 register(
     'LOGIN_REDIRECT_OVERRIDE',
     field_class=fields.CharField,
@@ -93,6 +52,7 @@ register(
     default='',
     label=_('Login redirect override URL'),
     help_text=_('URL to which unauthorized users will be redirected to log in.  If blank, users will be sent to the login page.'),
+    warning_text=_('Changing the redirect URL could impact the ability to login if local authentication is also disabled.'),
     category=_('Authentication'),
     category_slug='authentication',
 )
@@ -108,7 +68,7 @@ register(
 
 
 def authentication_validate(serializer, attrs):
-    if attrs.get('DISABLE_LOCAL_AUTH', False) and not is_remote_auth_enabled():
+    if attrs.get('DISABLE_LOCAL_AUTH', False):
         raise serializers.ValidationError(_("There are no remote authentication systems configured."))
     return attrs
 

awx/api/fields.py

@@ -9,7 +9,6 @@ from django.core.exceptions import ObjectDoesNotExist
 from rest_framework import serializers
 
 # AWX
-from awx.conf import fields
 from awx.main.models import Credential
 
 __all__ = ['BooleanNullField', 'CharNullField', 'ChoiceNullField', 'VerbatimField']
@@ -79,19 +78,6 @@ class VerbatimField(serializers.Field):
         return value
 
 
-class OAuth2ProviderField(fields.DictField):
-    default_error_messages = {'invalid_key_names': _('Invalid key names: {invalid_key_names}')}
-    valid_key_names = {'ACCESS_TOKEN_EXPIRE_SECONDS', 'AUTHORIZATION_CODE_EXPIRE_SECONDS', 'REFRESH_TOKEN_EXPIRE_SECONDS'}
-    child = fields.IntegerField(min_value=1)
-
-    def to_internal_value(self, data):
-        data = super(OAuth2ProviderField, self).to_internal_value(data)
-        invalid_flags = set(data.keys()) - self.valid_key_names
-        if invalid_flags:
-            self.fail('invalid_key_names', invalid_key_names=', '.join(list(invalid_flags)))
-        return data
-
-
 class DeprecatedCredentialField(serializers.IntegerField):
     def __init__(self, **kwargs):
         kwargs['allow_null'] = True

awx/api/filters.py

@@ -1,450 +0,0 @@
-# Copyright (c) 2015 Ansible, Inc.
-# All Rights Reserved.
-
-# Python
-import re
-import json
-from functools import reduce
-
-# Django
-from django.core.exceptions import FieldError, ValidationError, FieldDoesNotExist
-from django.db import models
-from django.db.models import Q, CharField, IntegerField, BooleanField, TextField, JSONField
-from django.db.models.fields.related import ForeignObjectRel, ManyToManyField, ForeignKey
-from django.db.models.functions import Cast
-from django.contrib.contenttypes.models import ContentType
-from django.contrib.contenttypes.fields import GenericForeignKey
-from django.utils.encoding import force_str
-from django.utils.translation import gettext_lazy as _
-
-# Django REST Framework
-from rest_framework.exceptions import ParseError, PermissionDenied
-from rest_framework.filters import BaseFilterBackend
-
-# AWX
-from awx.main.utils import get_type_for_model, to_python_boolean
-from awx.main.utils.db import get_all_field_names
-
-
-class TypeFilterBackend(BaseFilterBackend):
-    """
-    Filter on type field now returned with all objects.
-    """
-
-    def filter_queryset(self, request, queryset, view):
-        try:
-            types = None
-            for key, value in request.query_params.items():
-                if key == 'type':
-                    if ',' in value:
-                        types = value.split(',')
-                    else:
-                        types = (value,)
-            if types:
-                types_map = {}
-                for ct in ContentType.objects.filter(Q(app_label='main') | Q(app_label='auth', model='user')):
-                    ct_model = ct.model_class()
-                    if not ct_model:
-                        continue
-                    ct_type = get_type_for_model(ct_model)
-                    types_map[ct_type] = ct.pk
-                model = queryset.model
-                model_type = get_type_for_model(model)
-                if 'polymorphic_ctype' in get_all_field_names(model):
-                    types_pks = set([v for k, v in types_map.items() if k in types])
-                    queryset = queryset.filter(polymorphic_ctype_id__in=types_pks)
-                elif model_type in types:
-                    queryset = queryset
-                else:
-                    queryset = queryset.none()
-            return queryset
-        except FieldError as e:
-            # Return a 400 for invalid field names.
-            raise ParseError(*e.args)
-
-
-def get_fields_from_path(model, path):
-    """
-    Given a Django ORM lookup path (possibly over multiple models)
-    Returns the fields in the line, and also the revised lookup path
-    ex., given
-        model=Organization
-        path='project__timeout'
-    returns tuple of fields traversed as well and a corrected path,
-    for special cases we do substitutions
-        ([<IntegerField for timeout>], 'project__timeout')
-    """
-    # Store of all the fields used to detect repeats
-    field_list = []
-    new_parts = []
-    for name in path.split('__'):
-        if model is None:
-            raise ParseError(_('No related model for field {}.').format(name))
-        # HACK: Make project and inventory source filtering by old field names work for backwards compatibility.
-        if model._meta.object_name in ('Project', 'InventorySource'):
-            name = {'current_update': 'current_job', 'last_update': 'last_job', 'last_update_failed': 'last_job_failed', 'last_updated': 'last_job_run'}.get(
-                name, name
-            )
-
-        if name == 'type' and 'polymorphic_ctype' in get_all_field_names(model):
-            name = 'polymorphic_ctype'
-            new_parts.append('polymorphic_ctype__model')
-        else:
-            new_parts.append(name)
-
-        if name in getattr(model, 'PASSWORD_FIELDS', ()):
-            raise PermissionDenied(_('Filtering on password fields is not allowed.'))
-        elif name == 'pk':
-            field = model._meta.pk
-        else:
-            name_alt = name.replace("_", "")
-            if name_alt in model._meta.fields_map.keys():
-                field = model._meta.fields_map[name_alt]
-                new_parts.pop()
-                new_parts.append(name_alt)
-            else:
-                field = model._meta.get_field(name)
-            if isinstance(field, ForeignObjectRel) and getattr(field.field, '__prevent_search__', False):
-                raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
-            elif getattr(field, '__prevent_search__', False):
-                raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
-        if field in field_list:
-            # Field traversed twice, could create infinite JOINs, DoSing Tower
-            raise ParseError(_('Loops not allowed in filters, detected on field {}.').format(field.name))
-        field_list.append(field)
-        model = getattr(field, 'related_model', None)
-
-    return field_list, '__'.join(new_parts)
-
-
-def get_field_from_path(model, path):
-    """
-    Given a Django ORM lookup path (possibly over multiple models)
-    Returns the last field in the line, and the revised lookup path
-    ex.
-        (<IntegerField for timeout>, 'project__timeout')
-    """
-    field_list, new_path = get_fields_from_path(model, path)
-    return (field_list[-1], new_path)
-
-
-class FieldLookupBackend(BaseFilterBackend):
-    """
-    Filter using field lookups provided via query string parameters.
-    """
-
-    RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by', 'search', 'type', 'host_filter', 'count_disabled', 'no_truncate', 'limit')
-
-    SUPPORTED_LOOKUPS = (
-        'exact',
-        'iexact',
-        'contains',
-        'icontains',
-        'startswith',
-        'istartswith',
-        'endswith',
-        'iendswith',
-        'regex',
-        'iregex',
-        'gt',
-        'gte',
-        'lt',
-        'lte',
-        'in',
-        'isnull',
-        'search',
-    )
-
-    # A list of fields that we know can be filtered on without the possibility
-    # of introducing duplicates
-    NO_DUPLICATES_ALLOW_LIST = (CharField, IntegerField, BooleanField, TextField)
-
-    def get_fields_from_lookup(self, model, lookup):
-        if '__' in lookup and lookup.rsplit('__', 1)[-1] in self.SUPPORTED_LOOKUPS:
-            path, suffix = lookup.rsplit('__', 1)
-        else:
-            path = lookup
-            suffix = 'exact'
-
-        if not path:
-            raise ParseError(_('Query string field name not provided.'))
-
-        # FIXME: Could build up a list of models used across relationships, use
-        # those lookups combined with request.user.get_queryset(Model) to make
-        # sure user cannot query using objects he could not view.
-        field_list, new_path = get_fields_from_path(model, path)
-
-        new_lookup = new_path
-        new_lookup = '__'.join([new_path, suffix])
-        return field_list, new_lookup
-
-    def get_field_from_lookup(self, model, lookup):
-        '''Method to match return type of single field, if needed.'''
-        field_list, new_lookup = self.get_fields_from_lookup(model, lookup)
-        return (field_list[-1], new_lookup)
-
-    def to_python_related(self, value):
-        value = force_str(value)
-        if value.lower() in ('none', 'null'):
-            return None
-        else:
-            return int(value)
-
-    def value_to_python_for_field(self, field, value):
-        if isinstance(field, models.BooleanField):
-            return to_python_boolean(value)
-        elif isinstance(field, (ForeignObjectRel, ManyToManyField, GenericForeignKey, ForeignKey)):
-            try:
-                return self.to_python_related(value)
-            except ValueError:
-                raise ParseError(_('Invalid {field_name} id: {field_id}').format(field_name=getattr(field, 'name', 'related field'), field_id=value))
-        else:
-            return field.to_python(value)
-
-    def value_to_python(self, model, lookup, value):
-        try:
-            lookup.encode("ascii")
-        except UnicodeEncodeError:
-            raise ValueError("%r is not an allowed field name. Must be ascii encodable." % lookup)
-
-        field_list, new_lookup = self.get_fields_from_lookup(model, lookup)
-        field = field_list[-1]
-
-        needs_distinct = not all(isinstance(f, self.NO_DUPLICATES_ALLOW_LIST) for f in field_list)
-
-        # Type names are stored without underscores internally, but are presented and
-        # and serialized over the API containing underscores so we remove `_`
-        # for polymorphic_ctype__model lookups.
-        if new_lookup.startswith('polymorphic_ctype__model'):
-            value = value.replace('_', '')
-        elif new_lookup.endswith('__isnull'):
-            value = to_python_boolean(value)
-        elif new_lookup.endswith('__in'):
-            items = []
-            if not value:
-                raise ValueError('cannot provide empty value for __in')
-            for item in value.split(','):
-                items.append(self.value_to_python_for_field(field, item))
-            value = items
-        elif new_lookup.endswith('__regex') or new_lookup.endswith('__iregex'):
-            try:
-                re.compile(value)
-            except re.error as e:
-                raise ValueError(e.args[0])
-        elif new_lookup.endswith('__iexact'):
-            if not isinstance(field, (CharField, TextField)):
-                raise ValueError(f'{field.name} is not a text field and cannot be filtered by case-insensitive search')
-        elif new_lookup.endswith('__search'):
-            related_model = getattr(field, 'related_model', None)
-            if not related_model:
-                raise ValueError('%s is not searchable' % new_lookup[:-8])
-            new_lookups = []
-            for rm_field in related_model._meta.fields:
-                if rm_field.name in ('username', 'first_name', 'last_name', 'email', 'name', 'description', 'playbook'):
-                    new_lookups.append('{}__{}__icontains'.format(new_lookup[:-8], rm_field.name))
-            return value, new_lookups, needs_distinct
-        else:
-            if isinstance(field, JSONField):
-                new_lookup = new_lookup.replace(field.name, f'{field.name}_as_txt')
-            value = self.value_to_python_for_field(field, value)
-        return value, new_lookup, needs_distinct
-
-    def filter_queryset(self, request, queryset, view):
-        try:
-            # Apply filters specified via query_params. Each entry in the lists
-            # below is (negate, field, value).
-            and_filters = []
-            or_filters = []
-            chain_filters = []
-            role_filters = []
-            search_filters = {}
-            needs_distinct = False
-            # Can only have two values: 'AND', 'OR'
-            # If 'AND' is used, an item must satisfy all conditions to show up in the results.
-            # If 'OR' is used, an item just needs to satisfy one condition to appear in results.
-            search_filter_relation = 'OR'
-            for key, values in request.query_params.lists():
-                if key in self.RESERVED_NAMES:
-                    continue
-
-                # HACK: make `created` available via API for the Django User ORM model
-                # so it keep compatibility with other objects which exposes the `created` attr.
-                if queryset.model._meta.object_name == 'User' and key.startswith('created'):
-                    key = key.replace('created', 'date_joined')
-
-                # HACK: Make job event filtering by host name mostly work even
-                # when not capturing job event hosts M2M.
-                if queryset.model._meta.object_name == 'JobEvent' and key.startswith('hosts__name'):
-                    key = key.replace('hosts__name', 'or__host__name')
-                    or_filters.append((False, 'host__name__isnull', True))
-
-                # Custom __int filter suffix (internal use only).
-                q_int = False
-                if key.endswith('__int'):
-                    key = key[:-5]
-                    q_int = True
-
-                # RBAC filtering
-                if key == 'role_level':
-                    role_filters.append(values[0])
-                    continue
-
-                # Search across related objects.
-                if key.endswith('__search'):
-                    if values and ',' in values[0]:
-                        search_filter_relation = 'AND'
-                        values = reduce(lambda list1, list2: list1 + list2, [i.split(',') for i in values])
-                    for value in values:
-                        search_value, new_keys, _ = self.value_to_python(queryset.model, key, force_str(value))
-                        assert isinstance(new_keys, list)
-                        search_filters[search_value] = new_keys
-                    # by definition, search *only* joins across relations,
-                    # so it _always_ needs a .distinct()
-                    needs_distinct = True
-                    continue
-
-                # Custom chain__ and or__ filters, mutually exclusive (both can
-                # precede not__).
-                q_chain = False
-                q_or = False
-                if key.startswith('chain__'):
-                    key = key[7:]
-                    q_chain = True
-                elif key.startswith('or__'):
-                    key = key[4:]
-                    q_or = True
-
-                # Custom not__ filter prefix.
-                q_not = False
-                if key.startswith('not__'):
-                    key = key[5:]
-                    q_not = True
-
-                # Convert value(s) to python and add to the appropriate list.
-                for value in values:
-                    if q_int:
-                        value = int(value)
-                    value, new_key, distinct = self.value_to_python(queryset.model, key, value)
-                    if distinct:
-                        needs_distinct = True
-                    if '_as_txt' in new_key:
-                        fname = next(item for item in new_key.split('__') if item.endswith('_as_txt'))
-                        queryset = queryset.annotate(**{fname: Cast(fname[:-7], output_field=TextField())})
-                    if q_chain:
-                        chain_filters.append((q_not, new_key, value))
-                    elif q_or:
-                        or_filters.append((q_not, new_key, value))
-                    else:
-                        and_filters.append((q_not, new_key, value))
-
-            # Now build Q objects for database query filter.
-            if and_filters or or_filters or chain_filters or role_filters or search_filters:
-                args = []
-                for n, k, v in and_filters:
-                    if n:
-                        args.append(~Q(**{k: v}))
-                    else:
-                        args.append(Q(**{k: v}))
-                for role_name in role_filters:
-                    if not hasattr(queryset.model, 'accessible_pk_qs'):
-                        raise ParseError(_('Cannot apply role_level filter to this list because its model does not use roles for access control.'))
-                    args.append(Q(pk__in=queryset.model.accessible_pk_qs(request.user, role_name)))
-                if or_filters:
-                    q = Q()
-                    for n, k, v in or_filters:
-                        if n:
-                            q |= ~Q(**{k: v})
-                        else:
-                            q |= Q(**{k: v})
-                    args.append(q)
-                if search_filters and search_filter_relation == 'OR':
-                    q = Q()
-                    for term, constrains in search_filters.items():
-                        for constrain in constrains:
-                            q |= Q(**{constrain: term})
-                    args.append(q)
-                elif search_filters and search_filter_relation == 'AND':
-                    for term, constrains in search_filters.items():
-                        q_chain = Q()
-                        for constrain in constrains:
-                            q_chain |= Q(**{constrain: term})
-                        queryset = queryset.filter(q_chain)
-                for n, k, v in chain_filters:
-                    if n:
-                        q = ~Q(**{k: v})
-                    else:
-                        q = Q(**{k: v})
-                    queryset = queryset.filter(q)
-                queryset = queryset.filter(*args)
-                if needs_distinct:
-                    queryset = queryset.distinct()
-            return queryset
-        except (FieldError, FieldDoesNotExist, ValueError, TypeError) as e:
-            raise ParseError(e.args[0])
-        except ValidationError as e:
-            raise ParseError(json.dumps(e.messages, ensure_ascii=False))
-
-
-class OrderByBackend(BaseFilterBackend):
-    """
-    Filter to apply ordering based on query string parameters.
-    """
-
-    def filter_queryset(self, request, queryset, view):
-        try:
-            order_by = None
-            for key, value in request.query_params.items():
-                if key in ('order', 'order_by'):
-                    order_by = value
-                    if ',' in value:
-                        order_by = value.split(',')
-                    else:
-                        order_by = (value,)
-            default_order_by = self.get_default_ordering(view)
-            # glue the order by and default order by together so that the default is the backup option
-            order_by = list(order_by or []) + list(default_order_by or [])
-            if order_by:
-                order_by = self._validate_ordering_fields(queryset.model, order_by)
-                # Special handling of the type field for ordering. In this
-                # case, we're not sorting exactly on the type field, but
-                # given the limited number of views with multiple types,
-                # sorting on polymorphic_ctype.model is effectively the same.
-                new_order_by = []
-                if 'polymorphic_ctype' in get_all_field_names(queryset.model):
-                    for field in order_by:
-                        if field == 'type':
-                            new_order_by.append('polymorphic_ctype__model')
-                        elif field == '-type':
-                            new_order_by.append('-polymorphic_ctype__model')
-                        else:
-                            new_order_by.append(field)
-                else:
-                    for field in order_by:
-                        if field not in ('type', '-type'):
-                            new_order_by.append(field)
-                queryset = queryset.order_by(*new_order_by)
-            return queryset
-        except FieldError as e:
-            # Return a 400 for invalid field names.
-            raise ParseError(*e.args)
-
-    def get_default_ordering(self, view):
-        ordering = getattr(view, 'ordering', None)
-        if isinstance(ordering, str):
-            return (ordering,)
-        return ordering
-
-    def _validate_ordering_fields(self, model, order_by):
-        for field_name in order_by:
-            # strip off the negation prefix `-` if it exists
-            prefix = ''
-            path = field_name
-            if field_name[0] == '-':
-                prefix = field_name[0]
-                path = field_name[1:]
-            try:
-                field, new_path = get_field_from_path(model, path)
-                new_path = '{}{}'.format(prefix, new_path)
-            except (FieldError, FieldDoesNotExist) as e:
-                raise ParseError(e.args[0])
-            yield new_path

awx/api/generics.py

@@ -13,8 +13,8 @@ from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import FieldDoesNotExist
 from django.db import connection, transaction
 from django.db.models.fields.related import OneToOneRel
-from django.http import QueryDict
-from django.shortcuts import get_object_or_404
+from django.http import QueryDict, JsonResponse
+from django.shortcuts import get_object_or_404, redirect
 from django.template.loader import render_to_string
 from django.utils.encoding import smart_str
 from django.utils.safestring import mark_safe
@@ -30,13 +30,23 @@ from rest_framework.permissions import IsAuthenticated
 from rest_framework.renderers import StaticHTMLRenderer
 from rest_framework.negotiation import DefaultContentNegotiation
 
+# Shared code for the AWX platform
+from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name
+
+# django-ansible-base
+from ansible_base.rest_filters.rest_framework.field_lookup_backend import FieldLookupBackend
+from ansible_base.lib.utils.models import get_all_field_names
+from ansible_base.lib.utils.requests import get_remote_host, is_proxied_request
+from ansible_base.rbac.models import RoleEvaluation, RoleDefinition
+from ansible_base.rbac.permission_registry import permission_registry
+from ansible_base.jwt_consumer.common.util import validate_x_trusted_proxy_header
+
 # AWX
-from awx.api.filters import FieldLookupBackend
 from awx.main.models import UnifiedJob, UnifiedJobTemplate, User, Role, Credential, WorkflowJobTemplateNode, WorkflowApprovalTemplate
+from awx.main.models.rbac import give_creator_permissions
 from awx.main.access import optimize_queryset
 from awx.main.utils import camelcase_to_underscore, get_search_fields, getattrd, get_object_or_400, decrypt_field, get_awx_version
-from awx.main.utils.db import get_all_field_names
-from awx.main.utils.licensing import server_product_name
+from awx.main.utils.proxy import is_proxy_in_headers, delete_headers_starting_with_http
 from awx.main.views import ApiErrorView
 from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer
 from awx.api.versioning import URLPathVersioning
@@ -71,7 +81,14 @@ analytics_logger = logging.getLogger('awx.analytics.performance')
 
 
 class LoggedLoginView(auth_views.LoginView):
+
     def get(self, request, *args, **kwargs):
+        if is_proxied_request():
+            next = request.GET.get('next', "")
+            if next:
+                next = f"?next={next}"
+            return redirect(f"/{next}")
+
         # The django.auth.contrib login form doesn't perform the content
         # negotiation we've come to expect from DRF; add in code to catch
         # situations where Accept != text/html (or */*) and reply with
@@ -87,26 +104,46 @@ class LoggedLoginView(auth_views.LoginView):
         return super(LoggedLoginView, self).get(request, *args, **kwargs)
 
     def post(self, request, *args, **kwargs):
+        if is_proxied_request():
+            # Give a message, saying to login via AAP
+            return JsonResponse(
+                {
+                    'detail': _('Please log in via Platform Authentication.'),
+                },
+                status=status.HTTP_401_UNAUTHORIZED,
+            )
+
         ret = super(LoggedLoginView, self).post(request, *args, **kwargs)
+        ip = get_remote_host(request)  # request.META.get('REMOTE_ADDR', None)
         if request.user.is_authenticated:
-            logger.info(smart_str(u"User {} logged in from {}".format(self.request.user.username, request.META.get('REMOTE_ADDR', None))))
-            ret.set_cookie('userLoggedIn', 'true')
+            logger.info(smart_str(u"User {} logged in from {}".format(self.request.user.username, ip)))
+            ret.set_cookie(
+                'userLoggedIn', 'true', secure=getattr(settings, 'SESSION_COOKIE_SECURE', False), samesite=getattr(settings, 'USER_COOKIE_SAMESITE', 'Lax')
+            )
             ret.setdefault('X-API-Session-Cookie-Name', getattr(settings, 'SESSION_COOKIE_NAME', 'awx_sessionid'))
 
             return ret
         else:
             if 'username' in self.request.POST:
-                logger.warning(smart_str(u"Login failed for user {} from {}".format(self.request.POST.get('username'), request.META.get('REMOTE_ADDR', None))))
+                logger.warning(smart_str(u"Login failed for user {} from {}".format(self.request.POST.get('username'), ip)))
             ret.status_code = 401
             return ret
 
 
 class LoggedLogoutView(auth_views.LogoutView):
+    success_url_allowed_hosts = set(settings.LOGOUT_ALLOWED_HOSTS.split(",")) if settings.LOGOUT_ALLOWED_HOSTS else set()
+
     def dispatch(self, request, *args, **kwargs):
+        if is_proxied_request():
+            # 1) We intentionally don't obey ?next= here, just always redirect to platform login
+            # 2) Hack to prevent rewrites of Location header
+            qs = "?__gateway_no_rewrite__=1&next=/"
+            return redirect(f"/api/gateway/v1/logout/{qs}")
+
         original_user = getattr(request, 'user', None)
         ret = super(LoggedLogoutView, self).dispatch(request, *args, **kwargs)
         current_user = getattr(request, 'user', None)
-        ret.set_cookie('userLoggedIn', 'false')
+        ret.set_cookie('userLoggedIn', 'false', secure=getattr(settings, 'SESSION_COOKIE_SECURE', False))
         if (not current_user or not getattr(current_user, 'pk', True)) and current_user != original_user:
             logger.info("User {} logged out.".format(original_user.username))
         return ret
@@ -124,10 +161,10 @@ def get_view_description(view, html=False):
 
 
 def get_default_schema():
-    if settings.SETTINGS_MODULE == 'awx.settings.development':
-        from awx.api.swagger import AutoSchema
+    if settings.DYNACONF.is_development_mode:
+        from awx.api.swagger import schema_view
 
-        return AutoSchema()
+        return schema_view
     else:
         return views.APIView.schema
 
@@ -141,22 +178,23 @@ class APIView(views.APIView):
         Store the Django REST Framework Request object as an attribute on the
         normal Django request, store time the request started.
         """
+        remote_headers = ['REMOTE_ADDR', 'REMOTE_HOST']
+
         self.time_started = time.time()
         if getattr(settings, 'SQL_DEBUG', False):
             self.queries_before = len(connection.queries)
 
+        if 'HTTP_X_TRUSTED_PROXY' in request.environ:
+            if validate_x_trusted_proxy_header(request.environ['HTTP_X_TRUSTED_PROXY']):
+                remote_headers = settings.REMOTE_HOST_HEADERS
+            else:
+                logger.warning("Request appeared to be a trusted upstream proxy but failed to provide a matching shared secret.")
+
         # If there are any custom headers in REMOTE_HOST_HEADERS, make sure
         # they respect the allowed proxy list
-        if all(
-            [
-                settings.PROXY_IP_ALLOWED_LIST,
-                request.environ.get('REMOTE_ADDR') not in settings.PROXY_IP_ALLOWED_LIST,
-                request.environ.get('REMOTE_HOST') not in settings.PROXY_IP_ALLOWED_LIST,
-            ]
-        ):
-            for custom_header in settings.REMOTE_HOST_HEADERS:
-                if custom_header.startswith('HTTP_'):
-                    request.environ.pop(custom_header, None)
+        if settings.PROXY_IP_ALLOWED_LIST:
+            if not is_proxy_in_headers(self.request, settings.PROXY_IP_ALLOWED_LIST, remote_headers):
+                delete_headers_starting_with_http(request, settings.REMOTE_HOST_HEADERS)
 
         drf_request = super(APIView, self).initialize_request(request, *args, **kwargs)
         request.drf_request = drf_request
@@ -201,17 +239,21 @@ class APIView(views.APIView):
             return response
 
         if response.status_code >= 400:
+            ip = get_remote_host(request)  # request.META.get('REMOTE_ADDR', None)
             msg_data = {
                 'status_code': response.status_code,
                 'user_name': request.user,
                 'url_path': request.path,
-                'remote_addr': request.META.get('REMOTE_ADDR', None),
+                'remote_addr': ip,
             }
 
             if type(response.data) is dict:
                 msg_data['error'] = response.data.get('error', response.status_text)
             elif type(response.data) is list:
-                msg_data['error'] = ", ".join(list(map(lambda x: x.get('error', response.status_text), response.data)))
+                if len(response.data) > 0 and isinstance(response.data[0], str):
+                    msg_data['error'] = str(response.data[0])
+                else:
+                    msg_data['error'] = ", ".join(list(map(lambda x: x.get('error', response.status_text), response.data)))
             else:
                 msg_data['error'] = response.status_text
 
@@ -225,7 +267,8 @@ class APIView(views.APIView):
             if hasattr(self, '__init_request_error__'):
                 response = self.handle_exception(self.__init_request_error__)
             if response.status_code == 401:
-                response.data['detail'] += _(' To establish a login session, visit') + ' /api/login/.'
+                if response.data and 'detail' in response.data:
+                    response.data['detail'] += _(' To establish a login session, visit') + ' /api/login/.'
                 logger.info(status_msg)
             else:
                 logger.warning(status_msg)
@@ -234,7 +277,7 @@ class APIView(views.APIView):
         time_started = getattr(self, 'time_started', None)
         if request.user.is_authenticated:
             response['X-API-Product-Version'] = get_awx_version()
-        response['X-API-Product-Name'] = server_product_name()
+        response['X-API-Product-Name'] = detect_server_product_name()
 
         response['X-API-Node'] = settings.CLUSTER_HOST_ID
         if time_started:
@@ -331,12 +374,6 @@ class APIView(views.APIView):
                 kwargs.pop('version')
         return super(APIView, self).dispatch(request, *args, **kwargs)
 
-    def check_permissions(self, request):
-        if request.method not in ('GET', 'OPTIONS', 'HEAD'):
-            if 'write' not in getattr(request.user, 'oauth_scopes', ['write']):
-                raise PermissionDenied()
-        return super(APIView, self).check_permissions(request)
-
 
 class GenericAPIView(generics.GenericAPIView, APIView):
     # Base class for all model-based views.
@@ -471,7 +508,11 @@ class ListAPIView(generics.ListAPIView, GenericAPIView):
 
 class ListCreateAPIView(ListAPIView, generics.ListCreateAPIView):
     # Base class for a list view that allows creating new objects.
-    pass
+    def perform_create(self, serializer):
+        super().perform_create(serializer)
+        if serializer.Meta.model in permission_registry.all_registered_models:
+            if self.request and self.request.user:
+                give_creator_permissions(self.request.user, serializer.instance)
 
 
 class ParentMixin(object):
@@ -791,6 +832,7 @@ class RetrieveUpdateDestroyAPIView(RetrieveUpdateAPIView, DestroyAPIView):
 
 
 class ResourceAccessList(ParentMixin, ListAPIView):
+    deprecated = True
     serializer_class = ResourceAccessListElementSerializer
     ordering = ('username',)
 
@@ -798,6 +840,15 @@ class ResourceAccessList(ParentMixin, ListAPIView):
         obj = self.get_parent_object()
 
         content_type = ContentType.objects.get_for_model(obj)
+
+        if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+            ancestors = set(RoleEvaluation.objects.filter(content_type_id=content_type.id, object_id=obj.id).values_list('role_id', flat=True))
+            qs = User.objects.filter(has_roles__in=ancestors) | User.objects.filter(is_superuser=True)
+            auditor_role = RoleDefinition.objects.filter(name="Controller System Auditor").first()
+            if auditor_role:
+                qs |= User.objects.filter(role_assignments__role_definition=auditor_role)
+            return qs.distinct()
+
         roles = set(Role.objects.filter(content_type=content_type, object_id=obj.id))
 
         ancestors = set()
@@ -957,7 +1008,7 @@ class CopyAPIView(GenericAPIView):
             None, None, self.model, obj, request.user, create_kwargs=create_kwargs, copy_name=serializer.validated_data.get('name', '')
         )
         if hasattr(new_obj, 'admin_role') and request.user not in new_obj.admin_role.members.all():
-            new_obj.admin_role.members.add(request.user)
+            give_creator_permissions(request.user, new_obj)
         if sub_objs:
             permission_check_func = None
             if hasattr(type(self), 'deep_copy_permission_check_func'):

awx/api/metadata.py

@@ -36,11 +36,13 @@ class Metadata(metadata.SimpleMetadata):
         field_info = OrderedDict()
         field_info['type'] = self.label_lookup[field]
         field_info['required'] = getattr(field, 'required', False)
+        field_info['hidden'] = getattr(field, 'hidden', False)
 
         text_attrs = [
             'read_only',
             'label',
             'help_text',
+            'warning_text',
             'min_length',
             'max_length',
             'min_value',
@@ -101,7 +103,7 @@ class Metadata(metadata.SimpleMetadata):
             default = field.get_default()
             if type(default) is UUID:
                 default = 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
-            if field.field_name == 'TOWER_URL_BASE' and default == 'https://towerhost':
+            if field.field_name == 'TOWER_URL_BASE' and default == 'https://platformhost':
                 default = '{}://{}'.format(self.request.scheme, self.request.get_host())
             field_info['default'] = default
         except serializers.SkipField:

awx/api/swagger.py

@@ -1,62 +1,54 @@
 import warnings
 
 from rest_framework.permissions import AllowAny
-from rest_framework.schemas import SchemaGenerator, AutoSchema as DRFAuthSchema
-
-from drf_yasg.views import get_schema_view
 from drf_yasg import openapi
+from drf_yasg.inspectors import SwaggerAutoSchema
+from drf_yasg.views import get_schema_view
 
 
-class SuperUserSchemaGenerator(SchemaGenerator):
-    def has_view_permissions(self, path, method, view):
-        #
-        # Generate the Swagger schema as if you were a superuser and
-        # permissions didn't matter; this short-circuits the schema path
-        # discovery to include _all_ potential paths in the API.
-        #
-        return True
+class CustomSwaggerAutoSchema(SwaggerAutoSchema):
+    """Custom SwaggerAutoSchema to add swagger_topic to tags."""
 
-
-class AutoSchema(DRFAuthSchema):
-    def get_link(self, path, method, base_url):
-        link = super(AutoSchema, self).get_link(path, method, base_url)
+    def get_tags(self, operation_keys=None):
+        tags = []
         try:
-            serializer = self.view.get_serializer()
+            if hasattr(self.view, 'get_serializer'):
+                serializer = self.view.get_serializer()
+            else:
+                serializer = None
         except Exception:
             serializer = None
             warnings.warn(
                 '{}.get_serializer() raised an exception during '
                 'schema generation. Serializer fields will not be '
-                'generated for {} {}.'.format(self.view.__class__.__name__, method, path)
+                'generated for {}.'.format(self.view.__class__.__name__, operation_keys)
             )
-
-        link.__dict__['deprecated'] = getattr(self.view, 'deprecated', False)
-
-        # auto-generate a topic/tag for the serializer based on its model
         if hasattr(self.view, 'swagger_topic'):
-            link.__dict__['topic'] = str(self.view.swagger_topic).title()
+            tags.append(str(self.view.swagger_topic).title())
         elif serializer and hasattr(serializer, 'Meta'):
-            link.__dict__['topic'] = str(serializer.Meta.model._meta.verbose_name_plural).title()
+            tags.append(str(serializer.Meta.model._meta.verbose_name_plural).title())
         elif hasattr(self.view, 'model'):
-            link.__dict__['topic'] = str(self.view.model._meta.verbose_name_plural).title()
+            tags.append(str(self.view.model._meta.verbose_name_plural).title())
         else:
-            warnings.warn('Could not determine a Swagger tag for path {}'.format(path))
-        return link
+            tags = ['api']  # Fallback to default value
 
-    def get_description(self, path, method):
-        setattr(self.view.request, 'swagger_method', method)
-        description = super(AutoSchema, self).get_description(path, method)
-        return description
+        if not tags:
+            warnings.warn(f'Could not determine tags for {self.view.__class__.__name__}')
+        return tags
+
+    def is_deprecated(self):
+        """Return `True` if this operation is to be marked as deprecated."""
+        return getattr(self.view, 'deprecated', False)
 
 
 schema_view = get_schema_view(
     openapi.Info(
-        title="Snippets API",
-        default_version='v1',
-        description="Test description",
-        terms_of_service="https://www.google.com/policies/terms/",
-        contact=openapi.Contact(email="contact@snippets.local"),
-        license=openapi.License(name="BSD License"),
+        title='AWX API',
+        default_version='v2',
+        description='AWX API Documentation',
+        terms_of_service='https://www.google.com/policies/terms/',
+        contact=openapi.Contact(email='contact@snippets.local'),
+        license=openapi.License(name='Apache License'),
     ),
     public=True,
     permission_classes=[AllowAny],

awx/api/templates/api/api_o_auth_authorization_root_view.md

@@ -1,114 +0,0 @@
-# Token Handling using OAuth2
-
-This page lists OAuth 2 utility endpoints used for authorization, token refresh and revoke.
-Note endpoints other than `/api/o/authorize/` are not meant to be used in browsers and do not
-support HTTP GET. The endpoints here strictly follow
-[RFC specs for OAuth2](https://tools.ietf.org/html/rfc6749), so please use that for detailed
-reference. Note AWX net location default to `http://localhost:8013` in examples:
-
-
-## Create Token for an Application using Authorization code grant type
-Given an application "AuthCodeApp" of grant type `authorization-code`, 
-from the client app, the user makes a GET to the Authorize endpoint with 
-
-* `response_type`
-* `client_id`
-* `redirect_uris`
-* `scope`  
-
-AWX will respond with the authorization `code` and `state`
-to the redirect_uri specified in the application. The client application will then make a POST to the
-`api/o/token/` endpoint on AWX with
-
-* `code`
-* `client_id`
-* `client_secret`
-* `grant_type`
-* `redirect_uri`
-
-AWX will respond with the `access_token`, `token_type`, `refresh_token`, and `expires_in`. For more
-information on testing this flow, refer to [django-oauth-toolkit](http://django-oauth-toolkit.readthedocs.io/en/latest/tutorial/tutorial_01.html#test-your-authorization-server).
-
-
-## Create Token for an Application using Password grant type
-
-Log in is not required for `password` grant type, so a simple `curl` can be used to acquire a personal access token
-via `/api/o/token/` with 
-
-* `grant_type`: Required to be "password"
-* `username`
-* `password`
-* `client_id`: Associated application must have grant_type "password"
-* `client_secret`
-
-For example:
-
-```bash
-curl -X POST \
-  -H "Content-Type: application/x-www-form-urlencoded" \
-  -d "grant_type=password&username=<username>&password=<password>&scope=read" \
-  -u "gwSPoasWSdNkMDtBN3Hu2WYQpPWCO9SwUEsKK22l:fI6ZpfocHYBGfm1tP92r0yIgCyfRdDQt0Tos9L8a4fNsJjQQMwp9569e
-IaUBsaVDgt2eiwOGe0bg5m5vCSstClZmtdy359RVx2rQK5YlIWyPlrolpt2LEpVeKXWaiybo" \
-  http://localhost:8013/api/o/token/ -i
-```
-In the above post request, parameters `username` and `password` are username and password of the related
-AWX user of the underlying application, and the authentication information is of format
-`<client_id>:<client_secret>`, where `client_id` and `client_secret` are the corresponding fields of
-underlying application.
-
-Upon success, access token, refresh token and other information are given in the response body in JSON
-format:
-
-```text
-{
-"access_token": "9epHOqHhnXUcgYK8QanOmUQPSgX92g", 
-"token_type": "Bearer", 
-"expires_in": 31536000000, 
-"refresh_token": "jMRX6QvzOTf046KHee3TU5mT3nyXsz", 
-"scope": "read"
-}
-```
-
-
-## Refresh an existing access token
-
-The `/api/o/token/` endpoint is used for refreshing access token:
-```bash
-curl -X POST \
-  -H "Content-Type: application/x-www-form-urlencoded" \
-  -d "grant_type=refresh_token&refresh_token=AL0NK9TTpv0qp54dGbC4VUZtsZ9r8z" \
-  -u "gwSPoasWSdNkMDtBN3Hu2WYQpPWCO9SwUEsKK22l:fI6ZpfocHYBGfm1tP92r0yIgCyfRdDQt0Tos9L8a4fNsJjQQMwp9569eIaUBsaVDgt2eiwOGe0bg5m5vCSstClZmtdy359RVx2rQK5YlIWyPlrolpt2LEpVeKXWaiybo" \
-  http://localhost:8013/api/o/token/ -i
-```
-In the above post request, `refresh_token` is provided by `refresh_token` field of the access token
-above. The authentication information is of format `<client_id>:<client_secret>`, where `client_id`
-and `client_secret` are the corresponding fields of underlying related application of the access token.
-
-Upon success, the new (refreshed) access token with the same scope information as the previous one is
-given in the response body in JSON format:
-```text
-{
-"access_token": "NDInWxGJI4iZgqpsreujjbvzCfJqgR", 
-"token_type": "Bearer", 
-"expires_in": 31536000000, 
-"refresh_token": "DqOrmz8bx3srlHkZNKmDpqA86bnQkT", 
-"scope": "read write"
-}
-```
-Internally, the refresh operation deletes the existing token and a new token is created immediately
-after, with information like scope and related application identical to the original one. We can
-verify by checking the new token is present at the `api/v2/tokens` endpoint.  
-
-## Revoke an access token
-Revoking an access token is the same as deleting the token resource object. 
-Revoking is done by POSTing to `/api/o/revoke_token/` with the token to revoke as parameter:
-
-```bash
-curl -X POST -d "token=rQONsve372fQwuc2pn76k3IHDCYpi7" \
-  -H "Content-Type: application/x-www-form-urlencoded" \
-  -u "gwSPoasWSdNkMDtBN3Hu2WYQpPWCO9SwUEsKK22l:fI6ZpfocHYBGfm1tP92r0yIgCyfRdDQt0Tos9L8a4fNsJjQQMwp9569eIaUBsaVDgt2eiwOGe0bg5m5vCSstClZmtdy359RVx2rQK5YlIWyPlrolpt2LEpVeKXWaiybo" \
-  http://localhost:8013/api/o/revoke_token/ -i
-```
-`200 OK` means a successful delete.
-
-

awx/api/templates/api/bulk_host_delete_view.md

@@ -0,0 +1,22 @@
+# Bulk Host Delete
+
+This endpoint allows the client to delete multiple hosts from inventories.
+They may do this by providing a list of hosts ID's to be deleted.
+
+Example:
+
+    {
+        "hosts": [1, 2, 3, 4, 5]
+    }
+
+Return data:
+
+    {
+        "hosts": {
+            "1": "The host a1 was deleted",
+            "2": "The host a2 was deleted",
+            "3": "The host a3 was deleted",
+            "4": "The host a4 was deleted",
+            "5": "The host a5 was deleted",
+        }
+    }

awx/api/templates/instance_install_bundle/group_vars/all.yml

@@ -17,19 +17,18 @@ custom_worksign_public_keyfile: receptor/work_public_key.pem
 custom_tls_certfile: receptor/tls/receptor.crt
 custom_tls_keyfile: receptor/tls/receptor.key
 custom_ca_certfile: receptor/tls/ca/mesh-CA.crt
-receptor_protocol: 'tcp'
-{% if instance.listener_port %}
+{% if listener_port %}
+receptor_protocol: {{ listener_protocol }}
 receptor_listener: true
-receptor_port: {{ instance.listener_port }}
+receptor_port: {{ listener_port }}
 {% else %}
 receptor_listener: false
 {% endif %}
 {% if peers %}
 receptor_peers:
 {% for peer in peers %}
-  - host: {{ peer.host }}
-    port: {{ peer.port }}
-    protocol: tcp
+  - address: {{ peer.address }}
+    protocol: {{ peer.protocol }}
 {% endfor %}
 {% endif %}
 {% verbatim %}

awx/api/templates/instance_install_bundle/install_receptor.yml

@@ -2,6 +2,12 @@
 - hosts: all
   become: yes
   tasks:
+    - name: Create the receptor group
+      group:
+{% verbatim %}
+        name: "{{ receptor_group }}"
+{% endverbatim %}
+        state: present
     - name: Create the receptor user
       user:
 {% verbatim %}

awx/api/templates/instance_install_bundle/requirements.yml

@@ -1,4 +1,4 @@
 ---
 collections:
   - name: ansible.receptor
-    version: 2.0.0
+    version: 2.0.3

awx/api/urls/instance.py

@@ -10,6 +10,7 @@ from awx.api.views import (
     InstanceInstanceGroupsList,
     InstanceHealthCheck,
     InstancePeersList,
+    InstanceReceptorAddressesList,
 )
 from awx.api.views.instance_install_bundle import InstanceInstallBundle
 
@@ -21,6 +22,7 @@ urls = [
     re_path(r'^(?P<pk>[0-9]+)/instance_groups/$', InstanceInstanceGroupsList.as_view(), name='instance_instance_groups_list'),
     re_path(r'^(?P<pk>[0-9]+)/health_check/$', InstanceHealthCheck.as_view(), name='instance_health_check'),
     re_path(r'^(?P<pk>[0-9]+)/peers/$', InstancePeersList.as_view(), name='instance_peers_list'),
+    re_path(r'^(?P<pk>[0-9]+)/receptor_addresses/$', InstanceReceptorAddressesList.as_view(), name='instance_receptor_addresses_list'),
     re_path(r'^(?P<pk>[0-9]+)/install_bundle/$', InstanceInstallBundle.as_view(), name='instance_install_bundle'),
 ]
 

awx/api/urls/oauth2.py

@@ -1,27 +0,0 @@
-# Copyright (c) 2017 Ansible, Inc.
-# All Rights Reserved.
-
-from django.urls import re_path
-
-from awx.api.views import (
-    OAuth2ApplicationList,
-    OAuth2ApplicationDetail,
-    ApplicationOAuth2TokenList,
-    OAuth2ApplicationActivityStreamList,
-    OAuth2TokenList,
-    OAuth2TokenDetail,
-    OAuth2TokenActivityStreamList,
-)
-
-
-urls = [
-    re_path(r'^applications/$', OAuth2ApplicationList.as_view(), name='o_auth2_application_list'),
-    re_path(r'^applications/(?P<pk>[0-9]+)/$', OAuth2ApplicationDetail.as_view(), name='o_auth2_application_detail'),
-    re_path(r'^applications/(?P<pk>[0-9]+)/tokens/$', ApplicationOAuth2TokenList.as_view(), name='o_auth2_application_token_list'),
-    re_path(r'^applications/(?P<pk>[0-9]+)/activity_stream/$', OAuth2ApplicationActivityStreamList.as_view(), name='o_auth2_application_activity_stream_list'),
-    re_path(r'^tokens/$', OAuth2TokenList.as_view(), name='o_auth2_token_list'),
-    re_path(r'^tokens/(?P<pk>[0-9]+)/$', OAuth2TokenDetail.as_view(), name='o_auth2_token_detail'),
-    re_path(r'^tokens/(?P<pk>[0-9]+)/activity_stream/$', OAuth2TokenActivityStreamList.as_view(), name='o_auth2_token_activity_stream_list'),
-]
-
-__all__ = ['urls']

awx/api/urls/oauth2_root.py

@@ -1,45 +0,0 @@
-# Copyright (c) 2017 Ansible, Inc.
-# All Rights Reserved.
-from datetime import timedelta
-
-from django.utils.timezone import now
-from django.conf import settings
-from django.urls import re_path
-
-from oauthlib import oauth2
-from oauth2_provider import views
-
-from awx.main.models import RefreshToken
-from awx.api.views.root import ApiOAuthAuthorizationRootView
-
-
-class TokenView(views.TokenView):
-    def create_token_response(self, request):
-        # Django OAuth2 Toolkit has a bug whereby refresh tokens are *never*
-        # properly expired (ugh):
-        #
-        # https://github.com/jazzband/django-oauth-toolkit/issues/746
-        #
-        # This code detects and auto-expires them on refresh grant
-        # requests.
-        if request.POST.get('grant_type') == 'refresh_token' and 'refresh_token' in request.POST:
-            refresh_token = RefreshToken.objects.filter(token=request.POST['refresh_token']).first()
-            if refresh_token:
-                expire_seconds = settings.OAUTH2_PROVIDER.get('REFRESH_TOKEN_EXPIRE_SECONDS', 0)
-                if refresh_token.created + timedelta(seconds=expire_seconds) < now():
-                    return request.build_absolute_uri(), {}, 'The refresh token has expired.', '403'
-        try:
-            return super(TokenView, self).create_token_response(request)
-        except oauth2.AccessDeniedError as e:
-            return request.build_absolute_uri(), {}, str(e), '403'
-
-
-urls = [
-    re_path(r'^$', ApiOAuthAuthorizationRootView.as_view(), name='oauth_authorization_root_view'),
-    re_path(r"^authorize/$", views.AuthorizationView.as_view(), name="authorize"),
-    re_path(r"^token/$", TokenView.as_view(), name="token"),
-    re_path(r"^revoke_token/$", views.RevokeTokenView.as_view(), name="revoke-token"),
-]
-
-
-__all__ = ['urls']

awx/api/urls/organization.py

@@ -25,7 +25,7 @@ from awx.api.views.organization import (
     OrganizationObjectRolesList,
     OrganizationAccessList,
 )
-from awx.api.views import OrganizationCredentialList, OrganizationApplicationList
+from awx.api.views import OrganizationCredentialList
 
 
 urls = [
@@ -66,7 +66,6 @@ urls = [
     re_path(r'^(?P<pk>[0-9]+)/galaxy_credentials/$', OrganizationGalaxyCredentialsList.as_view(), name='organization_galaxy_credentials_list'),
     re_path(r'^(?P<pk>[0-9]+)/object_roles/$', OrganizationObjectRolesList.as_view(), name='organization_object_roles_list'),
     re_path(r'^(?P<pk>[0-9]+)/access_list/$', OrganizationAccessList.as_view(), name='organization_access_list'),
-    re_path(r'^(?P<pk>[0-9]+)/applications/$', OrganizationApplicationList.as_view(), name='organization_applications_list'),
 ]
 
 __all__ = ['urls']

awx/api/urls/receptor_address.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.urls import re_path
+
+from awx.api.views import (
+    ReceptorAddressesList,
+    ReceptorAddressDetail,
+)
+
+
+urls = [
+    re_path(r'^$', ReceptorAddressesList.as_view(), name='receptor_addresses_list'),
+    re_path(r'^(?P<pk>[0-9]+)/$', ReceptorAddressDetail.as_view(), name='receptor_address_detail'),
+]
+
+__all__ = ['urls']

awx/api/urls/urls.py

@@ -15,7 +15,6 @@ from awx.api.views.root import (
     ApiV2AttachView,
 )
 from awx.api.views import (
-    AuthView,
     UserMeList,
     DashboardView,
     DashboardJobsGraphView,
@@ -26,16 +25,13 @@ from awx.api.views import (
     JobTemplateCredentialsList,
     SchedulePreview,
     ScheduleZoneInfo,
-    OAuth2ApplicationList,
-    OAuth2TokenList,
-    ApplicationOAuth2TokenList,
-    OAuth2ApplicationDetail,
     HostMetricSummaryMonthlyList,
 )
 
 from awx.api.views.bulk import (
     BulkView,
     BulkHostCreateView,
+    BulkHostDeleteView,
     BulkJobLaunchView,
 )
 
@@ -79,11 +75,10 @@ from .schedule import urls as schedule_urls
 from .activity_stream import urls as activity_stream_urls
 from .instance import urls as instance_urls
 from .instance_group import urls as instance_group_urls
-from .oauth2 import urls as oauth2_urls
-from .oauth2_root import urls as oauth2_root_urls
 from .workflow_approval_template import urls as workflow_approval_template_urls
 from .workflow_approval import urls as workflow_approval_urls
 from .analytics import urls as analytics_urls
+from .receptor_address import urls as receptor_address_urls
 
 v2_urls = [
     re_path(r'^$', ApiV2RootView.as_view(), name='api_v2_root_view'),
@@ -94,17 +89,11 @@ v2_urls = [
     re_path(r'^job_templates/(?P<pk>[0-9]+)/credentials/$', JobTemplateCredentialsList.as_view(), name='job_template_credentials_list'),
     re_path(r'^schedules/preview/$', SchedulePreview.as_view(), name='schedule_rrule'),
     re_path(r'^schedules/zoneinfo/$', ScheduleZoneInfo.as_view(), name='schedule_zoneinfo'),
-    re_path(r'^applications/$', OAuth2ApplicationList.as_view(), name='o_auth2_application_list'),
-    re_path(r'^applications/(?P<pk>[0-9]+)/$', OAuth2ApplicationDetail.as_view(), name='o_auth2_application_detail'),
-    re_path(r'^applications/(?P<pk>[0-9]+)/tokens/$', ApplicationOAuth2TokenList.as_view(), name='application_o_auth2_token_list'),
-    re_path(r'^tokens/$', OAuth2TokenList.as_view(), name='o_auth2_token_list'),
-    re_path(r'^', include(oauth2_urls)),
     re_path(r'^metrics/$', MetricsView.as_view(), name='metrics_view'),
     re_path(r'^ping/$', ApiV2PingView.as_view(), name='api_v2_ping_view'),
     re_path(r'^config/$', ApiV2ConfigView.as_view(), name='api_v2_config_view'),
     re_path(r'^config/subscriptions/$', ApiV2SubscriptionView.as_view(), name='api_v2_subscription_view'),
     re_path(r'^config/attach/$', ApiV2AttachView.as_view(), name='api_v2_attach_view'),
-    re_path(r'^auth/$', AuthView.as_view()),
     re_path(r'^me/$', UserMeList.as_view(), name='user_me_list'),
     re_path(r'^dashboard/$', DashboardView.as_view(), name='dashboard_view'),
     re_path(r'^dashboard/graphs/jobs/$', DashboardJobsGraphView.as_view(), name='dashboard_jobs_graph_view'),
@@ -152,7 +141,9 @@ v2_urls = [
     re_path(r'^workflow_approvals/', include(workflow_approval_urls)),
     re_path(r'^bulk/$', BulkView.as_view(), name='bulk'),
     re_path(r'^bulk/host_create/$', BulkHostCreateView.as_view(), name='bulk_host_create'),
+    re_path(r'^bulk/host_delete/$', BulkHostDeleteView.as_view(), name='bulk_host_delete'),
     re_path(r'^bulk/job_launch/$', BulkJobLaunchView.as_view(), name='bulk_job_launch'),
+    re_path(r'^receptor_addresses/', include(receptor_address_urls)),
 ]
 
 
@@ -162,7 +153,6 @@ urlpatterns = [
     re_path(r'^(?P<version>(v2))/', include(v2_urls)),
     re_path(r'^login/$', LoggedLoginView.as_view(template_name='rest_framework/login.html', extra_context={'inside_login_context': True}), name='login'),
     re_path(r'^logout/$', LoggedLogoutView.as_view(next_page='/api/', redirect_field_name='next'), name='logout'),
-    re_path(r'^o/', include(oauth2_root_urls)),
 ]
 if MODE == 'development':
     # Only include these if we are in the development environment

awx/api/urls/user.py

@@ -14,10 +14,6 @@ from awx.api.views import (
     UserRolesList,
     UserActivityStreamList,
     UserAccessList,
-    OAuth2ApplicationList,
-    OAuth2UserTokenList,
-    UserPersonalTokenList,
-    UserAuthorizedTokenList,
 )
 
 urls = [
@@ -31,10 +27,6 @@ urls = [
     re_path(r'^(?P<pk>[0-9]+)/roles/$', UserRolesList.as_view(), name='user_roles_list'),
     re_path(r'^(?P<pk>[0-9]+)/activity_stream/$', UserActivityStreamList.as_view(), name='user_activity_stream_list'),
     re_path(r'^(?P<pk>[0-9]+)/access_list/$', UserAccessList.as_view(), name='user_access_list'),
-    re_path(r'^(?P<pk>[0-9]+)/applications/$', OAuth2ApplicationList.as_view(), name='o_auth2_application_list'),
-    re_path(r'^(?P<pk>[0-9]+)/tokens/$', OAuth2UserTokenList.as_view(), name='o_auth2_token_list'),
-    re_path(r'^(?P<pk>[0-9]+)/authorized_tokens/$', UserAuthorizedTokenList.as_view(), name='user_authorized_token_list'),
-    re_path(r'^(?P<pk>[0-9]+)/personal_tokens/$', UserPersonalTokenList.as_view(), name='user_personal_token_list'),
 ]
 
 __all__ = ['urls']

awx/api/urls/webhooks.py

@@ -1,10 +1,11 @@
 from django.urls import re_path
 
-from awx.api.views.webhooks import WebhookKeyView, GithubWebhookReceiver, GitlabWebhookReceiver
+from awx.api.views.webhooks import WebhookKeyView, GithubWebhookReceiver, GitlabWebhookReceiver, BitbucketDcWebhookReceiver
 
 
 urlpatterns = [
     re_path(r'^webhook_key/$', WebhookKeyView.as_view(), name='webhook_key'),
     re_path(r'^github/$', GithubWebhookReceiver.as_view(), name='webhook_receiver_github'),
     re_path(r'^gitlab/$', GitlabWebhookReceiver.as_view(), name='webhook_receiver_gitlab'),
+    re_path(r'^bitbucket_dc/$', BitbucketDcWebhookReceiver.as_view(), name='webhook_receiver_bitbucket_dc'),
 ]

awx/api/versioning.py

@@ -2,28 +2,21 @@
 # All Rights Reserved.
 
 from django.conf import settings
-from django.urls import NoReverseMatch
 
-from rest_framework.reverse import _reverse
+from rest_framework.reverse import reverse as drf_reverse
 from rest_framework.versioning import URLPathVersioning as BaseVersioning
 
 
-def drf_reverse(viewname, args=None, kwargs=None, request=None, format=None, **extra):
-    """
-    Copy and monkey-patch `rest_framework.reverse.reverse` to prevent adding unwarranted
-    query string parameters.
-    """
-    scheme = getattr(request, 'versioning_scheme', None)
-    if scheme is not None:
-        try:
-            url = scheme.reverse(viewname, args, kwargs, request, format, **extra)
-        except NoReverseMatch:
-            # In case the versioning scheme reversal fails, fallback to the
-            # default implementation
-            url = _reverse(viewname, args, kwargs, request, format, **extra)
-    else:
-        url = _reverse(viewname, args, kwargs, request, format, **extra)
+def is_optional_api_urlpattern_prefix_request(request):
+    if settings.OPTIONAL_API_URLPATTERN_PREFIX and request:
+        if request.path.startswith(f"/api/{settings.OPTIONAL_API_URLPATTERN_PREFIX}"):
+            return True
+    return False
 
+
+def transform_optional_api_urlpattern_prefix_url(request, url):
+    if is_optional_api_urlpattern_prefix_request(request):
+        url = url.replace('/api', f"/api/{settings.OPTIONAL_API_URLPATTERN_PREFIX}")
     return url
 
 

awx/api/views/__init__.py

@@ -33,11 +33,10 @@ from django.http import HttpResponse, HttpResponseRedirect
 from django.contrib.contenttypes.models import ContentType
 from django.utils.translation import gettext_lazy as _
 
-
 # Django REST Framework
 from rest_framework.exceptions import APIException, PermissionDenied, ParseError, NotFound
 from rest_framework.parsers import FormParser
-from rest_framework.permissions import AllowAny, IsAuthenticated
+from rest_framework.permissions import IsAuthenticated
 from rest_framework.renderers import JSONRenderer, StaticHTMLRenderer
 from rest_framework.response import Response
 from rest_framework.settings import api_settings
@@ -48,18 +47,17 @@ from rest_framework import status
 from rest_framework_yaml.parsers import YAMLParser
 from rest_framework_yaml.renderers import YAMLRenderer
 
-# ANSIConv
-import ansiconv
-
-# Python Social Auth
-from social_core.backends.utils import load_backends
-
-# Django OAuth Toolkit
-from oauth2_provider.models import get_access_token_model
+# ansi2html
+from ansi2html import Ansi2HTMLConverter
 
 import pytz
 from wsgiref.util import FileWrapper
 
+# django-ansible-base
+from ansible_base.lib.utils.requests import get_remote_hosts
+from ansible_base.rbac.models import RoleEvaluation, ObjectRole
+from ansible_base.resource_registry.shared_types import OrganizationType, TeamType, UserType
+
 # AWX
 from awx.main.tasks.system import send_notifications, update_inventory_computed_fields
 from awx.main.access import get_user_queryset
@@ -87,6 +85,7 @@ from awx.api.generics import (
 from awx.api.views.labels import LabelSubListCreateAttachDetachView
 from awx.api.versioning import reverse
 from awx.main import models
+from awx.main.models.rbac import get_role_definition
 from awx.main.utils import (
     camelcase_to_underscore,
     extract_ansible_vars,
@@ -98,6 +97,7 @@ from awx.main.utils import (
 )
 from awx.main.utils.encryption import encrypt_value
 from awx.main.utils.filters import SmartFilter
+from awx.main.utils.plugins import compute_cloud_inventory_sources
 from awx.main.redact import UriCleaner
 from awx.api.permissions import (
     JobTemplateCallbackPermission,
@@ -128,6 +128,10 @@ logger = logging.getLogger('awx.api.views')
 
 
 def unpartitioned_event_horizon(cls):
+    with connection.cursor() as cursor:
+        cursor.execute(f"SELECT 1 FROM INFORMATION_SCHEMA.TABLES WHERE table_name = '_unpartitioned_{cls._meta.db_table}';")
+        if not cursor.fetchone():
+            return 0
     with connection.cursor() as cursor:
         try:
             cursor.execute(f'SELECT MAX(id) FROM _unpartitioned_{cls._meta.db_table}')
@@ -268,16 +272,24 @@ class DashboardJobsGraphView(APIView):
 
         success_query = user_unified_jobs.filter(status='successful')
         failed_query = user_unified_jobs.filter(status='failed')
+        canceled_query = user_unified_jobs.filter(status='canceled')
+        error_query = user_unified_jobs.filter(status='error')
 
         if job_type == 'inv_sync':
             success_query = success_query.filter(instance_of=models.InventoryUpdate)
             failed_query = failed_query.filter(instance_of=models.InventoryUpdate)
+            canceled_query = canceled_query.filter(instance_of=models.InventoryUpdate)
+            error_query = error_query.filter(instance_of=models.InventoryUpdate)
         elif job_type == 'playbook_run':
             success_query = success_query.filter(instance_of=models.Job)
             failed_query = failed_query.filter(instance_of=models.Job)
+            canceled_query = canceled_query.filter(instance_of=models.Job)
+            error_query = error_query.filter(instance_of=models.Job)
         elif job_type == 'scm_update':
             success_query = success_query.filter(instance_of=models.ProjectUpdate)
             failed_query = failed_query.filter(instance_of=models.ProjectUpdate)
+            canceled_query = canceled_query.filter(instance_of=models.ProjectUpdate)
+            error_query = error_query.filter(instance_of=models.ProjectUpdate)
 
         end = now()
         interval = 'day'
@@ -293,10 +305,12 @@ class DashboardJobsGraphView(APIView):
         else:
             return Response({'error': _('Unknown period "%s"') % str(period)}, status=status.HTTP_400_BAD_REQUEST)
 
-        dashboard_data = {"jobs": {"successful": [], "failed": []}}
+        dashboard_data = {"jobs": {"successful": [], "failed": [], "canceled": [], "error": []}}
 
         succ_list = dashboard_data['jobs']['successful']
         fail_list = dashboard_data['jobs']['failed']
+        canceled_list = dashboard_data['jobs']['canceled']
+        error_list = dashboard_data['jobs']['error']
 
         qs_s = (
             success_query.filter(finished__range=(start, end))
@@ -314,6 +328,22 @@ class DashboardJobsGraphView(APIView):
             .annotate(agg=Count('id', distinct=True))
         )
         data_f = {item['d']: item['agg'] for item in qs_f}
+        qs_c = (
+            canceled_query.filter(finished__range=(start, end))
+            .annotate(d=Trunc('finished', interval, tzinfo=end.tzinfo))
+            .order_by()
+            .values('d')
+            .annotate(agg=Count('id', distinct=True))
+        )
+        data_c = {item['d']: item['agg'] for item in qs_c}
+        qs_e = (
+            error_query.filter(finished__range=(start, end))
+            .annotate(d=Trunc('finished', interval, tzinfo=end.tzinfo))
+            .order_by()
+            .values('d')
+            .annotate(agg=Count('id', distinct=True))
+        )
+        data_e = {item['d']: item['agg'] for item in qs_e}
 
         start_date = start.replace(hour=0, minute=0, second=0, microsecond=0)
         for d in itertools.count():
@@ -322,6 +352,8 @@ class DashboardJobsGraphView(APIView):
                 break
             succ_list.append([time.mktime(date.timetuple()), data_s.get(date, 0)])
             fail_list.append([time.mktime(date.timetuple()), data_f.get(date, 0)])
+            canceled_list.append([time.mktime(date.timetuple()), data_c.get(date, 0)])
+            error_list.append([time.mktime(date.timetuple()), data_e.get(date, 0)])
 
         return Response(dashboard_data)
 
@@ -333,12 +365,20 @@ class InstanceList(ListCreateAPIView):
     search_fields = ('hostname',)
     ordering = ('id',)
 
+    def get_queryset(self):
+        qs = super().get_queryset().prefetch_related('receptor_addresses')
+        return qs
+
 
 class InstanceDetail(RetrieveUpdateAPIView):
     name = _("Instance Detail")
     model = models.Instance
     serializer_class = serializers.InstanceSerializer
 
+    def get_queryset(self):
+        qs = super().get_queryset().prefetch_related('receptor_addresses')
+        return qs
+
     def update_raw_data(self, data):
         # these fields are only valid on creation of an instance, so they unwanted on detail view
         data.pop('node_type', None)
@@ -371,13 +411,37 @@ class InstanceUnifiedJobsList(SubListAPIView):
 
 
 class InstancePeersList(SubListAPIView):
-    name = _("Instance Peers")
+    name = _("Peers")
+    model = models.ReceptorAddress
+    serializer_class = serializers.ReceptorAddressSerializer
     parent_model = models.Instance
-    model = models.Instance
-    serializer_class = serializers.InstanceSerializer
     parent_access = 'read'
-    search_fields = {'hostname'}
     relationship = 'peers'
+    search_fields = ('address',)
+
+
+class InstanceReceptorAddressesList(SubListAPIView):
+    name = _("Receptor Addresses")
+    model = models.ReceptorAddress
+    parent_key = 'instance'
+    parent_model = models.Instance
+    serializer_class = serializers.ReceptorAddressSerializer
+    search_fields = ('address',)
+
+
+class ReceptorAddressesList(ListAPIView):
+    name = _("Receptor Addresses")
+    model = models.ReceptorAddress
+    serializer_class = serializers.ReceptorAddressSerializer
+    search_fields = ('address',)
+
+
+class ReceptorAddressDetail(RetrieveAPIView):
+    name = _("Receptor Address Detail")
+    model = models.ReceptorAddress
+    serializer_class = serializers.ReceptorAddressSerializer
+    parent_model = models.Instance
+    relationship = 'receptor_addresses'
 
 
 class InstanceInstanceGroupsList(InstanceGroupMembershipMixin, SubListCreateAttachDetachAPIView):
@@ -472,6 +536,7 @@ class InstanceGroupAccessList(ResourceAccessList):
 
 
 class InstanceGroupObjectRolesList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.InstanceGroup
@@ -606,51 +671,81 @@ class ScheduleUnifiedJobsList(SubListAPIView):
     name = _('Schedule Jobs List')
 
 
-class AuthView(APIView):
-    '''List enabled single-sign-on endpoints'''
+def immutablesharedfields(cls):
+    '''
+    Class decorator to prevent modifying shared resources when ALLOW_LOCAL_RESOURCE_MANAGEMENT setting is set to False.
 
-    authentication_classes = []
-    permission_classes = (AllowAny,)
-    swagger_topic = 'System Configuration'
+    Works by overriding these view methods:
+    - create
+    - delete
+    - perform_update
+    create and delete are overridden to raise a PermissionDenied exception.
+    perform_update is overridden to check if any shared fields are being modified,
+    and raise a PermissionDenied exception if so.
+    '''
+    # create instead of perform_create because some of our views
+    # override create instead of perform_create
+    if hasattr(cls, 'create'):
+        cls.original_create = cls.create
 
-    def get(self, request):
-        from rest_framework.reverse import reverse
+        @functools.wraps(cls.create)
+        def create_wrapper(*args, **kwargs):
+            if settings.ALLOW_LOCAL_RESOURCE_MANAGEMENT:
+                return cls.original_create(*args, **kwargs)
+            raise PermissionDenied({'detail': _('Creation of this resource is not allowed. Create this resource via the platform ingress.')})
 
-        data = OrderedDict()
-        err_backend, err_message = request.session.get('social_auth_error', (None, None))
-        auth_backends = list(load_backends(settings.AUTHENTICATION_BACKENDS, force_load=True).items())
-        # Return auth backends in consistent order: Google, GitHub, SAML.
-        auth_backends.sort(key=lambda x: 'g' if x[0] == 'google-oauth2' else x[0])
-        for name, backend in auth_backends:
-            login_url = reverse('social:begin', args=(name,))
-            complete_url = request.build_absolute_uri(reverse('social:complete', args=(name,)))
-            backend_data = {'login_url': login_url, 'complete_url': complete_url}
-            if name == 'saml':
-                backend_data['metadata_url'] = reverse('sso:saml_metadata')
-                for idp in sorted(settings.SOCIAL_AUTH_SAML_ENABLED_IDPS.keys()):
-                    saml_backend_data = dict(backend_data.items())
-                    saml_backend_data['login_url'] = '%s?idp=%s' % (login_url, idp)
-                    full_backend_name = '%s:%s' % (name, idp)
-                    if (err_backend == full_backend_name or err_backend == name) and err_message:
-                        saml_backend_data['error'] = err_message
-                    data[full_backend_name] = saml_backend_data
-            else:
-                if err_backend == name and err_message:
-                    backend_data['error'] = err_message
-                data[name] = backend_data
-        return Response(data)
+        cls.create = create_wrapper
+
+    if hasattr(cls, 'delete'):
+        cls.original_delete = cls.delete
+
+        @functools.wraps(cls.delete)
+        def delete_wrapper(*args, **kwargs):
+            if settings.ALLOW_LOCAL_RESOURCE_MANAGEMENT:
+                return cls.original_delete(*args, **kwargs)
+            raise PermissionDenied({'detail': _('Deletion of this resource is not allowed. Delete this resource via the platform ingress.')})
+
+        cls.delete = delete_wrapper
+
+    if hasattr(cls, 'perform_update'):
+        cls.original_perform_update = cls.perform_update
+
+        @functools.wraps(cls.perform_update)
+        def update_wrapper(*args, **kwargs):
+            if not settings.ALLOW_LOCAL_RESOURCE_MANAGEMENT:
+                view, serializer = args
+                instance = view.get_object()
+                if instance:
+                    if isinstance(instance, models.Organization):
+                        shared_fields = OrganizationType._declared_fields.keys()
+                    elif isinstance(instance, models.User):
+                        shared_fields = UserType._declared_fields.keys()
+                    elif isinstance(instance, models.Team):
+                        shared_fields = TeamType._declared_fields.keys()
+                    attrs = serializer.validated_data
+                    for field in shared_fields:
+                        if field in attrs and getattr(instance, field) != attrs[field]:
+                            raise PermissionDenied({field: _(f"Cannot change shared field '{field}'. Alter this field via the platform ingress.")})
+            return cls.original_perform_update(*args, **kwargs)
+
+        cls.perform_update = update_wrapper
+
+    return cls
 
 
+@immutablesharedfields
 class TeamList(ListCreateAPIView):
     model = models.Team
     serializer_class = serializers.TeamSerializer
 
 
+@immutablesharedfields
 class TeamDetail(RetrieveUpdateDestroyAPIView):
     model = models.Team
     serializer_class = serializers.TeamSerializer
 
 
+@immutablesharedfields
 class TeamUsersList(BaseUsersList):
     model = models.User
     serializer_class = serializers.UserSerializer
@@ -660,6 +755,7 @@ class TeamUsersList(BaseUsersList):
 
 
 class TeamRolesList(SubListAttachDetachAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializerWithParentAccess
     metadata_class = RoleMetadata
@@ -699,10 +795,12 @@ class TeamRolesList(SubListAttachDetachAPIView):
 
 
 class TeamObjectRolesList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.Team
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()
@@ -720,8 +818,15 @@ class TeamProjectsList(SubListAPIView):
         self.check_parent_access(team)
         model_ct = ContentType.objects.get_for_model(self.model)
         parent_ct = ContentType.objects.get_for_model(self.parent_model)
-        proj_roles = models.Role.objects.filter(Q(ancestors__content_type=parent_ct) & Q(ancestors__object_id=team.pk), content_type=model_ct)
-        return self.model.accessible_objects(self.request.user, 'read_role').filter(pk__in=[t.content_object.pk for t in proj_roles])
+
+        rd = get_role_definition(team.member_role)
+        role = ObjectRole.objects.filter(object_id=team.id, content_type=parent_ct, role_definition=rd).first()
+        if role is None:
+            # Team has no permissions, therefore team has no projects
+            return self.model.objects.none()
+        else:
+            project_qs = self.model.accessible_objects(self.request.user, 'read_role')
+            return project_qs.filter(id__in=RoleEvaluation.objects.filter(content_type_id=model_ct.id, role=role).values_list('object_id'))
 
 
 class TeamActivityStreamList(SubListAPIView):
@@ -736,10 +841,23 @@ class TeamActivityStreamList(SubListAPIView):
         self.check_parent_access(parent)
 
         qs = self.request.user.get_queryset(self.model)
+
         return qs.filter(
             Q(team=parent)
-            | Q(project__in=models.Project.accessible_objects(parent, 'read_role'))
-            | Q(credential__in=models.Credential.accessible_objects(parent, 'read_role'))
+            | Q(
+                project__in=RoleEvaluation.objects.filter(
+                    role__in=parent.has_roles.all(), content_type_id=ContentType.objects.get_for_model(models.Project).id, codename='view_project'
+                )
+                .values_list('object_id')
+                .distinct()
+            )
+            | Q(
+                credential__in=RoleEvaluation.objects.filter(
+                    role__in=parent.has_roles.all(), content_type_id=ContentType.objects.get_for_model(models.Credential).id, codename='view_credential'
+                )
+                .values_list('object_id')
+                .distinct()
+            )
         )
 
 
@@ -991,10 +1109,12 @@ class ProjectAccessList(ResourceAccessList):
 
 
 class ProjectObjectRolesList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.Project
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()
@@ -1007,6 +1127,7 @@ class ProjectCopy(CopyAPIView):
     copy_return_serializer_class = serializers.ProjectSerializer
 
 
+@immutablesharedfields
 class UserList(ListCreateAPIView):
     model = models.User
     serializer_class = serializers.UserSerializer
@@ -1024,121 +1145,6 @@ class UserMeList(ListAPIView):
         return self.model.objects.filter(pk=self.request.user.pk)
 
 
-class OAuth2ApplicationList(ListCreateAPIView):
-    name = _("OAuth 2 Applications")
-
-    model = models.OAuth2Application
-    serializer_class = serializers.OAuth2ApplicationSerializer
-    swagger_topic = 'Authentication'
-
-
-class OAuth2ApplicationDetail(RetrieveUpdateDestroyAPIView):
-    name = _("OAuth 2 Application Detail")
-
-    model = models.OAuth2Application
-    serializer_class = serializers.OAuth2ApplicationSerializer
-    swagger_topic = 'Authentication'
-
-    def update_raw_data(self, data):
-        data.pop('client_secret', None)
-        return super(OAuth2ApplicationDetail, self).update_raw_data(data)
-
-
-class ApplicationOAuth2TokenList(SubListCreateAPIView):
-    name = _("OAuth 2 Application Tokens")
-
-    model = models.OAuth2AccessToken
-    serializer_class = serializers.OAuth2TokenSerializer
-    parent_model = models.OAuth2Application
-    relationship = 'oauth2accesstoken_set'
-    parent_key = 'application'
-    swagger_topic = 'Authentication'
-
-
-class OAuth2ApplicationActivityStreamList(SubListAPIView):
-    model = models.ActivityStream
-    serializer_class = serializers.ActivityStreamSerializer
-    parent_model = models.OAuth2Application
-    relationship = 'activitystream_set'
-    swagger_topic = 'Authentication'
-    search_fields = ('changes',)
-
-
-class OAuth2TokenList(ListCreateAPIView):
-    name = _("OAuth2 Tokens")
-
-    model = models.OAuth2AccessToken
-    serializer_class = serializers.OAuth2TokenSerializer
-    swagger_topic = 'Authentication'
-
-
-class OAuth2UserTokenList(SubListCreateAPIView):
-    name = _("OAuth2 User Tokens")
-
-    model = models.OAuth2AccessToken
-    serializer_class = serializers.OAuth2TokenSerializer
-    parent_model = models.User
-    relationship = 'main_oauth2accesstoken'
-    parent_key = 'user'
-    swagger_topic = 'Authentication'
-
-
-class UserAuthorizedTokenList(SubListCreateAPIView):
-    name = _("OAuth2 User Authorized Access Tokens")
-
-    model = models.OAuth2AccessToken
-    serializer_class = serializers.UserAuthorizedTokenSerializer
-    parent_model = models.User
-    relationship = 'oauth2accesstoken_set'
-    parent_key = 'user'
-    swagger_topic = 'Authentication'
-
-    def get_queryset(self):
-        return get_access_token_model().objects.filter(application__isnull=False, user=self.request.user)
-
-
-class OrganizationApplicationList(SubListCreateAPIView):
-    name = _("Organization OAuth2 Applications")
-
-    model = models.OAuth2Application
-    serializer_class = serializers.OAuth2ApplicationSerializer
-    parent_model = models.Organization
-    relationship = 'applications'
-    parent_key = 'organization'
-    swagger_topic = 'Authentication'
-
-
-class UserPersonalTokenList(SubListCreateAPIView):
-    name = _("OAuth2 Personal Access Tokens")
-
-    model = models.OAuth2AccessToken
-    serializer_class = serializers.UserPersonalTokenSerializer
-    parent_model = models.User
-    relationship = 'main_oauth2accesstoken'
-    parent_key = 'user'
-    swagger_topic = 'Authentication'
-
-    def get_queryset(self):
-        return get_access_token_model().objects.filter(application__isnull=True, user=self.request.user)
-
-
-class OAuth2TokenDetail(RetrieveUpdateDestroyAPIView):
-    name = _("OAuth Token Detail")
-
-    model = models.OAuth2AccessToken
-    serializer_class = serializers.OAuth2TokenDetailSerializer
-    swagger_topic = 'Authentication'
-
-
-class OAuth2TokenActivityStreamList(SubListAPIView):
-    model = models.ActivityStream
-    serializer_class = serializers.ActivityStreamSerializer
-    parent_model = models.OAuth2AccessToken
-    relationship = 'activitystream_set'
-    swagger_topic = 'Authentication'
-    search_fields = ('changes',)
-
-
 class UserTeamsList(SubListAPIView):
     model = models.Team
     serializer_class = serializers.TeamSerializer
@@ -1152,6 +1158,7 @@ class UserTeamsList(SubListAPIView):
 
 
 class UserRolesList(SubListAttachDetachAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializerWithParentAccess
     metadata_class = RoleMetadata
@@ -1176,7 +1183,16 @@ class UserRolesList(SubListAttachDetachAPIView):
         user = get_object_or_400(models.User, pk=self.kwargs['pk'])
         role = get_object_or_400(models.Role, pk=sub_id)
 
-        credential_content_type = ContentType.objects.get_for_model(models.Credential)
+        content_types = ContentType.objects.get_for_models(models.Organization, models.Team, models.Credential)  # dict of {model: content_type}
+        # Prevent user to be associated with team/org when ALLOW_LOCAL_RESOURCE_MANAGEMENT is False
+        if not settings.ALLOW_LOCAL_RESOURCE_MANAGEMENT:
+            for model in [models.Organization, models.Team]:
+                ct = content_types[model]
+                if role.content_type == ct and role.role_field in ['member_role', 'admin_role']:
+                    data = dict(msg=_(f"Cannot directly modify user membership to {ct.model}. Direct shared resource management disabled"))
+                    return Response(data, status=status.HTTP_403_FORBIDDEN)
+
+        credential_content_type = content_types[models.Credential]
         if role.content_type == credential_content_type:
             if 'disassociate' not in request.data and role.content_object.organization and user not in role.content_object.organization.member_role:
                 data = dict(msg=_("You cannot grant credential access to a user not in the credentials' organization"))
@@ -1248,6 +1264,7 @@ class UserActivityStreamList(SubListAPIView):
         return qs.filter(Q(actor=parent) | Q(user__in=[parent]))
 
 
+@immutablesharedfields
 class UserDetail(RetrieveUpdateDestroyAPIView):
     model = models.User
     serializer_class = serializers.UserSerializer
@@ -1393,7 +1410,7 @@ class OrganizationCredentialList(SubListCreateAPIView):
         self.check_parent_access(organization)
 
         user_visible = models.Credential.accessible_objects(self.request.user, 'read_role').all()
-        org_set = models.Credential.accessible_objects(organization.admin_role, 'read_role').all()
+        org_set = models.Credential.objects.filter(organization=organization)
 
         if self.request.user.is_superuser or self.request.user.is_system_auditor:
             return org_set
@@ -1426,10 +1443,12 @@ class CredentialAccessList(ResourceAccessList):
 
 
 class CredentialObjectRolesList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.Credential
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()
@@ -2060,9 +2079,9 @@ class InventorySourceNotificationTemplatesAnyList(SubListCreateAttachDetachAPIVi
 
     def post(self, request, *args, **kwargs):
         parent = self.get_parent_object()
-        if parent.source not in models.CLOUD_INVENTORY_SOURCES:
+        if parent.source not in compute_cloud_inventory_sources():
             return Response(
-                dict(msg=_("Notification Templates can only be assigned when source is one of {}.").format(models.CLOUD_INVENTORY_SOURCES, parent.source)),
+                dict(msg=_("Notification Templates can only be assigned when source is one of {}.").format(compute_cloud_inventory_sources(), parent.source)),
                 status=status.HTTP_400_BAD_REQUEST,
             )
         return super(InventorySourceNotificationTemplatesAnyList, self).post(request, *args, **kwargs)
@@ -2216,12 +2235,16 @@ class JobTemplateList(ListCreateAPIView):
     serializer_class = serializers.JobTemplateSerializer
     always_allow_superuser = False
 
-    def post(self, request, *args, **kwargs):
-        ret = super(JobTemplateList, self).post(request, *args, **kwargs)
-        if ret.status_code == 201:
-            job_template = models.JobTemplate.objects.get(id=ret.data['id'])
-            job_template.admin_role.members.add(request.user)
-        return ret
+    def check_permissions(self, request):
+        if request.method == 'POST':
+            if request.user.is_anonymous:
+                self.permission_denied(request)
+            else:
+                can_access, messages = request.user.can_access_with_errors(self.model, 'add', request.data)
+                if not can_access:
+                    self.permission_denied(request, message=messages)
+
+        super(JobTemplateList, self).check_permissions(request)
 
 
 class JobTemplateDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIView):
@@ -2602,12 +2625,7 @@ class JobTemplateCallback(GenericAPIView):
         host for the current request.
         """
         # Find the list of remote host names/IPs to check.
-        remote_hosts = set()
-        for header in settings.REMOTE_HOST_HEADERS:
-            for value in self.request.META.get(header, '').split(','):
-                value = value.strip()
-                if value:
-                    remote_hosts.add(value)
+        remote_hosts = set(get_remote_hosts(self.request))
         # Add the reverse lookup of IP addresses.
         for rh in list(remote_hosts):
             try:
@@ -2768,10 +2786,12 @@ class JobTemplateAccessList(ResourceAccessList):
 
 
 class JobTemplateObjectRolesList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.JobTemplate
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()
@@ -2945,6 +2965,17 @@ class WorkflowJobTemplateList(ListCreateAPIView):
     serializer_class = serializers.WorkflowJobTemplateSerializer
     always_allow_superuser = False
 
+    def check_permissions(self, request):
+        if request.method == 'POST':
+            if request.user.is_anonymous:
+                self.permission_denied(request)
+            else:
+                can_access, messages = request.user.can_access_with_errors(self.model, 'add', request.data)
+                if not can_access:
+                    self.permission_denied(request, message=messages)
+
+        super(WorkflowJobTemplateList, self).check_permissions(request)
+
 
 class WorkflowJobTemplateDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIView):
     model = models.WorkflowJobTemplate
@@ -3154,10 +3185,12 @@ class WorkflowJobTemplateAccessList(ResourceAccessList):
 
 
 class WorkflowJobTemplateObjectRolesList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.WorkflowJobTemplate
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()
@@ -3402,6 +3435,7 @@ class JobRelaunch(RetrieveAPIView):
 
         copy_kwargs = {}
         retry_hosts = serializer.validated_data.get('hosts', None)
+        job_type = serializer.validated_data.get('job_type', None)
         if retry_hosts and retry_hosts != 'all':
             if obj.status in ACTIVE_STATES:
                 return Response(
@@ -3422,6 +3456,8 @@ class JobRelaunch(RetrieveAPIView):
                 )
             copy_kwargs['limit'] = ','.join(retry_host_list)
 
+        if job_type:
+            copy_kwargs['job_type'] = job_type
         new_job = obj.copy_unified_job(**copy_kwargs)
         result = new_job.signal_start(**serializer.validated_data['credential_passwords'])
         if not result:
@@ -4021,7 +4057,8 @@ class UnifiedJobStdout(RetrieveAPIView):
                 # Remove any ANSI escape sequences containing job event data.
                 content = re.sub(r'\x1b\[K(?:[A-Za-z0-9+/=]+\x1b\[\d+D)+\x1b\[K', '', content)
 
-                body = ansiconv.to_html(html.escape(content))
+                conv = Ansi2HTMLConverter()
+                body = conv.convert(html.escape(content))
 
                 context = {'title': get_view_name(self.__class__), 'body': mark_safe(body), 'dark': dark_bg, 'content_only': content_only}
                 data = render_to_string('api/stdout.html', context).strip()
@@ -4166,6 +4203,7 @@ class ActivityStreamDetail(RetrieveAPIView):
 
 
 class RoleList(ListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     permission_classes = (IsAuthenticated,)
@@ -4173,11 +4211,13 @@ class RoleList(ListAPIView):
 
 
 class RoleDetail(RetrieveAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
 
 
 class RoleUsersList(SubListAttachDetachAPIView):
+    deprecated = True
     model = models.User
     serializer_class = serializers.UserSerializer
     parent_model = models.Role
@@ -4198,7 +4238,15 @@ class RoleUsersList(SubListAttachDetachAPIView):
         user = get_object_or_400(models.User, pk=sub_id)
         role = self.get_parent_object()
 
-        credential_content_type = ContentType.objects.get_for_model(models.Credential)
+        content_types = ContentType.objects.get_for_models(models.Organization, models.Team, models.Credential)  # dict of {model: content_type}
+        if not settings.ALLOW_LOCAL_RESOURCE_MANAGEMENT:
+            for model in [models.Organization, models.Team]:
+                ct = content_types[model]
+                if role.content_type == ct and role.role_field in ['member_role', 'admin_role']:
+                    data = dict(msg=_(f"Cannot directly modify user membership to {ct.model}. Direct shared resource management disabled"))
+                    return Response(data, status=status.HTTP_403_FORBIDDEN)
+
+        credential_content_type = content_types[models.Credential]
         if role.content_type == credential_content_type:
             if 'disassociate' not in request.data and role.content_object.organization and user not in role.content_object.organization.member_role:
                 data = dict(msg=_("You cannot grant credential access to a user not in the credentials' organization"))
@@ -4212,6 +4260,7 @@ class RoleUsersList(SubListAttachDetachAPIView):
 
 
 class RoleTeamsList(SubListAttachDetachAPIView):
+    deprecated = True
     model = models.Team
     serializer_class = serializers.TeamSerializer
     parent_model = models.Role
@@ -4256,10 +4305,12 @@ class RoleTeamsList(SubListAttachDetachAPIView):
             team.member_role.children.remove(role)
         else:
             team.member_role.children.add(role)
+
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
 class RoleParentsList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.Role
@@ -4273,6 +4324,7 @@ class RoleParentsList(SubListAPIView):
 
 
 class RoleChildrenList(SubListAPIView):
+    deprecated = True
     model = models.Role
     serializer_class = serializers.RoleSerializer
     parent_model = models.Role

awx/api/views/analytics.py

@@ -10,6 +10,7 @@ from awx.api.generics import APIView, Response
 from awx.api.permissions import AnalyticsPermission
 from awx.api.versioning import reverse
 from awx.main.utils import get_awx_version
+from awx.main.utils.analytics_proxy import OIDCClient
 from rest_framework import status
 
 from collections import OrderedDict
@@ -48,23 +49,23 @@ class AnalyticsRootView(APIView):
 
     def get(self, request, format=None):
         data = OrderedDict()
-        data['authorized'] = reverse('api:analytics_authorized')
-        data['reports'] = reverse('api:analytics_reports_list')
-        data['report_options'] = reverse('api:analytics_report_options_list')
-        data['adoption_rate'] = reverse('api:analytics_adoption_rate')
-        data['adoption_rate_options'] = reverse('api:analytics_adoption_rate_options')
-        data['event_explorer'] = reverse('api:analytics_event_explorer')
-        data['event_explorer_options'] = reverse('api:analytics_event_explorer_options')
-        data['host_explorer'] = reverse('api:analytics_host_explorer')
-        data['host_explorer_options'] = reverse('api:analytics_host_explorer_options')
-        data['job_explorer'] = reverse('api:analytics_job_explorer')
-        data['job_explorer_options'] = reverse('api:analytics_job_explorer_options')
-        data['probe_templates'] = reverse('api:analytics_probe_templates_explorer')
-        data['probe_templates_options'] = reverse('api:analytics_probe_templates_options')
-        data['probe_template_for_hosts'] = reverse('api:analytics_probe_template_for_hosts_explorer')
-        data['probe_template_for_hosts_options'] = reverse('api:analytics_probe_template_for_hosts_options')
-        data['roi_templates'] = reverse('api:analytics_roi_templates_explorer')
-        data['roi_templates_options'] = reverse('api:analytics_roi_templates_options')
+        data['authorized'] = reverse('api:analytics_authorized', request=request)
+        data['reports'] = reverse('api:analytics_reports_list', request=request)
+        data['report_options'] = reverse('api:analytics_report_options_list', request=request)
+        data['adoption_rate'] = reverse('api:analytics_adoption_rate', request=request)
+        data['adoption_rate_options'] = reverse('api:analytics_adoption_rate_options', request=request)
+        data['event_explorer'] = reverse('api:analytics_event_explorer', request=request)
+        data['event_explorer_options'] = reverse('api:analytics_event_explorer_options', request=request)
+        data['host_explorer'] = reverse('api:analytics_host_explorer', request=request)
+        data['host_explorer_options'] = reverse('api:analytics_host_explorer_options', request=request)
+        data['job_explorer'] = reverse('api:analytics_job_explorer', request=request)
+        data['job_explorer_options'] = reverse('api:analytics_job_explorer_options', request=request)
+        data['probe_templates'] = reverse('api:analytics_probe_templates_explorer', request=request)
+        data['probe_templates_options'] = reverse('api:analytics_probe_templates_options', request=request)
+        data['probe_template_for_hosts'] = reverse('api:analytics_probe_template_for_hosts_explorer', request=request)
+        data['probe_template_for_hosts_options'] = reverse('api:analytics_probe_template_for_hosts_options', request=request)
+        data['roi_templates'] = reverse('api:analytics_roi_templates_explorer', request=request)
+        data['roi_templates_options'] = reverse('api:analytics_roi_templates_options', request=request)
         return Response(data)
 
 
@@ -179,28 +180,48 @@ class AnalyticsGenericView(APIView):
 
         return Response(response.content, status=response.status_code)
 
+    @staticmethod
+    def _base_auth_request(request: requests.Request, method: str, url: str, user: str, pw: str, headers: dict[str, str]) -> requests.Response:
+        response = requests.request(
+            method,
+            url,
+            auth=(user, pw),
+            verify=settings.INSIGHTS_CERT_PATH,
+            params=getattr(request, 'query_params', {}),
+            headers=headers,
+            json=getattr(request, 'data', {}),
+            timeout=(31, 31),
+        )
+        return response
+
     def _send_to_analytics(self, request, method):
         try:
             headers = self._request_headers(request)
 
             self._get_setting('INSIGHTS_TRACKING_STATE', False, ERROR_UPLOAD_NOT_ENABLED)
-            url = self._get_analytics_url(request.path)
-            rh_user = self._get_setting('REDHAT_USERNAME', None, ERROR_MISSING_USER)
-            rh_password = self._get_setting('REDHAT_PASSWORD', None, ERROR_MISSING_PASSWORD)
-
             if method not in ["GET", "POST", "OPTIONS"]:
                 return self._error_response(ERROR_UNSUPPORTED_METHOD, method, remote=False, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
-            else:
-                response = requests.request(
+            url = self._get_analytics_url(request.path)
+            try:
+                rh_user = self._get_setting('REDHAT_USERNAME', None, ERROR_MISSING_USER)
+                rh_password = self._get_setting('REDHAT_PASSWORD', None, ERROR_MISSING_PASSWORD)
+                client = OIDCClient(rh_user, rh_password)
+                response = client.make_request(
                     method,
                     url,
-                    auth=(rh_user, rh_password),
-                    verify=settings.INSIGHTS_CERT_PATH,
-                    params=request.query_params,
                     headers=headers,
-                    json=request.data,
+                    verify=settings.INSIGHTS_CERT_PATH,
+                    params=getattr(request, 'query_params', {}),
+                    json=getattr(request, 'data', {}),
                     timeout=(31, 31),
                 )
+            except requests.RequestException:
+                logger.error("Automation Analytics API request failed, trying base auth method")
+                response = self._base_auth_request(request, method, url, rh_user, rh_password, headers)
+            except MissingSettings:
+                rh_user = self._get_setting('SUBSCRIPTIONS_CLIENT_ID', None, ERROR_MISSING_USER)
+                rh_password = self._get_setting('SUBSCRIPTIONS_CLIENT_SECRET', None, ERROR_MISSING_PASSWORD)
+                response = self._base_auth_request(request, method, url, rh_user, rh_password, headers)
             #
             # Missing or wrong user/pass
             #

awx/api/views/bulk.py

@@ -34,6 +34,7 @@ class BulkView(APIView):
         '''List top level resources'''
         data = OrderedDict()
         data['host_create'] = reverse('api:bulk_host_create', request=request)
+        data['host_delete'] = reverse('api:bulk_host_delete', request=request)
         data['job_launch'] = reverse('api:bulk_job_launch', request=request)
         return Response(data)
 
@@ -72,3 +73,20 @@ class BulkHostCreateView(GenericAPIView):
             result = serializer.create(serializer.validated_data)
             return Response(result, status=status.HTTP_201_CREATED)
         return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+
+
+class BulkHostDeleteView(GenericAPIView):
+    permission_classes = [IsAuthenticated]
+    model = Host
+    serializer_class = serializers.BulkHostDeleteSerializer
+    allowed_methods = ['GET', 'POST', 'OPTIONS']
+
+    def get(self, request):
+        return Response({"detail": "Bulk delete hosts with this endpoint"}, status=status.HTTP_200_OK)
+
+    def post(self, request):
+        serializer = serializers.BulkHostDeleteSerializer(data=request.data, context={'request': request})
+        if serializer.is_valid():
+            result = serializer.delete(serializer.validated_data)
+            return Response(result, status=status.HTTP_201_CREATED)
+        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

awx/api/views/instance_install_bundle.py

@@ -124,10 +124,19 @@ def generate_inventory_yml(instance_obj):
 
 
 def generate_group_vars_all_yml(instance_obj):
+    # get peers
     peers = []
-    for instance in instance_obj.peers.all():
-        peers.append(dict(host=instance.hostname, port=instance.listener_port))
-    all_yaml = render_to_string("instance_install_bundle/group_vars/all.yml", context=dict(instance=instance_obj, peers=peers))
+    for addr in instance_obj.peers.select_related('instance'):
+        peers.append(dict(address=addr.get_full_address(), protocol=addr.protocol))
+    context = dict(instance=instance_obj, peers=peers)
+
+    canonical_addr = instance_obj.canonical_address
+    if canonical_addr:
+        context['listener_port'] = canonical_addr.port
+        protocol = canonical_addr.protocol if canonical_addr.protocol != 'wss' else 'ws'
+        context['listener_protocol'] = protocol
+
+    all_yaml = render_to_string("instance_install_bundle/group_vars/all.yml", context=context)
     # convert consecutive newlines with a single newline
     return re.sub(r'\n+', '\n', all_yaml)
 

awx/api/views/inventory.py

@@ -152,6 +152,7 @@ class InventoryObjectRolesList(SubListAPIView):
     serializer_class = RoleSerializer
     parent_model = Inventory
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()

awx/api/views/mesh_visualizer.py

@@ -17,7 +17,7 @@ class MeshVisualizer(APIView):
     def get(self, request, format=None):
         data = {
             'nodes': InstanceNodeSerializer(Instance.objects.all(), many=True).data,
-            'links': InstanceLinkSerializer(InstanceLink.objects.select_related('target', 'source'), many=True).data,
+            'links': InstanceLinkSerializer(InstanceLink.objects.select_related('target__instance', 'source'), many=True).data,
         }
 
         return Response(data)

awx/api/views/mixin.py

@@ -15,6 +15,7 @@ from rest_framework.response import Response
 from rest_framework import status
 
 from awx.main.constants import ACTIVE_STATES
+from awx.main.models import Organization
 from awx.main.utils import get_object_or_400
 from awx.main.models.ha import Instance, InstanceGroup, schedule_policy_task
 from awx.main.models.organization import Team
@@ -60,6 +61,21 @@ class UnifiedJobDeletionMixin(object):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
+class OrganizationInstanceGroupMembershipMixin(object):
+    """
+    This mixin overloads attach/detach so that it calls Organization.save(),
+    to ensure instance group updates are persisted
+    """
+
+    def unattach(self, request, *args, **kwargs):
+        with transaction.atomic():
+            organization_queryset = Organization.objects.select_for_update()
+            organization = organization_queryset.get(pk=self.get_parent_object().id)
+            response = super(OrganizationInstanceGroupMembershipMixin, self).unattach(request, *args, **kwargs)
+            organization.save()
+            return response
+
+
 class InstanceGroupMembershipMixin(object):
     """
     This mixin overloads attach/detach so that it calls InstanceGroup.save(),

awx/api/views/organization.py

@@ -52,16 +52,19 @@ from awx.api.serializers import (
     WorkflowJobTemplateSerializer,
     CredentialSerializer,
 )
-from awx.api.views.mixin import RelatedJobsPreventDeleteMixin, OrganizationCountsMixin
+from awx.api.views.mixin import RelatedJobsPreventDeleteMixin, OrganizationCountsMixin, OrganizationInstanceGroupMembershipMixin
+from awx.api.views import immutablesharedfields
 
 logger = logging.getLogger('awx.api.views.organization')
 
 
+@immutablesharedfields
 class OrganizationList(OrganizationCountsMixin, ListCreateAPIView):
     model = Organization
     serializer_class = OrganizationSerializer
 
 
+@immutablesharedfields
 class OrganizationDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIView):
     model = Organization
     serializer_class = OrganizationSerializer
@@ -104,6 +107,7 @@ class OrganizationInventoriesList(SubListAPIView):
     relationship = 'inventories'
 
 
+@immutablesharedfields
 class OrganizationUsersList(BaseUsersList):
     model = User
     serializer_class = UserSerializer
@@ -112,6 +116,7 @@ class OrganizationUsersList(BaseUsersList):
     ordering = ('username',)
 
 
+@immutablesharedfields
 class OrganizationAdminsList(BaseUsersList):
     model = User
     serializer_class = UserSerializer
@@ -150,6 +155,7 @@ class OrganizationWorkflowJobTemplatesList(SubListCreateAPIView):
     parent_key = 'organization'
 
 
+@immutablesharedfields
 class OrganizationTeamsList(SubListCreateAttachDetachAPIView):
     model = Team
     serializer_class = TeamSerializer
@@ -196,7 +202,7 @@ class OrganizationNotificationTemplatesApprovalList(OrganizationNotificationTemp
     relationship = 'notification_templates_approvals'
 
 
-class OrganizationInstanceGroupsList(SubListAttachDetachAPIView):
+class OrganizationInstanceGroupsList(OrganizationInstanceGroupMembershipMixin, SubListAttachDetachAPIView):
     model = InstanceGroup
     serializer_class = InstanceGroupSerializer
     parent_model = Organization
@@ -226,6 +232,7 @@ class OrganizationObjectRolesList(SubListAPIView):
     serializer_class = RoleSerializer
     parent_model = Organization
     search_fields = ('role_field', 'content_type__model')
+    deprecated = True
 
     def get_queryset(self):
         po = self.get_parent_object()

awx/api/views/root.py

@@ -13,6 +13,7 @@ from django.utils.decorators import method_decorator
 from django.views.decorators.csrf import ensure_csrf_cookie
 from django.template.loader import render_to_string
 from django.utils.translation import gettext_lazy as _
+from django.urls import reverse as django_reverse
 
 from rest_framework.permissions import AllowAny, IsAuthenticated
 from rest_framework.response import Response
@@ -27,10 +28,11 @@ from awx.main.analytics import all_collectors
 from awx.main.ha import is_ha_environment
 from awx.main.utils import get_awx_version, get_custom_venv_choices
 from awx.main.utils.licensing import validate_entitlement_manifest
-from awx.api.versioning import reverse, drf_reverse
+from awx.api.versioning import URLPathVersioning, reverse, drf_reverse
 from awx.main.constants import PRIVILEGE_ESCALATION_METHODS
 from awx.main.models import Project, Organization, Instance, InstanceGroup, JobTemplate
 from awx.main.utils import set_environ
+from awx.main.utils.analytics_proxy import TokenError
 from awx.main.utils.licensing import get_licenser
 
 logger = logging.getLogger('awx.api.views.root')
@@ -39,19 +41,17 @@ logger = logging.getLogger('awx.api.views.root')
 class ApiRootView(APIView):
     permission_classes = (AllowAny,)
     name = _('REST API')
-    versioning_class = None
+    versioning_class = URLPathVersioning
     swagger_topic = 'Versioning'
 
     @method_decorator(ensure_csrf_cookie)
     def get(self, request, format=None):
         '''List supported API versions'''
-
-        v2 = reverse('api:api_v2_root_view', kwargs={'version': 'v2'})
+        v2 = reverse('api:api_v2_root_view', request=request, kwargs={'version': 'v2'})
         data = OrderedDict()
         data['description'] = _('AWX REST API')
         data['current_version'] = v2
         data['available_versions'] = dict(v2=v2)
-        data['oauth2'] = drf_reverse('api:oauth_authorization_root_view')
         data['custom_logo'] = settings.CUSTOM_LOGO
         data['custom_login_info'] = settings.CUSTOM_LOGIN_INFO
         data['login_redirect_override'] = settings.LOGIN_REDIRECT_OVERRIDE
@@ -60,20 +60,6 @@ class ApiRootView(APIView):
         return Response(data)
 
 
-class ApiOAuthAuthorizationRootView(APIView):
-    permission_classes = (AllowAny,)
-    name = _("API OAuth 2 Authorization Root")
-    versioning_class = None
-    swagger_topic = 'Authentication'
-
-    def get(self, request, format=None):
-        data = OrderedDict()
-        data['authorize'] = drf_reverse('api:authorize')
-        data['token'] = drf_reverse('api:token')
-        data['revoke_token'] = drf_reverse('api:revoke-token')
-        return Response(data)
-
-
 class ApiVersionRootView(APIView):
     permission_classes = (AllowAny,)
     swagger_topic = 'Versioning'
@@ -84,6 +70,7 @@ class ApiVersionRootView(APIView):
         data['ping'] = reverse('api:api_v2_ping_view', request=request)
         data['instances'] = reverse('api:instance_list', request=request)
         data['instance_groups'] = reverse('api:instance_group_list', request=request)
+        data['receptor_addresses'] = reverse('api:receptor_addresses_list', request=request)
         data['config'] = reverse('api:api_v2_config_view', request=request)
         data['settings'] = reverse('api:setting_category_list', request=request)
         data['me'] = reverse('api:user_me_list', request=request)
@@ -97,8 +84,6 @@ class ApiVersionRootView(APIView):
         data['credentials'] = reverse('api:credential_list', request=request)
         data['credential_types'] = reverse('api:credential_type_list', request=request)
         data['credential_input_sources'] = reverse('api:credential_input_source_list', request=request)
-        data['applications'] = reverse('api:o_auth2_application_list', request=request)
-        data['tokens'] = reverse('api:o_auth2_token_list', request=request)
         data['metrics'] = reverse('api:metrics_view', request=request)
         data['inventory'] = reverse('api:inventory_list', request=request)
         data['constructed_inventory'] = reverse('api:constructed_inventory_list', request=request)
@@ -129,6 +114,10 @@ class ApiVersionRootView(APIView):
         data['mesh_visualizer'] = reverse('api:mesh_visualizer_view', request=request)
         data['bulk'] = reverse('api:bulk', request=request)
         data['analytics'] = reverse('api:analytics_root_view', request=request)
+        data['service_index'] = django_reverse('service-index-root')
+        data['role_definitions'] = django_reverse('roledefinition-list')
+        data['role_user_assignments'] = django_reverse('roleuserassignment-list')
+        data['role_team_assignments'] = django_reverse('roleteamassignment-list')
         return Response(data)
 
 
@@ -188,19 +177,21 @@ class ApiV2SubscriptionView(APIView):
 
     def post(self, request):
         data = request.data.copy()
-        if data.get('subscriptions_password') == '$encrypted$':
-            data['subscriptions_password'] = settings.SUBSCRIPTIONS_PASSWORD
+        if data.get('subscriptions_client_secret') == '$encrypted$':
+            data['subscriptions_client_secret'] = settings.SUBSCRIPTIONS_CLIENT_SECRET
         try:
-            user, pw = data.get('subscriptions_username'), data.get('subscriptions_password')
+            user, pw = data.get('subscriptions_client_id'), data.get('subscriptions_client_secret')
             with set_environ(**settings.AWX_TASK_ENV):
                 validated = get_licenser().validate_rh(user, pw)
             if user:
-                settings.SUBSCRIPTIONS_USERNAME = data['subscriptions_username']
+                settings.SUBSCRIPTIONS_CLIENT_ID = data['subscriptions_client_id']
             if pw:
-                settings.SUBSCRIPTIONS_PASSWORD = data['subscriptions_password']
+                settings.SUBSCRIPTIONS_CLIENT_SECRET = data['subscriptions_client_secret']
         except Exception as exc:
             msg = _("Invalid Subscription")
-            if isinstance(exc, requests.exceptions.HTTPError) and getattr(getattr(exc, 'response', None), 'status_code', None) == 401:
+            if isinstance(exc, TokenError) or (
+                isinstance(exc, requests.exceptions.HTTPError) and getattr(getattr(exc, 'response', None), 'status_code', None) == 401
+            ):
                 msg = _("The provided credentials are invalid (HTTP 401).")
             elif isinstance(exc, requests.exceptions.ProxyError):
                 msg = _("Unable to connect to proxy server.")
@@ -227,12 +218,12 @@ class ApiV2AttachView(APIView):
 
     def post(self, request):
         data = request.data.copy()
-        pool_id = data.get('pool_id', None)
-        if not pool_id:
-            return Response({"error": _("No subscription pool ID provided.")}, status=status.HTTP_400_BAD_REQUEST)
-        user = getattr(settings, 'SUBSCRIPTIONS_USERNAME', None)
-        pw = getattr(settings, 'SUBSCRIPTIONS_PASSWORD', None)
-        if pool_id and user and pw:
+        subscription_id = data.get('subscription_id', None)
+        if not subscription_id:
+            return Response({"error": _("No subscription ID provided.")}, status=status.HTTP_400_BAD_REQUEST)
+        user = getattr(settings, 'SUBSCRIPTIONS_CLIENT_ID', None)
+        pw = getattr(settings, 'SUBSCRIPTIONS_CLIENT_SECRET', None)
+        if subscription_id and user and pw:
             data = request.data.copy()
             try:
                 with set_environ(**settings.AWX_TASK_ENV):
@@ -251,7 +242,7 @@ class ApiV2AttachView(APIView):
                     logger.exception(smart_str(u"Invalid subscription submitted."), extra=dict(actor=request.user.username))
                 return Response({"error": msg}, status=status.HTTP_400_BAD_REQUEST)
         for sub in validated:
-            if sub['pool_id'] == pool_id:
+            if sub['subscription_id'] == subscription_id:
                 sub['valid_key'] = True
                 settings.LICENSE = sub
                 return Response(sub)
@@ -279,9 +270,6 @@ class ApiV2ConfigView(APIView):
 
         pendo_state = settings.PENDO_TRACKING_STATE if settings.PENDO_TRACKING_STATE in ('off', 'anonymous', 'detailed') else 'off'
 
-        # Guarding against settings.UI_NEXT being set to a non-boolean value
-        ui_next_state = settings.UI_NEXT if settings.UI_NEXT in (True, False) else False
-
         data = dict(
             time_zone=settings.TIME_ZONE,
             license_info=license_data,
@@ -290,18 +278,8 @@ class ApiV2ConfigView(APIView):
             analytics_status=pendo_state,
             analytics_collectors=all_collectors(),
             become_methods=PRIVILEGE_ESCALATION_METHODS,
-            ui_next=ui_next_state,
         )
 
-        # If LDAP is enabled, user_ldap_fields will return a list of field
-        # names that are managed by LDAP and should be read-only for users with
-        # a non-empty ldap_dn attribute.
-        if getattr(settings, 'AUTH_LDAP_SERVER_URI', None):
-            user_ldap_fields = ['username', 'password']
-            user_ldap_fields.extend(getattr(settings, 'AUTH_LDAP_USER_ATTR_MAP', {}).keys())
-            user_ldap_fields.extend(getattr(settings, 'AUTH_LDAP_USER_FLAGS_BY_GROUP', {}).keys())
-            data['user_ldap_fields'] = user_ldap_fields
-
         if (
             request.user.is_superuser
             or request.user.is_system_auditor

awx/api/views/webhooks.py

@@ -1,4 +1,4 @@
-from hashlib import sha1
+from hashlib import sha1, sha256
 import hmac
 import logging
 import urllib.parse
@@ -99,14 +99,31 @@ class WebhookReceiverBase(APIView):
     def get_signature(self):
         raise NotImplementedError
 
+    def must_check_signature(self):
+        return True
+
+    def is_ignored_request(self):
+        return False
+
     def check_signature(self, obj):
         if not obj.webhook_key:
             raise PermissionDenied
+        if not self.must_check_signature():
+            logger.debug("skipping signature validation")
+            return
 
-        mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha1)
-        logger.debug("header signature: %s", self.get_signature())
+        hash_alg, expected_digest = self.get_signature()
+        if hash_alg == 'sha1':
+            mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha1)
+        elif hash_alg == 'sha256':
+            mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha256)
+        else:
+            logger.debug("Unsupported signature type, supported: sha1, sha256, received: {}".format(hash_alg))
+            raise PermissionDenied
+
+        logger.debug("header signature: %s", expected_digest)
         logger.debug("calculated signature: %s", force_bytes(mac.hexdigest()))
-        if not hmac.compare_digest(force_bytes(mac.hexdigest()), self.get_signature()):
+        if not hmac.compare_digest(force_bytes(mac.hexdigest()), expected_digest):
             raise PermissionDenied
 
     @csrf_exempt
@@ -118,6 +135,10 @@ class WebhookReceiverBase(APIView):
         obj = self.get_object()
         self.check_signature(obj)
 
+        if self.is_ignored_request():
+            # This was an ignored request type (e.g. ping), don't act on it
+            return Response({'message': _("Webhook ignored")}, status=status.HTTP_200_OK)
+
         event_type = self.get_event_type()
         event_guid = self.get_event_guid()
         event_ref = self.get_event_ref()
@@ -186,7 +207,7 @@ class GithubWebhookReceiver(WebhookReceiverBase):
         if hash_alg != 'sha1':
             logger.debug("Unsupported signature type, expected: sha1, received: {}".format(hash_alg))
             raise PermissionDenied
-        return force_bytes(signature)
+        return hash_alg, force_bytes(signature)
 
 
 class GitlabWebhookReceiver(WebhookReceiverBase):
@@ -214,15 +235,73 @@ class GitlabWebhookReceiver(WebhookReceiverBase):
 
         return "{}://{}/api/v4/projects/{}/statuses/{}".format(parsed.scheme, parsed.netloc, project['id'], self.get_event_ref())
 
-    def get_signature(self):
-        return force_bytes(self.request.META.get('HTTP_X_GITLAB_TOKEN') or '')
-
     def check_signature(self, obj):
         if not obj.webhook_key:
             raise PermissionDenied
 
+        token_from_request = force_bytes(self.request.META.get('HTTP_X_GITLAB_TOKEN') or '')
+
         # GitLab only returns the secret token, not an hmac hash.  Use
         # the hmac `compare_digest` helper function to prevent timing
         # analysis by attackers.
-        if not hmac.compare_digest(force_bytes(obj.webhook_key), self.get_signature()):
+        if not hmac.compare_digest(force_bytes(obj.webhook_key), token_from_request):
             raise PermissionDenied
+
+
+class BitbucketDcWebhookReceiver(WebhookReceiverBase):
+    service = 'bitbucket_dc'
+
+    ref_keys = {
+        'repo:refs_changed': 'changes.0.toHash',
+        'mirror:repo_synchronized': 'changes.0.toHash',
+        'pr:opened': 'pullRequest.toRef.latestCommit',
+        'pr:from_ref_updated': 'pullRequest.toRef.latestCommit',
+        'pr:modified': 'pullRequest.toRef.latestCommit',
+    }
+
+    def get_event_type(self):
+        return self.request.META.get('HTTP_X_EVENT_KEY')
+
+    def get_event_guid(self):
+        return self.request.META.get('HTTP_X_REQUEST_ID')
+
+    def get_event_status_api(self):
+        # https://<bitbucket-base-url>/rest/build-status/1.0/commits/<commit-hash>
+        if self.get_event_type() not in self.ref_keys.keys():
+            return
+        if self.get_event_ref() is None:
+            return
+        any_url = None
+        if 'actor' in self.request.data:
+            any_url = self.request.data['actor'].get('links', {}).get('self')
+        if any_url is None and 'repository' in self.request.data:
+            any_url = self.request.data['repository'].get('links', {}).get('self')
+        if any_url is None:
+            return
+        any_url = any_url[0].get('href')
+        if any_url is None:
+            return
+        parsed = urllib.parse.urlparse(any_url)
+
+        return "{}://{}/rest/build-status/1.0/commits/{}".format(parsed.scheme, parsed.netloc, self.get_event_ref())
+
+    def is_ignored_request(self):
+        return self.get_event_type() not in [
+            'repo:refs_changed',
+            'mirror:repo_synchronized',
+            'pr:opened',
+            'pr:from_ref_updated',
+            'pr:modified',
+        ]
+
+    def must_check_signature(self):
+        # Bitbucket does not sign ping requests...
+        return self.get_event_type() != 'diagnostics:ping'
+
+    def get_signature(self):
+        header_sig = self.request.META.get('HTTP_X_HUB_SIGNATURE')
+        if not header_sig:
+            logger.debug("Expected signature missing from header key HTTP_X_HUB_SIGNATURE")
+            raise PermissionDenied
+        hash_alg, signature = header_sig.split('=')
+        return hash_alg, force_bytes(signature)

awx/conf/conf.py

@@ -55,6 +55,7 @@ register(
     # Optional; category_slug will be slugified version of category if not
     # explicitly provided.
     category_slug='cows',
+    hidden=True,
 )
 
 

awx/conf/fields.py

@@ -61,6 +61,10 @@ class StringListBooleanField(ListField):
 
     def to_representation(self, value):
         try:
+            if isinstance(value, str):
+                # https://github.com/encode/django-rest-framework/commit/a180bde0fd965915718b070932418cabc831cee1
+                # DRF changed truthy and falsy lists to be capitalized
+                value = value.lower()
             if isinstance(value, (list, tuple)):
                 return super(StringListBooleanField, self).to_representation(value)
             elif value in BooleanField.TRUE_VALUES:
@@ -78,6 +82,8 @@ class StringListBooleanField(ListField):
 
     def to_internal_value(self, data):
         try:
+            if isinstance(data, str):
+                data = data.lower()
             if isinstance(data, (list, tuple)):
                 return super(StringListBooleanField, self).to_internal_value(data)
             elif data in BooleanField.TRUE_VALUES:
@@ -201,7 +207,8 @@ class URLField(CharField):
         if self.allow_plain_hostname:
             try:
                 url_parts = urlparse.urlsplit(value)
-                if url_parts.hostname and '.' not in url_parts.hostname:
+                looks_like_ipv6 = bool(url_parts.netloc and url_parts.netloc.startswith('[') and url_parts.netloc.endswith(']'))
+                if not looks_like_ipv6 and url_parts.hostname and '.' not in url_parts.hostname:
                     netloc = '{}.local'.format(url_parts.hostname)
                     if url_parts.port:
                         netloc = '{}:{}'.format(netloc, url_parts.port)

awx/conf/migrations/0006_v331_ldap_group_type.py

@@ -1,13 +1,11 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-# AWX
-from awx.conf.migrations._ldap_group_type import fill_ldap_group_type_params
-
 from django.db import migrations
 
 
 class Migration(migrations.Migration):
     dependencies = [('conf', '0005_v330_rename_two_session_settings')]
 
-    operations = [migrations.RunPython(fill_ldap_group_type_params)]
+    # this migration is doing nothing, and is here to preserve migrations files integrity
+    operations = []

awx/conf/migrations/0011_remove_ldap_auth_conf.py

@@ -0,0 +1,115 @@
+from django.db import migrations
+
+LDAP_AUTH_CONF_KEYS = [
+    'AUTH_LDAP_SERVER_URI',
+    'AUTH_LDAP_BIND_DN',
+    'AUTH_LDAP_BIND_PASSWORD',
+    'AUTH_LDAP_START_TLS',
+    'AUTH_LDAP_CONNECTION_OPTIONS',
+    'AUTH_LDAP_USER_SEARCH',
+    'AUTH_LDAP_USER_DN_TEMPLATE',
+    'AUTH_LDAP_USER_ATTR_MAP',
+    'AUTH_LDAP_GROUP_SEARCH',
+    'AUTH_LDAP_GROUP_TYPE',
+    'AUTH_LDAP_GROUP_TYPE_PARAMS',
+    'AUTH_LDAP_REQUIRE_GROUP',
+    'AUTH_LDAP_DENY_GROUP',
+    'AUTH_LDAP_USER_FLAGS_BY_GROUP',
+    'AUTH_LDAP_ORGANIZATION_MAP',
+    'AUTH_LDAP_TEAM_MAP',
+    'AUTH_LDAP_1_SERVER_URI',
+    'AUTH_LDAP_1_BIND_DN',
+    'AUTH_LDAP_1_BIND_PASSWORD',
+    'AUTH_LDAP_1_START_TLS',
+    'AUTH_LDAP_1_CONNECTION_OPTIONS',
+    'AUTH_LDAP_1_USER_SEARCH',
+    'AUTH_LDAP_1_USER_DN_TEMPLATE',
+    'AUTH_LDAP_1_USER_ATTR_MAP',
+    'AUTH_LDAP_1_GROUP_SEARCH',
+    'AUTH_LDAP_1_GROUP_TYPE',
+    'AUTH_LDAP_1_GROUP_TYPE_PARAMS',
+    'AUTH_LDAP_1_REQUIRE_GROUP',
+    'AUTH_LDAP_1_DENY_GROUP',
+    'AUTH_LDAP_1_USER_FLAGS_BY_GROUP',
+    'AUTH_LDAP_1_ORGANIZATION_MAP',
+    'AUTH_LDAP_1_TEAM_MAP',
+    'AUTH_LDAP_2_SERVER_URI',
+    'AUTH_LDAP_2_BIND_DN',
+    'AUTH_LDAP_2_BIND_PASSWORD',
+    'AUTH_LDAP_2_START_TLS',
+    'AUTH_LDAP_2_CONNECTION_OPTIONS',
+    'AUTH_LDAP_2_USER_SEARCH',
+    'AUTH_LDAP_2_USER_DN_TEMPLATE',
+    'AUTH_LDAP_2_USER_ATTR_MAP',
+    'AUTH_LDAP_2_GROUP_SEARCH',
+    'AUTH_LDAP_2_GROUP_TYPE',
+    'AUTH_LDAP_2_GROUP_TYPE_PARAMS',
+    'AUTH_LDAP_2_REQUIRE_GROUP',
+    'AUTH_LDAP_2_DENY_GROUP',
+    'AUTH_LDAP_2_USER_FLAGS_BY_GROUP',
+    'AUTH_LDAP_2_ORGANIZATION_MAP',
+    'AUTH_LDAP_2_TEAM_MAP',
+    'AUTH_LDAP_3_SERVER_URI',
+    'AUTH_LDAP_3_BIND_DN',
+    'AUTH_LDAP_3_BIND_PASSWORD',
+    'AUTH_LDAP_3_START_TLS',
+    'AUTH_LDAP_3_CONNECTION_OPTIONS',
+    'AUTH_LDAP_3_USER_SEARCH',
+    'AUTH_LDAP_3_USER_DN_TEMPLATE',
+    'AUTH_LDAP_3_USER_ATTR_MAP',
+    'AUTH_LDAP_3_GROUP_SEARCH',
+    'AUTH_LDAP_3_GROUP_TYPE',
+    'AUTH_LDAP_3_GROUP_TYPE_PARAMS',
+    'AUTH_LDAP_3_REQUIRE_GROUP',
+    'AUTH_LDAP_3_DENY_GROUP',
+    'AUTH_LDAP_3_USER_FLAGS_BY_GROUP',
+    'AUTH_LDAP_3_ORGANIZATION_MAP',
+    'AUTH_LDAP_3_TEAM_MAP',
+    'AUTH_LDAP_4_SERVER_URI',
+    'AUTH_LDAP_4_BIND_DN',
+    'AUTH_LDAP_4_BIND_PASSWORD',
+    'AUTH_LDAP_4_START_TLS',
+    'AUTH_LDAP_4_CONNECTION_OPTIONS',
+    'AUTH_LDAP_4_USER_SEARCH',
+    'AUTH_LDAP_4_USER_DN_TEMPLATE',
+    'AUTH_LDAP_4_USER_ATTR_MAP',
+    'AUTH_LDAP_4_GROUP_SEARCH',
+    'AUTH_LDAP_4_GROUP_TYPE',
+    'AUTH_LDAP_4_GROUP_TYPE_PARAMS',
+    'AUTH_LDAP_4_REQUIRE_GROUP',
+    'AUTH_LDAP_4_DENY_GROUP',
+    'AUTH_LDAP_4_USER_FLAGS_BY_GROUP',
+    'AUTH_LDAP_4_ORGANIZATION_MAP',
+    'AUTH_LDAP_4_TEAM_MAP',
+    'AUTH_LDAP_5_SERVER_URI',
+    'AUTH_LDAP_5_BIND_DN',
+    'AUTH_LDAP_5_BIND_PASSWORD',
+    'AUTH_LDAP_5_START_TLS',
+    'AUTH_LDAP_5_CONNECTION_OPTIONS',
+    'AUTH_LDAP_5_USER_SEARCH',
+    'AUTH_LDAP_5_USER_DN_TEMPLATE',
+    'AUTH_LDAP_5_USER_ATTR_MAP',
+    'AUTH_LDAP_5_GROUP_SEARCH',
+    'AUTH_LDAP_5_GROUP_TYPE',
+    'AUTH_LDAP_5_GROUP_TYPE_PARAMS',
+    'AUTH_LDAP_5_REQUIRE_GROUP',
+    'AUTH_LDAP_5_DENY_GROUP',
+    'AUTH_LDAP_5_USER_FLAGS_BY_GROUP',
+    'AUTH_LDAP_5_ORGANIZATION_MAP',
+    'AUTH_LDAP_5_TEAM_MAP',
+]
+
+
+def remove_ldap_auth_conf(apps, scheme_editor):
+    setting = apps.get_model('conf', 'Setting')
+    setting.objects.filter(key__in=LDAP_AUTH_CONF_KEYS).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('conf', '0010_change_to_JSONField'),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_ldap_auth_conf),
+    ]

awx/conf/migrations/0012_remove_oidc_auth_conf.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.10 on 2024-08-27 19:31
+
+from django.db import migrations
+
+OIDC_AUTH_CONF_KEYS = ['SOCIAL_AUTH_OIDC_KEY', 'SOCIAL_AUTH_OIDC_SECRET', 'SOCIAL_AUTH_OIDC_OIDC_ENDPOINT', 'SOCIAL_AUTH_OIDC_VERIFY_SSL']
+
+
+def remove_oidc_auth_conf(apps, scheme_editor):
+    setting = apps.get_model('conf', 'Setting')
+    setting.objects.filter(key__in=OIDC_AUTH_CONF_KEYS).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('conf', '0011_remove_ldap_auth_conf'),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_oidc_auth_conf),
+    ]

awx/conf/migrations/0013_remove_radius_auth_conf.py

@@ -0,0 +1,22 @@
+from django.db import migrations
+
+RADIUS_AUTH_CONF_KEYS = [
+    'RADIUS_SERVER',
+    'RADIUS_PORT',
+    'RADIUS_SECRET',
+]
+
+
+def remove_radius_auth_conf(apps, scheme_editor):
+    setting = apps.get_model('conf', 'Setting')
+    setting.objects.filter(key__in=RADIUS_AUTH_CONF_KEYS).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('conf', '0012_remove_oidc_auth_conf'),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_radius_auth_conf),
+    ]

awx/conf/migrations/0014_remove_saml_auth_conf.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.2.10 on 2024-08-27 14:20
+
+from django.db import migrations
+
+SAML_AUTH_CONF_KEYS = [
+    'SAML_AUTO_CREATE_OBJECTS',
+    'SOCIAL_AUTH_SAML_CALLBACK_URL',
+    'SOCIAL_AUTH_SAML_METADATA_URL',
+    'SOCIAL_AUTH_SAML_SP_ENTITY_ID',
+    'SOCIAL_AUTH_SAML_SP_PUBLIC_CERT',
+    'SOCIAL_AUTH_SAML_SP_PRIVATE_KEY',
+    'SOCIAL_AUTH_SAML_ORG_INFO',
+    'SOCIAL_AUTH_SAML_TECHNICAL_CONTACT',
+    'SOCIAL_AUTH_SAML_SUPPORT_CONTACT',
+    'SOCIAL_AUTH_SAML_ENABLED_IDPS',
+    'SOCIAL_AUTH_SAML_SECURITY_CONFIG',
+    'SOCIAL_AUTH_SAML_SP_EXTRA',
+    'SOCIAL_AUTH_SAML_EXTRA_DATA',
+    'SOCIAL_AUTH_SAML_ORGANIZATION_MAP',
+    'SOCIAL_AUTH_SAML_TEAM_MAP',
+    'SOCIAL_AUTH_SAML_ORGANIZATION_ATTR',
+    'SOCIAL_AUTH_SAML_TEAM_ATTR',
+    'SOCIAL_AUTH_SAML_USER_FLAGS_BY_ATTR',
+]
+
+
+def remove_saml_auth_conf(apps, scheme_editor):
+    setting = apps.get_model('conf', 'Setting')
+    setting.objects.filter(key__in=SAML_AUTH_CONF_KEYS).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('conf', '0013_remove_radius_auth_conf'),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_saml_auth_conf),
+    ]

awx/conf/migrations/0015_remove_social_oauth_conf.py

@@ -0,0 +1,81 @@
+# Generated by Django 4.2.10 on 2024-08-13 11:14
+
+from django.db import migrations
+
+SOCIAL_OAUTH_CONF_KEYS = [
+    # MICROSOFT AZURE ACTIVE DIRECTORY SETTINGS
+    'SOCIAL_AUTH_AZUREAD_OAUTH2_CALLBACK_URL',
+    'SOCIAL_AUTH_AZUREAD_OAUTH2_KEY',
+    'SOCIAL_AUTH_AZUREAD_OAUTH2_SECRET',
+    'SOCIAL_AUTH_AZUREAD_OAUTH2_ORGANIZATION_MAP',
+    'SOCIAL_AUTH_AZUREAD_OAUTH2_TEAM_MAP',
+    # GOOGLE OAUTH2 AUTHENTICATION SETTINGS
+    'SOCIAL_AUTH_GOOGLE_OAUTH2_CALLBACK_URL',
+    'SOCIAL_AUTH_GOOGLE_OAUTH2_KEY',
+    'SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET',
+    'SOCIAL_AUTH_GOOGLE_OAUTH2_WHITELISTED_DOMAINS',
+    'SOCIAL_AUTH_GOOGLE_OAUTH2_AUTH_EXTRA_ARGUMENTS',
+    'SOCIAL_AUTH_GOOGLE_OAUTH2_ORGANIZATION_MAP',
+    'SOCIAL_AUTH_GOOGLE_OAUTH2_TEAM_MAP',
+    # GITHUB OAUTH2 AUTHENTICATION SETTINGS
+    'SOCIAL_AUTH_GITHUB_CALLBACK_URL',
+    'SOCIAL_AUTH_GITHUB_KEY',
+    'SOCIAL_AUTH_GITHUB_SECRET',
+    'SOCIAL_AUTH_GITHUB_ORGANIZATION_MAP',
+    'SOCIAL_AUTH_GITHUB_TEAM_MAP',
+    # GITHUB ORG OAUTH2 AUTHENTICATION SETTINGS
+    'SOCIAL_AUTH_GITHUB_ORG_CALLBACK_URL',
+    'SOCIAL_AUTH_GITHUB_ORG_KEY',
+    'SOCIAL_AUTH_GITHUB_ORG_SECRET',
+    'SOCIAL_AUTH_GITHUB_ORG_NAME',
+    'SOCIAL_AUTH_GITHUB_ORG_ORGANIZATION_MAP',
+    'SOCIAL_AUTH_GITHUB_ORG_TEAM_MAP',
+    # GITHUB TEAM OAUTH2 AUTHENTICATION SETTINGS
+    'SOCIAL_AUTH_GITHUB_TEAM_CALLBACK_URL',
+    'SOCIAL_AUTH_GITHUB_TEAM_KEY',
+    'SOCIAL_AUTH_GITHUB_TEAM_SECRET',
+    'SOCIAL_AUTH_GITHUB_TEAM_ID',
+    'SOCIAL_AUTH_GITHUB_TEAM_ORGANIZATION_MAP',
+    'SOCIAL_AUTH_GITHUB_TEAM_TEAM_MAP',
+    # GITHUB ENTERPRISE OAUTH2 AUTHENTICATION SETTINGS
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_CALLBACK_URL',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_URL',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_API_URL',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_KEY',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_SECRET',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_ORGANIZATION_MAP',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_TEAM_MAP',
+    # GITHUB ENTERPRISE ORG OAUTH2 AUTHENTICATION SETTINGS
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_ORG_CALLBACK_URL',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_ORG_URL',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_ORG_API_URL',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_ORG_KEY',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_ORG_SECRET',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_ORG_NAME',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_ORG_ORGANIZATION_MAP',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_ORG_TEAM_MAP',
+    # GITHUB ENTERPRISE TEAM OAUTH2 AUTHENTICATION SETTINGS
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_TEAM_CALLBACK_URL',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_TEAM_URL',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_TEAM_API_URL',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_TEAM_KEY',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_TEAM_SECRET',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_TEAM_ID',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_TEAM_ORGANIZATION_MAP',
+    'SOCIAL_AUTH_GITHUB_ENTERPRISE_TEAM_TEAM_MAP',
+]
+
+
+def remove_social_oauth_conf(apps, scheme_editor):
+    setting = apps.get_model('conf', 'Setting')
+    setting.objects.filter(key__in=SOCIAL_OAUTH_CONF_KEYS).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('conf', '0014_remove_saml_auth_conf'),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_social_oauth_conf),
+    ]

awx/conf/migrations/0016_remove_tacacs_plus_auth_conf.py

@@ -0,0 +1,25 @@
+from django.db import migrations
+
+TACACS_PLUS_AUTH_CONF_KEYS = [
+    'TACACSPLUS_HOST',
+    'TACACSPLUS_PORT',
+    'TACACSPLUS_SECRET',
+    'TACACSPLUS_SESSION_TIMEOUT',
+    'TACACSPLUS_AUTH_PROTOCOL',
+    'TACACSPLUS_REM_ADDR',
+]
+
+
+def remove_tacacs_plus_auth_conf(apps, scheme_editor):
+    setting = apps.get_model('conf', 'Setting')
+    setting.objects.filter(key__in=TACACS_PLUS_AUTH_CONF_KEYS).delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('conf', '0015_remove_social_oauth_conf'),
+    ]
+
+    operations = [
+        migrations.RunPython(remove_tacacs_plus_auth_conf),
+    ]

awx/conf/migrations/_ldap_group_type.py

@@ -1,31 +0,0 @@
-import inspect
-
-from django.conf import settings
-
-import logging
-
-
-logger = logging.getLogger('awx.conf.migrations')
-
-
-def fill_ldap_group_type_params(apps, schema_editor):
-    group_type = getattr(settings, 'AUTH_LDAP_GROUP_TYPE', None)
-    Setting = apps.get_model('conf', 'Setting')
-
-    group_type_params = {'name_attr': 'cn', 'member_attr': 'member'}
-    qs = Setting.objects.filter(key='AUTH_LDAP_GROUP_TYPE_PARAMS')
-    entry = None
-    if qs.exists():
-        entry = qs[0]
-        group_type_params = entry.value
-    else:
-        return  # for new installs we prefer to use the default value
-
-    init_attrs = set(inspect.getfullargspec(group_type.__init__).args[1:])
-    for k in list(group_type_params.keys()):
-        if k not in init_attrs:
-            del group_type_params[k]
-
-    entry.value = group_type_params
-    logger.warning(f'Migration updating AUTH_LDAP_GROUP_TYPE_PARAMS with value {entry.value}')
-    entry.save()

awx/conf/migrations/_subscriptions.py

@@ -27,5 +27,5 @@ def _migrate_setting(apps, old_key, new_key, encrypted=False):
 
 
 def prefill_rh_credentials(apps, schema_editor):
-    _migrate_setting(apps, 'REDHAT_USERNAME', 'SUBSCRIPTIONS_USERNAME', encrypted=False)
-    _migrate_setting(apps, 'REDHAT_PASSWORD', 'SUBSCRIPTIONS_PASSWORD', encrypted=True)
+    _migrate_setting(apps, 'REDHAT_USERNAME', 'SUBSCRIPTIONS_CLIENT_ID', encrypted=False)
+    _migrate_setting(apps, 'REDHAT_PASSWORD', 'SUBSCRIPTIONS_CLIENT_SECRET', encrypted=True)

awx/conf/models.py

@@ -7,8 +7,10 @@ import json
 # Django
 from django.db import models
 
+from ansible_base.lib.utils.models import prevent_search
+
 # AWX
-from awx.main.models.base import CreatedModifiedModel, prevent_search
+from awx.main.models.base import CreatedModifiedModel
 from awx.main.utils import encrypt_field
 from awx.conf import settings_registry
 

awx/conf/registry.py

@@ -127,6 +127,8 @@ class SettingsRegistry(object):
         encrypted = bool(field_kwargs.pop('encrypted', False))
         defined_in_file = bool(field_kwargs.pop('defined_in_file', False))
         unit = field_kwargs.pop('unit', None)
+        hidden = field_kwargs.pop('hidden', False)
+        warning_text = field_kwargs.pop('warning_text', None)
         if getattr(field_kwargs.get('child', None), 'source', None) is not None:
             field_kwargs['child'].source = None
         field_instance = field_class(**field_kwargs)
@@ -134,12 +136,14 @@ class SettingsRegistry(object):
         field_instance.category = category
         field_instance.depends_on = depends_on
         field_instance.unit = unit
+        field_instance.hidden = hidden
         if placeholder is not empty:
             field_instance.placeholder = placeholder
         field_instance.defined_in_file = defined_in_file
         if field_instance.defined_in_file:
             field_instance.help_text = str(_('This value has been set manually in a settings file.')) + '\n\n' + str(field_instance.help_text)
         field_instance.encrypted = encrypted
+        field_instance.warning_text = warning_text
         original_field_instance = field_instance
         if field_class != original_field_class:
             original_field_instance = original_field_class(**field_kwargs)

awx/conf/settings.py

@@ -1,6 +1,7 @@
 # Python
 import contextlib
 import logging
+import psycopg
 import threading
 import time
 import os
@@ -13,7 +14,7 @@ from django.conf import settings, UserSettingsHolder
 from django.core.cache import cache as django_cache
 from django.core.exceptions import ImproperlyConfigured, SynchronousOnlyOperation
 from django.db import transaction, connection
-from django.db.utils import Error as DBError, ProgrammingError
+from django.db.utils import DatabaseError, ProgrammingError
 from django.utils.functional import cached_property
 
 # Django REST Framework
@@ -80,18 +81,29 @@ def _ctit_db_wrapper(trans_safe=False):
                     logger.debug('Obtaining database settings in spite of broken transaction.')
                     transaction.set_rollback(False)
         yield
-    except DBError as exc:
+    except ProgrammingError as e:
+        # Exception raised for programming errors
+        # Examples may be table not found or already exists,
+        # this generally means we can't fetch Tower configuration
+        # because the database hasn't actually finished migrating yet;
+        # this is usually a sign that a service in a container (such as ws_broadcast)
+        # has come up *before* the database has finished migrating, and
+        # especially that the conf.settings table doesn't exist yet
+        # syntax error in the SQL statement, wrong number of parameters specified, etc.
         if trans_safe:
-            level = logger.warning
-            if isinstance(exc, ProgrammingError):
-                if 'relation' in str(exc) and 'does not exist' in str(exc):
-                    # this generally means we can't fetch Tower configuration
-                    # because the database hasn't actually finished migrating yet;
-                    # this is usually a sign that a service in a container (such as ws_broadcast)
-                    # has come up *before* the database has finished migrating, and
-                    # especially that the conf.settings table doesn't exist yet
-                    level = logger.debug
-            level(f'Database settings are not available, using defaults. error: {str(exc)}')
+            logger.debug(f'Database settings are not available, using defaults. error: {str(e)}')
+        else:
+            logger.exception('Error modifying something related to database settings.')
+    except DatabaseError as e:
+        if trans_safe:
+            cause = e.__cause__
+            sqlstate = getattr(cause, 'sqlstate', None)
+            if cause and sqlstate:
+                sqlstate = cause.sqlstate
+                sqlstate_str = psycopg.errors.lookup(sqlstate)
+                logger.error('SQL Error state: {} - {}'.format(sqlstate, sqlstate_str))
+            else:
+                logger.error(f'Error reading something related to database settings: {str(e)}.')
         else:
             logger.exception('Error modifying something related to database settings.')
     finally:
@@ -418,6 +430,10 @@ class SettingsWrapper(UserSettingsHolder):
         """Get value while accepting the in-memory cache if key is available"""
         with _ctit_db_wrapper(trans_safe=True):
             return self._get_local(name)
+        # If the last line did not return, that means we hit a database error
+        # in that case, we should not have a local cache value
+        # thus, return empty as a signal to use the default
+        return empty
 
     def __getattr__(self, name):
         value = empty

awx/conf/signals.py

@@ -61,18 +61,3 @@ def on_post_delete_setting(sender, **kwargs):
     key = getattr(instance, '_saved_key_', None)
     if key:
         handle_setting_change(key, True)
-
-
-@receiver(setting_changed)
-def disable_local_auth(**kwargs):
-    if (kwargs['setting'], kwargs['value']) == ('DISABLE_LOCAL_AUTH', True):
-        from django.contrib.auth.models import User
-        from oauth2_provider.models import RefreshToken
-        from awx.main.models.oauth import OAuth2AccessToken
-        from awx.main.management.commands.revoke_oauth2_tokens import revoke_tokens
-
-        logger.warning("Triggering token invalidation for local users.")
-
-        qs = User.objects.filter(profile__ldap_dn='', enterprise_auth__isnull=True, social_auth__isnull=True)
-        revoke_tokens(RefreshToken.objects.filter(revoked=None, user__in=qs))
-        revoke_tokens(OAuth2AccessToken.objects.filter(user__in=qs))

awx/conf/tests/functional/test_api.py

@@ -8,7 +8,6 @@ from awx.main.utils.encryption import decrypt_field
 from awx.conf import fields
 from awx.conf.registry import settings_registry
 from awx.conf.models import Setting
-from awx.sso import fields as sso_fields
 
 
 @pytest.fixture
@@ -103,24 +102,6 @@ def test_setting_singleton_update(api_request, dummy_setting):
         assert response.data['FOO_BAR'] == 4
 
 
-@pytest.mark.django_db
-def test_setting_singleton_update_hybriddictfield_with_forbidden(api_request, dummy_setting):
-    # Some HybridDictField subclasses have a child of _Forbidden,
-    # indicating that only the defined fields can be filled in.  Make
-    # sure that the _Forbidden validator doesn't get used for the
-    # fields.  See also https://github.com/ansible/awx/issues/4099.
-    with dummy_setting('FOO_BAR', field_class=sso_fields.SAMLOrgAttrField, category='FooBar', category_slug='foobar'), mock.patch(
-        'awx.conf.views.clear_setting_cache'
-    ):
-        api_request(
-            'patch',
-            reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}),
-            data={'FOO_BAR': {'saml_admin_attr': 'Admins', 'saml_attr': 'Orgs'}},
-        )
-        response = api_request('get', reverse('api:setting_singleton_detail', kwargs={'category_slug': 'foobar'}))
-        assert response.data['FOO_BAR'] == {'saml_admin_attr': 'Admins', 'saml_attr': 'Orgs'}
-
-
 @pytest.mark.django_db
 def test_setting_singleton_update_dont_change_readonly_fields(api_request, dummy_setting):
     with dummy_setting('FOO_BAR', field_class=fields.IntegerField, read_only=True, default=4, category='FooBar', category_slug='foobar'), mock.patch(

awx/conf/tests/functional/test_migrations.py

@@ -1,25 +0,0 @@
-import pytest
-
-from awx.conf.migrations._ldap_group_type import fill_ldap_group_type_params
-from awx.conf.models import Setting
-
-from django.apps import apps
-
-
-@pytest.mark.django_db
-def test_fill_group_type_params_no_op():
-    fill_ldap_group_type_params(apps, 'dont-use-me')
-    assert Setting.objects.count() == 0
-
-
-@pytest.mark.django_db
-def test_keep_old_setting_with_default_value():
-    Setting.objects.create(key='AUTH_LDAP_GROUP_TYPE', value={'name_attr': 'cn', 'member_attr': 'member'})
-    fill_ldap_group_type_params(apps, 'dont-use-me')
-    assert Setting.objects.count() == 1
-    s = Setting.objects.first()
-    assert s.value == {'name_attr': 'cn', 'member_attr': 'member'}
-
-
-# NOTE: would be good to test the removal of attributes by migration
-# but this requires fighting with the validator and is not done here

awx/conf/tests/unit/test_fields.py

@@ -111,7 +111,6 @@ class TestURLField:
     @pytest.mark.parametrize(
         "url,schemes,regex, allow_numbers_in_top_level_domain, expect_no_error",
         [
-            ("ldap://www.example.org42", "ldap", None, True, True),
             ("https://www.example.org42", "https", None, False, False),
             ("https://www.example.org", None, regex, None, True),
             ("https://www.example3.org", None, regex, None, False),
@@ -129,3 +128,41 @@ class TestURLField:
         else:
             with pytest.raises(ValidationError):
                 field.run_validators(url)
+
+    @pytest.mark.parametrize(
+        "url, expect_error",
+        [
+            ("https://[1:2:3]", True),
+            ("http://[1:2:3]", True),
+            ("https://[2001:db8:3333:4444:5555:6666:7777:8888", True),
+            ("https://2001:db8:3333:4444:5555:6666:7777:8888", True),
+            ("https://[2001:db8:3333:4444:5555:6666:7777:8888]", False),
+            ("https://[::1]", False),
+            ("https://[::]", False),
+            ("https://[2001:db8::1]", False),
+            ("https://[2001:db8:0:0:0:0:1:1]", False),
+            ("https://[fe80::2%eth0]", True),  # ipv6 scope identifier
+            ("https://[fe80:0:0:0:200:f8ff:fe21:67cf]", False),
+            ("https://[::ffff:192.168.1.10]", False),
+            ("https://[0:0:0:0:0:ffff:c000:0201]", False),
+            ("https://[2001:0db8:000a:0001:0000:0000:0000:0000]", False),
+            ("https://[2001:db8:a:1::]", False),
+            ("https://[ff02::1]", False),
+            ("https://[ff02:0:0:0:0:0:0:1]", False),
+            ("https://[fc00::1]", False),
+            ("https://[fd12:3456:789a:1::1]", False),
+            ("https://[2001:db8::abcd:ef12:3456:7890]", False),
+            ("https://[2001:db8:0000:abcd:0000:ef12:0000:3456]", False),
+            ("https://[::ffff:10.0.0.1]", False),
+            ("https://[2001:db8:cafe::]", False),
+            ("https://[2001:db8:cafe:0:0:0:0:0]", False),
+            ("https://[fe80::210:f3ff:fedf:4567%3]", True),  # ipv6 scope identifier, numerical interface
+        ],
+    )
+    def test_ipv6_urls(self, url, expect_error):
+        field = URLField()
+        if expect_error:
+            with pytest.raises(ValidationError, match="Enter a valid URL"):
+                field.run_validators(url)
+        else:
+            field.run_validators(url)

awx/conf/tests/unit/test_settings.py

@@ -13,6 +13,7 @@ from unittest import mock
 from django.conf import LazySettings
 from django.core.cache.backends.locmem import LocMemCache
 from django.core.exceptions import ImproperlyConfigured
+from django.db.utils import Error as DBError
 from django.utils.translation import gettext_lazy as _
 import pytest
 
@@ -129,9 +130,9 @@ def test_default_setting(settings, mocker):
     settings.registry.register('AWX_SOME_SETTING', field_class=fields.CharField, category=_('System'), category_slug='system', default='DEFAULT')
 
     settings_to_cache = mocker.Mock(**{'order_by.return_value': []})
-    with mocker.patch('awx.conf.models.Setting.objects.filter', return_value=settings_to_cache):
-        assert settings.AWX_SOME_SETTING == 'DEFAULT'
-        assert settings.cache.get('AWX_SOME_SETTING') == 'DEFAULT'
+    mocker.patch('awx.conf.models.Setting.objects.filter', return_value=settings_to_cache)
+    assert settings.AWX_SOME_SETTING == 'DEFAULT'
+    assert settings.cache.get('AWX_SOME_SETTING') == 'DEFAULT'
 
 
 @pytest.mark.defined_in_file(AWX_SOME_SETTING='DEFAULT')
@@ -145,9 +146,9 @@ def test_setting_is_not_from_setting_file(settings, mocker):
     settings.registry.register('AWX_SOME_SETTING', field_class=fields.CharField, category=_('System'), category_slug='system', default='DEFAULT')
 
     settings_to_cache = mocker.Mock(**{'order_by.return_value': []})
-    with mocker.patch('awx.conf.models.Setting.objects.filter', return_value=settings_to_cache):
-        assert settings.AWX_SOME_SETTING == 'DEFAULT'
-        assert settings.registry.get_setting_field('AWX_SOME_SETTING').defined_in_file is False
+    mocker.patch('awx.conf.models.Setting.objects.filter', return_value=settings_to_cache)
+    assert settings.AWX_SOME_SETTING == 'DEFAULT'
+    assert settings.registry.get_setting_field('AWX_SOME_SETTING').defined_in_file is False
 
 
 def test_empty_setting(settings, mocker):
@@ -155,10 +156,10 @@ def test_empty_setting(settings, mocker):
     settings.registry.register('AWX_SOME_SETTING', field_class=fields.CharField, category=_('System'), category_slug='system')
 
     mocks = mocker.Mock(**{'order_by.return_value': mocker.Mock(**{'__iter__': lambda self: iter([]), 'first.return_value': None})})
-    with mocker.patch('awx.conf.models.Setting.objects.filter', return_value=mocks):
-        with pytest.raises(AttributeError):
-            settings.AWX_SOME_SETTING
-        assert settings.cache.get('AWX_SOME_SETTING') == SETTING_CACHE_NOTSET
+    mocker.patch('awx.conf.models.Setting.objects.filter', return_value=mocks)
+    with pytest.raises(AttributeError):
+        settings.AWX_SOME_SETTING
+    assert settings.cache.get('AWX_SOME_SETTING') == SETTING_CACHE_NOTSET
 
 
 def test_setting_from_db(settings, mocker):
@@ -167,9 +168,9 @@ def test_setting_from_db(settings, mocker):
 
     setting_from_db = mocker.Mock(key='AWX_SOME_SETTING', value='FROM_DB')
     mocks = mocker.Mock(**{'order_by.return_value': mocker.Mock(**{'__iter__': lambda self: iter([setting_from_db]), 'first.return_value': setting_from_db})})
-    with mocker.patch('awx.conf.models.Setting.objects.filter', return_value=mocks):
-        assert settings.AWX_SOME_SETTING == 'FROM_DB'
-        assert settings.cache.get('AWX_SOME_SETTING') == 'FROM_DB'
+    mocker.patch('awx.conf.models.Setting.objects.filter', return_value=mocks)
+    assert settings.AWX_SOME_SETTING == 'FROM_DB'
+    assert settings.cache.get('AWX_SOME_SETTING') == 'FROM_DB'
 
 
 @pytest.mark.defined_in_file(AWX_SOME_SETTING='DEFAULT')
@@ -204,8 +205,8 @@ def test_db_setting_update(settings, mocker):
 
     existing_setting = mocker.Mock(key='AWX_SOME_SETTING', value='FROM_DB')
     setting_list = mocker.Mock(**{'order_by.return_value.first.return_value': existing_setting})
-    with mocker.patch('awx.conf.models.Setting.objects.filter', return_value=setting_list):
-        settings.AWX_SOME_SETTING = 'NEW-VALUE'
+    mocker.patch('awx.conf.models.Setting.objects.filter', return_value=setting_list)
+    settings.AWX_SOME_SETTING = 'NEW-VALUE'
 
     assert existing_setting.value == 'NEW-VALUE'
     existing_setting.save.assert_called_with(update_fields=['value'])
@@ -216,8 +217,8 @@ def test_db_setting_deletion(settings, mocker):
     settings.registry.register('AWX_SOME_SETTING', field_class=fields.CharField, category=_('System'), category_slug='system')
 
     existing_setting = mocker.Mock(key='AWX_SOME_SETTING', value='FROM_DB')
-    with mocker.patch('awx.conf.models.Setting.objects.filter', return_value=[existing_setting]):
-        del settings.AWX_SOME_SETTING
+    mocker.patch('awx.conf.models.Setting.objects.filter', return_value=[existing_setting])
+    del settings.AWX_SOME_SETTING
 
     assert existing_setting.delete.call_count == 1
 
@@ -282,10 +283,10 @@ def test_sensitive_cache_data_is_encrypted(settings, mocker):
     # use its primary key as part of the encryption key
     setting_from_db = mocker.Mock(pk=123, key='AWX_ENCRYPTED', value='SECRET!')
     mocks = mocker.Mock(**{'order_by.return_value': mocker.Mock(**{'__iter__': lambda self: iter([setting_from_db]), 'first.return_value': setting_from_db})})
-    with mocker.patch('awx.conf.models.Setting.objects.filter', return_value=mocks):
-        cache.set('AWX_ENCRYPTED', 'SECRET!')
-        assert cache.get('AWX_ENCRYPTED') == 'SECRET!'
-        assert native_cache.get('AWX_ENCRYPTED') == 'FRPERG!'
+    mocker.patch('awx.conf.models.Setting.objects.filter', return_value=mocks)
+    cache.set('AWX_ENCRYPTED', 'SECRET!')
+    assert cache.get('AWX_ENCRYPTED') == 'SECRET!'
+    assert native_cache.get('AWX_ENCRYPTED') == 'FRPERG!'
 
 
 def test_readonly_sensitive_cache_data_is_encrypted(settings):
@@ -331,3 +332,18 @@ def test_in_memory_cache_works(settings):
     with mock.patch.object(settings, '_get_local') as mock_get:
         assert settings.AWX_VAR == 'DEFAULT'
         mock_get.assert_not_called()
+
+
+@pytest.mark.defined_in_file(AWX_VAR=[])
+def test_getattr_with_database_error(settings):
+    """
+    If a setting is defined via the registry and has a null-ish default which is not None
+    then referencing that setting during a database outage should give that default
+    this is regression testing for a bug where it would return None
+    """
+    settings.registry.register('AWX_VAR', field_class=fields.StringListField, default=[], category=_('System'), category_slug='system')
+    settings._awx_conf_memoizedcache.clear()
+
+    with mock.patch('django.db.backends.base.base.BaseDatabaseWrapper.ensure_connection') as mock_ensure:
+        mock_ensure.side_effect = DBError('for test')
+        assert settings.AWX_VAR == []

awx/main/access.py

@@ -17,14 +17,15 @@ from django.core.exceptions import ObjectDoesNotExist, FieldDoesNotExist
 # Django REST Framework
 from rest_framework.exceptions import ParseError, PermissionDenied
 
-# Django OAuth Toolkit
-from awx.main.models.oauth import OAuth2Application, OAuth2AccessToken
+# django-ansible-base
+from ansible_base.lib.utils.validation import to_python_boolean
+from ansible_base.rbac.models import RoleEvaluation
+from ansible_base.rbac import permission_registry
 
 # AWX
 from awx.main.utils import (
     get_object_or_400,
     get_pk_from_dict,
-    to_python_boolean,
     get_licenser,
 )
 from awx.main.models import (
@@ -56,6 +57,7 @@ from awx.main.models import (
     Project,
     ProjectUpdate,
     ProjectUpdateEvent,
+    ReceptorAddress,
     Role,
     Schedule,
     SystemJob,
@@ -70,8 +72,6 @@ from awx.main.models import (
     WorkflowJobTemplateNode,
     WorkflowApproval,
     WorkflowApprovalTemplate,
-    ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
-    ROLE_SINGLETON_SYSTEM_AUDITOR,
 )
 from awx.main.models.mixins import ResourceMixin
 
@@ -79,7 +79,6 @@ __all__ = [
     'get_user_queryset',
     'check_user_access',
     'check_user_access_with_errors',
-    'user_accessible_objects',
     'consumer_access',
 ]
 
@@ -136,10 +135,6 @@ def register_access(model_class, access_class):
     access_registry[model_class] = access_class
 
 
-def user_accessible_objects(user, role_name):
-    return ResourceMixin._accessible_objects(User, user, role_name)
-
-
 def get_user_queryset(user, model_class):
     """
     Return a queryset for the given model_class containing only the instances
@@ -244,9 +239,10 @@ class BaseAccess(object):
         return qs
 
     def filtered_queryset(self):
-        # Override in subclasses
-        # filter objects according to user's read access
-        return self.model.objects.none()
+        if permission_registry.is_registered(self.model):
+            return self.model.access_qs(self.user, 'view')
+        else:
+            raise NotImplementedError('Filtered queryset for model is not written')
 
     def can_read(self, obj):
         return bool(obj and self.get_queryset().filter(pk=obj.pk).exists())
@@ -267,7 +263,11 @@ class BaseAccess(object):
         return self.can_change(obj, data)
 
     def can_delete(self, obj):
-        return self.user.is_superuser
+        if self.user.is_superuser:
+            return True
+        if obj._meta.model_name in [cls._meta.model_name for cls in permission_registry.all_registered_models]:
+            return self.user.has_obj_perm(obj, 'delete')
+        return False
 
     def can_copy(self, obj):
         return self.can_add({'reference_obj': obj})
@@ -438,10 +438,7 @@ class BaseAccess(object):
 
             # Actions not possible for reason unrelated to RBAC
             # Cannot copy with validation errors, or update a manual group/project
-            if 'write' not in getattr(self.user, 'oauth_scopes', ['write']):
-                user_capabilities[display_method] = False  # Read tokens cannot take any actions
-                continue
-            elif display_method in ['copy', 'start', 'schedule'] and isinstance(obj, JobTemplate):
+            if display_method in ['copy', 'start', 'schedule'] and isinstance(obj, JobTemplate):
                 if obj.validation_errors:
                     user_capabilities[display_method] = False
                     continue
@@ -596,7 +593,7 @@ class InstanceGroupAccess(BaseAccess):
        - a superuser
        - admin role on the Instance group
     I can add/delete Instance Groups:
-       - a superuser(system administrator)
+       - a superuser(system administrator), because these are not org-scoped
     I can use Instance Groups when I have:
        - use_role on the instance group
     """
@@ -604,9 +601,6 @@ class InstanceGroupAccess(BaseAccess):
     model = InstanceGroup
     prefetch_related = ('instances',)
 
-    def filtered_queryset(self):
-        return self.model.accessible_objects(self.user, 'read_role')
-
     @check_superuser
     def can_use(self, obj):
         return self.user in obj.use_role
@@ -625,7 +619,7 @@ class InstanceGroupAccess(BaseAccess):
     def can_delete(self, obj):
         if obj.name in [settings.DEFAULT_EXECUTION_QUEUE_NAME, settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME]:
             return False
-        return self.user.is_superuser
+        return self.user.has_obj_perm(obj, 'delete')
 
 
 class UserAccess(BaseAccess):
@@ -642,18 +636,16 @@ class UserAccess(BaseAccess):
     """
 
     model = User
-    prefetch_related = ('profile',)
+    prefetch_related = ('resource',)
 
     def filtered_queryset(self):
         if settings.ORG_ADMINS_CAN_SEE_ALL_USERS and (self.user.admin_of_organizations.exists() or self.user.auditor_of_organizations.exists()):
             qs = User.objects.all()
         else:
             qs = (
-                User.objects.filter(pk__in=Organization.accessible_objects(self.user, 'read_role').values('member_role__members'))
+                User.objects.filter(pk__in=Organization.access_qs(self.user, 'view').values('member_role__members'))
                 | User.objects.filter(pk=self.user.id)
-                | User.objects.filter(
-                    pk__in=Role.objects.filter(singleton_name__in=[ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR]).values('members')
-                )
+                | User.objects.filter(is_superuser=True)
             ).distinct()
         return qs
 
@@ -668,7 +660,7 @@ class UserAccess(BaseAccess):
             return True
         if not settings.MANAGE_ORGANIZATION_AUTH:
             return False
-        return Organization.accessible_objects(self.user, 'admin_role').exists()
+        return Organization.access_qs(self.user, 'change').exists()
 
     def can_change(self, obj, data):
         if data is not None and ('is_superuser' in data or 'is_system_auditor' in data):
@@ -688,7 +680,7 @@ class UserAccess(BaseAccess):
         """
         Returns all organizations that count `u` as a member
         """
-        return Organization.accessible_objects(u, 'member_role')
+        return Organization.access_qs(u, 'member')
 
     def is_all_org_admin(self, u):
         """
@@ -711,6 +703,15 @@ class UserAccess(BaseAccess):
             if not allow_orphans:
                 # in these cases only superusers can modify orphan users
                 return False
+            if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+                # Permission granted if the user has all permissions that the target user has
+                target_perms = set(
+                    RoleEvaluation.objects.filter(role__in=obj.has_roles.all()).values_list('object_id', 'content_type_id', 'codename').distinct()
+                )
+                user_perms = set(
+                    RoleEvaluation.objects.filter(role__in=self.user.has_roles.all()).values_list('object_id', 'content_type_id', 'codename').distinct()
+                )
+                return not (target_perms - user_perms)
             return not obj.roles.all().exclude(ancestors__in=self.user.roles.all()).exists()
         else:
             return self.is_all_org_admin(obj)
@@ -746,82 +747,6 @@ class UserAccess(BaseAccess):
         return False
 
 
-class OAuth2ApplicationAccess(BaseAccess):
-    """
-    I can read, change or delete OAuth 2 applications when:
-     - I am a superuser.
-     - I am the admin of the organization of the user of the application.
-     - I am a user in the organization of the application.
-    I can create OAuth 2 applications when:
-     - I am a superuser.
-     - I am the admin of the organization of the application.
-    """
-
-    model = OAuth2Application
-    select_related = ('user',)
-    prefetch_related = ('organization', 'oauth2accesstoken_set')
-
-    def filtered_queryset(self):
-        org_access_qs = Organization.accessible_objects(self.user, 'member_role')
-        return self.model.objects.filter(organization__in=org_access_qs)
-
-    def can_change(self, obj, data):
-        return self.user.is_superuser or self.check_related('organization', Organization, data, obj=obj, role_field='admin_role', mandatory=True)
-
-    def can_delete(self, obj):
-        return self.user.is_superuser or obj.organization in self.user.admin_of_organizations
-
-    def can_add(self, data):
-        if self.user.is_superuser:
-            return True
-        if not data:
-            return Organization.accessible_objects(self.user, 'admin_role').exists()
-        return self.check_related('organization', Organization, data, role_field='admin_role', mandatory=True)
-
-
-class OAuth2TokenAccess(BaseAccess):
-    """
-    I can read, change or delete an app token when:
-     - I am a superuser.
-     - I am the admin of the organization of the application of the token.
-     - I am the user of the token.
-    I can create an OAuth2 app token when:
-     - I have the read permission of the related application.
-    I can read, change or delete a personal token when:
-     - I am the user of the token
-     - I am the superuser
-    I can create an OAuth2 Personal Access Token when:
-     - I am a user.  But I can only create a PAT for myself.
-    """
-
-    model = OAuth2AccessToken
-
-    select_related = ('user', 'application')
-    prefetch_related = ('refresh_token',)
-
-    def filtered_queryset(self):
-        org_access_qs = Organization.objects.filter(Q(admin_role__members=self.user) | Q(auditor_role__members=self.user))
-        return self.model.objects.filter(application__organization__in=org_access_qs) | self.model.objects.filter(user__id=self.user.pk)
-
-    def can_delete(self, obj):
-        if (self.user.is_superuser) | (obj.user == self.user):
-            return True
-        elif not obj.application:
-            return False
-        return self.user in obj.application.organization.admin_role
-
-    def can_change(self, obj, data):
-        return self.can_delete(obj)
-
-    def can_add(self, data):
-        if 'application' in data:
-            app = get_object_from_data('application', OAuth2Application, data)
-            if app is None:
-                return True
-            return OAuth2ApplicationAccess(self.user).can_read(app)
-        return True
-
-
 class OrganizationAccess(NotificationAttachMixin, BaseAccess):
     """
     I can see organizations when:
@@ -838,13 +763,11 @@ class OrganizationAccess(NotificationAttachMixin, BaseAccess):
     prefetch_related = (
         'created_by',
         'modified_by',
+        'resource',  # dab_resource_registry
     )
     # organization admin_role is not a parent of organization auditor_role
     notification_attach_roles = ['admin_role', 'auditor_role']
 
-    def filtered_queryset(self):
-        return self.model.accessible_objects(self.user, 'read_role')
-
     @check_superuser
     def can_change(self, obj, data):
         if data and data.get('default_environment'):
@@ -912,9 +835,6 @@ class InventoryAccess(BaseAccess):
         Prefetch('labels', queryset=Label.objects.all().order_by('name')),
     )
 
-    def filtered_queryset(self, allowed=None, ad_hoc=None):
-        return self.model.accessible_objects(self.user, 'read_role')
-
     @check_superuser
     def can_use(self, obj):
         return self.user in obj.use_role
@@ -923,7 +843,7 @@ class InventoryAccess(BaseAccess):
     def can_add(self, data):
         # If no data is specified, just checking for generic add permission?
         if not data:
-            return Organization.accessible_objects(self.user, 'inventory_admin_role').exists()
+            return Organization.access_qs(self.user, 'add_inventory').exists()
         return self.check_related('organization', Organization, data, role_field='inventory_admin_role')
 
     @check_superuser
@@ -948,9 +868,6 @@ class InventoryAccess(BaseAccess):
     def can_update(self, obj):
         return self.user in obj.update_role
 
-    def can_delete(self, obj):
-        return self.can_admin(obj, None)
-
     def can_run_ad_hoc_commands(self, obj):
         return self.user in obj.adhoc_role
 
@@ -988,7 +905,7 @@ class HostAccess(BaseAccess):
 
     def can_add(self, data):
         if not data:  # So the browseable API will work
-            return Inventory.accessible_objects(self.user, 'admin_role').exists()
+            return Inventory.access_qs(self.user, 'change').exists()
 
         # Checks for admin or change permission on inventory.
         if not self.check_related('inventory', Inventory, data):
@@ -1050,7 +967,7 @@ class GroupAccess(BaseAccess):
 
     def can_add(self, data):
         if not data:  # So the browseable API will work
-            return Inventory.accessible_objects(self.user, 'admin_role').exists()
+            return Inventory.access_qs(self.user, 'change').exists()
         if 'inventory' not in data:
             return False
         # Checks for admin or change permission on inventory.
@@ -1092,7 +1009,7 @@ class InventorySourceAccess(NotificationAttachMixin, UnifiedCredentialsMixin, Ba
 
     def can_add(self, data):
         if not data or 'inventory' not in data:
-            return Inventory.accessible_objects(self.user, 'admin_role').exists()
+            return Inventory.access_qs(self.user, 'change').exists()
 
         if not self.check_related('source_project', Project, data, role_field='use_role'):
             return False
@@ -1206,9 +1123,6 @@ class CredentialAccess(BaseAccess):
     )
     prefetch_related = ('admin_role', 'use_role', 'read_role', 'admin_role__parents', 'admin_role__members', 'credential_type', 'organization')
 
-    def filtered_queryset(self):
-        return self.model.accessible_objects(self.user, 'read_role')
-
     @check_superuser
     def can_add(self, data):
         if not data:  # So the browseable API will work
@@ -1306,6 +1220,7 @@ class TeamAccess(BaseAccess):
         'created_by',
         'modified_by',
         'organization',
+        'resource',  # dab_resource_registry
     )
 
     def filtered_queryset(self):
@@ -1318,7 +1233,7 @@ class TeamAccess(BaseAccess):
     @check_superuser
     def can_add(self, data):
         if not data:  # So the browseable API will work
-            return Organization.accessible_objects(self.user, 'admin_role').exists()
+            return Organization.access_qs(self.user, 'view').exists()
         if not settings.MANAGE_ORGANIZATION_AUTH:
             return False
         return self.check_related('organization', Organization, data)
@@ -1376,12 +1291,11 @@ class TeamAccess(BaseAccess):
 class ExecutionEnvironmentAccess(BaseAccess):
     """
     I can see an execution environment when:
-     - I'm a superuser
-     - I'm a member of the same organization
-     - it is a global ExecutionEnvironment
+     - I can see its organization
+     - It is a global ExecutionEnvironment
     I can create/change an execution environment when:
      - I'm a superuser
-     - I'm an admin for the organization(s)
+     - I have an organization or object role that gives access
     """
 
     model = ExecutionEnvironment
@@ -1390,26 +1304,34 @@ class ExecutionEnvironmentAccess(BaseAccess):
 
     def filtered_queryset(self):
         return ExecutionEnvironment.objects.filter(
-            Q(organization__in=Organization.accessible_pk_qs(self.user, 'read_role')) | Q(organization__isnull=True)
+            Q(organization__in=Organization.access_ids_qs(self.user, 'view'))
+            | Q(organization__isnull=True)
+            | Q(id__in=ExecutionEnvironment.access_ids_qs(self.user, 'change'))
         ).distinct()
 
     @check_superuser
     def can_add(self, data):
         if not data:  # So the browseable API will work
-            return Organization.accessible_objects(self.user, 'execution_environment_admin_role').exists()
+            return Organization.access_qs(self.user, 'add_executionenvironment').exists()
         return self.check_related('organization', Organization, data, mandatory=True, role_field='execution_environment_admin_role')
 
     @check_superuser
     def can_change(self, obj, data):
         if obj and obj.organization_id is None:
             raise PermissionDenied
-        if self.user not in obj.organization.execution_environment_admin_role:
-            raise PermissionDenied
-        if data and 'organization' in data:
-            new_org = get_object_from_data('organization', Organization, data, obj=obj)
-            if not new_org or self.user not in new_org.execution_environment_admin_role:
+        if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+            if not self.user.has_obj_perm(obj, 'change'):
                 return False
-        return self.check_related('organization', Organization, data, obj=obj, mandatory=True, role_field='execution_environment_admin_role')
+        else:
+            if self.user not in obj.organization.execution_environment_admin_role:
+                raise PermissionDenied
+        if not self.check_related('organization', Organization, data, obj=obj, role_field='execution_environment_admin_role'):
+            return False
+        # Special case that check_related does not catch, org users can not remove the organization from the EE
+        if data and ('organization' in data or 'organization_id' in data):
+            if (not data.get('organization')) and (not data.get('organization_id')):
+                return False
+        return True
 
     def can_delete(self, obj):
         if obj.managed:
@@ -1439,13 +1361,10 @@ class ProjectAccess(NotificationAttachMixin, BaseAccess):
     prefetch_related = ('modified_by', 'created_by', 'organization', 'last_job', 'current_job')
     notification_attach_roles = ['admin_role']
 
-    def filtered_queryset(self):
-        return self.model.accessible_objects(self.user, 'read_role')
-
     @check_superuser
     def can_add(self, data):
         if not data:  # So the browseable API will work
-            return Organization.accessible_objects(self.user, 'project_admin_role').exists()
+            return Organization.access_qs(self.user, 'add_project').exists()
 
         if data.get('default_environment'):
             ee = get_object_from_data('default_environment', ExecutionEnvironment, data)
@@ -1541,9 +1460,6 @@ class JobTemplateAccess(NotificationAttachMixin, UnifiedCredentialsMixin, BaseAc
         Prefetch('last_job', queryset=UnifiedJob.objects.non_polymorphic()),
     )
 
-    def filtered_queryset(self):
-        return self.model.accessible_objects(self.user, 'read_role')
-
     def can_add(self, data):
         """
         a user can create a job template if
@@ -1556,7 +1472,7 @@ class JobTemplateAccess(NotificationAttachMixin, UnifiedCredentialsMixin, BaseAc
         Users who are able to create deploy jobs can also run normal and check (dry run) jobs.
         """
         if not data:  # So the browseable API will work
-            return Project.accessible_objects(self.user, 'use_role').exists()
+            return Project.access_qs(self.user, 'use_project').exists()
 
         # if reference_obj is provided, determine if it can be copied
         reference_obj = data.get('reference_obj', None)
@@ -1581,6 +1497,8 @@ class JobTemplateAccess(NotificationAttachMixin, UnifiedCredentialsMixin, BaseAc
         inventory = get_value(Inventory, 'inventory')
         if inventory:
             if self.user not in inventory.use_role:
+                if self.save_messages:
+                    self.messages['inventory'] = [_('You do not have use permission on Inventory')]
                 return False
 
         if not self.check_related('execution_environment', ExecutionEnvironment, data, role_field='read_role'):
@@ -1589,11 +1507,16 @@ class JobTemplateAccess(NotificationAttachMixin, UnifiedCredentialsMixin, BaseAc
         project = get_value(Project, 'project')
         # If the user has admin access to the project (as an org admin), should
         # be able to proceed without additional checks.
-        if project:
-            return self.user in project.use_role
-        else:
+        if not project:
             return False
 
+        if self.user not in project.use_role:
+            if self.save_messages:
+                self.messages['project'] = [_('You do not have use permission on Project')]
+            return False
+
+        return True
+
     @check_superuser
     def can_copy_related(self, obj):
         """
@@ -1740,13 +1663,13 @@ class JobAccess(BaseAccess):
     def filtered_queryset(self):
         qs = self.model.objects
 
-        qs_jt = qs.filter(job_template__in=JobTemplate.accessible_objects(self.user, 'read_role'))
+        qs_jt = qs.filter(job_template__in=JobTemplate.access_qs(self.user, 'view'))
 
         org_access_qs = Organization.objects.filter(Q(admin_role__members=self.user) | Q(auditor_role__members=self.user))
         if not org_access_qs.exists():
             return qs_jt
 
-        return qs.filter(Q(job_template__in=JobTemplate.accessible_objects(self.user, 'read_role')) | Q(organization__in=org_access_qs)).distinct()
+        return qs.filter(Q(job_template__in=JobTemplate.access_qs(self.user, 'view')) | Q(organization__in=org_access_qs)).distinct()
 
     def can_add(self, data, validate_license=True):
         raise NotImplementedError('Direct job creation not possible in v2 API')
@@ -1835,6 +1758,11 @@ class SystemJobTemplateAccess(BaseAccess):
 
     model = SystemJobTemplate
 
+    def filtered_queryset(self):
+        if self.user.is_superuser or self.user.is_system_auditor:
+            return self.model.objects.all()
+        return self.model.objects.none()
+
     @check_superuser
     def can_start(self, obj, validate_license=True):
         '''Only a superuser can start a job from a SystemJobTemplate'''
@@ -1848,6 +1776,11 @@ class SystemJobAccess(BaseAccess):
 
     model = SystemJob
 
+    def filtered_queryset(self):
+        if self.user.is_superuser or self.user.is_system_auditor:
+            return self.model.objects.all()
+        return self.model.objects.none()
+
     def can_start(self, obj, validate_license=True):
         return False  # no relaunching of system jobs
 
@@ -1947,7 +1880,7 @@ class WorkflowJobTemplateNodeAccess(UnifiedCredentialsMixin, BaseAccess):
     prefetch_related = ('success_nodes', 'failure_nodes', 'always_nodes', 'unified_job_template', 'workflow_job_template')
 
     def filtered_queryset(self):
-        return self.model.objects.filter(workflow_job_template__in=WorkflowJobTemplate.accessible_objects(self.user, 'read_role'))
+        return self.model.objects.filter(workflow_job_template__in=WorkflowJobTemplate.access_qs(self.user, 'view'))
 
     @check_superuser
     def can_add(self, data):
@@ -2062,9 +1995,6 @@ class WorkflowJobTemplateAccess(NotificationAttachMixin, BaseAccess):
         'read_role',
     )
 
-    def filtered_queryset(self):
-        return self.model.accessible_objects(self.user, 'read_role')
-
     @check_superuser
     def can_add(self, data):
         """
@@ -2075,13 +2005,25 @@ class WorkflowJobTemplateAccess(NotificationAttachMixin, BaseAccess):
         Users who are able to create deploy jobs can also run normal and check (dry run) jobs.
         """
         if not data:  # So the browseable API will work
-            return Organization.accessible_objects(self.user, 'workflow_admin_role').exists()
+            return Organization.access_qs(self.user, 'add_workflowjobtemplate').exists()
 
-        return bool(
-            self.check_related('organization', Organization, data, role_field='workflow_admin_role', mandatory=True)
-            and self.check_related('inventory', Inventory, data, role_field='use_role')
-            and self.check_related('execution_environment', ExecutionEnvironment, data, role_field='read_role')
-        )
+        if not self.check_related('organization', Organization, data, role_field='workflow_admin_role', mandatory=True):
+            if data.get('organization', None) is None:
+                if self.save_messages:
+                    self.messages['organization'] = [_('An organization is required to create a workflow job template for normal user')]
+            return False
+
+        if not self.check_related('inventory', Inventory, data, role_field='use_role'):
+            if self.save_messages:
+                self.messages['inventory'] = [_('You do not have use_role to the inventory')]
+            return False
+
+        if not self.check_related('execution_environment', ExecutionEnvironment, data, role_field='read_role'):
+            if self.save_messages:
+                self.messages['execution_environment'] = [_('You do not have read_role to the execution environment')]
+            return False
+
+        return True
 
     def can_copy(self, obj):
         if self.save_messages:
@@ -2156,7 +2098,7 @@ class WorkflowJobAccess(BaseAccess):
     def filtered_queryset(self):
         return WorkflowJob.objects.filter(
             Q(unified_job_template__in=UnifiedJobTemplate.accessible_pk_qs(self.user, 'read_role'))
-            | Q(organization__in=Organization.objects.filter(Q(admin_role__members=self.user)), is_bulk_job=True)
+            | Q(organization__in=Organization.accessible_pk_qs(self.user, 'auditor_role'))
         )
 
     def can_read(self, obj):
@@ -2434,6 +2376,29 @@ class InventoryUpdateEventAccess(BaseAccess):
         return False
 
 
+class ReceptorAddressAccess(BaseAccess):
+    """
+    I can see receptor address records whenever I can access the instance
+    """
+
+    model = ReceptorAddress
+
+    def filtered_queryset(self):
+        return self.model.objects.filter(Q(instance__in=Instance.accessible_pk_qs(self.user, 'read_role')))
+
+    @check_superuser
+    def can_add(self, data):
+        return False
+
+    @check_superuser
+    def can_change(self, obj, data):
+        return False
+
+    @check_superuser
+    def can_delete(self, obj):
+        return False
+
+
 class SystemJobEventAccess(BaseAccess):
     """
     I can only see manage System Jobs events if I'm a super user
@@ -2531,12 +2496,11 @@ class UnifiedJobAccess(BaseAccess):
 
     def filtered_queryset(self):
         inv_pk_qs = Inventory._accessible_pk_qs(Inventory, self.user, 'read_role')
-        org_auditor_qs = Organization.objects.filter(Q(admin_role__members=self.user) | Q(auditor_role__members=self.user))
         qs = self.model.objects.filter(
             Q(unified_job_template_id__in=UnifiedJobTemplate.accessible_pk_qs(self.user, 'read_role'))
             | Q(inventoryupdate__inventory_source__inventory__id__in=inv_pk_qs)
             | Q(adhoccommand__inventory__id__in=inv_pk_qs)
-            | Q(organization__in=org_auditor_qs)
+            | Q(organization__in=Organization.accessible_pk_qs(self.user, 'auditor_role'))
         )
         return qs
 
@@ -2567,6 +2531,8 @@ class ScheduleAccess(UnifiedCredentialsMixin, BaseAccess):
         if not JobLaunchConfigAccess(self.user).can_add(data):
             return False
         if not data:
+            if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+                return self.user.has_roles.filter(permission_partials__codename__in=['execute_jobtemplate', 'update_project', 'update_inventory']).exists()
             return Role.objects.filter(role_field__in=['update_role', 'execute_role'], ancestors__in=self.user.roles.all()).exists()
 
         return self.check_related('unified_job_template', UnifiedJobTemplate, data, role_field='execute_role', mandatory=True)
@@ -2588,29 +2554,28 @@ class ScheduleAccess(UnifiedCredentialsMixin, BaseAccess):
 
 class NotificationTemplateAccess(BaseAccess):
     """
-    I can see/use a notification_template if I have permission to
+    Run standard logic from DAB RBAC
     """
 
     model = NotificationTemplate
     prefetch_related = ('created_by', 'modified_by', 'organization')
 
     def filtered_queryset(self):
+        if settings.ANSIBLE_BASE_ROLE_SYSTEM_ACTIVATED:
+            return self.model.access_qs(self.user, 'view')
         return self.model.objects.filter(
-            Q(organization__in=Organization.accessible_objects(self.user, 'notification_admin_role')) | Q(organization__in=self.user.auditor_of_organizations)
+            Q(organization__in=Organization.access_qs(self.user, 'add_notificationtemplate')) | Q(organization__in=self.user.auditor_of_organizations)
         ).distinct()
 
     @check_superuser
     def can_add(self, data):
         if not data:
-            return Organization.accessible_objects(self.user, 'notification_admin_role').exists()
+            return Organization.access_qs(self.user, 'add_notificationtemplate').exists()
         return self.check_related('organization', Organization, data, role_field='notification_admin_role', mandatory=True)
 
     @check_superuser
     def can_change(self, obj, data):
-        if obj.organization is None:
-            # only superusers are allowed to edit orphan notification templates
-            return False
-        return self.check_related('organization', Organization, data, obj=obj, role_field='notification_admin_role', mandatory=True)
+        return self.user.has_obj_perm(obj, 'change') and self.check_related('organization', Organization, data, obj=obj, role_field='notification_admin_role')
 
     def can_admin(self, obj, data):
         return self.can_change(obj, data)
@@ -2620,9 +2585,7 @@ class NotificationTemplateAccess(BaseAccess):
 
     @check_superuser
     def can_start(self, obj, validate_license=True):
-        if obj.organization is None:
-            return False
-        return self.user in obj.organization.notification_admin_role
+        return self.can_change(obj, None)
 
 
 class NotificationAccess(BaseAccess):
@@ -2635,7 +2598,7 @@ class NotificationAccess(BaseAccess):
 
     def filtered_queryset(self):
         return self.model.objects.filter(
-            Q(notification_template__organization__in=Organization.accessible_objects(self.user, 'notification_admin_role'))
+            Q(notification_template__organization__in=Organization.access_qs(self.user, 'add_notificationtemplate'))
             | Q(notification_template__organization__in=self.user.auditor_of_organizations)
         ).distinct()
 
@@ -2695,8 +2658,6 @@ class ActivityStreamAccess(BaseAccess):
         'credential_type',
         'team',
         'ad_hoc_command',
-        'o_auth2_application',
-        'o_auth2_access_token',
         'notification_template',
         'notification',
         'label',
@@ -2751,11 +2712,7 @@ class ActivityStreamAccess(BaseAccess):
         if credential_set:
             q |= Q(credential__in=credential_set)
 
-        auditing_orgs = (
-            (Organization.accessible_objects(self.user, 'admin_role') | Organization.accessible_objects(self.user, 'auditor_role'))
-            .distinct()
-            .values_list('id', flat=True)
-        )
+        auditing_orgs = (Organization.access_qs(self.user, 'change') | Organization.access_qs(self.user, 'audit')).distinct().values_list('id', flat=True)
         if auditing_orgs:
             q |= (
                 Q(user__in=auditing_orgs.values('member_role__members'))
@@ -2763,7 +2720,7 @@ class ActivityStreamAccess(BaseAccess):
                 | Q(notification_template__organization__in=auditing_orgs)
                 | Q(notification__notification_template__organization__in=auditing_orgs)
                 | Q(label__organization__in=auditing_orgs)
-                | Q(role__in=Role.objects.filter(ancestors__in=self.user.roles.all()) if auditing_orgs else [])
+                | Q(role__in=Role.visible_roles(self.user) if auditing_orgs else [])
             )
 
         project_set = Project.accessible_pk_qs(self.user, 'read_role')
@@ -2786,14 +2743,6 @@ class ActivityStreamAccess(BaseAccess):
         if team_set:
             q |= Q(team__in=team_set)
 
-        app_set = OAuth2ApplicationAccess(self.user).filtered_queryset()
-        if app_set:
-            q |= Q(o_auth2_application__in=app_set)
-
-        token_set = OAuth2TokenAccess(self.user).filtered_queryset()
-        if token_set:
-            q |= Q(o_auth2_access_token__in=token_set)
-
         return qs.filter(q).distinct()
 
     def can_add(self, data):
@@ -2820,13 +2769,10 @@ class RoleAccess(BaseAccess):
 
     def filtered_queryset(self):
         result = Role.visible_roles(self.user)
-        # Sanity check: is the requesting user an orphaned non-admin/auditor?
-        # if yes, make system admin/auditor mandatorily visible.
-        if not self.user.is_superuser and not self.user.is_system_auditor and not self.user.organizations.exists():
-            mandatories = ('system_administrator', 'system_auditor')
-            super_qs = Role.objects.filter(singleton_name__in=mandatories)
-            result = result | super_qs
-        return result
+        # Make system admin/auditor mandatorily visible.
+        mandatories = ('system_administrator', 'system_auditor')
+        super_qs = Role.objects.filter(singleton_name__in=mandatories)
+        return result | super_qs
 
     def can_add(self, obj, data):
         # Unsupported for now

awx/main/analytics/analytics_tasks.py

@@ -2,7 +2,7 @@
 import logging
 
 # AWX
-from awx.main.analytics.subsystem_metrics import Metrics
+from awx.main.analytics.subsystem_metrics import DispatcherMetrics, CallbackReceiverMetrics
 from awx.main.dispatch.publish import task
 from awx.main.dispatch import get_task_queuename
 
@@ -11,4 +11,5 @@ logger = logging.getLogger('awx.main.scheduler')
 
 @task(queue=get_task_queuename)
 def send_subsystem_metrics():
-    Metrics().send_metrics()
+    DispatcherMetrics().send_metrics()
+    CallbackReceiverMetrics().send_metrics()

awx/main/analytics/broadcast_websocket.py

@@ -66,10 +66,8 @@ class FixedSlidingWindow:
 
 
 class RelayWebsocketStatsManager:
-    def __init__(self, event_loop, local_hostname):
+    def __init__(self, local_hostname):
         self._local_hostname = local_hostname
-
-        self._event_loop = event_loop
         self._stats = dict()
         self._redis_key = BROADCAST_WEBSOCKET_REDIS_KEY_NAME
 
@@ -94,7 +92,10 @@ class RelayWebsocketStatsManager:
             self.start()
 
     def start(self):
-        self.async_task = self._event_loop.create_task(self.run_loop())
+        self.async_task = asyncio.get_running_loop().create_task(
+            self.run_loop(),
+            name='RelayWebsocketStatsManager.run_loop',
+        )
         return self.async_task
 
     @classmethod

awx/main/analytics/collectors.py

@@ -419,7 +419,7 @@ def _events_table(since, full_path, until, tbl, where_column, project_job_create
                           resolved_action,
                           resolved_role,
                           -- '-' operator listed here:
-                          -- https://www.postgresql.org/docs/12/functions-json.html
+                          -- https://www.postgresql.org/docs/15/functions-json.html
                           -- note that operator is only supported by jsonb objects
                           -- https://www.postgresql.org/docs/current/datatype-json.html
                           (CASE WHEN event = 'playbook_on_stats' THEN {event_data} - 'artifact_data' END) as playbook_on_stats,
@@ -444,11 +444,6 @@ def _events_table(since, full_path, until, tbl, where_column, project_job_create
     return _copy_table(table='events', query=query(fr"replace({tbl}.event_data, '\u', '\u005cu')::jsonb"), path=full_path)
 
 
-@register('events_table', '1.5', format='csv', description=_('Automation task records'), expensive=four_hour_slicing)
-def events_table_unpartitioned(since, full_path, until, **kwargs):
-    return _events_table(since, full_path, until, '_unpartitioned_main_jobevent', 'created', **kwargs)
-
-
 @register('events_table', '1.5', format='csv', description=_('Automation task records'), expensive=four_hour_slicing)
 def events_table_partitioned_modified(since, full_path, until, **kwargs):
     return _events_table(since, full_path, until, 'main_jobevent', 'modified', project_job_created=True, **kwargs)

awx/main/analytics/core.py

@@ -16,10 +16,13 @@ from rest_framework.exceptions import PermissionDenied
 import requests
 
 from awx.conf.license import get_license
+
+from ansible_base.lib.utils.db import advisory_lock
+
 from awx.main.models import Job
 from awx.main.access import access_registry
 from awx.main.utils import get_awx_http_client_headers, set_environ, datetime_hook
-from awx.main.utils.pglock import advisory_lock
+from awx.main.utils.analytics_proxy import OIDCClient
 
 __all__ = ['register', 'gather', 'ship']
 
@@ -181,7 +184,10 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti
             logger.log(log_level, "Automation Analytics not enabled. Use --dry-run to gather locally without sending.")
             return None
 
-        if not (settings.AUTOMATION_ANALYTICS_URL and settings.REDHAT_USERNAME and settings.REDHAT_PASSWORD):
+        if not (
+            settings.AUTOMATION_ANALYTICS_URL
+            and ((settings.REDHAT_USERNAME and settings.REDHAT_PASSWORD) or (settings.SUBSCRIPTIONS_CLIENT_ID and settings.SUBSCRIPTIONS_CLIENT_SECRET))
+        ):
             logger.log(log_level, "Not gathering analytics, configuration is invalid. Use --dry-run to gather locally without sending.")
             return None
 
@@ -361,21 +367,35 @@ def ship(path):
     if not url:
         logger.error('AUTOMATION_ANALYTICS_URL is not set')
         return False
-    rh_user = getattr(settings, 'REDHAT_USERNAME', None)
-    rh_password = getattr(settings, 'REDHAT_PASSWORD', None)
-    if not rh_user:
-        logger.error('REDHAT_USERNAME is not set')
+
+    rh_id = getattr(settings, 'REDHAT_USERNAME', None)
+    rh_secret = getattr(settings, 'REDHAT_PASSWORD', None)
+
+    if not (rh_id and rh_secret):
+        rh_id = getattr(settings, 'SUBSCRIPTIONS_CLIENT_ID', None)
+        rh_secret = getattr(settings, 'SUBSCRIPTIONS_CLIENT_SECRET', None)
+
+    if not rh_id:
+        logger.error('Neither REDHAT_USERNAME nor SUBSCRIPTIONS_CLIENT_ID are set')
         return False
-    if not rh_password:
-        logger.error('REDHAT_PASSWORD is not set')
+
+    if not rh_secret:
+        logger.error('Neither REDHAT_PASSWORD nor SUBSCRIPTIONS_CLIENT_SECRET are set')
         return False
+
     with open(path, 'rb') as f:
         files = {'file': (os.path.basename(path), f, settings.INSIGHTS_AGENT_MIME)}
         s = requests.Session()
         s.headers = get_awx_http_client_headers()
         s.headers.pop('Content-Type')
         with set_environ(**settings.AWX_TASK_ENV):
-            response = s.post(url, files=files, verify=settings.INSIGHTS_CERT_PATH, auth=(rh_user, rh_password), headers=s.headers, timeout=(31, 31))
+            try:
+                client = OIDCClient(rh_id, rh_secret)
+                response = client.make_request("POST", url, headers=s.headers, files=files, verify=settings.INSIGHTS_CERT_PATH, timeout=(31, 31))
+            except requests.RequestException:
+                logger.error("Automation Analytics API request failed, trying base auth method")
+                response = s.post(url, files=files, verify=settings.INSIGHTS_CERT_PATH, auth=(rh_id, rh_secret), headers=s.headers, timeout=(31, 31))
+
         # Accept 2XX status_codes
         if response.status_code >= 300:
             logger.error('Upload failed with status {}, {}'.format(response.status_code, response.text))

awx/main/analytics/subsystem_metrics.py

@@ -1,10 +1,16 @@
+import itertools
 import redis
 import json
 import time
 import logging
 
+import prometheus_client
+from prometheus_client.core import GaugeMetricFamily, HistogramMetricFamily
+from prometheus_client.registry import CollectorRegistry
 from django.conf import settings
-from django.apps import apps
+from django.http import HttpRequest
+import redis.exceptions
+from rest_framework.request import Request
 
 from awx.main.consumers import emit_channel_notification
 from awx.main.utils import is_testing
@@ -13,6 +19,30 @@ root_key = settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX
 logger = logging.getLogger('awx.main.analytics')
 
 
+class MetricsNamespace:
+    def __init__(self, namespace):
+        self._namespace = namespace
+
+
+class MetricsServerSettings(MetricsNamespace):
+    def port(self):
+        return settings.METRICS_SUBSYSTEM_CONFIG['server'][self._namespace]['port']
+
+
+class MetricsServer(MetricsServerSettings):
+    def __init__(self, namespace, registry):
+        MetricsNamespace.__init__(self, namespace)
+        self._registry = registry
+
+    def start(self):
+        try:
+            # TODO: addr for ipv6 ?
+            prometheus_client.start_http_server(self.port(), addr='localhost', registry=self._registry)
+        except Exception:
+            logger.error(f"MetricsServer failed to start for service '{self._namespace}.")
+            raise
+
+
 class BaseM:
     def __init__(self, field, help_text):
         self.field = field
@@ -148,76 +178,40 @@ class HistogramM(BaseM):
         return output_text
 
 
-class Metrics:
-    def __init__(self, auto_pipe_execute=False, instance_name=None):
+class Metrics(MetricsNamespace):
+    # metric name, help_text
+    METRICSLIST = []
+    _METRICSLIST = [
+        FloatM('subsystem_metrics_pipe_execute_seconds', 'Time spent saving metrics to redis'),
+        IntM('subsystem_metrics_pipe_execute_calls', 'Number of calls to pipe_execute'),
+        FloatM('subsystem_metrics_send_metrics_seconds', 'Time spent sending metrics to other nodes'),
+    ]
+
+    def __init__(self, namespace, auto_pipe_execute=False, instance_name=None, metrics_have_changed=True, **kwargs):
+        MetricsNamespace.__init__(self, namespace)
+
         self.pipe = redis.Redis.from_url(settings.BROKER_URL).pipeline()
         self.conn = redis.Redis.from_url(settings.BROKER_URL)
         self.last_pipe_execute = time.time()
         # track if metrics have been modified since last saved to redis
         # start with True so that we get an initial save to redis
-        self.metrics_have_changed = True
+        self.metrics_have_changed = metrics_have_changed
         self.pipe_execute_interval = settings.SUBSYSTEM_METRICS_INTERVAL_SAVE_TO_REDIS
         self.send_metrics_interval = settings.SUBSYSTEM_METRICS_INTERVAL_SEND_METRICS
         # auto pipe execute will commit transaction of metric data to redis
         # at a regular interval (pipe_execute_interval). If set to False,
         # the calling function should call .pipe_execute() explicitly
         self.auto_pipe_execute = auto_pipe_execute
-        Instance = apps.get_model('main', 'Instance')
         if instance_name:
             self.instance_name = instance_name
         elif is_testing():
             self.instance_name = "awx_testing"
         else:
-            self.instance_name = Instance.objects.my_hostname()
+            self.instance_name = settings.CLUSTER_HOST_ID  # Same as Instance.objects.my_hostname() BUT we do not need to import Instance
 
-        # metric name, help_text
-        METRICSLIST = [
-            SetIntM('callback_receiver_events_queue_size_redis', 'Current number of events in redis queue'),
-            IntM('callback_receiver_events_popped_redis', 'Number of events popped from redis'),
-            IntM('callback_receiver_events_in_memory', 'Current number of events in memory (in transfer from redis to db)'),
-            IntM('callback_receiver_batch_events_errors', 'Number of times batch insertion failed'),
-            FloatM('callback_receiver_events_insert_db_seconds', 'Total time spent saving events to database'),
-            IntM('callback_receiver_events_insert_db', 'Number of events batch inserted into database'),
-            IntM('callback_receiver_events_broadcast', 'Number of events broadcast to other control plane nodes'),
-            HistogramM(
-                'callback_receiver_batch_events_insert_db', 'Number of events batch inserted into database', settings.SUBSYSTEM_METRICS_BATCH_INSERT_BUCKETS
-            ),
-            SetFloatM('callback_receiver_event_processing_avg_seconds', 'Average processing time per event per callback receiver batch'),
-            FloatM('subsystem_metrics_pipe_execute_seconds', 'Time spent saving metrics to redis'),
-            IntM('subsystem_metrics_pipe_execute_calls', 'Number of calls to pipe_execute'),
-            FloatM('subsystem_metrics_send_metrics_seconds', 'Time spent sending metrics to other nodes'),
-            SetFloatM('task_manager_get_tasks_seconds', 'Time spent in loading tasks from db'),
-            SetFloatM('task_manager_start_task_seconds', 'Time spent starting task'),
-            SetFloatM('task_manager_process_running_tasks_seconds', 'Time spent processing running tasks'),
-            SetFloatM('task_manager_process_pending_tasks_seconds', 'Time spent processing pending tasks'),
-            SetFloatM('task_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
-            IntM('task_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
-            SetFloatM('task_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
-            SetIntM('task_manager_tasks_started', 'Number of tasks started'),
-            SetIntM('task_manager_running_processed', 'Number of running tasks processed'),
-            SetIntM('task_manager_pending_processed', 'Number of pending tasks processed'),
-            SetIntM('task_manager_tasks_blocked', 'Number of tasks blocked from running'),
-            SetFloatM('task_manager_commit_seconds', 'Time spent in db transaction, including on_commit calls'),
-            SetFloatM('dependency_manager_get_tasks_seconds', 'Time spent loading pending tasks from db'),
-            SetFloatM('dependency_manager_generate_dependencies_seconds', 'Time spent generating dependencies for pending tasks'),
-            SetFloatM('dependency_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
-            IntM('dependency_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
-            SetFloatM('dependency_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
-            SetIntM('dependency_manager_pending_processed', 'Number of pending tasks processed'),
-            SetFloatM('workflow_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
-            IntM('workflow_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
-            SetFloatM('workflow_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
-            SetFloatM('workflow_manager_spawn_workflow_graph_jobs_seconds', 'Time spent spawning workflow tasks'),
-            SetFloatM('workflow_manager_get_tasks_seconds', 'Time spent loading workflow tasks from db'),
-            # dispatcher subsystem metrics
-            SetIntM('dispatcher_pool_scale_up_events', 'Number of times local dispatcher scaled up a worker since startup'),
-            SetIntM('dispatcher_pool_active_task_count', 'Number of active tasks in the worker pool when last task was submitted'),
-            SetIntM('dispatcher_pool_max_worker_count', 'Highest number of workers in worker pool in last collection interval, about 20s'),
-            SetFloatM('dispatcher_availability', 'Fraction of time (in last collection interval) dispatcher was able to receive messages'),
-        ]
         # turn metric list into dictionary with the metric name as a key
         self.METRICS = {}
-        for m in METRICSLIST:
+        for m in itertools.chain(self.METRICSLIST, self._METRICSLIST):
             self.METRICS[m.field] = m
 
         # track last time metrics were sent to other nodes
@@ -230,7 +224,7 @@ class Metrics:
             m.reset_value(self.conn)
         self.metrics_have_changed = True
         self.conn.delete(root_key + "_lock")
-        for m in self.conn.scan_iter(root_key + '_instance_*'):
+        for m in self.conn.scan_iter(root_key + '-' + self._namespace + '_instance_*'):
             self.conn.delete(m)
 
     def inc(self, field, value):
@@ -297,8 +291,12 @@ class Metrics:
     def send_metrics(self):
         # more than one thread could be calling this at the same time, so should
         # acquire redis lock before sending metrics
-        lock = self.conn.lock(root_key + '_lock')
-        if not lock.acquire(blocking=False):
+        try:
+            lock = self.conn.lock(root_key + '-' + self._namespace + '_lock')
+            if not lock.acquire(blocking=False):
+                return
+        except redis.exceptions.ConnectionError as exc:
+            logger.warning(f'Connection error in send_metrics: {exc}')
             return
         try:
             current_time = time.time()
@@ -307,9 +305,10 @@ class Metrics:
                 payload = {
                     'instance': self.instance_name,
                     'metrics': serialized_metrics,
+                    'metrics_namespace': self._namespace,
                 }
                 # store the serialized data locally as well, so that load_other_metrics will read it
-                self.conn.set(root_key + '_instance_' + self.instance_name, serialized_metrics)
+                self.conn.set(root_key + '-' + self._namespace + '_instance_' + self.instance_name, serialized_metrics)
                 emit_channel_notification("metrics", payload)
 
                 self.previous_send_metrics.set(current_time)
@@ -331,14 +330,14 @@ class Metrics:
         instances_filter = request.query_params.getlist("node")
         # get a sorted list of instance names
         instance_names = [self.instance_name]
-        for m in self.conn.scan_iter(root_key + '_instance_*'):
+        for m in self.conn.scan_iter(root_key + '-' + self._namespace + '_instance_*'):
             instance_names.append(m.decode('UTF-8').split('_instance_')[1])
         instance_names.sort()
         # load data, including data from the this local instance
         instance_data = {}
         for instance in instance_names:
             if len(instances_filter) == 0 or instance in instances_filter:
-                instance_data_from_redis = self.conn.get(root_key + '_instance_' + instance)
+                instance_data_from_redis = self.conn.get(root_key + '-' + self._namespace + '_instance_' + instance)
                 # data from other instances may not be available. That is OK.
                 if instance_data_from_redis:
                     instance_data[instance] = json.loads(instance_data_from_redis.decode('UTF-8'))
@@ -357,6 +356,120 @@ class Metrics:
         return output_text
 
 
+class DispatcherMetrics(Metrics):
+    METRICSLIST = [
+        SetFloatM('task_manager_get_tasks_seconds', 'Time spent in loading tasks from db'),
+        SetFloatM('task_manager_start_task_seconds', 'Time spent starting task'),
+        SetFloatM('task_manager_process_running_tasks_seconds', 'Time spent processing running tasks'),
+        SetFloatM('task_manager_process_pending_tasks_seconds', 'Time spent processing pending tasks'),
+        SetFloatM('task_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
+        IntM('task_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
+        SetFloatM('task_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
+        SetIntM('task_manager_tasks_started', 'Number of tasks started'),
+        SetIntM('task_manager_running_processed', 'Number of running tasks processed'),
+        SetIntM('task_manager_pending_processed', 'Number of pending tasks processed'),
+        SetIntM('task_manager_tasks_blocked', 'Number of tasks blocked from running'),
+        SetFloatM('task_manager_commit_seconds', 'Time spent in db transaction, including on_commit calls'),
+        SetFloatM('dependency_manager_get_tasks_seconds', 'Time spent loading pending tasks from db'),
+        SetFloatM('dependency_manager_generate_dependencies_seconds', 'Time spent generating dependencies for pending tasks'),
+        SetFloatM('dependency_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
+        IntM('dependency_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
+        SetFloatM('dependency_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
+        SetIntM('dependency_manager_pending_processed', 'Number of pending tasks processed'),
+        SetFloatM('workflow_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
+        IntM('workflow_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
+        SetFloatM('workflow_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
+        SetFloatM('workflow_manager_spawn_workflow_graph_jobs_seconds', 'Time spent spawning workflow tasks'),
+        SetFloatM('workflow_manager_get_tasks_seconds', 'Time spent loading workflow tasks from db'),
+        # dispatcher subsystem metrics
+        SetIntM('dispatcher_pool_scale_up_events', 'Number of times local dispatcher scaled up a worker since startup'),
+        SetIntM('dispatcher_pool_active_task_count', 'Number of active tasks in the worker pool when last task was submitted'),
+        SetIntM('dispatcher_pool_max_worker_count', 'Highest number of workers in worker pool in last collection interval, about 20s'),
+        SetFloatM('dispatcher_availability', 'Fraction of time (in last collection interval) dispatcher was able to receive messages'),
+    ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(settings.METRICS_SERVICE_DISPATCHER, *args, **kwargs)
+
+
+class CallbackReceiverMetrics(Metrics):
+    METRICSLIST = [
+        SetIntM('callback_receiver_events_queue_size_redis', 'Current number of events in redis queue'),
+        IntM('callback_receiver_events_popped_redis', 'Number of events popped from redis'),
+        IntM('callback_receiver_events_in_memory', 'Current number of events in memory (in transfer from redis to db)'),
+        IntM('callback_receiver_batch_events_errors', 'Number of times batch insertion failed'),
+        FloatM('callback_receiver_events_insert_db_seconds', 'Total time spent saving events to database'),
+        IntM('callback_receiver_events_insert_db', 'Number of events batch inserted into database'),
+        IntM('callback_receiver_events_broadcast', 'Number of events broadcast to other control plane nodes'),
+        HistogramM(
+            'callback_receiver_batch_events_insert_db', 'Number of events batch inserted into database', settings.SUBSYSTEM_METRICS_BATCH_INSERT_BUCKETS
+        ),
+        SetFloatM('callback_receiver_event_processing_avg_seconds', 'Average processing time per event per callback receiver batch'),
+    ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(settings.METRICS_SERVICE_CALLBACK_RECEIVER, *args, **kwargs)
+
+
 def metrics(request):
-    m = Metrics()
-    return m.generate_metrics(request)
+    output_text = ''
+    for m in [DispatcherMetrics(), CallbackReceiverMetrics()]:
+        output_text += m.generate_metrics(request)
+    return output_text
+
+
+class CustomToPrometheusMetricsCollector(prometheus_client.registry.Collector):
+    """
+    Takes the metric data from redis -> our custom metric fields -> prometheus
+    library metric fields.
+
+    The plan is to get rid of the use of redis, our custom metric fields, and
+    to switch fully to the prometheus library. At that point, this translation
+    code will be deleted.
+    """
+
+    def __init__(self, metrics_obj, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._metrics = metrics_obj
+
+    def collect(self):
+        my_hostname = settings.CLUSTER_HOST_ID
+
+        instance_data = self._metrics.load_other_metrics(Request(HttpRequest()))
+        if not instance_data:
+            logger.debug(f"No metric data not found in redis for metric namespace '{self._metrics._namespace}'")
+            return None
+
+        host_metrics = instance_data.get(my_hostname)
+        for _, metric in self._metrics.METRICS.items():
+            entry = host_metrics.get(metric.field)
+            if not entry:
+                logger.debug(f"{self._metrics._namespace} metric '{metric.field}' not found in redis data payload {json.dumps(instance_data, indent=2)}")
+                continue
+            if isinstance(metric, HistogramM):
+                buckets = list(zip(metric.buckets, entry['counts']))
+                buckets = [[str(i[0]), str(i[1])] for i in buckets]
+                yield HistogramMetricFamily(metric.field, metric.help_text, buckets=buckets, sum_value=entry['sum'])
+            else:
+                yield GaugeMetricFamily(metric.field, metric.help_text, value=entry)
+
+
+class CallbackReceiverMetricsServer(MetricsServer):
+    def __init__(self):
+        registry = CollectorRegistry(auto_describe=True)
+        registry.register(CustomToPrometheusMetricsCollector(CallbackReceiverMetrics(metrics_have_changed=False)))
+        super().__init__(settings.METRICS_SERVICE_CALLBACK_RECEIVER, registry)
+
+
+class DispatcherMetricsServer(MetricsServer):
+    def __init__(self):
+        registry = CollectorRegistry(auto_describe=True)
+        registry.register(CustomToPrometheusMetricsCollector(DispatcherMetrics(metrics_have_changed=False)))
+        super().__init__(settings.METRICS_SERVICE_DISPATCHER, registry)
+
+
+class WebsocketsMetricsServer(MetricsServer):
+    def __init__(self):
+        registry = CollectorRegistry(auto_describe=True)
+        # registry.register()
+        super().__init__(settings.METRICS_SERVICE_WEBSOCKETS, registry)

awx/main/apps.py

@@ -1,7 +1,90 @@
+import os
+
 from django.apps import AppConfig
 from django.utils.translation import gettext_lazy as _
+from awx.main.utils.common import bypass_in_test, load_all_entry_points_for
+from awx.main.utils.migration import is_database_synchronized
+from awx.main.utils.named_url_graph import _customize_graph, generate_graph
+from awx.conf import register, fields
+
+from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name
 
 
 class MainConfig(AppConfig):
     name = 'awx.main'
     verbose_name = _('Main')
+
+    def load_named_url_feature(self):
+        models = [m for m in self.get_models() if hasattr(m, 'get_absolute_url')]
+        generate_graph(models)
+        _customize_graph()
+        register(
+            'NAMED_URL_FORMATS',
+            field_class=fields.DictField,
+            read_only=True,
+            label=_('Formats of all available named urls'),
+            help_text=_('Read-only list of key-value pairs that shows the standard format of all available named URLs.'),
+            category=_('Named URL'),
+            category_slug='named-url',
+        )
+        register(
+            'NAMED_URL_GRAPH_NODES',
+            field_class=fields.DictField,
+            read_only=True,
+            label=_('List of all named url graph nodes.'),
+            help_text=_(
+                'Read-only list of key-value pairs that exposes named URL graph topology.'
+                ' Use this list to programmatically generate named URLs for resources'
+            ),
+            category=_('Named URL'),
+            category_slug='named-url',
+        )
+
+    def _load_credential_types_feature(self):
+        """
+        Create CredentialType records for any discovered credentials.
+
+        Note that Django docs advise _against_ interacting with the database using
+        the ORM models in the ready() path. Specifically, during testing.
+        However, we explicitly use the @bypass_in_test decorator to avoid calling this
+        method during testing.
+
+        Django also advises against running pattern because it runs everywhere i.e.
+        every management command. We use an advisory lock to ensure correctness and
+        we will deal performance if it becomes an issue.
+        """
+        from awx.main.models.credential import CredentialType
+
+        if is_database_synchronized():
+            CredentialType.setup_tower_managed_defaults(app_config=self)
+
+    @bypass_in_test
+    def load_credential_types_feature(self):
+        from awx.main.models.credential import load_credentials
+
+        load_credentials()
+        return self._load_credential_types_feature()
+
+    def load_inventory_plugins(self):
+        from awx.main.models.inventory import InventorySourceOptions
+
+        is_awx = detect_server_product_name() == 'AWX'
+        extra_entry_point_groups = () if is_awx else ('inventory.supported',)
+        entry_points = load_all_entry_points_for(['inventory', *extra_entry_point_groups])
+
+        for entry_point_name, entry_point in entry_points.items():
+            cls = entry_point.load()
+            InventorySourceOptions.injectors[entry_point_name] = cls
+
+    def ready(self):
+        super().ready()
+
+        """
+        Credential loading triggers database operations. There are cases we want to call
+        awx-manage collectstatic without a database. All management commands invoke the ready() code
+        path. Using settings.AWX_SKIP_CREDENTIAL_TYPES_DISCOVER _could_ invoke a database operation.
+        """
+        if not os.environ.get('AWX_SKIP_CREDENTIAL_TYPES_DISCOVER', None):
+            self.load_credential_types_feature()
+        self.load_named_url_feature()
+        self.load_inventory_plugins()

awx/main/conf.py

@@ -2,6 +2,7 @@
 import logging
 
 # Django
+from django.core.checks import Error
 from django.utils.translation import gettext_lazy as _
 
 # Django REST Framework
@@ -45,10 +46,7 @@ register(
     'MANAGE_ORGANIZATION_AUTH',
     field_class=fields.BooleanField,
     label=_('Organization Admins Can Manage Users and Teams'),
-    help_text=_(
-        'Controls whether any Organization Admin has the privileges to create and manage users and teams. '
-        'You may want to disable this ability if you are using an LDAP or SAML integration.'
-    ),
+    help_text=_('Controls whether any Organization Admin has the privileges to create and manage users and teams.'),
     category=_('System'),
     category_slug='system',
 )
@@ -92,6 +90,7 @@ register(
     ),
     category=_('System'),
     category_slug='system',
+    required=False,
 )
 
 register(
@@ -125,8 +124,8 @@ register(
     allow_blank=True,
     encrypted=False,
     read_only=False,
-    label=_('Red Hat customer username'),
-    help_text=_('This username is used to send data to Automation Analytics'),
+    label=_('Red Hat Client ID for Analytics'),
+    help_text=_('Client ID used to send data to Automation Analytics'),
     category=_('System'),
     category_slug='system',
 )
@@ -138,34 +137,34 @@ register(
     allow_blank=True,
     encrypted=True,
     read_only=False,
-    label=_('Red Hat customer password'),
-    help_text=_('This password is used to send data to Automation Analytics'),
+    label=_('Red Hat Client Secret for Analytics'),
+    help_text=_('Client secret used to send data to Automation Analytics'),
     category=_('System'),
     category_slug='system',
 )
 
 register(
-    'SUBSCRIPTIONS_USERNAME',
+    'SUBSCRIPTIONS_CLIENT_ID',
     field_class=fields.CharField,
     default='',
     allow_blank=True,
     encrypted=False,
     read_only=False,
-    label=_('Red Hat or Satellite username'),
-    help_text=_('This username is used to retrieve subscription and content information'),  # noqa
+    label=_('Red Hat Client ID for Subscriptions'),
+    help_text=_('Client ID used to retrieve subscription and content information'),  # noqa
     category=_('System'),
     category_slug='system',
 )
 
 register(
-    'SUBSCRIPTIONS_PASSWORD',
+    'SUBSCRIPTIONS_CLIENT_SECRET',
     field_class=fields.CharField,
     default='',
     allow_blank=True,
     encrypted=True,
     read_only=False,
-    label=_('Red Hat or Satellite password'),
-    help_text=_('This password is used to retrieve subscription and content information'),  # noqa
+    label=_('Red Hat Client Secret for Subscriptions'),
+    help_text=_('Client secret used to retrieve subscription and content information'),  # noqa
     category=_('System'),
     category_slug='system',
 )
@@ -593,7 +592,7 @@ register(
 register(
     'LOG_AGGREGATOR_LOGGERS',
     field_class=fields.StringListField,
-    default=['awx', 'activity_stream', 'job_events', 'system_tracking', 'broadcast_websocket'],
+    default=['awx', 'activity_stream', 'job_events', 'system_tracking', 'broadcast_websocket', 'job_lifecycle'],
     label=_('Loggers Sending Data to Log Aggregator Form'),
     help_text=_(
         'List of loggers that will send HTTP logs to the collector, these can '
@@ -603,6 +602,7 @@ register(
         'job_events - callback data from Ansible job events\n'
         'system_tracking - facts gathered from scan jobs\n'
         'broadcast_websocket - errors pertaining to websockets broadcast metrics\n'
+        'job_lifecycle - logs related to processing of a job\n'
     ),
     category=_('Logging'),
     category_slug='logging',
@@ -694,16 +694,18 @@ register(
     category_slug='logging',
 )
 register(
-    'LOG_AGGREGATOR_MAX_DISK_USAGE_GB',
+    'LOG_AGGREGATOR_ACTION_QUEUE_SIZE',
     field_class=fields.IntegerField,
-    default=1,
+    default=131072,
     min_value=1,
-    label=_('Maximum disk persistence for external log aggregation (in GB)'),
+    label=_('Maximum number of messages that can be stored in the log action queue'),
     help_text=_(
-        'Amount of data to store (in gigabytes) during an outage of '
-        'the external log aggregator (defaults to 1). '
-        'Equivalent to the rsyslogd queue.maxdiskspace setting for main_queue. '
-        'Notably, this is used for the rsyslogd main queue (for input messages).'
+        'Defines how large the rsyslog action queue can grow in number of messages '
+        'stored. This can have an impact on memory utilization. When the queue '
+        'reaches 75% of this number, the queue will start writing to disk '
+        '(queue.highWatermark in rsyslog). When it reaches 90%, NOTICE, INFO, and '
+        'DEBUG messages will start to be discarded (queue.discardMark with '
+        'queue.discardSeverity=5).'
     ),
     category=_('Logging'),
     category_slug='logging',
@@ -718,8 +720,7 @@ register(
         'Amount of data to store (in gigabytes) if an rsyslog action takes time '
         'to process an incoming message (defaults to 1). '
         'Equivalent to the rsyslogd queue.maxdiskspace setting on the action (e.g. omhttp). '
-        'Like LOG_AGGREGATOR_MAX_DISK_USAGE_GB, it stores files in the directory specified '
-        'by LOG_AGGREGATOR_MAX_DISK_USAGE_PATH.'
+        'It stores files in the directory specified by LOG_AGGREGATOR_MAX_DISK_USAGE_PATH.'
     ),
     category=_('Logging'),
     category_slug='logging',
@@ -773,6 +774,8 @@ register(
     allow_null=True,
     category=_('System'),
     category_slug='system',
+    required=False,
+    hidden=True,
 )
 register(
     'AUTOMATION_ANALYTICS_LAST_ENTRIES',
@@ -814,6 +817,7 @@ register(
     help_text=_('Max jobs to allow bulk jobs to launch'),
     category=_('Bulk Actions'),
     category_slug='bulk',
+    hidden=True,
 )
 
 register(
@@ -824,23 +828,26 @@ register(
     help_text=_('Max number of hosts to allow to be created in a single bulk action'),
     category=_('Bulk Actions'),
     category_slug='bulk',
+    hidden=True,
 )
 
 register(
-    'UI_NEXT',
-    field_class=fields.BooleanField,
-    default=False,
-    label=_('Enable Preview of New User Interface'),
-    help_text=_('Enable preview of new user interface.'),
-    category=_('System'),
-    category_slug='system',
+    'BULK_HOST_MAX_DELETE',
+    field_class=fields.IntegerField,
+    default=250,
+    label=_('Max number of hosts to allow to be deleted in a single bulk action'),
+    help_text=_('Max number of hosts to allow to be deleted in a single bulk action'),
+    category=_('Bulk Actions'),
+    category_slug='bulk',
+    hidden=True,
 )
 
+
 register(
     'SUBSCRIPTION_USAGE_MODEL',
     field_class=fields.ChoiceField,
     choices=[
-        ('', _('Default model for AWX - no subscription. Deletion of host_metrics will not be considered for purposes of managed host counting')),
+        ('', _('No subscription. Deletion of host_metrics will not be considered for purposes of managed host counting')),
         (
             SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS,
             _('Usage based on unique managed nodes in a large historical time frame and delete functionality for no longer used managed nodes'),
@@ -860,6 +867,7 @@ register(
     allow_null=True,
     category=_('System'),
     category_slug='system',
+    hidden=True,
 )
 
 register(
@@ -869,6 +877,7 @@ register(
     allow_null=True,
     category=_('System'),
     category_slug='system',
+    hidden=True,
 )
 
 register(
@@ -911,6 +920,16 @@ register(
     category_slug='debug',
 )
 
+register(
+    'RECEPTOR_KEEP_WORK_ON_ERROR',
+    field_class=fields.BooleanField,
+    label=_('Keep receptor work on error'),
+    default=False,
+    help_text=_('Prevent receptor work from being released on when error is detected'),
+    category=('Debug'),
+    category_slug='debug',
+)
+
 
 def logging_validate(serializer, attrs):
     if not serializer.instance or not hasattr(serializer.instance, 'LOG_AGGREGATOR_HOST') or not hasattr(serializer.instance, 'LOG_AGGREGATOR_TYPE'):
@@ -937,3 +956,27 @@ def logging_validate(serializer, attrs):
 
 
 register_validate('logging', logging_validate)
+
+
+def csrf_trusted_origins_validate(serializer, attrs):
+    if not serializer.instance or not hasattr(serializer.instance, 'CSRF_TRUSTED_ORIGINS'):
+        return attrs
+    if 'CSRF_TRUSTED_ORIGINS' not in attrs:
+        return attrs
+    errors = []
+    for origin in attrs['CSRF_TRUSTED_ORIGINS']:
+        if "://" not in origin:
+            errors.append(
+                Error(
+                    "As of Django 4.0, the values in the CSRF_TRUSTED_ORIGINS "
+                    "setting must start with a scheme (usually http:// or "
+                    "https://) but found %s. See the release notes for details." % origin,
+                )
+            )
+    if errors:
+        error_messages = [error.msg for error in errors]
+        raise serializers.ValidationError(_('\n'.join(error_messages)))
+    return attrs
+
+
+register_validate('system', csrf_trusted_origins_validate)

awx/main/constants.py

@@ -6,7 +6,6 @@ import re
 from django.utils.translation import gettext_lazy as _
 
 __all__ = [
-    'CLOUD_PROVIDERS',
     'PRIVILEGE_ESCALATION_METHODS',
     'ANSI_SGR_PATTERN',
     'CAN_CANCEL',
@@ -14,7 +13,6 @@ __all__ = [
     'STANDARD_INVENTORY_UPDATE_ENV',
 ]
 
-CLOUD_PROVIDERS = ('azure_rm', 'ec2', 'gce', 'vmware', 'openstack', 'rhv', 'satellite6', 'controller', 'insights')
 PRIVILEGE_ESCALATION_METHODS = [
     ('sudo', _('Sudo')),
     ('su', _('Su')),
@@ -43,6 +41,7 @@ STANDARD_INVENTORY_UPDATE_ENV = {
 }
 CAN_CANCEL = ('new', 'pending', 'waiting', 'running')
 ACTIVE_STATES = CAN_CANCEL
+ERROR_STATES = ('error',)
 MINIMAL_EVENTS = set(['playbook_on_play_start', 'playbook_on_task_start', 'playbook_on_stats', 'EOF'])
 CENSOR_VALUE = '************'
 ENV_BLOCKLIST = frozenset(
@@ -114,3 +113,28 @@ SUBSCRIPTION_USAGE_MODEL_UNIQUE_HOSTS = 'unique_managed_hosts'
 
 # Shared prefetch to use for creating a queryset for the purpose of writing or saving facts
 HOST_FACTS_FIELDS = ('name', 'ansible_facts', 'ansible_facts_modified', 'modified', 'inventory_id')
+
+# Data for RBAC compatibility layer
+role_name_to_perm_mapping = {
+    'adhoc_role': ['adhoc_'],
+    'approval_role': ['approve_'],
+    'auditor_role': ['audit_'],
+    'admin_role': ['change_', 'add_', 'delete_'],
+    'execute_role': ['execute_'],
+    'read_role': ['view_'],
+    'update_role': ['update_'],
+    'member_role': ['member_'],
+    'use_role': ['use_'],
+}
+
+org_role_to_permission = {
+    'notification_admin_role': 'add_notificationtemplate',
+    'project_admin_role': 'add_project',
+    'execute_role': 'execute_jobtemplate',
+    'inventory_admin_role': 'add_inventory',
+    'credential_admin_role': 'add_credential',
+    'workflow_admin_role': 'add_workflowjobtemplate',
+    'job_template_admin_role': 'change_jobtemplate',  # TODO: this doesnt really work, solution not clear
+    'execution_environment_admin_role': 'add_executionenvironment',
+    'auditor_role': 'view_project',  # TODO: also doesnt really work
+}

awx/main/consumers.py

@@ -106,7 +106,7 @@ class RelayConsumer(AsyncJsonWebsocketConsumer):
         if group == "metrics":
             message = json.loads(message['text'])
             conn = redis.Redis.from_url(settings.BROKER_URL)
-            conn.set(settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX + "_instance_" + message['instance'], message['metrics'])
+            conn.set(settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX + "-" + message['metrics_namespace'] + "_instance_" + message['instance'], message['metrics'])
         else:
             await self.channel_layer.group_send(group, message)
 

awx/main/credential_plugins/aim.py

@@ -1,122 +0,0 @@
-from .plugin import CredentialPlugin, CertFiles, raise_for_status
-
-from urllib.parse import quote, urlencode, urljoin
-
-from django.utils.translation import gettext_lazy as _
-import requests
-
-aim_inputs = {
-    'fields': [
-        {
-            'id': 'url',
-            'label': _('CyberArk CCP URL'),
-            'type': 'string',
-            'format': 'url',
-        },
-        {
-            'id': 'webservice_id',
-            'label': _('Web Service ID'),
-            'type': 'string',
-            'help_text': _('The CCP Web Service ID. Leave blank to default to AIMWebService.'),
-        },
-        {
-            'id': 'app_id',
-            'label': _('Application ID'),
-            'type': 'string',
-            'secret': True,
-        },
-        {
-            'id': 'client_key',
-            'label': _('Client Key'),
-            'type': 'string',
-            'secret': True,
-            'multiline': True,
-        },
-        {
-            'id': 'client_cert',
-            'label': _('Client Certificate'),
-            'type': 'string',
-            'secret': True,
-            'multiline': True,
-        },
-        {
-            'id': 'verify',
-            'label': _('Verify SSL Certificates'),
-            'type': 'boolean',
-            'default': True,
-        },
-    ],
-    'metadata': [
-        {
-            'id': 'object_query',
-            'label': _('Object Query'),
-            'type': 'string',
-            'help_text': _('Lookup query for the object. Ex: Safe=TestSafe;Object=testAccountName123'),
-        },
-        {'id': 'object_query_format', 'label': _('Object Query Format'), 'type': 'string', 'default': 'Exact', 'choices': ['Exact', 'Regexp']},
-        {
-            'id': 'object_property',
-            'label': _('Object Property'),
-            'type': 'string',
-            'help_text': _('The property of the object to return. Default: Content Ex: Username, Address, etc.'),
-        },
-        {
-            'id': 'reason',
-            'label': _('Reason'),
-            'type': 'string',
-            'help_text': _('Object request reason. This is only needed if it is required by the object\'s policy.'),
-        },
-    ],
-    'required': ['url', 'app_id', 'object_query'],
-}
-
-
-def aim_backend(**kwargs):
-    url = kwargs['url']
-    client_cert = kwargs.get('client_cert', None)
-    client_key = kwargs.get('client_key', None)
-    verify = kwargs['verify']
-    webservice_id = kwargs.get('webservice_id', '')
-    app_id = kwargs['app_id']
-    object_query = kwargs['object_query']
-    object_query_format = kwargs['object_query_format']
-    object_property = kwargs.get('object_property', '')
-    reason = kwargs.get('reason', None)
-    if webservice_id == '':
-        webservice_id = 'AIMWebService'
-
-    query_params = {
-        'AppId': app_id,
-        'Query': object_query,
-        'QueryFormat': object_query_format,
-    }
-    if reason:
-        query_params['reason'] = reason
-
-    request_qs = '?' + urlencode(query_params, quote_via=quote)
-    request_url = urljoin(url, '/'.join([webservice_id, 'api', 'Accounts']))
-
-    with CertFiles(client_cert, client_key) as cert:
-        res = requests.get(
-            request_url + request_qs,
-            timeout=30,
-            cert=cert,
-            verify=verify,
-            allow_redirects=False,
-        )
-    raise_for_status(res)
-    # CCP returns the property name capitalized, username is camel case
-    # so we need to handle that case
-    if object_property == '':
-        object_property = 'Content'
-    elif object_property.lower() == 'username':
-        object_property = 'UserName'
-    elif object_property not in res:
-        raise KeyError('Property {} not found in object'.format(object_property))
-    else:
-        object_property = object_property.capitalize()
-
-    return res.json()[object_property]
-
-
-aim_plugin = CredentialPlugin('CyberArk Central Credential Provider Lookup', inputs=aim_inputs, backend=aim_backend)

awx/main/credential_plugins/aws_secretsmanager.py

@@ -1,65 +0,0 @@
-import boto3
-from botocore.exceptions import ClientError
-
-from .plugin import CredentialPlugin
-from django.utils.translation import gettext_lazy as _
-
-
-secrets_manager_inputs = {
-    'fields': [
-        {
-            'id': 'aws_access_key',
-            'label': _('AWS Access Key'),
-            'type': 'string',
-        },
-        {
-            'id': 'aws_secret_key',
-            'label': _('AWS Secret Key'),
-            'type': 'string',
-            'secret': True,
-        },
-    ],
-    'metadata': [
-        {
-            'id': 'region_name',
-            'label': _('AWS Secrets Manager Region'),
-            'type': 'string',
-            'help_text': _('Region which the secrets manager is located'),
-        },
-        {
-            'id': 'secret_name',
-            'label': _('AWS Secret Name'),
-            'type': 'string',
-        },
-    ],
-    'required': ['aws_access_key', 'aws_secret_key', 'region_name', 'secret_name'],
-}
-
-
-def aws_secretsmanager_backend(**kwargs):
-    secret_name = kwargs['secret_name']
-    region_name = kwargs['region_name']
-    aws_secret_access_key = kwargs['aws_secret_key']
-    aws_access_key_id = kwargs['aws_access_key']
-
-    session = boto3.session.Session()
-    client = session.client(
-        service_name='secretsmanager', region_name=region_name, aws_secret_access_key=aws_secret_access_key, aws_access_key_id=aws_access_key_id
-    )
-
-    try:
-        get_secret_value_response = client.get_secret_value(SecretId=secret_name)
-    except ClientError as e:
-        raise e
-    # Secrets Manager decrypts the secret value using the associated KMS CMK
-    # Depending on whether the secret was a string or binary, only one of these fields will be populated
-    if 'SecretString' in get_secret_value_response:
-        secret = get_secret_value_response['SecretString']
-
-    else:
-        secret = get_secret_value_response['SecretBinary']
-
-    return secret
-
-
-aws_secretmanager_plugin = CredentialPlugin('AWS Secrets Manager lookup', inputs=secrets_manager_inputs, backend=aws_secretsmanager_backend)