add licenses

Signed-off-by: jessicamack <jmack@redhat.com>

.github/actions/awx_devel_image/action.yml

@@ -11,12 +11,6 @@ runs:
       shell: bash
       run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
 
-    - name: Set lower case owner name
-      shell: bash
-      run: echo "OWNER_LC=${OWNER,,}" >> $GITHUB_ENV
-      env:
-        OWNER: '${{ github.repository_owner }}'
-
     - name: Log in to registry
       shell: bash
       run: |
@@ -24,11 +18,11 @@ runs:
 
     - name: Pre-pull latest devel image to warm cache
       shell: bash
-      run: docker pull ghcr.io/${OWNER_LC}/awx_devel:${{ github.base_ref }}
+      run: docker pull ghcr.io/${{ github.repository_owner }}/awx_devel:${{ github.base_ref }}
 
     - name: Build image for current source checkout
       shell: bash
       run: |
-        DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} \
+        DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} \
         COMPOSE_TAG=${{ github.base_ref }} \
         make docker-compose-build

.github/actions/issue_metrics/issue_metrics.yml

@@ -1,40 +0,0 @@
-name: Monthly issue metrics
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: '3 2 1 * *'
-
-permissions:
-  issues: write
-  pull-requests: read
-
-jobs:
-  build:
-    name: issue metrics
-    runs-on: ubuntu-latest
-    steps:
-    - name: Get dates for last month
-      shell: bash
-      run: |
-        # Calculate the first day of the previous month
-        first_day=$(date -d "last month" +%Y-%m-01)
-
-        # Calculate the last day of the previous month
-        last_day=$(date -d "$first_day +1 month -1 day" +%Y-%m-%d)
-
-        #Set an environment variable with the date range
-        echo "$first_day..$last_day"
-        echo "last_month=$first_day..$last_day" >> "$GITHUB_ENV"
-
-    - name: Run issue-metrics tool
-      uses: github/issue-metrics@v2
-      env:
-        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        SEARCH_QUERY: 'repo:ansible/awx is:issue created:${{ env.last_month }} -reason:"not planned"'
-
-    - name: Create issue
-      uses: peter-evans/create-issue-from-file@v4
-      with:
-        title: Monthly issue metrics report
-        token: ${{ secrets.GITHUB_TOKEN }}
-        content-filepath: ./issue_metrics.md

.github/actions/run_awx_devel/action.yml

@@ -35,7 +35,7 @@ runs:
     - name: Start AWX
       shell: bash
       run: |
-        DEV_DOCKER_OWNER=${{ github.repository_owner }} \
+        DEV_DOCKER_TAG_BASE=ghcr.io/${{ github.repository_owner }} \
         COMPOSE_TAG=${{ github.base_ref }} \
         COMPOSE_UP_OPTS="-d" \
         make docker-compose
@@ -71,7 +71,7 @@ runs:
       id: data
       shell: bash
       run: |
-        AWX_IP=$(docker inspect -f '{{.NetworkSettings.Networks._sources_awx.IPAddress}}' tools_awx_1)
+        AWX_IP=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' tools_awx_1)
         ADMIN_TOKEN=$(docker exec -i tools_awx_1 awx-manage create_oauth2_token --user admin)
         echo "ip=$AWX_IP" >> $GITHUB_OUTPUT
         echo "admin_token=$ADMIN_TOKEN" >> $GITHUB_OUTPUT

.github/dependabot.yml

@@ -1,10 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "pip"
-    directory: "docs/docsite/"
-    schedule:
-      interval: "weekly"
-    open-pull-requests-limit: 2
-    labels:
-      - "docs"
-      - "dependencies"

.github/pr_labeler.yml

@@ -15,4 +15,5 @@
 
 "dependencies":
   - any: ["awx/ui/package.json"]
-  - any: ["requirements/*"]
+  - any: ["requirements/*.txt"]
+  - any: ["requirements/requirements.in"]

.github/workflows/devel_images.yml

@@ -9,43 +9,22 @@ on:
       - release_*
       - feature_*
 jobs:
-  push-development-images:
+  push:
+    if: endsWith(github.repository, '/awx') || startsWith(github.ref, 'refs/heads/release_')
     runs-on: ubuntu-latest
-    timeout-minutes: 120
+    timeout-minutes: 60
     permissions:
       packages: write
       contents: read
-    strategy:
-      fail-fast: false
-      matrix:
-        build-targets:
-          - image-name: awx_devel
-            make-target: docker-compose-buildx
-          - image-name: awx_kube_devel
-            make-target: awx-kube-dev-buildx
-          - image-name: awx
-            make-target: awx-kube-buildx
     steps:
-
-      - name: Skipping build of awx image for non-awx repository
-        run: |
-          echo "Skipping build of awx image for non-awx repository"
-          exit 0
-        if: matrix.build-targets.image-name == 'awx' && !endsWith(github.repository, '/awx')
-
       - uses: actions/checkout@v3
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+      - name: Get python version from Makefile
+        run: echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Set GITHUB_ENV variables
+      - name: Set lower case owner name
         run: |
-          echo "DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER,,}" >> $GITHUB_ENV
-          echo "COMPOSE_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV
-          echo py_version=`make PYTHON_VERSION` >> $GITHUB_ENV
+          echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV}
         env:
           OWNER: '${{ github.repository_owner }}'
 
@@ -58,19 +37,23 @@ jobs:
         run: |
           echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
-      - name: Setup node and npm
-        uses: actions/setup-node@v2
-        with:
-          node-version: '16.13.1'
-        if: matrix.build-targets.image-name == 'awx'
-
-      - name: Prebuild UI for awx image (to speed up build process)
+      - name: Pre-pull image to warm build cache
         run: |
-          sudo apt-get install gettext
-          make ui-release
-          make ui-next
-        if: matrix.build-targets.image-name == 'awx'
+          docker pull ghcr.io/${OWNER_LC}/awx_devel:${GITHUB_REF##*/} || :
+          docker pull ghcr.io/${OWNER_LC}/awx_kube_devel:${GITHUB_REF##*/} || :
+          docker pull ghcr.io/${OWNER_LC}/awx:${GITHUB_REF##*/} || :
 
-      - name: Build and push AWX devel images
+      - name: Build images
         run: |
-          make ${{ matrix.build-targets.make-target }}
+          DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make docker-compose-build
+          DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-dev-build
+          DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-build
+
+      - name: Push development images
+        run: |
+          docker push ghcr.io/${OWNER_LC}/awx_devel:${GITHUB_REF##*/}
+          docker push ghcr.io/${OWNER_LC}/awx_kube_devel:${GITHUB_REF##*/}
+
+      - name: Push AWX k8s image, only for upstream and feature branches
+        run: docker push ghcr.io/${OWNER_LC}/awx:${GITHUB_REF##*/}
+        if: endsWith(github.repository, '/awx')

.github/workflows/feature_branch_deletion.yml

@@ -2,10 +2,12 @@
 name: Feature branch deletion cleanup
 env:
   LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
-on: delete
+on:
+  delete:
+    branches:
+      - feature_**
 jobs:
-  branch_delete:
-    if: ${{ github.event.ref_type == 'branch' && startsWith(github.event.ref, 'feature_') }}
+  push:
     runs-on: ubuntu-latest
     timeout-minutes: 20
     permissions:
@@ -20,4 +22,6 @@ jobs:
         run: |
           ansible localhost -c local, -m command -a "{{ ansible_python_interpreter + ' -m pip install boto3'}}"
           ansible localhost -c local -m aws_s3 \
-            -a "bucket=awx-public-ci-files object=${GITHUB_REF##*/}/schema.json mode=delobj permission=public-read"
+            -a "bucket=awx-public-ci-files object=${GITHUB_REF##*/}/schema.json mode=delete permission=public-read"
+
+

.github/workflows/promote.yml

@@ -66,7 +66,7 @@ jobs:
       - name: Build awxkit and upload to pypi
         run: |
           git reset --hard
-          cd awxkit && python3 setup.py sdist bdist_wheel
+          cd awxkit && python3 setup.py bdist_wheel
           twine upload \
             -r ${{ env.pypi_repo }} \
             -u ${{ secrets.PYPI_USERNAME }} \

.github/workflows/stage.yml

@@ -86,19 +86,13 @@ jobs:
             -e push=yes \
             -e awx_official=yes
 
-      - name: Log into registry ghcr.io
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d    # v3.0.0
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Log in to GHCR
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
 
-      - name: Log into registry quay.io
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d    # v3.0.0
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_USER }}
-          password: ${{ secrets.QUAY_TOKEN }}
+      - name: Log in to Quay
+        run: |
+          echo ${{ secrets.QUAY_TOKEN }} | docker login quay.io -u ${{ secrets.QUAY_USER }} --password-stdin
 
       - name: tag awx-ee:latest with version input
         run: |
@@ -106,13 +100,13 @@ jobs:
           docker tag quay.io/ansible/awx-ee:latest ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.inputs.version }}
           docker push ghcr.io/${{ github.repository_owner }}/awx-ee:${{ github.event.inputs.version }}
 
-      - name: Stage awx-operator image
+      - name: Build and stage awx-operator
         working-directory: awx-operator
         run: |
-          BUILD_ARGS="--build-arg DEFAULT_AWX_VERSION=${{ github.event.inputs.version}} \
-              --build-arg OPERATOR_VERSION=${{ github.event.inputs.operator_version }}" \
-          IMG=ghcr.io/${{ github.repository_owner }}/awx-operator:${{ github.event.inputs.operator_version }} \
-          make docker-buildx
+          BUILD_ARGS="--build-arg DEFAULT_AWX_VERSION=${{ github.event.inputs.version }} \
+                      --build-arg OPERATOR_VERSION=${{ github.event.inputs.operator_version }}" \
+          IMAGE_TAG_BASE=ghcr.io/${{ github.repository_owner }}/awx-operator \
+          VERSION=${{ github.event.inputs.operator_version }} make docker-build docker-push
 
       - name: Run test deployment with awx-operator
         working-directory: awx-operator

.gitignore

@@ -169,6 +169,3 @@ awx/ui_next/build
 # Docs build stuff
 docs/docsite/build/
 _readthedocs/
-
-# Pyenv
-.python-version

MANIFEST.in

@@ -22,7 +22,7 @@ recursive-exclude awx/settings local_settings.py*
 include tools/scripts/request_tower_configuration.sh
 include tools/scripts/request_tower_configuration.ps1
 include tools/scripts/automation-controller-service
-include tools/scripts/rsyslog-4xx-recovery
+include tools/scripts/failure-event-handler
 include tools/scripts/awx-python
 include awx/playbooks/library/mkfifo.py
 include tools/sosreport/*

Makefile

@@ -10,7 +10,7 @@ KIND_BIN ?= $(shell which kind)
 CHROMIUM_BIN=/tmp/chrome-linux/chrome
 GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 MANAGEMENT_COMMAND ?= awx-manage
-VERSION ?= $(shell $(PYTHON) tools/scripts/scm_version.py 2> /dev/null)
+VERSION ?= $(shell $(PYTHON) tools/scripts/scm_version.py)
 
 # ansible-test requires semver compatable version, so we allow overrides to hack it
 COLLECTION_VERSION ?= $(shell $(PYTHON) tools/scripts/scm_version.py | cut -d . -f 1-3)
@@ -43,8 +43,6 @@ PROMETHEUS ?= false
 GRAFANA ?= false
 # If set to true docker-compose will also start a hashicorp vault instance
 VAULT ?= false
-# If set to true docker-compose will also start a hashicorp vault instance with TLS enabled
-VAULT_TLS ?= false
 # If set to true docker-compose will also start a tacacs+ instance
 TACACS ?= false
 
@@ -63,7 +61,7 @@ RECEPTOR_IMAGE ?= quay.io/ansible/receptor:devel
 SRC_ONLY_PKGS ?= cffi,pycparser,psycopg,twilio
 # These should be upgraded in the AWX and Ansible venv before attempting
 # to install the actual requirements
-VENV_BOOTSTRAP ?= pip==21.2.4 setuptools==65.6.3 setuptools_scm[toml]==8.0.4 wheel==0.38.4
+VENV_BOOTSTRAP ?= pip==21.2.4 setuptools==65.6.3 setuptools_scm[toml]==7.0.5 wheel==0.38.4
 
 NAME ?= awx
 
@@ -75,9 +73,6 @@ SDIST_TAR_FILE ?= $(SDIST_TAR_NAME).tar.gz
 
 I18N_FLAG_FILE = .i18n_built
 
-## PLATFORMS defines the target platforms for  the manager image be build to provide support to multiple
-PLATFORMS ?= linux/amd64,linux/arm64  # linux/ppc64le,linux/s390x
-
 .PHONY: awx-link clean clean-tmp clean-venv requirements requirements_dev \
 	develop refresh adduser migrate dbchange \
 	receiver test test_unit test_coverage coverage_html \
@@ -533,16 +528,13 @@ docker-compose-sources: .git/hooks/pre-commit
 	    -e enable_prometheus=$(PROMETHEUS) \
 	    -e enable_grafana=$(GRAFANA) \
 	    -e enable_vault=$(VAULT) \
-	    -e vault_tls=$(VAULT_TLS) \
 	    -e enable_tacacs=$(TACACS) \
-	    $(EXTRA_SOURCES_ANSIBLE_OPTS)
+            $(EXTRA_SOURCES_ANSIBLE_OPTS)
 
 docker-compose: awx/projects docker-compose-sources
 	ansible-galaxy install --ignore-certs -r tools/docker-compose/ansible/requirements.yml;
 	ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/initialize_containers.yml \
-	    -e enable_vault=$(VAULT) \
-	    -e vault_tls=$(VAULT_TLS) \
-	    -e enable_ldap=$(LDAP);
+	  -e enable_vault=$(VAULT);
 	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans
 
 docker-compose-credential-plugins: awx/projects docker-compose-sources
@@ -589,27 +581,12 @@ docker-compose-build: Dockerfile.dev
 		--build-arg BUILDKIT_INLINE_CACHE=1 \
 		--cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) .
 
-
-.PHONY: docker-compose-buildx
-## Build awx_devel image for docker compose development environment for multiple architectures
-docker-compose-buildx: Dockerfile.dev
-	- docker buildx create --name docker-compose-buildx
-	docker buildx use docker-compose-buildx
-	- docker buildx build \
-		--push \
-		--build-arg BUILDKIT_INLINE_CACHE=1 \
-		--cache-from=$(DEV_DOCKER_TAG_BASE)/awx_devel:$(COMPOSE_TAG) \
-		--platform=$(PLATFORMS) \
-		--tag $(DEVEL_IMAGE_NAME) \
-		-f Dockerfile.dev .
-	- docker buildx rm docker-compose-buildx
-
 docker-clean:
 	-$(foreach container_id,$(shell docker ps -f name=tools_awx -aq && docker ps -f name=tools_receptor -aq),docker stop $(container_id); docker rm -f $(container_id);)
 	-$(foreach image_id,$(shell docker images --filter=reference='*/*/*awx_devel*' --filter=reference='*/*awx_devel*' --filter=reference='*awx_devel*' -aq),docker rmi --force $(image_id);)
 
 docker-clean-volumes: docker-compose-clean docker-compose-container-group-clean
-	docker volume rm -f tools_awx_db tools_vault_1 tools_ldap_1 tools_grafana_storage tools_prometheus_storage $(docker volume ls --filter name=tools_redis_socket_ -q)
+	docker volume rm -f tools_awx_db tools_vault_1 tools_grafana_storage tools_prometheus_storage $(docker volume ls --filter name=tools_redis_socket_ -q)
 
 docker-refresh: docker-clean docker-compose
 
@@ -666,21 +643,6 @@ awx-kube-build: Dockerfile
 		--build-arg HEADLESS=$(HEADLESS) \
 		-t $(DEV_DOCKER_TAG_BASE)/awx:$(COMPOSE_TAG) .
 
-## Build multi-arch awx image for deployment on Kubernetes environment.
-awx-kube-buildx: Dockerfile
-	- docker buildx create --name awx-kube-buildx
-	docker buildx use awx-kube-buildx
-	- docker buildx build \
-		--push \
-		--build-arg VERSION=$(VERSION) \
-		--build-arg SETUPTOOLS_SCM_PRETEND_VERSION=$(VERSION) \
-		--build-arg HEADLESS=$(HEADLESS) \
-		--platform=$(PLATFORMS) \
-		--tag $(DEV_DOCKER_TAG_BASE)/awx:$(COMPOSE_TAG) \
-		-f Dockerfile .
-	- docker buildx rm awx-kube-buildx
-
-
 .PHONY: Dockerfile.kube-dev
 ## Generate Docker.kube-dev for awx_kube_devel image
 Dockerfile.kube-dev: tools/ansible/roles/dockerfile/templates/Dockerfile.j2
@@ -697,18 +659,6 @@ awx-kube-dev-build: Dockerfile.kube-dev
 	    --cache-from=$(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG) \
 	    -t $(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG) .
 
-## Build and push multi-arch awx_kube_devel image for development on local Kubernetes environment.
-awx-kube-dev-buildx: Dockerfile.kube-dev
-	- docker buildx create --name awx-kube-dev-buildx
-	docker buildx use awx-kube-dev-buildx
-	- docker buildx build \
-		--push \
-		--build-arg BUILDKIT_INLINE_CACHE=1 \
-		--cache-from=$(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG) \
-		--platform=$(PLATFORMS) \
-		--tag $(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG) \
-		-f Dockerfile.kube-dev .
-	- docker buildx rm awx-kube-dev-buildx
 
 kind-dev-load: awx-kube-dev-build
 	$(KIND_BIN) load docker-image $(DEV_DOCKER_TAG_BASE)/awx_kube_devel:$(COMPOSE_TAG)

README.md

@@ -7,7 +7,7 @@ AWX provides a web-based user interface, REST API, and task engine built on top
 
 To install AWX, please view the [Install guide](./INSTALL.md).
 
-To learn more about using AWX, view the [AWX docs site](https://ansible.readthedocs.io/projects/awx/en/latest/).
+To learn more about using AWX, and Tower, view the [Tower docs site](http://docs.ansible.com/ansible-tower/index.html).
 
 The AWX Project Frequently Asked Questions can be found [here](https://www.ansible.com/awx-project-faq).
 

awx/api/filters.py

@@ -0,0 +1,450 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved.
+
+# Python
+import re
+import json
+from functools import reduce
+
+# Django
+from django.core.exceptions import FieldError, ValidationError, FieldDoesNotExist
+from django.db import models
+from django.db.models import Q, CharField, IntegerField, BooleanField, TextField, JSONField
+from django.db.models.fields.related import ForeignObjectRel, ManyToManyField, ForeignKey
+from django.db.models.functions import Cast
+from django.contrib.contenttypes.models import ContentType
+from django.contrib.contenttypes.fields import GenericForeignKey
+from django.utils.encoding import force_str
+from django.utils.translation import gettext_lazy as _
+
+# Django REST Framework
+from rest_framework.exceptions import ParseError, PermissionDenied
+from rest_framework.filters import BaseFilterBackend
+
+# AWX
+from awx.main.utils import get_type_for_model, to_python_boolean
+from awx.main.utils.db import get_all_field_names
+
+
+class TypeFilterBackend(BaseFilterBackend):
+    """
+    Filter on type field now returned with all objects.
+    """
+
+    def filter_queryset(self, request, queryset, view):
+        try:
+            types = None
+            for key, value in request.query_params.items():
+                if key == 'type':
+                    if ',' in value:
+                        types = value.split(',')
+                    else:
+                        types = (value,)
+            if types:
+                types_map = {}
+                for ct in ContentType.objects.filter(Q(app_label='main') | Q(app_label='auth', model='user')):
+                    ct_model = ct.model_class()
+                    if not ct_model:
+                        continue
+                    ct_type = get_type_for_model(ct_model)
+                    types_map[ct_type] = ct.pk
+                model = queryset.model
+                model_type = get_type_for_model(model)
+                if 'polymorphic_ctype' in get_all_field_names(model):
+                    types_pks = set([v for k, v in types_map.items() if k in types])
+                    queryset = queryset.filter(polymorphic_ctype_id__in=types_pks)
+                elif model_type in types:
+                    queryset = queryset
+                else:
+                    queryset = queryset.none()
+            return queryset
+        except FieldError as e:
+            # Return a 400 for invalid field names.
+            raise ParseError(*e.args)
+
+
+def get_fields_from_path(model, path):
+    """
+    Given a Django ORM lookup path (possibly over multiple models)
+    Returns the fields in the line, and also the revised lookup path
+    ex., given
+        model=Organization
+        path='project__timeout'
+    returns tuple of fields traversed as well and a corrected path,
+    for special cases we do substitutions
+        ([<IntegerField for timeout>], 'project__timeout')
+    """
+    # Store of all the fields used to detect repeats
+    field_list = []
+    new_parts = []
+    for name in path.split('__'):
+        if model is None:
+            raise ParseError(_('No related model for field {}.').format(name))
+        # HACK: Make project and inventory source filtering by old field names work for backwards compatibility.
+        if model._meta.object_name in ('Project', 'InventorySource'):
+            name = {'current_update': 'current_job', 'last_update': 'last_job', 'last_update_failed': 'last_job_failed', 'last_updated': 'last_job_run'}.get(
+                name, name
+            )
+
+        if name == 'type' and 'polymorphic_ctype' in get_all_field_names(model):
+            name = 'polymorphic_ctype'
+            new_parts.append('polymorphic_ctype__model')
+        else:
+            new_parts.append(name)
+
+        if name in getattr(model, 'PASSWORD_FIELDS', ()):
+            raise PermissionDenied(_('Filtering on password fields is not allowed.'))
+        elif name == 'pk':
+            field = model._meta.pk
+        else:
+            name_alt = name.replace("_", "")
+            if name_alt in model._meta.fields_map.keys():
+                field = model._meta.fields_map[name_alt]
+                new_parts.pop()
+                new_parts.append(name_alt)
+            else:
+                field = model._meta.get_field(name)
+            if isinstance(field, ForeignObjectRel) and getattr(field.field, '__prevent_search__', False):
+                raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
+            elif getattr(field, '__prevent_search__', False):
+                raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
+        if field in field_list:
+            # Field traversed twice, could create infinite JOINs, DoSing Tower
+            raise ParseError(_('Loops not allowed in filters, detected on field {}.').format(field.name))
+        field_list.append(field)
+        model = getattr(field, 'related_model', None)
+
+    return field_list, '__'.join(new_parts)
+
+
+def get_field_from_path(model, path):
+    """
+    Given a Django ORM lookup path (possibly over multiple models)
+    Returns the last field in the line, and the revised lookup path
+    ex.
+        (<IntegerField for timeout>, 'project__timeout')
+    """
+    field_list, new_path = get_fields_from_path(model, path)
+    return (field_list[-1], new_path)
+
+
+class FieldLookupBackend(BaseFilterBackend):
+    """
+    Filter using field lookups provided via query string parameters.
+    """
+
+    RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by', 'search', 'type', 'host_filter', 'count_disabled', 'no_truncate', 'limit')
+
+    SUPPORTED_LOOKUPS = (
+        'exact',
+        'iexact',
+        'contains',
+        'icontains',
+        'startswith',
+        'istartswith',
+        'endswith',
+        'iendswith',
+        'regex',
+        'iregex',
+        'gt',
+        'gte',
+        'lt',
+        'lte',
+        'in',
+        'isnull',
+        'search',
+    )
+
+    # A list of fields that we know can be filtered on without the possibility
+    # of introducing duplicates
+    NO_DUPLICATES_ALLOW_LIST = (CharField, IntegerField, BooleanField, TextField)
+
+    def get_fields_from_lookup(self, model, lookup):
+        if '__' in lookup and lookup.rsplit('__', 1)[-1] in self.SUPPORTED_LOOKUPS:
+            path, suffix = lookup.rsplit('__', 1)
+        else:
+            path = lookup
+            suffix = 'exact'
+
+        if not path:
+            raise ParseError(_('Query string field name not provided.'))
+
+        # FIXME: Could build up a list of models used across relationships, use
+        # those lookups combined with request.user.get_queryset(Model) to make
+        # sure user cannot query using objects he could not view.
+        field_list, new_path = get_fields_from_path(model, path)
+
+        new_lookup = new_path
+        new_lookup = '__'.join([new_path, suffix])
+        return field_list, new_lookup
+
+    def get_field_from_lookup(self, model, lookup):
+        '''Method to match return type of single field, if needed.'''
+        field_list, new_lookup = self.get_fields_from_lookup(model, lookup)
+        return (field_list[-1], new_lookup)
+
+    def to_python_related(self, value):
+        value = force_str(value)
+        if value.lower() in ('none', 'null'):
+            return None
+        else:
+            return int(value)
+
+    def value_to_python_for_field(self, field, value):
+        if isinstance(field, models.BooleanField):
+            return to_python_boolean(value)
+        elif isinstance(field, (ForeignObjectRel, ManyToManyField, GenericForeignKey, ForeignKey)):
+            try:
+                return self.to_python_related(value)
+            except ValueError:
+                raise ParseError(_('Invalid {field_name} id: {field_id}').format(field_name=getattr(field, 'name', 'related field'), field_id=value))
+        else:
+            return field.to_python(value)
+
+    def value_to_python(self, model, lookup, value):
+        try:
+            lookup.encode("ascii")
+        except UnicodeEncodeError:
+            raise ValueError("%r is not an allowed field name. Must be ascii encodable." % lookup)
+
+        field_list, new_lookup = self.get_fields_from_lookup(model, lookup)
+        field = field_list[-1]
+
+        needs_distinct = not all(isinstance(f, self.NO_DUPLICATES_ALLOW_LIST) for f in field_list)
+
+        # Type names are stored without underscores internally, but are presented and
+        # and serialized over the API containing underscores so we remove `_`
+        # for polymorphic_ctype__model lookups.
+        if new_lookup.startswith('polymorphic_ctype__model'):
+            value = value.replace('_', '')
+        elif new_lookup.endswith('__isnull'):
+            value = to_python_boolean(value)
+        elif new_lookup.endswith('__in'):
+            items = []
+            if not value:
+                raise ValueError('cannot provide empty value for __in')
+            for item in value.split(','):
+                items.append(self.value_to_python_for_field(field, item))
+            value = items
+        elif new_lookup.endswith('__regex') or new_lookup.endswith('__iregex'):
+            try:
+                re.compile(value)
+            except re.error as e:
+                raise ValueError(e.args[0])
+        elif new_lookup.endswith('__iexact'):
+            if not isinstance(field, (CharField, TextField)):
+                raise ValueError(f'{field.name} is not a text field and cannot be filtered by case-insensitive search')
+        elif new_lookup.endswith('__search'):
+            related_model = getattr(field, 'related_model', None)
+            if not related_model:
+                raise ValueError('%s is not searchable' % new_lookup[:-8])
+            new_lookups = []
+            for rm_field in related_model._meta.fields:
+                if rm_field.name in ('username', 'first_name', 'last_name', 'email', 'name', 'description', 'playbook'):
+                    new_lookups.append('{}__{}__icontains'.format(new_lookup[:-8], rm_field.name))
+            return value, new_lookups, needs_distinct
+        else:
+            if isinstance(field, JSONField):
+                new_lookup = new_lookup.replace(field.name, f'{field.name}_as_txt')
+            value = self.value_to_python_for_field(field, value)
+        return value, new_lookup, needs_distinct
+
+    def filter_queryset(self, request, queryset, view):
+        try:
+            # Apply filters specified via query_params. Each entry in the lists
+            # below is (negate, field, value).
+            and_filters = []
+            or_filters = []
+            chain_filters = []
+            role_filters = []
+            search_filters = {}
+            needs_distinct = False
+            # Can only have two values: 'AND', 'OR'
+            # If 'AND' is used, an item must satisfy all conditions to show up in the results.
+            # If 'OR' is used, an item just needs to satisfy one condition to appear in results.
+            search_filter_relation = 'OR'
+            for key, values in request.query_params.lists():
+                if key in self.RESERVED_NAMES:
+                    continue
+
+                # HACK: make `created` available via API for the Django User ORM model
+                # so it keep compatibility with other objects which exposes the `created` attr.
+                if queryset.model._meta.object_name == 'User' and key.startswith('created'):
+                    key = key.replace('created', 'date_joined')
+
+                # HACK: Make job event filtering by host name mostly work even
+                # when not capturing job event hosts M2M.
+                if queryset.model._meta.object_name == 'JobEvent' and key.startswith('hosts__name'):
+                    key = key.replace('hosts__name', 'or__host__name')
+                    or_filters.append((False, 'host__name__isnull', True))
+
+                # Custom __int filter suffix (internal use only).
+                q_int = False
+                if key.endswith('__int'):
+                    key = key[:-5]
+                    q_int = True
+
+                # RBAC filtering
+                if key == 'role_level':
+                    role_filters.append(values[0])
+                    continue
+
+                # Search across related objects.
+                if key.endswith('__search'):
+                    if values and ',' in values[0]:
+                        search_filter_relation = 'AND'
+                        values = reduce(lambda list1, list2: list1 + list2, [i.split(',') for i in values])
+                    for value in values:
+                        search_value, new_keys, _ = self.value_to_python(queryset.model, key, force_str(value))
+                        assert isinstance(new_keys, list)
+                        search_filters[search_value] = new_keys
+                    # by definition, search *only* joins across relations,
+                    # so it _always_ needs a .distinct()
+                    needs_distinct = True
+                    continue
+
+                # Custom chain__ and or__ filters, mutually exclusive (both can
+                # precede not__).
+                q_chain = False
+                q_or = False
+                if key.startswith('chain__'):
+                    key = key[7:]
+                    q_chain = True
+                elif key.startswith('or__'):
+                    key = key[4:]
+                    q_or = True
+
+                # Custom not__ filter prefix.
+                q_not = False
+                if key.startswith('not__'):
+                    key = key[5:]
+                    q_not = True
+
+                # Convert value(s) to python and add to the appropriate list.
+                for value in values:
+                    if q_int:
+                        value = int(value)
+                    value, new_key, distinct = self.value_to_python(queryset.model, key, value)
+                    if distinct:
+                        needs_distinct = True
+                    if '_as_txt' in new_key:
+                        fname = next(item for item in new_key.split('__') if item.endswith('_as_txt'))
+                        queryset = queryset.annotate(**{fname: Cast(fname[:-7], output_field=TextField())})
+                    if q_chain:
+                        chain_filters.append((q_not, new_key, value))
+                    elif q_or:
+                        or_filters.append((q_not, new_key, value))
+                    else:
+                        and_filters.append((q_not, new_key, value))
+
+            # Now build Q objects for database query filter.
+            if and_filters or or_filters or chain_filters or role_filters or search_filters:
+                args = []
+                for n, k, v in and_filters:
+                    if n:
+                        args.append(~Q(**{k: v}))
+                    else:
+                        args.append(Q(**{k: v}))
+                for role_name in role_filters:
+                    if not hasattr(queryset.model, 'accessible_pk_qs'):
+                        raise ParseError(_('Cannot apply role_level filter to this list because its model does not use roles for access control.'))
+                    args.append(Q(pk__in=queryset.model.accessible_pk_qs(request.user, role_name)))
+                if or_filters:
+                    q = Q()
+                    for n, k, v in or_filters:
+                        if n:
+                            q |= ~Q(**{k: v})
+                        else:
+                            q |= Q(**{k: v})
+                    args.append(q)
+                if search_filters and search_filter_relation == 'OR':
+                    q = Q()
+                    for term, constrains in search_filters.items():
+                        for constrain in constrains:
+                            q |= Q(**{constrain: term})
+                    args.append(q)
+                elif search_filters and search_filter_relation == 'AND':
+                    for term, constrains in search_filters.items():
+                        q_chain = Q()
+                        for constrain in constrains:
+                            q_chain |= Q(**{constrain: term})
+                        queryset = queryset.filter(q_chain)
+                for n, k, v in chain_filters:
+                    if n:
+                        q = ~Q(**{k: v})
+                    else:
+                        q = Q(**{k: v})
+                    queryset = queryset.filter(q)
+                queryset = queryset.filter(*args)
+                if needs_distinct:
+                    queryset = queryset.distinct()
+            return queryset
+        except (FieldError, FieldDoesNotExist, ValueError, TypeError) as e:
+            raise ParseError(e.args[0])
+        except ValidationError as e:
+            raise ParseError(json.dumps(e.messages, ensure_ascii=False))
+
+
+class OrderByBackend(BaseFilterBackend):
+    """
+    Filter to apply ordering based on query string parameters.
+    """
+
+    def filter_queryset(self, request, queryset, view):
+        try:
+            order_by = None
+            for key, value in request.query_params.items():
+                if key in ('order', 'order_by'):
+                    order_by = value
+                    if ',' in value:
+                        order_by = value.split(',')
+                    else:
+                        order_by = (value,)
+            default_order_by = self.get_default_ordering(view)
+            # glue the order by and default order by together so that the default is the backup option
+            order_by = list(order_by or []) + list(default_order_by or [])
+            if order_by:
+                order_by = self._validate_ordering_fields(queryset.model, order_by)
+                # Special handling of the type field for ordering. In this
+                # case, we're not sorting exactly on the type field, but
+                # given the limited number of views with multiple types,
+                # sorting on polymorphic_ctype.model is effectively the same.
+                new_order_by = []
+                if 'polymorphic_ctype' in get_all_field_names(queryset.model):
+                    for field in order_by:
+                        if field == 'type':
+                            new_order_by.append('polymorphic_ctype__model')
+                        elif field == '-type':
+                            new_order_by.append('-polymorphic_ctype__model')
+                        else:
+                            new_order_by.append(field)
+                else:
+                    for field in order_by:
+                        if field not in ('type', '-type'):
+                            new_order_by.append(field)
+                queryset = queryset.order_by(*new_order_by)
+            return queryset
+        except FieldError as e:
+            # Return a 400 for invalid field names.
+            raise ParseError(*e.args)
+
+    def get_default_ordering(self, view):
+        ordering = getattr(view, 'ordering', None)
+        if isinstance(ordering, str):
+            return (ordering,)
+        return ordering
+
+    def _validate_ordering_fields(self, model, order_by):
+        for field_name in order_by:
+            # strip off the negation prefix `-` if it exists
+            prefix = ''
+            path = field_name
+            if field_name[0] == '-':
+                prefix = field_name[0]
+                path = field_name[1:]
+            try:
+                field, new_path = get_field_from_path(model, path)
+                new_path = '{}{}'.format(prefix, new_path)
+            except (FieldError, FieldDoesNotExist) as e:
+                raise ParseError(e.args[0])
+            yield new_path

awx/api/generics.py

@@ -30,13 +30,12 @@ from rest_framework.permissions import IsAuthenticated
 from rest_framework.renderers import StaticHTMLRenderer
 from rest_framework.negotiation import DefaultContentNegotiation
 
-from ansible_base.rest_filters.rest_framework.field_lookup_backend import FieldLookupBackend
-from ansible_base.lib.utils.models import get_all_field_names
-
 # AWX
+from awx.api.filters import FieldLookupBackend
 from awx.main.models import UnifiedJob, UnifiedJobTemplate, User, Role, Credential, WorkflowJobTemplateNode, WorkflowApprovalTemplate
 from awx.main.access import optimize_queryset
 from awx.main.utils import camelcase_to_underscore, get_search_fields, getattrd, get_object_or_400, decrypt_field, get_awx_version
+from awx.main.utils.db import get_all_field_names
 from awx.main.utils.licensing import server_product_name
 from awx.main.views import ApiErrorView
 from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer
@@ -91,7 +90,7 @@ class LoggedLoginView(auth_views.LoginView):
         ret = super(LoggedLoginView, self).post(request, *args, **kwargs)
         if request.user.is_authenticated:
             logger.info(smart_str(u"User {} logged in from {}".format(self.request.user.username, request.META.get('REMOTE_ADDR', None))))
-            ret.set_cookie('userLoggedIn', 'true', secure=getattr(settings, 'SESSION_COOKIE_SECURE', False))
+            ret.set_cookie('userLoggedIn', 'true')
             ret.setdefault('X-API-Session-Cookie-Name', getattr(settings, 'SESSION_COOKIE_NAME', 'awx_sessionid'))
 
             return ret
@@ -107,7 +106,7 @@ class LoggedLogoutView(auth_views.LogoutView):
         original_user = getattr(request, 'user', None)
         ret = super(LoggedLogoutView, self).dispatch(request, *args, **kwargs)
         current_user = getattr(request, 'user', None)
-        ret.set_cookie('userLoggedIn', 'false', secure=getattr(settings, 'SESSION_COOKIE_SECURE', False))
+        ret.set_cookie('userLoggedIn', 'false')
         if (not current_user or not getattr(current_user, 'pk', True)) and current_user != original_user:
             logger.info("User {} logged out.".format(original_user.username))
         return ret

awx/api/serializers.py

@@ -6,7 +6,7 @@ import copy
 import json
 import logging
 import re
-from collections import Counter, OrderedDict
+from collections import OrderedDict
 from datetime import timedelta
 from uuid import uuid4
 
@@ -43,8 +43,6 @@ from rest_framework.utils.serializer_helpers import ReturnList
 # Django-Polymorphic
 from polymorphic.models import PolymorphicModel
 
-from ansible_base.lib.utils.models import get_type_for_model
-
 # AWX
 from awx.main.access import get_user_capabilities
 from awx.main.constants import ACTIVE_STATES, CENSOR_VALUE
@@ -82,7 +80,6 @@ from awx.main.models import (
     Project,
     ProjectUpdate,
     ProjectUpdateEvent,
-    ReceptorAddress,
     RefreshToken,
     Role,
     Schedule,
@@ -102,9 +99,10 @@ from awx.main.models import (
     CLOUD_INVENTORY_SOURCES,
 )
 from awx.main.models.base import VERBOSITY_CHOICES, NEW_JOB_TYPE_CHOICES
-from awx.main.models.rbac import role_summary_fields_generator, RoleAncestorEntry
+from awx.main.models.rbac import get_roles_on_resource, role_summary_fields_generator
 from awx.main.fields import ImplicitRoleField
 from awx.main.utils import (
+    get_type_for_model,
     get_model_for_type,
     camelcase_to_underscore,
     getattrd,
@@ -637,7 +635,7 @@ class BaseSerializer(serializers.ModelSerializer, metaclass=BaseSerializerMetacl
             exclusions = self.get_validation_exclusions(self.instance)
             obj = self.instance or self.Meta.model()
             for k, v in attrs.items():
-                if k not in exclusions and k != 'canonical_address_port':
+                if k not in exclusions:
                     setattr(obj, k, v)
             obj.full_clean(exclude=exclusions)
             # full_clean may modify values on the instance; copy those changes
@@ -2203,99 +2201,6 @@ class BulkHostCreateSerializer(serializers.Serializer):
         return return_data
 
 
-class BulkHostDeleteSerializer(serializers.Serializer):
-    hosts = serializers.ListField(
-        allow_empty=False,
-        max_length=100000,
-        write_only=True,
-        help_text=_('List of hosts ids to be deleted, e.g. [105, 130, 131, 200]'),
-    )
-
-    class Meta:
-        model = Host
-        fields = ('hosts',)
-
-    def validate(self, attrs):
-        request = self.context.get('request', None)
-        max_hosts = settings.BULK_HOST_MAX_DELETE
-        # Validating the number of hosts to be deleted
-        if len(attrs['hosts']) > max_hosts:
-            raise serializers.ValidationError(
-                {
-                    "ERROR": 'Number of hosts exceeds system setting BULK_HOST_MAX_DELETE',
-                    "BULK_HOST_MAX_DELETE": max_hosts,
-                    "Hosts_count": len(attrs['hosts']),
-                }
-            )
-
-        # Getting list of all host objects, filtered by the list of the hosts to delete
-        attrs['host_qs'] = Host.objects.get_queryset().filter(pk__in=attrs['hosts']).only('id', 'inventory_id', 'name')
-
-        # Converting the queryset data in a dict. to reduce the number of queries when
-        # manipulating the data
-        attrs['hosts_data'] = attrs['host_qs'].values()
-
-        if len(attrs['host_qs']) == 0:
-            error_hosts = {host: "Hosts do not exist or you lack permission to delete it" for host in attrs['hosts']}
-            raise serializers.ValidationError({'hosts': error_hosts})
-
-        if len(attrs['host_qs']) < len(attrs['hosts']):
-            hosts_exists = [host['id'] for host in attrs['hosts_data']]
-            failed_hosts = list(set(attrs['hosts']).difference(hosts_exists))
-            error_hosts = {host: "Hosts do not exist or you lack permission to delete it" for host in failed_hosts}
-            raise serializers.ValidationError({'hosts': error_hosts})
-
-        # Getting all inventories that the hosts can be in
-        inv_list = list(set([host['inventory_id'] for host in attrs['hosts_data']]))
-
-        # Checking that the user have permission to all inventories
-        errors = dict()
-        for inv in Inventory.objects.get_queryset().filter(pk__in=inv_list):
-            if request and not request.user.is_superuser:
-                if request.user not in inv.admin_role:
-                    errors[inv.name] = "Lack permissions to delete hosts from this inventory."
-        if errors != {}:
-            raise PermissionDenied({"inventories": errors})
-
-        # check the inventory type only if the user have permission to it.
-        errors = dict()
-        for inv in Inventory.objects.get_queryset().filter(pk__in=inv_list):
-            if inv.kind != '':
-                errors[inv.name] = "Hosts can only be deleted from manual inventories."
-        if errors != {}:
-            raise serializers.ValidationError({"inventories": errors})
-        attrs['inventories'] = inv_list
-        return attrs
-
-    def delete(self, validated_data):
-        result = {"hosts": dict()}
-        changes = {'deleted_hosts': dict()}
-        for inventory in validated_data['inventories']:
-            changes['deleted_hosts'][inventory] = list()
-
-        for host in validated_data['hosts_data']:
-            result["hosts"][host["id"]] = f"The host {host['name']} was deleted"
-            changes['deleted_hosts'][host["inventory_id"]].append({"host_id": host["id"], "host_name": host["name"]})
-
-        try:
-            validated_data['host_qs'].delete()
-        except Exception as e:
-            raise serializers.ValidationError({"detail": _(f"cannot delete hosts, host deletion error {e}")})
-
-        request = self.context.get('request', None)
-
-        for inventory in validated_data['inventories']:
-            activity_entry = ActivityStream.objects.create(
-                operation='update',
-                object1='inventory',
-                changes=json.dumps(changes['deleted_hosts'][inventory]),
-                actor=request.user,
-            )
-            activity_entry.inventory.add(inventory)
-
-        return result
-
-
 class GroupTreeSerializer(GroupSerializer):
     children = serializers.SerializerMethodField()
 
@@ -2759,17 +2664,6 @@ class ResourceAccessListElementSerializer(UserSerializer):
         if 'summary_fields' not in ret:
             ret['summary_fields'] = {}
 
-        team_content_type = ContentType.objects.get_for_model(Team)
-        content_type = ContentType.objects.get_for_model(obj)
-
-        def get_roles_on_resource(parent_role):
-            "Returns a string list of the roles a parent_role has for current obj."
-            return list(
-                RoleAncestorEntry.objects.filter(ancestor=parent_role, content_type_id=content_type.id, object_id=obj.id)
-                .values_list('role_field', flat=True)
-                .distinct()
-            )
-
         def format_role_perm(role):
             role_dict = {'id': role.id, 'name': role.name, 'description': role.description}
             try:
@@ -2785,7 +2679,7 @@ class ResourceAccessListElementSerializer(UserSerializer):
             else:
                 # Singleton roles should not be managed from this view, as per copy/edit rework spec
                 role_dict['user_capabilities'] = {'unattach': False}
-            return {'role': role_dict, 'descendant_roles': get_roles_on_resource(role)}
+            return {'role': role_dict, 'descendant_roles': get_roles_on_resource(obj, role)}
 
         def format_team_role_perm(naive_team_role, permissive_role_ids):
             ret = []
@@ -2811,9 +2705,12 @@ class ResourceAccessListElementSerializer(UserSerializer):
                 else:
                     # Singleton roles should not be managed from this view, as per copy/edit rework spec
                     role_dict['user_capabilities'] = {'unattach': False}
-                ret.append({'role': role_dict, 'descendant_roles': get_roles_on_resource(team_role)})
+                ret.append({'role': role_dict, 'descendant_roles': get_roles_on_resource(obj, team_role)})
             return ret
 
+        team_content_type = ContentType.objects.get_for_model(Team)
+        content_type = ContentType.objects.get_for_model(obj)
+
         direct_permissive_role_ids = Role.objects.filter(content_type=content_type, object_id=obj.id).values_list('id', flat=True)
         all_permissive_role_ids = Role.objects.filter(content_type=content_type, object_id=obj.id).values_list('ancestors__id', flat=True)
 
@@ -5177,21 +5074,16 @@ class NotificationTemplateSerializer(BaseSerializer):
                 body = messages[event].get('body', {})
                 if body:
                     try:
-                        sandbox.ImmutableSandboxedEnvironment(undefined=DescriptiveUndefined).from_string(body).render(JobNotificationMixin.context_stub())
-
-                        # https://github.com/ansible/awx/issues/14410
-
-                        # When rendering something such as "{{ job.id }}"
-                        # the return type is not a dict, unlike "{{ job_metadata }}" which is a dict
-
-                        # potential_body = json.loads(rendered_body)
-
-                        # if not isinstance(potential_body, dict):
-                        #     error_list.append(
-                        #         _("Webhook body for '{}' should be a json dictionary. Found type '{}'.".format(event, type(potential_body).__name__))
-                        #     )
-                    except Exception as exc:
-                        error_list.append(_("Webhook body for '{}' is not valid. The following gave an error ({}).".format(event, exc)))
+                        rendered_body = (
+                            sandbox.ImmutableSandboxedEnvironment(undefined=DescriptiveUndefined).from_string(body).render(JobNotificationMixin.context_stub())
+                        )
+                        potential_body = json.loads(rendered_body)
+                        if not isinstance(potential_body, dict):
+                            error_list.append(
+                                _("Webhook body for '{}' should be a json dictionary. Found type '{}'.".format(event, type(potential_body).__name__))
+                            )
+                    except json.JSONDecodeError as exc:
+                        error_list.append(_("Webhook body for '{}' is not a valid json dictionary ({}).".format(event, exc)))
 
         if error_list:
             raise serializers.ValidationError(error_list)
@@ -5464,25 +5356,17 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria
 class InstanceLinkSerializer(BaseSerializer):
     class Meta:
         model = InstanceLink
-        fields = ('id', 'related', 'source', 'target', 'target_full_address', 'link_state')
+        fields = ('id', 'url', 'related', 'source', 'target', 'link_state')
 
     source = serializers.SlugRelatedField(slug_field="hostname", queryset=Instance.objects.all())
-
-    target = serializers.SerializerMethodField()
-    target_full_address = serializers.SerializerMethodField()
+    target = serializers.SlugRelatedField(slug_field="hostname", queryset=Instance.objects.all())
 
     def get_related(self, obj):
         res = super(InstanceLinkSerializer, self).get_related(obj)
         res['source_instance'] = self.reverse('api:instance_detail', kwargs={'pk': obj.source.id})
-        res['target_address'] = self.reverse('api:receptor_address_detail', kwargs={'pk': obj.target.id})
+        res['target_instance'] = self.reverse('api:instance_detail', kwargs={'pk': obj.target.id})
         return res
 
-    def get_target(self, obj):
-        return obj.target.instance.hostname
-
-    def get_target_full_address(self, obj):
-        return obj.target.get_full_address()
-
 
 class InstanceNodeSerializer(BaseSerializer):
     class Meta:
@@ -5490,29 +5374,6 @@ class InstanceNodeSerializer(BaseSerializer):
         fields = ('id', 'hostname', 'node_type', 'node_state', 'enabled')
 
 
-class ReceptorAddressSerializer(BaseSerializer):
-    full_address = serializers.SerializerMethodField()
-
-    class Meta:
-        model = ReceptorAddress
-        fields = (
-            'id',
-            'url',
-            'address',
-            'port',
-            'protocol',
-            'websocket_path',
-            'is_internal',
-            'canonical',
-            'instance',
-            'peers_from_control_nodes',
-            'full_address',
-        )
-
-    def get_full_address(self, obj):
-        return obj.get_full_address()
-
-
 class InstanceSerializer(BaseSerializer):
     show_capabilities = ['edit']
 
@@ -5521,17 +5382,11 @@ class InstanceSerializer(BaseSerializer):
     jobs_running = serializers.IntegerField(help_text=_('Count of jobs in the running or waiting state that are targeted for this instance'), read_only=True)
     jobs_total = serializers.IntegerField(help_text=_('Count of all jobs that target this instance'), read_only=True)
     health_check_pending = serializers.SerializerMethodField()
-    peers = serializers.PrimaryKeyRelatedField(
-        help_text=_('Primary keys of receptor addresses to peer to.'), many=True, required=False, queryset=ReceptorAddress.objects.all()
-    )
-    reverse_peers = serializers.SerializerMethodField()
-    listener_port = serializers.IntegerField(source='canonical_address_port', required=False, allow_null=True)
-    peers_from_control_nodes = serializers.BooleanField(source='canonical_address_peers_from_control_nodes', required=False)
-    protocol = serializers.SerializerMethodField()
+    peers = serializers.SlugRelatedField(many=True, required=False, slug_field="hostname", queryset=Instance.objects.all())
 
     class Meta:
         model = Instance
-        read_only_fields = ('ip_address', 'uuid', 'version', 'managed', 'reverse_peers')
+        read_only_fields = ('ip_address', 'uuid', 'version')
         fields = (
             'id',
             'hostname',
@@ -5562,13 +5417,10 @@ class InstanceSerializer(BaseSerializer):
             'managed_by_policy',
             'node_type',
             'node_state',
-            'managed',
             'ip_address',
-            'peers',
-            'reverse_peers',
             'listener_port',
+            'peers',
             'peers_from_control_nodes',
-            'protocol',
         )
         extra_kwargs = {
             'node_type': {'initial': Instance.Types.EXECUTION, 'default': Instance.Types.EXECUTION},
@@ -5590,54 +5442,16 @@ class InstanceSerializer(BaseSerializer):
 
     def get_related(self, obj):
         res = super(InstanceSerializer, self).get_related(obj)
-        res['receptor_addresses'] = self.reverse('api:instance_receptor_addresses_list', kwargs={'pk': obj.pk})
         res['jobs'] = self.reverse('api:instance_unified_jobs_list', kwargs={'pk': obj.pk})
-        res['peers'] = self.reverse('api:instance_peers_list', kwargs={"pk": obj.pk})
         res['instance_groups'] = self.reverse('api:instance_instance_groups_list', kwargs={'pk': obj.pk})
-        if obj.node_type in [Instance.Types.EXECUTION, Instance.Types.HOP] and not obj.managed:
+        if obj.node_type in [Instance.Types.EXECUTION, Instance.Types.HOP]:
             res['install_bundle'] = self.reverse('api:instance_install_bundle', kwargs={'pk': obj.pk})
+        res['peers'] = self.reverse('api:instance_peers_list', kwargs={"pk": obj.pk})
         if self.context['request'].user.is_superuser or self.context['request'].user.is_system_auditor:
             if obj.node_type == 'execution':
                 res['health_check'] = self.reverse('api:instance_health_check', kwargs={'pk': obj.pk})
         return res
 
-    def create_or_update(self, validated_data, obj=None, create=True):
-        # create a managed receptor address if listener port is defined
-        port = validated_data.pop('listener_port', -1)
-        peers_from_control_nodes = validated_data.pop('peers_from_control_nodes', -1)
-
-        # delete the receptor address if the port is explicitly set to None
-        if obj and port == None:
-            obj.receptor_addresses.filter(address=obj.hostname).delete()
-
-        if create:
-            instance = super(InstanceSerializer, self).create(validated_data)
-        else:
-            instance = super(InstanceSerializer, self).update(obj, validated_data)
-            instance.refresh_from_db()  # instance canonical address lookup is deferred, so needs to be reloaded
-
-        # only create or update if port is defined in validated_data or already exists in the
-        # canonical address
-        # this prevents creating a receptor address if peers_from_control_nodes is in
-        # validated_data but a port is not set
-        if (port != None and port != -1) or instance.canonical_address_port:
-            kwargs = {}
-            if port != -1:
-                kwargs['port'] = port
-            if peers_from_control_nodes != -1:
-                kwargs['peers_from_control_nodes'] = peers_from_control_nodes
-            if kwargs:
-                kwargs['canonical'] = True
-                instance.receptor_addresses.update_or_create(address=instance.hostname, defaults=kwargs)
-
-        return instance
-
-    def create(self, validated_data):
-        return self.create_or_update(validated_data, create=True)
-
-    def update(self, obj, validated_data):
-        return self.create_or_update(validated_data, obj, create=False)
-
     def get_summary_fields(self, obj):
         summary = super().get_summary_fields(obj)
 
@@ -5647,16 +5461,6 @@ class InstanceSerializer(BaseSerializer):
 
         return summary
 
-    def get_reverse_peers(self, obj):
-        return Instance.objects.prefetch_related('peers').filter(peers__in=obj.receptor_addresses.all()).values_list('id', flat=True)
-
-    def get_protocol(self, obj):
-        # note: don't create a different query for receptor addresses, as this is prefetched on the View for optimization
-        for addr in obj.receptor_addresses.all():
-            if addr.canonical:
-                return addr.protocol
-        return ""
-
     def get_consumed_capacity(self, obj):
         return obj.consumed_capacity
 
@@ -5670,20 +5474,47 @@ class InstanceSerializer(BaseSerializer):
         return obj.health_check_pending
 
     def validate(self, attrs):
-        # Oddly, using 'source' on a DRF field populates attrs with the source name, so we should rename it back
-        if 'canonical_address_port' in attrs:
-            attrs['listener_port'] = attrs.pop('canonical_address_port')
-        if 'canonical_address_peers_from_control_nodes' in attrs:
-            attrs['peers_from_control_nodes'] = attrs.pop('canonical_address_peers_from_control_nodes')
+        def get_field_from_model_or_attrs(fd):
+            return attrs.get(fd, self.instance and getattr(self.instance, fd) or None)
+
+        def check_peers_changed():
+            '''
+            return True if
+            - 'peers' in attrs
+            - instance peers matches peers in attrs
+            '''
+            return self.instance and 'peers' in attrs and set(self.instance.peers.all()) != set(attrs['peers'])
 
         if not self.instance and not settings.IS_K8S:
             raise serializers.ValidationError(_("Can only create instances on Kubernetes or OpenShift."))
 
-        # cannot enable peers_from_control_nodes if listener_port is not set
-        if attrs.get('peers_from_control_nodes'):
-            port = attrs.get('listener_port', -1)  # -1 denotes missing, None denotes explicit null
-            if (port is None) or (port == -1 and self.instance and self.instance.canonical_address is None):
-                raise serializers.ValidationError(_("Cannot enable peers_from_control_nodes if listener_port is not set."))
+        node_type = get_field_from_model_or_attrs("node_type")
+        peers_from_control_nodes = get_field_from_model_or_attrs("peers_from_control_nodes")
+        listener_port = get_field_from_model_or_attrs("listener_port")
+        peers = attrs.get('peers', [])
+
+        if peers_from_control_nodes and node_type not in (Instance.Types.EXECUTION, Instance.Types.HOP):
+            raise serializers.ValidationError(_("peers_from_control_nodes can only be enabled for execution or hop nodes."))
+
+        if node_type in [Instance.Types.CONTROL, Instance.Types.HYBRID]:
+            if check_peers_changed():
+                raise serializers.ValidationError(
+                    _("Setting peers manually for control nodes is not allowed. Enable peers_from_control_nodes on the hop and execution nodes instead.")
+                )
+
+        if not listener_port and peers_from_control_nodes:
+            raise serializers.ValidationError(_("Field listener_port must be a valid integer when peers_from_control_nodes is enabled."))
+
+        if not listener_port and self.instance and self.instance.peers_from.exists():
+            raise serializers.ValidationError(_("Field listener_port must be a valid integer when other nodes peer to it."))
+
+        for peer in peers:
+            if peer.listener_port is None:
+                raise serializers.ValidationError(_("Field listener_port must be set on peer ") + peer.hostname + ".")
+
+        if not settings.IS_K8S:
+            if check_peers_changed():
+                raise serializers.ValidationError(_("Cannot change peers."))
 
         return super().validate(attrs)
 
@@ -5703,8 +5534,8 @@ class InstanceSerializer(BaseSerializer):
                     raise serializers.ValidationError(_("Can only change the state on Kubernetes or OpenShift."))
                 if value != Instance.States.DEPROVISIONING:
                     raise serializers.ValidationError(_("Can only change instances to the 'deprovisioning' state."))
-                if self.instance.managed:
-                    raise serializers.ValidationError(_("Cannot deprovision managed nodes."))
+                if self.instance.node_type not in (Instance.Types.EXECUTION, Instance.Types.HOP):
+                    raise serializers.ValidationError(_("Can only deprovision execution or hop nodes."))
         else:
             if value and value != Instance.States.INSTALLED:
                 raise serializers.ValidationError(_("Can only create instances in the 'installed' state."))
@@ -5723,48 +5554,18 @@ class InstanceSerializer(BaseSerializer):
     def validate_listener_port(self, value):
         """
         Cannot change listener port, unless going from none to integer, and vice versa
-        If instance is managed, cannot change listener port at all
         """
-        if self.instance:
-            canonical_address_port = self.instance.canonical_address_port
-            if value and canonical_address_port and canonical_address_port != value:
-                raise serializers.ValidationError(_("Cannot change listener port."))
-            if self.instance.managed and value != canonical_address_port:
-                raise serializers.ValidationError(_("Cannot change listener port for managed nodes."))
-        return value
-
-    def validate_peers(self, value):
-        # cannot peer to an instance more than once
-        peers_instances = Counter(p.instance_id for p in value)
-        if any(count > 1 for count in peers_instances.values()):
-            raise serializers.ValidationError(_("Cannot peer to the same instance more than once."))
-
-        if self.instance:
-            instance_addresses = set(self.instance.receptor_addresses.all())
-            setting_peers = set(value)
-            peers_changed = set(self.instance.peers.all()) != setting_peers
-
-            if not settings.IS_K8S and peers_changed:
-                raise serializers.ValidationError(_("Cannot change peers."))
-
-            if self.instance.managed and peers_changed:
-                raise serializers.ValidationError(_("Setting peers manually for managed nodes is not allowed."))
-
-            # cannot peer to self
-            if instance_addresses & setting_peers:
-                raise serializers.ValidationError(_("Instance cannot peer to its own address."))
-
-            # cannot peer to an instance that is already peered to this instance
-            if instance_addresses:
-                for p in setting_peers:
-                    if set(p.instance.peers.all()) & instance_addresses:
-                        raise serializers.ValidationError(_(f"Instance {p.instance.hostname} is already peered to this instance."))
+        if value and self.instance and self.instance.listener_port and self.instance.listener_port != value:
+            raise serializers.ValidationError(_("Cannot change listener port."))
 
         return value
 
     def validate_peers_from_control_nodes(self, value):
-        if self.instance and self.instance.managed and self.instance.canonical_address_peers_from_control_nodes != value:
-            raise serializers.ValidationError(_("Cannot change peers_from_control_nodes for managed nodes."))
+        """
+        Can only enable for K8S based deployments
+        """
+        if value and not settings.IS_K8S:
+            raise serializers.ValidationError(_("Can only be enabled on Kubernetes or Openshift."))
 
         return value
 

awx/api/templates/api/bulk_host_delete_view.md

@@ -1,22 +0,0 @@
-# Bulk Host Delete
-
-This endpoint allows the client to delete multiple hosts from inventories.
-They may do this by providing a list of hosts ID's to be deleted.
-
-Example:
-
-    {
-        "hosts": [1, 2, 3, 4, 5]
-    }
-
-Return data:
-
-    {
-        "hosts": {
-            "1": "The host a1 was deleted",
-            "2": "The host a2 was deleted",
-            "3": "The host a3 was deleted",
-            "4": "The host a4 was deleted",
-            "5": "The host a5 was deleted",
-        }
-    }

awx/api/templates/instance_install_bundle/group_vars/all.yml

@@ -17,18 +17,19 @@ custom_worksign_public_keyfile: receptor/work_public_key.pem
 custom_tls_certfile: receptor/tls/receptor.crt
 custom_tls_keyfile: receptor/tls/receptor.key
 custom_ca_certfile: receptor/tls/ca/mesh-CA.crt
-{% if listener_port %}
-receptor_protocol: {{ listener_protocol }}
+receptor_protocol: 'tcp'
+{% if instance.listener_port %}
 receptor_listener: true
-receptor_port: {{ listener_port }}
+receptor_port: {{ instance.listener_port }}
 {% else %}
 receptor_listener: false
 {% endif %}
 {% if peers %}
 receptor_peers:
 {% for peer in peers %}
-  - address: {{ peer.address }}
-    protocol: {{ peer.protocol }}
+  - host: {{ peer.host }}
+    port: {{ peer.port }}
+    protocol: tcp
 {% endfor %}
 {% endif %}
 {% verbatim %}

awx/api/templates/instance_install_bundle/requirements.yml

@@ -1,4 +1,4 @@
 ---
 collections:
   - name: ansible.receptor
-    version: 2.0.3
+    version: 2.0.2

awx/api/urls/instance.py

@@ -10,7 +10,6 @@ from awx.api.views import (
     InstanceInstanceGroupsList,
     InstanceHealthCheck,
     InstancePeersList,
-    InstanceReceptorAddressesList,
 )
 from awx.api.views.instance_install_bundle import InstanceInstallBundle
 
@@ -22,7 +21,6 @@ urls = [
     re_path(r'^(?P<pk>[0-9]+)/instance_groups/$', InstanceInstanceGroupsList.as_view(), name='instance_instance_groups_list'),
     re_path(r'^(?P<pk>[0-9]+)/health_check/$', InstanceHealthCheck.as_view(), name='instance_health_check'),
     re_path(r'^(?P<pk>[0-9]+)/peers/$', InstancePeersList.as_view(), name='instance_peers_list'),
-    re_path(r'^(?P<pk>[0-9]+)/receptor_addresses/$', InstanceReceptorAddressesList.as_view(), name='instance_receptor_addresses_list'),
     re_path(r'^(?P<pk>[0-9]+)/install_bundle/$', InstanceInstallBundle.as_view(), name='instance_install_bundle'),
 ]
 

awx/api/urls/receptor_address.py

@@ -1,17 +0,0 @@
-# Copyright (c) 2017 Ansible, Inc.
-# All Rights Reserved.
-
-from django.urls import re_path
-
-from awx.api.views import (
-    ReceptorAddressesList,
-    ReceptorAddressDetail,
-)
-
-
-urls = [
-    re_path(r'^$', ReceptorAddressesList.as_view(), name='receptor_addresses_list'),
-    re_path(r'^(?P<pk>[0-9]+)/$', ReceptorAddressDetail.as_view(), name='receptor_address_detail'),
-]
-
-__all__ = ['urls']

awx/api/urls/urls.py

@@ -36,7 +36,6 @@ from awx.api.views import (
 from awx.api.views.bulk import (
     BulkView,
     BulkHostCreateView,
-    BulkHostDeleteView,
     BulkJobLaunchView,
 )
 
@@ -85,7 +84,6 @@ from .oauth2_root import urls as oauth2_root_urls
 from .workflow_approval_template import urls as workflow_approval_template_urls
 from .workflow_approval import urls as workflow_approval_urls
 from .analytics import urls as analytics_urls
-from .receptor_address import urls as receptor_address_urls
 
 v2_urls = [
     re_path(r'^$', ApiV2RootView.as_view(), name='api_v2_root_view'),
@@ -154,9 +152,7 @@ v2_urls = [
     re_path(r'^workflow_approvals/', include(workflow_approval_urls)),
     re_path(r'^bulk/$', BulkView.as_view(), name='bulk'),
     re_path(r'^bulk/host_create/$', BulkHostCreateView.as_view(), name='bulk_host_create'),
-    re_path(r'^bulk/host_delete/$', BulkHostDeleteView.as_view(), name='bulk_host_delete'),
     re_path(r'^bulk/job_launch/$', BulkJobLaunchView.as_view(), name='bulk_job_launch'),
-    re_path(r'^receptor_addresses/', include(receptor_address_urls)),
 ]
 
 

awx/api/urls/webhooks.py

@@ -1,11 +1,10 @@
 from django.urls import re_path
 
-from awx.api.views.webhooks import WebhookKeyView, GithubWebhookReceiver, GitlabWebhookReceiver, BitbucketDcWebhookReceiver
+from awx.api.views.webhooks import WebhookKeyView, GithubWebhookReceiver, GitlabWebhookReceiver
 
 
 urlpatterns = [
     re_path(r'^webhook_key/$', WebhookKeyView.as_view(), name='webhook_key'),
     re_path(r'^github/$', GithubWebhookReceiver.as_view(), name='webhook_receiver_github'),
     re_path(r'^gitlab/$', GitlabWebhookReceiver.as_view(), name='webhook_receiver_gitlab'),
-    re_path(r'^bitbucket_dc/$', BitbucketDcWebhookReceiver.as_view(), name='webhook_receiver_bitbucket_dc'),
 ]

awx/api/views/__init__.py

@@ -272,24 +272,16 @@ class DashboardJobsGraphView(APIView):
 
         success_query = user_unified_jobs.filter(status='successful')
         failed_query = user_unified_jobs.filter(status='failed')
-        canceled_query = user_unified_jobs.filter(status='canceled')
-        error_query = user_unified_jobs.filter(status='error')
 
         if job_type == 'inv_sync':
             success_query = success_query.filter(instance_of=models.InventoryUpdate)
             failed_query = failed_query.filter(instance_of=models.InventoryUpdate)
-            canceled_query = canceled_query.filter(instance_of=models.InventoryUpdate)
-            error_query = error_query.filter(instance_of=models.InventoryUpdate)
         elif job_type == 'playbook_run':
             success_query = success_query.filter(instance_of=models.Job)
             failed_query = failed_query.filter(instance_of=models.Job)
-            canceled_query = canceled_query.filter(instance_of=models.Job)
-            error_query = error_query.filter(instance_of=models.Job)
         elif job_type == 'scm_update':
             success_query = success_query.filter(instance_of=models.ProjectUpdate)
             failed_query = failed_query.filter(instance_of=models.ProjectUpdate)
-            canceled_query = canceled_query.filter(instance_of=models.ProjectUpdate)
-            error_query = error_query.filter(instance_of=models.ProjectUpdate)
 
         end = now()
         interval = 'day'
@@ -305,12 +297,10 @@ class DashboardJobsGraphView(APIView):
         else:
             return Response({'error': _('Unknown period "%s"') % str(period)}, status=status.HTTP_400_BAD_REQUEST)
 
-        dashboard_data = {"jobs": {"successful": [], "failed": [], "canceled": [], "error": []}}
+        dashboard_data = {"jobs": {"successful": [], "failed": []}}
 
         succ_list = dashboard_data['jobs']['successful']
         fail_list = dashboard_data['jobs']['failed']
-        canceled_list = dashboard_data['jobs']['canceled']
-        error_list = dashboard_data['jobs']['error']
 
         qs_s = (
             success_query.filter(finished__range=(start, end))
@@ -328,22 +318,6 @@ class DashboardJobsGraphView(APIView):
             .annotate(agg=Count('id', distinct=True))
         )
         data_f = {item['d']: item['agg'] for item in qs_f}
-        qs_c = (
-            canceled_query.filter(finished__range=(start, end))
-            .annotate(d=Trunc('finished', interval, tzinfo=end.tzinfo))
-            .order_by()
-            .values('d')
-            .annotate(agg=Count('id', distinct=True))
-        )
-        data_c = {item['d']: item['agg'] for item in qs_c}
-        qs_e = (
-            error_query.filter(finished__range=(start, end))
-            .annotate(d=Trunc('finished', interval, tzinfo=end.tzinfo))
-            .order_by()
-            .values('d')
-            .annotate(agg=Count('id', distinct=True))
-        )
-        data_e = {item['d']: item['agg'] for item in qs_e}
 
         start_date = start.replace(hour=0, minute=0, second=0, microsecond=0)
         for d in itertools.count():
@@ -352,8 +326,6 @@ class DashboardJobsGraphView(APIView):
                 break
             succ_list.append([time.mktime(date.timetuple()), data_s.get(date, 0)])
             fail_list.append([time.mktime(date.timetuple()), data_f.get(date, 0)])
-            canceled_list.append([time.mktime(date.timetuple()), data_c.get(date, 0)])
-            error_list.append([time.mktime(date.timetuple()), data_e.get(date, 0)])
 
         return Response(dashboard_data)
 
@@ -365,20 +337,12 @@ class InstanceList(ListCreateAPIView):
     search_fields = ('hostname',)
     ordering = ('id',)
 
-    def get_queryset(self):
-        qs = super().get_queryset().prefetch_related('receptor_addresses')
-        return qs
-
 
 class InstanceDetail(RetrieveUpdateAPIView):
     name = _("Instance Detail")
     model = models.Instance
     serializer_class = serializers.InstanceSerializer
 
-    def get_queryset(self):
-        qs = super().get_queryset().prefetch_related('receptor_addresses')
-        return qs
-
     def update_raw_data(self, data):
         # these fields are only valid on creation of an instance, so they unwanted on detail view
         data.pop('node_type', None)
@@ -411,37 +375,13 @@ class InstanceUnifiedJobsList(SubListAPIView):
 
 
 class InstancePeersList(SubListAPIView):
-    name = _("Peers")
-    model = models.ReceptorAddress
-    serializer_class = serializers.ReceptorAddressSerializer
+    name = _("Instance Peers")
     parent_model = models.Instance
+    model = models.Instance
+    serializer_class = serializers.InstanceSerializer
     parent_access = 'read'
+    search_fields = {'hostname'}
     relationship = 'peers'
-    search_fields = ('address',)
-
-
-class InstanceReceptorAddressesList(SubListAPIView):
-    name = _("Receptor Addresses")
-    model = models.ReceptorAddress
-    parent_key = 'instance'
-    parent_model = models.Instance
-    serializer_class = serializers.ReceptorAddressSerializer
-    search_fields = ('address',)
-
-
-class ReceptorAddressesList(ListAPIView):
-    name = _("Receptor Addresses")
-    model = models.ReceptorAddress
-    serializer_class = serializers.ReceptorAddressSerializer
-    search_fields = ('address',)
-
-
-class ReceptorAddressDetail(RetrieveAPIView):
-    name = _("Receptor Address Detail")
-    model = models.ReceptorAddress
-    serializer_class = serializers.ReceptorAddressSerializer
-    parent_model = models.Instance
-    relationship = 'receptor_addresses'
 
 
 class InstanceInstanceGroupsList(InstanceGroupMembershipMixin, SubListCreateAttachDetachAPIView):
@@ -802,8 +742,8 @@ class TeamActivityStreamList(SubListAPIView):
         qs = self.request.user.get_queryset(self.model)
         return qs.filter(
             Q(team=parent)
-            | Q(project__in=models.Project.accessible_objects(parent.member_role, 'read_role'))
-            | Q(credential__in=models.Credential.accessible_objects(parent.member_role, 'read_role'))
+            | Q(project__in=models.Project.accessible_objects(parent, 'read_role'))
+            | Q(credential__in=models.Credential.accessible_objects(parent, 'read_role'))
         )
 
 
@@ -1457,7 +1397,7 @@ class OrganizationCredentialList(SubListCreateAPIView):
         self.check_parent_access(organization)
 
         user_visible = models.Credential.accessible_objects(self.request.user, 'read_role').all()
-        org_set = models.Credential.objects.filter(organization=organization)
+        org_set = models.Credential.accessible_objects(organization.admin_role, 'read_role').all()
 
         if self.request.user.is_superuser or self.request.user.is_system_auditor:
             return org_set

awx/api/views/bulk.py

@@ -34,7 +34,6 @@ class BulkView(APIView):
         '''List top level resources'''
         data = OrderedDict()
         data['host_create'] = reverse('api:bulk_host_create', request=request)
-        data['host_delete'] = reverse('api:bulk_host_delete', request=request)
         data['job_launch'] = reverse('api:bulk_job_launch', request=request)
         return Response(data)
 
@@ -73,20 +72,3 @@ class BulkHostCreateView(GenericAPIView):
             result = serializer.create(serializer.validated_data)
             return Response(result, status=status.HTTP_201_CREATED)
         return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
-
-
-class BulkHostDeleteView(GenericAPIView):
-    permission_classes = [IsAuthenticated]
-    model = Host
-    serializer_class = serializers.BulkHostDeleteSerializer
-    allowed_methods = ['GET', 'POST', 'OPTIONS']
-
-    def get(self, request):
-        return Response({"detail": "Bulk delete hosts with this endpoint"}, status=status.HTTP_200_OK)
-
-    def post(self, request):
-        serializer = serializers.BulkHostDeleteSerializer(data=request.data, context={'request': request})
-        if serializer.is_valid():
-            result = serializer.delete(serializer.validated_data)
-            return Response(result, status=status.HTTP_201_CREATED)
-        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

awx/api/views/instance_install_bundle.py

@@ -124,19 +124,10 @@ def generate_inventory_yml(instance_obj):
 
 
 def generate_group_vars_all_yml(instance_obj):
-    # get peers
     peers = []
-    for addr in instance_obj.peers.select_related('instance'):
-        peers.append(dict(address=addr.get_full_address(), protocol=addr.protocol))
-    context = dict(instance=instance_obj, peers=peers)
-
-    canonical_addr = instance_obj.canonical_address
-    if canonical_addr:
-        context['listener_port'] = canonical_addr.port
-        protocol = canonical_addr.protocol if canonical_addr.protocol != 'wss' else 'ws'
-        context['listener_protocol'] = protocol
-
-    all_yaml = render_to_string("instance_install_bundle/group_vars/all.yml", context=context)
+    for instance in instance_obj.peers.all():
+        peers.append(dict(host=instance.hostname, port=instance.listener_port))
+    all_yaml = render_to_string("instance_install_bundle/group_vars/all.yml", context=dict(instance=instance_obj, peers=peers))
     # convert consecutive newlines with a single newline
     return re.sub(r'\n+', '\n', all_yaml)
 

awx/api/views/mesh_visualizer.py

@@ -17,7 +17,7 @@ class MeshVisualizer(APIView):
     def get(self, request, format=None):
         data = {
             'nodes': InstanceNodeSerializer(Instance.objects.all(), many=True).data,
-            'links': InstanceLinkSerializer(InstanceLink.objects.select_related('target__instance', 'source'), many=True).data,
+            'links': InstanceLinkSerializer(InstanceLink.objects.select_related('target', 'source'), many=True).data,
         }
 
         return Response(data)

awx/api/views/root.py

@@ -84,7 +84,6 @@ class ApiVersionRootView(APIView):
         data['ping'] = reverse('api:api_v2_ping_view', request=request)
         data['instances'] = reverse('api:instance_list', request=request)
         data['instance_groups'] = reverse('api:instance_group_list', request=request)
-        data['receptor_addresses'] = reverse('api:receptor_addresses_list', request=request)
         data['config'] = reverse('api:api_v2_config_view', request=request)
         data['settings'] = reverse('api:setting_category_list', request=request)
         data['me'] = reverse('api:user_me_list', request=request)

awx/api/views/webhooks.py

@@ -1,4 +1,4 @@
-from hashlib import sha1, sha256
+from hashlib import sha1
 import hmac
 import logging
 import urllib.parse
@@ -99,31 +99,14 @@ class WebhookReceiverBase(APIView):
     def get_signature(self):
         raise NotImplementedError
 
-    def must_check_signature(self):
-        return True
-
-    def is_ignored_request(self):
-        return False
-
     def check_signature(self, obj):
         if not obj.webhook_key:
             raise PermissionDenied
-        if not self.must_check_signature():
-            logger.debug("skipping signature validation")
-            return
 
-        hash_alg, expected_digest = self.get_signature()
-        if hash_alg == 'sha1':
-            mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha1)
-        elif hash_alg == 'sha256':
-            mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha256)
-        else:
-            logger.debug("Unsupported signature type, supported: sha1, sha256, received: {}".format(hash_alg))
-            raise PermissionDenied
-
-        logger.debug("header signature: %s", expected_digest)
+        mac = hmac.new(force_bytes(obj.webhook_key), msg=force_bytes(self.request.body), digestmod=sha1)
+        logger.debug("header signature: %s", self.get_signature())
         logger.debug("calculated signature: %s", force_bytes(mac.hexdigest()))
-        if not hmac.compare_digest(force_bytes(mac.hexdigest()), expected_digest):
+        if not hmac.compare_digest(force_bytes(mac.hexdigest()), self.get_signature()):
             raise PermissionDenied
 
     @csrf_exempt
@@ -135,10 +118,6 @@ class WebhookReceiverBase(APIView):
         obj = self.get_object()
         self.check_signature(obj)
 
-        if self.is_ignored_request():
-            # This was an ignored request type (e.g. ping), don't act on it
-            return Response({'message': _("Webhook ignored")}, status=status.HTTP_200_OK)
-
         event_type = self.get_event_type()
         event_guid = self.get_event_guid()
         event_ref = self.get_event_ref()
@@ -207,7 +186,7 @@ class GithubWebhookReceiver(WebhookReceiverBase):
         if hash_alg != 'sha1':
             logger.debug("Unsupported signature type, expected: sha1, received: {}".format(hash_alg))
             raise PermissionDenied
-        return hash_alg, force_bytes(signature)
+        return force_bytes(signature)
 
 
 class GitlabWebhookReceiver(WebhookReceiverBase):
@@ -235,73 +214,15 @@ class GitlabWebhookReceiver(WebhookReceiverBase):
 
         return "{}://{}/api/v4/projects/{}/statuses/{}".format(parsed.scheme, parsed.netloc, project['id'], self.get_event_ref())
 
+    def get_signature(self):
+        return force_bytes(self.request.META.get('HTTP_X_GITLAB_TOKEN') or '')
+
     def check_signature(self, obj):
         if not obj.webhook_key:
             raise PermissionDenied
 
-        token_from_request = force_bytes(self.request.META.get('HTTP_X_GITLAB_TOKEN') or '')
-
         # GitLab only returns the secret token, not an hmac hash.  Use
         # the hmac `compare_digest` helper function to prevent timing
         # analysis by attackers.
-        if not hmac.compare_digest(force_bytes(obj.webhook_key), token_from_request):
+        if not hmac.compare_digest(force_bytes(obj.webhook_key), self.get_signature()):
             raise PermissionDenied
-
-
-class BitbucketDcWebhookReceiver(WebhookReceiverBase):
-    service = 'bitbucket_dc'
-
-    ref_keys = {
-        'repo:refs_changed': 'changes.0.toHash',
-        'mirror:repo_synchronized': 'changes.0.toHash',
-        'pr:opened': 'pullRequest.toRef.latestCommit',
-        'pr:from_ref_updated': 'pullRequest.toRef.latestCommit',
-        'pr:modified': 'pullRequest.toRef.latestCommit',
-    }
-
-    def get_event_type(self):
-        return self.request.META.get('HTTP_X_EVENT_KEY')
-
-    def get_event_guid(self):
-        return self.request.META.get('HTTP_X_REQUEST_ID')
-
-    def get_event_status_api(self):
-        # https://<bitbucket-base-url>/rest/build-status/1.0/commits/<commit-hash>
-        if self.get_event_type() not in self.ref_keys.keys():
-            return
-        if self.get_event_ref() is None:
-            return
-        any_url = None
-        if 'actor' in self.request.data:
-            any_url = self.request.data['actor'].get('links', {}).get('self')
-        if any_url is None and 'repository' in self.request.data:
-            any_url = self.request.data['repository'].get('links', {}).get('self')
-        if any_url is None:
-            return
-        any_url = any_url[0].get('href')
-        if any_url is None:
-            return
-        parsed = urllib.parse.urlparse(any_url)
-
-        return "{}://{}/rest/build-status/1.0/commits/{}".format(parsed.scheme, parsed.netloc, self.get_event_ref())
-
-    def is_ignored_request(self):
-        return self.get_event_type() not in [
-            'repo:refs_changed',
-            'mirror:repo_synchronized',
-            'pr:opened',
-            'pr:from_ref_updated',
-            'pr:modified',
-        ]
-
-    def must_check_signature(self):
-        # Bitbucket does not sign ping requests...
-        return self.get_event_type() != 'diagnostics:ping'
-
-    def get_signature(self):
-        header_sig = self.request.META.get('HTTP_X_HUB_SIGNATURE')
-        if not header_sig:
-            logger.debug("Expected signature missing from header key HTTP_X_HUB_SIGNATURE")
-            raise PermissionDenied
-        hash_alg, signature = header_sig.split('=')
-        return hash_alg, force_bytes(signature)

awx/conf/models.py

@@ -7,10 +7,8 @@ import json
 # Django
 from django.db import models
 
-from ansible_base.lib.utils.models import prevent_search
-
 # AWX
-from awx.main.models.base import CreatedModifiedModel
+from awx.main.models.base import CreatedModifiedModel, prevent_search
 from awx.main.utils import encrypt_field
 from awx.conf import settings_registry
 

awx/main/access.py

@@ -20,12 +20,11 @@ from rest_framework.exceptions import ParseError, PermissionDenied
 # Django OAuth Toolkit
 from awx.main.models.oauth import OAuth2Application, OAuth2AccessToken
 
-from ansible_base.lib.utils.validation import to_python_boolean
-
 # AWX
 from awx.main.utils import (
     get_object_or_400,
     get_pk_from_dict,
+    to_python_boolean,
     get_licenser,
 )
 from awx.main.models import (
@@ -57,7 +56,6 @@ from awx.main.models import (
     Project,
     ProjectUpdate,
     ProjectUpdateEvent,
-    ReceptorAddress,
     Role,
     Schedule,
     SystemJob,
@@ -2431,29 +2429,6 @@ class InventoryUpdateEventAccess(BaseAccess):
         return False
 
 
-class ReceptorAddressAccess(BaseAccess):
-    """
-    I can see receptor address records whenever I can access the instance
-    """
-
-    model = ReceptorAddress
-
-    def filtered_queryset(self):
-        return self.model.objects.filter(Q(instance__in=Instance.accessible_pk_qs(self.user, 'read_role')))
-
-    @check_superuser
-    def can_add(self, data):
-        return False
-
-    @check_superuser
-    def can_change(self, obj, data):
-        return False
-
-    @check_superuser
-    def can_delete(self, obj):
-        return False
-
-
 class SystemJobEventAccess(BaseAccess):
     """
     I can only see manage System Jobs events if I'm a super user

awx/main/analytics/analytics_tasks.py

@@ -2,7 +2,7 @@
 import logging
 
 # AWX
-from awx.main.analytics.subsystem_metrics import DispatcherMetrics, CallbackReceiverMetrics
+from awx.main.analytics.subsystem_metrics import Metrics
 from awx.main.dispatch.publish import task
 from awx.main.dispatch import get_task_queuename
 
@@ -11,5 +11,4 @@ logger = logging.getLogger('awx.main.scheduler')
 
 @task(queue=get_task_queuename)
 def send_subsystem_metrics():
-    DispatcherMetrics().send_metrics()
-    CallbackReceiverMetrics().send_metrics()
+    Metrics().send_metrics()

awx/main/analytics/subsystem_metrics.py

@@ -1,15 +1,10 @@
-import itertools
 import redis
 import json
 import time
 import logging
 
-import prometheus_client
-from prometheus_client.core import GaugeMetricFamily, HistogramMetricFamily
-from prometheus_client.registry import CollectorRegistry
 from django.conf import settings
-from django.http import HttpRequest
-from rest_framework.request import Request
+from django.apps import apps
 
 from awx.main.consumers import emit_channel_notification
 from awx.main.utils import is_testing
@@ -18,30 +13,6 @@ root_key = settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX
 logger = logging.getLogger('awx.main.analytics')
 
 
-class MetricsNamespace:
-    def __init__(self, namespace):
-        self._namespace = namespace
-
-
-class MetricsServerSettings(MetricsNamespace):
-    def port(self):
-        return settings.METRICS_SUBSYSTEM_CONFIG['server'][self._namespace]['port']
-
-
-class MetricsServer(MetricsServerSettings):
-    def __init__(self, namespace, registry):
-        MetricsNamespace.__init__(self, namespace)
-        self._registry = registry
-
-    def start(self):
-        try:
-            # TODO: addr for ipv6 ?
-            prometheus_client.start_http_server(self.port(), addr='localhost', registry=self._registry)
-        except Exception:
-            logger.error(f"MetricsServer failed to start for service '{self._namespace}.")
-            raise
-
-
 class BaseM:
     def __init__(self, field, help_text):
         self.field = field
@@ -177,40 +148,76 @@ class HistogramM(BaseM):
         return output_text
 
 
-class Metrics(MetricsNamespace):
-    # metric name, help_text
-    METRICSLIST = []
-    _METRICSLIST = [
-        FloatM('subsystem_metrics_pipe_execute_seconds', 'Time spent saving metrics to redis'),
-        IntM('subsystem_metrics_pipe_execute_calls', 'Number of calls to pipe_execute'),
-        FloatM('subsystem_metrics_send_metrics_seconds', 'Time spent sending metrics to other nodes'),
-    ]
-
-    def __init__(self, namespace, auto_pipe_execute=False, instance_name=None, metrics_have_changed=True, **kwargs):
-        MetricsNamespace.__init__(self, namespace)
-
+class Metrics:
+    def __init__(self, auto_pipe_execute=False, instance_name=None):
         self.pipe = redis.Redis.from_url(settings.BROKER_URL).pipeline()
         self.conn = redis.Redis.from_url(settings.BROKER_URL)
         self.last_pipe_execute = time.time()
         # track if metrics have been modified since last saved to redis
         # start with True so that we get an initial save to redis
-        self.metrics_have_changed = metrics_have_changed
+        self.metrics_have_changed = True
         self.pipe_execute_interval = settings.SUBSYSTEM_METRICS_INTERVAL_SAVE_TO_REDIS
         self.send_metrics_interval = settings.SUBSYSTEM_METRICS_INTERVAL_SEND_METRICS
         # auto pipe execute will commit transaction of metric data to redis
         # at a regular interval (pipe_execute_interval). If set to False,
         # the calling function should call .pipe_execute() explicitly
         self.auto_pipe_execute = auto_pipe_execute
+        Instance = apps.get_model('main', 'Instance')
         if instance_name:
             self.instance_name = instance_name
         elif is_testing():
             self.instance_name = "awx_testing"
         else:
-            self.instance_name = settings.CLUSTER_HOST_ID  # Same as Instance.objects.my_hostname() BUT we do not need to import Instance
+            self.instance_name = Instance.objects.my_hostname()
 
+        # metric name, help_text
+        METRICSLIST = [
+            SetIntM('callback_receiver_events_queue_size_redis', 'Current number of events in redis queue'),
+            IntM('callback_receiver_events_popped_redis', 'Number of events popped from redis'),
+            IntM('callback_receiver_events_in_memory', 'Current number of events in memory (in transfer from redis to db)'),
+            IntM('callback_receiver_batch_events_errors', 'Number of times batch insertion failed'),
+            FloatM('callback_receiver_events_insert_db_seconds', 'Total time spent saving events to database'),
+            IntM('callback_receiver_events_insert_db', 'Number of events batch inserted into database'),
+            IntM('callback_receiver_events_broadcast', 'Number of events broadcast to other control plane nodes'),
+            HistogramM(
+                'callback_receiver_batch_events_insert_db', 'Number of events batch inserted into database', settings.SUBSYSTEM_METRICS_BATCH_INSERT_BUCKETS
+            ),
+            SetFloatM('callback_receiver_event_processing_avg_seconds', 'Average processing time per event per callback receiver batch'),
+            FloatM('subsystem_metrics_pipe_execute_seconds', 'Time spent saving metrics to redis'),
+            IntM('subsystem_metrics_pipe_execute_calls', 'Number of calls to pipe_execute'),
+            FloatM('subsystem_metrics_send_metrics_seconds', 'Time spent sending metrics to other nodes'),
+            SetFloatM('task_manager_get_tasks_seconds', 'Time spent in loading tasks from db'),
+            SetFloatM('task_manager_start_task_seconds', 'Time spent starting task'),
+            SetFloatM('task_manager_process_running_tasks_seconds', 'Time spent processing running tasks'),
+            SetFloatM('task_manager_process_pending_tasks_seconds', 'Time spent processing pending tasks'),
+            SetFloatM('task_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
+            IntM('task_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
+            SetFloatM('task_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
+            SetIntM('task_manager_tasks_started', 'Number of tasks started'),
+            SetIntM('task_manager_running_processed', 'Number of running tasks processed'),
+            SetIntM('task_manager_pending_processed', 'Number of pending tasks processed'),
+            SetIntM('task_manager_tasks_blocked', 'Number of tasks blocked from running'),
+            SetFloatM('task_manager_commit_seconds', 'Time spent in db transaction, including on_commit calls'),
+            SetFloatM('dependency_manager_get_tasks_seconds', 'Time spent loading pending tasks from db'),
+            SetFloatM('dependency_manager_generate_dependencies_seconds', 'Time spent generating dependencies for pending tasks'),
+            SetFloatM('dependency_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
+            IntM('dependency_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
+            SetFloatM('dependency_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
+            SetIntM('dependency_manager_pending_processed', 'Number of pending tasks processed'),
+            SetFloatM('workflow_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
+            IntM('workflow_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
+            SetFloatM('workflow_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
+            SetFloatM('workflow_manager_spawn_workflow_graph_jobs_seconds', 'Time spent spawning workflow tasks'),
+            SetFloatM('workflow_manager_get_tasks_seconds', 'Time spent loading workflow tasks from db'),
+            # dispatcher subsystem metrics
+            SetIntM('dispatcher_pool_scale_up_events', 'Number of times local dispatcher scaled up a worker since startup'),
+            SetIntM('dispatcher_pool_active_task_count', 'Number of active tasks in the worker pool when last task was submitted'),
+            SetIntM('dispatcher_pool_max_worker_count', 'Highest number of workers in worker pool in last collection interval, about 20s'),
+            SetFloatM('dispatcher_availability', 'Fraction of time (in last collection interval) dispatcher was able to receive messages'),
+        ]
         # turn metric list into dictionary with the metric name as a key
         self.METRICS = {}
-        for m in itertools.chain(self.METRICSLIST, self._METRICSLIST):
+        for m in METRICSLIST:
             self.METRICS[m.field] = m
 
         # track last time metrics were sent to other nodes
@@ -223,7 +230,7 @@ class Metrics(MetricsNamespace):
             m.reset_value(self.conn)
         self.metrics_have_changed = True
         self.conn.delete(root_key + "_lock")
-        for m in self.conn.scan_iter(root_key + '-' + self._namespace + '_instance_*'):
+        for m in self.conn.scan_iter(root_key + '_instance_*'):
             self.conn.delete(m)
 
     def inc(self, field, value):
@@ -290,7 +297,7 @@ class Metrics(MetricsNamespace):
     def send_metrics(self):
         # more than one thread could be calling this at the same time, so should
         # acquire redis lock before sending metrics
-        lock = self.conn.lock(root_key + '-' + self._namespace + '_lock')
+        lock = self.conn.lock(root_key + '_lock')
         if not lock.acquire(blocking=False):
             return
         try:
@@ -300,10 +307,9 @@ class Metrics(MetricsNamespace):
                 payload = {
                     'instance': self.instance_name,
                     'metrics': serialized_metrics,
-                    'metrics_namespace': self._namespace,
                 }
                 # store the serialized data locally as well, so that load_other_metrics will read it
-                self.conn.set(root_key + '-' + self._namespace + '_instance_' + self.instance_name, serialized_metrics)
+                self.conn.set(root_key + '_instance_' + self.instance_name, serialized_metrics)
                 emit_channel_notification("metrics", payload)
 
                 self.previous_send_metrics.set(current_time)
@@ -325,14 +331,14 @@ class Metrics(MetricsNamespace):
         instances_filter = request.query_params.getlist("node")
         # get a sorted list of instance names
         instance_names = [self.instance_name]
-        for m in self.conn.scan_iter(root_key + '-' + self._namespace + '_instance_*'):
+        for m in self.conn.scan_iter(root_key + '_instance_*'):
             instance_names.append(m.decode('UTF-8').split('_instance_')[1])
         instance_names.sort()
         # load data, including data from the this local instance
         instance_data = {}
         for instance in instance_names:
             if len(instances_filter) == 0 or instance in instances_filter:
-                instance_data_from_redis = self.conn.get(root_key + '-' + self._namespace + '_instance_' + instance)
+                instance_data_from_redis = self.conn.get(root_key + '_instance_' + instance)
                 # data from other instances may not be available. That is OK.
                 if instance_data_from_redis:
                     instance_data[instance] = json.loads(instance_data_from_redis.decode('UTF-8'))
@@ -351,120 +357,6 @@ class Metrics(MetricsNamespace):
         return output_text
 
 
-class DispatcherMetrics(Metrics):
-    METRICSLIST = [
-        SetFloatM('task_manager_get_tasks_seconds', 'Time spent in loading tasks from db'),
-        SetFloatM('task_manager_start_task_seconds', 'Time spent starting task'),
-        SetFloatM('task_manager_process_running_tasks_seconds', 'Time spent processing running tasks'),
-        SetFloatM('task_manager_process_pending_tasks_seconds', 'Time spent processing pending tasks'),
-        SetFloatM('task_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
-        IntM('task_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
-        SetFloatM('task_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
-        SetIntM('task_manager_tasks_started', 'Number of tasks started'),
-        SetIntM('task_manager_running_processed', 'Number of running tasks processed'),
-        SetIntM('task_manager_pending_processed', 'Number of pending tasks processed'),
-        SetIntM('task_manager_tasks_blocked', 'Number of tasks blocked from running'),
-        SetFloatM('task_manager_commit_seconds', 'Time spent in db transaction, including on_commit calls'),
-        SetFloatM('dependency_manager_get_tasks_seconds', 'Time spent loading pending tasks from db'),
-        SetFloatM('dependency_manager_generate_dependencies_seconds', 'Time spent generating dependencies for pending tasks'),
-        SetFloatM('dependency_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
-        IntM('dependency_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
-        SetFloatM('dependency_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
-        SetIntM('dependency_manager_pending_processed', 'Number of pending tasks processed'),
-        SetFloatM('workflow_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
-        IntM('workflow_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
-        SetFloatM('workflow_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
-        SetFloatM('workflow_manager_spawn_workflow_graph_jobs_seconds', 'Time spent spawning workflow tasks'),
-        SetFloatM('workflow_manager_get_tasks_seconds', 'Time spent loading workflow tasks from db'),
-        # dispatcher subsystem metrics
-        SetIntM('dispatcher_pool_scale_up_events', 'Number of times local dispatcher scaled up a worker since startup'),
-        SetIntM('dispatcher_pool_active_task_count', 'Number of active tasks in the worker pool when last task was submitted'),
-        SetIntM('dispatcher_pool_max_worker_count', 'Highest number of workers in worker pool in last collection interval, about 20s'),
-        SetFloatM('dispatcher_availability', 'Fraction of time (in last collection interval) dispatcher was able to receive messages'),
-    ]
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(settings.METRICS_SERVICE_DISPATCHER, *args, **kwargs)
-
-
-class CallbackReceiverMetrics(Metrics):
-    METRICSLIST = [
-        SetIntM('callback_receiver_events_queue_size_redis', 'Current number of events in redis queue'),
-        IntM('callback_receiver_events_popped_redis', 'Number of events popped from redis'),
-        IntM('callback_receiver_events_in_memory', 'Current number of events in memory (in transfer from redis to db)'),
-        IntM('callback_receiver_batch_events_errors', 'Number of times batch insertion failed'),
-        FloatM('callback_receiver_events_insert_db_seconds', 'Total time spent saving events to database'),
-        IntM('callback_receiver_events_insert_db', 'Number of events batch inserted into database'),
-        IntM('callback_receiver_events_broadcast', 'Number of events broadcast to other control plane nodes'),
-        HistogramM(
-            'callback_receiver_batch_events_insert_db', 'Number of events batch inserted into database', settings.SUBSYSTEM_METRICS_BATCH_INSERT_BUCKETS
-        ),
-        SetFloatM('callback_receiver_event_processing_avg_seconds', 'Average processing time per event per callback receiver batch'),
-    ]
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(settings.METRICS_SERVICE_CALLBACK_RECEIVER, *args, **kwargs)
-
-
 def metrics(request):
-    output_text = ''
-    for m in [DispatcherMetrics(), CallbackReceiverMetrics()]:
-        output_text += m.generate_metrics(request)
-    return output_text
-
-
-class CustomToPrometheusMetricsCollector(prometheus_client.registry.Collector):
-    """
-    Takes the metric data from redis -> our custom metric fields -> prometheus
-    library metric fields.
-
-    The plan is to get rid of the use of redis, our custom metric fields, and
-    to switch fully to the prometheus library. At that point, this translation
-    code will be deleted.
-    """
-
-    def __init__(self, metrics_obj, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._metrics = metrics_obj
-
-    def collect(self):
-        my_hostname = settings.CLUSTER_HOST_ID
-
-        instance_data = self._metrics.load_other_metrics(Request(HttpRequest()))
-        if not instance_data:
-            logger.debug(f"No metric data not found in redis for metric namespace '{self._metrics._namespace}'")
-            return None
-
-        host_metrics = instance_data.get(my_hostname)
-        for _, metric in self._metrics.METRICS.items():
-            entry = host_metrics.get(metric.field)
-            if not entry:
-                logger.debug(f"{self._metrics._namespace} metric '{metric.field}' not found in redis data payload {json.dumps(instance_data, indent=2)}")
-                continue
-            if isinstance(metric, HistogramM):
-                buckets = list(zip(metric.buckets, entry['counts']))
-                buckets = [[str(i[0]), str(i[1])] for i in buckets]
-                yield HistogramMetricFamily(metric.field, metric.help_text, buckets=buckets, sum_value=entry['sum'])
-            else:
-                yield GaugeMetricFamily(metric.field, metric.help_text, value=entry)
-
-
-class CallbackReceiverMetricsServer(MetricsServer):
-    def __init__(self):
-        registry = CollectorRegistry(auto_describe=True)
-        registry.register(CustomToPrometheusMetricsCollector(DispatcherMetrics(metrics_have_changed=False)))
-        super().__init__(settings.METRICS_SERVICE_CALLBACK_RECEIVER, registry)
-
-
-class DispatcherMetricsServer(MetricsServer):
-    def __init__(self):
-        registry = CollectorRegistry(auto_describe=True)
-        registry.register(CustomToPrometheusMetricsCollector(CallbackReceiverMetrics(metrics_have_changed=False)))
-        super().__init__(settings.METRICS_SERVICE_DISPATCHER, registry)
-
-
-class WebsocketsMetricsServer(MetricsServer):
-    def __init__(self):
-        registry = CollectorRegistry(auto_describe=True)
-        # registry.register()
-        super().__init__(settings.METRICS_SERVICE_WEBSOCKETS, registry)
+    m = Metrics()
+    return m.generate_metrics(request)

awx/main/conf.py

@@ -827,16 +827,6 @@ register(
     category_slug='bulk',
 )
 
-register(
-    'BULK_HOST_MAX_DELETE',
-    field_class=fields.IntegerField,
-    default=250,
-    label=_('Max number of hosts to allow to be deleted in a single bulk action'),
-    help_text=_('Max number of hosts to allow to be deleted in a single bulk action'),
-    category=_('Bulk Actions'),
-    category_slug='bulk',
-)
-
 register(
     'UI_NEXT',
     field_class=fields.BooleanField,

awx/main/consumers.py

@@ -106,7 +106,7 @@ class RelayConsumer(AsyncJsonWebsocketConsumer):
         if group == "metrics":
             message = json.loads(message['text'])
             conn = redis.Redis.from_url(settings.BROKER_URL)
-            conn.set(settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX + "-" + message['metrics_namespace'] + "_instance_" + message['instance'], message['metrics'])
+            conn.set(settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX + "_instance_" + message['instance'], message['metrics'])
         else:
             await self.channel_layer.group_send(group, message)
 

awx/main/credential_plugins/aim.py

@@ -58,7 +58,7 @@ aim_inputs = {
             'id': 'object_property',
             'label': _('Object Property'),
             'type': 'string',
-            'help_text': _('The property of the object to return. Available properties: Username, Password and Address.'),
+            'help_text': _('The property of the object to return. Default: Content Ex: Username, Address, etc.'),
         },
         {
             'id': 'reason',
@@ -111,12 +111,8 @@ def aim_backend(**kwargs):
         object_property = 'Content'
     elif object_property.lower() == 'username':
         object_property = 'UserName'
-    elif object_property.lower() == 'password':
-        object_property = 'Content'
-    elif object_property.lower() == 'address':
-        object_property = 'Address'
     elif object_property not in res:
-        raise KeyError('Property {} not found in object, available properties: Username, Password and Address'.format(object_property))
+        raise KeyError('Property {} not found in object'.format(object_property))
     else:
         object_property = object_property.capitalize()
 

awx/main/credential_plugins/hashivault.py

@@ -41,34 +41,6 @@ base_inputs = {
             'secret': True,
             'help_text': _('The Secret ID for AppRole Authentication'),
         },
-        {
-            'id': 'client_cert_public',
-            'label': _('Client Certificate'),
-            'type': 'string',
-            'multiline': True,
-            'help_text': _(
-                'The PEM-encoded client certificate used for TLS client authentication.'
-                ' This should include the certificate and any intermediate certififcates.'
-            ),
-        },
-        {
-            'id': 'client_cert_private',
-            'label': _('Client Certificate Key'),
-            'type': 'string',
-            'multiline': True,
-            'secret': True,
-            'help_text': _('The certificate private key used for TLS client authentication.'),
-        },
-        {
-            'id': 'client_cert_role',
-            'label': _('TLS Authentication Role'),
-            'type': 'string',
-            'multiline': False,
-            'help_text': _(
-                'The role configured in Hashicorp Vault for TLS client authentication.'
-                ' If not provided, Hashicorp Vault may assign roles based on the certificate used.'
-            ),
-        },
         {
             'id': 'namespace',
             'label': _('Namespace name (Vault Enterprise only)'),
@@ -87,20 +59,6 @@ base_inputs = {
                 ' see https://www.vaultproject.io/docs/auth/kubernetes#configuration'
             ),
         },
-        {
-            'id': 'username',
-            'label': _('Username'),
-            'type': 'string',
-            'secret': False,
-            'help_text': _('Username for user authentication.'),
-        },
-        {
-            'id': 'password',
-            'label': _('Password'),
-            'type': 'string',
-            'secret': True,
-            'help_text': _('Password for user authentication.'),
-        },
         {
             'id': 'default_auth_path',
             'label': _('Path to Auth'),
@@ -199,25 +157,19 @@ hashi_ssh_inputs['required'].extend(['public_key', 'role'])
 
 def handle_auth(**kwargs):
     token = None
+
     if kwargs.get('token'):
         token = kwargs['token']
-    elif kwargs.get('username') and kwargs.get('password'):
-        token = method_auth(**kwargs, auth_param=userpass_auth(**kwargs))
     elif kwargs.get('role_id') and kwargs.get('secret_id'):
         token = method_auth(**kwargs, auth_param=approle_auth(**kwargs))
     elif kwargs.get('kubernetes_role'):
         token = method_auth(**kwargs, auth_param=kubernetes_auth(**kwargs))
-    elif kwargs.get('client_cert_public') and kwargs.get('client_cert_private'):
-        token = method_auth(**kwargs, auth_param=client_cert_auth(**kwargs))
     else:
-        raise Exception('Token, Username/Password, AppRole, Kubernetes, or TLS authentication parameters must be set')
+        raise Exception('Either token or AppRole/Kubernetes authentication parameters must be set')
+
     return token
 
 
-def userpass_auth(**kwargs):
-    return {'username': kwargs['username'], 'password': kwargs['password']}
-
-
 def approle_auth(**kwargs):
     return {'role_id': kwargs['role_id'], 'secret_id': kwargs['secret_id']}
 
@@ -229,10 +181,6 @@ def kubernetes_auth(**kwargs):
     return {'role': kwargs['kubernetes_role'], 'jwt': jwt}
 
 
-def client_cert_auth(**kwargs):
-    return {'name': kwargs.get('client_cert_role')}
-
-
 def method_auth(**kwargs):
     # get auth method specific params
     request_kwargs = {'json': kwargs['auth_param'], 'timeout': 30}
@@ -245,25 +193,13 @@ def method_auth(**kwargs):
     cacert = kwargs.get('cacert', None)
 
     sess = requests.Session()
-    sess.mount(url, requests.adapters.HTTPAdapter(max_retries=5))
-
     # Namespace support
     if kwargs.get('namespace'):
         sess.headers['X-Vault-Namespace'] = kwargs['namespace']
     request_url = '/'.join([url, 'auth', auth_path, 'login']).rstrip('/')
-    if kwargs['auth_param'].get('username'):
-        request_url = request_url + '/' + (kwargs['username'])
     with CertFiles(cacert) as cert:
         request_kwargs['verify'] = cert
-        # TLS client certificate support
-        if kwargs.get('client_cert_public') and kwargs.get('client_cert_private'):
-            # Add client cert to requests Session before making call
-            with CertFiles(kwargs['client_cert_public'], key=kwargs['client_cert_private']) as client_cert:
-                sess.cert = client_cert
-                resp = sess.post(request_url, **request_kwargs)
-        else:
-            # Make call without client certificate
-            resp = sess.post(request_url, **request_kwargs)
+        resp = sess.post(request_url, **request_kwargs)
     resp.raise_for_status()
     token = resp.json()['auth']['client_token']
     return token
@@ -284,7 +220,6 @@ def kv_backend(**kwargs):
     }
 
     sess = requests.Session()
-    sess.mount(url, requests.adapters.HTTPAdapter(max_retries=5))
     sess.headers['Authorization'] = 'Bearer {}'.format(token)
     # Compatibility header for older installs of Hashicorp Vault
     sess.headers['X-Vault-Token'] = token
@@ -355,7 +290,6 @@ def ssh_backend(**kwargs):
         request_kwargs['json']['valid_principals'] = kwargs['valid_principals']
 
     sess = requests.Session()
-    sess.mount(url, requests.adapters.HTTPAdapter(max_retries=5))
     sess.headers['Authorization'] = 'Bearer {}'.format(token)
     if kwargs.get('namespace'):
         sess.headers['X-Vault-Namespace'] = kwargs['namespace']

awx/main/dispatch/__init__.py

@@ -93,26 +93,6 @@ class PubSub(object):
         self.conn.close()
 
 
-def create_listener_connection():
-    conf = settings.DATABASES['default'].copy()
-    conf['OPTIONS'] = conf.get('OPTIONS', {}).copy()
-    # Modify the application name to distinguish from other connections the process might use
-    conf['OPTIONS']['application_name'] = get_application_name(settings.CLUSTER_HOST_ID, function='listener')
-
-    # Apply overrides specifically for the listener connection
-    for k, v in settings.LISTENER_DATABASES.get('default', {}).items():
-        conf[k] = v
-    for k, v in settings.LISTENER_DATABASES.get('default', {}).get('OPTIONS', {}).items():
-        conf['OPTIONS'][k] = v
-
-    # Allow password-less authentication
-    if 'PASSWORD' in conf:
-        conf['OPTIONS']['password'] = conf.pop('PASSWORD')
-
-    connection_data = f"dbname={conf['NAME']} host={conf['HOST']} user={conf['USER']} port={conf['PORT']}"
-    return psycopg.connect(connection_data, autocommit=True, **conf['OPTIONS'])
-
-
 @contextmanager
 def pg_bus_conn(new_connection=False, select_timeout=None):
     '''
@@ -126,7 +106,12 @@ def pg_bus_conn(new_connection=False, select_timeout=None):
     '''
 
     if new_connection:
-        conn = create_listener_connection()
+        conf = settings.DATABASES['default'].copy()
+        conf['OPTIONS'] = conf.get('OPTIONS', {}).copy()
+        # Modify the application name to distinguish from other connections the process might use
+        conf['OPTIONS']['application_name'] = get_application_name(settings.CLUSTER_HOST_ID, function='listener')
+        connection_data = f"dbname={conf['NAME']} host={conf['HOST']} user={conf['USER']} password={conf['PASSWORD']} port={conf['PORT']}"
+        conn = psycopg.connect(connection_data, autocommit=True, **conf['OPTIONS'])
     else:
         if pg_connection.connection is None:
             pg_connection.connect()

awx/main/dispatch/worker/base.py

@@ -162,13 +162,13 @@ class AWXConsumerRedis(AWXConsumerBase):
 class AWXConsumerPG(AWXConsumerBase):
     def __init__(self, *args, schedule=None, **kwargs):
         super().__init__(*args, **kwargs)
-        self.pg_max_wait = getattr(settings, 'DISPATCHER_DB_DOWNTOWN_TOLLERANCE', settings.DISPATCHER_DB_DOWNTIME_TOLERANCE)
+        self.pg_max_wait = settings.DISPATCHER_DB_DOWNTIME_TOLERANCE
         # if no successful loops have ran since startup, then we should fail right away
         self.pg_is_down = True  # set so that we fail if we get database errors on startup
         init_time = time.time()
         self.pg_down_time = init_time - self.pg_max_wait  # allow no grace period
         self.last_cleanup = init_time
-        self.subsystem_metrics = s_metrics.DispatcherMetrics(auto_pipe_execute=False)
+        self.subsystem_metrics = s_metrics.Metrics(auto_pipe_execute=False)
         self.last_metrics_gather = init_time
         self.listen_cumulative_time = 0.0
         if schedule:
@@ -214,10 +214,7 @@ class AWXConsumerPG(AWXConsumerBase):
                 # bypasses pg_notify for scheduled tasks
                 self.dispatch_task(body)
 
-        if self.pg_is_down:
-            logger.info('Dispatcher listener connection established')
-            self.pg_is_down = False
-
+        self.pg_is_down = False
         self.listen_start = time.time()
 
         return self.scheduler.time_until_next_run()

awx/main/dispatch/worker/callback.py

@@ -72,7 +72,7 @@ class CallbackBrokerWorker(BaseWorker):
     def __init__(self):
         self.buff = {}
         self.redis = redis.Redis.from_url(settings.BROKER_URL)
-        self.subsystem_metrics = s_metrics.CallbackReceiverMetrics(auto_pipe_execute=False)
+        self.subsystem_metrics = s_metrics.Metrics(auto_pipe_execute=False)
         self.queue_pop = 0
         self.queue_name = settings.CALLBACK_QUEUE
         self.prof = AWXProfiler("CallbackBrokerWorker")

awx/main/fields.py

@@ -5,7 +5,6 @@
 import copy
 import json
 import re
-import sys
 import urllib.parse
 
 from jinja2 import sandbox, StrictUndefined
@@ -407,13 +406,11 @@ class SmartFilterField(models.TextField):
         # https://docs.python.org/2/library/stdtypes.html#truth-value-testing
         if not value:
             return None
-        # avoid doing too much during migrations
-        if 'migrate' not in sys.argv:
-            value = urllib.parse.unquote(value)
-            try:
-                SmartFilter().query_from_string(value)
-            except RuntimeError as e:
-                raise models.base.ValidationError(e)
+        value = urllib.parse.unquote(value)
+        try:
+            SmartFilter().query_from_string(value)
+        except RuntimeError as e:
+            raise models.base.ValidationError(e)
         return super(SmartFilterField, self).get_prep_value(value)
 
 

awx/main/management/commands/add_receptor_address.py

@@ -1,53 +0,0 @@
-# Copyright (c) 2015 Ansible, Inc.
-# All Rights Reserved
-
-from django.core.management.base import BaseCommand
-
-from awx.main.models import Instance, ReceptorAddress
-
-
-def add_address(**kwargs):
-    try:
-        instance = Instance.objects.get(hostname=kwargs.pop('instance'))
-        kwargs['instance'] = instance
-
-        if kwargs.get('canonical') and instance.receptor_addresses.filter(canonical=True).exclude(address=kwargs['address']).exists():
-            print(f"Instance {instance.hostname} already has a canonical address, skipping")
-            return False
-        # if ReceptorAddress already exists with address, just update
-        # otherwise, create new ReceptorAddress
-        addr, _ = ReceptorAddress.objects.update_or_create(address=kwargs.pop('address'), defaults=kwargs)
-        print(f"Successfully added receptor address {addr.get_full_address()}")
-        return True
-    except Exception as e:
-        print(f"Error adding receptor address: {e}")
-        return False
-
-
-class Command(BaseCommand):
-    """
-    Internal controller command.
-    Register receptor address to an already-registered instance.
-    """
-
-    help = "Add receptor address to an instance."
-
-    def add_arguments(self, parser):
-        parser.add_argument('--instance', dest='instance', required=True, type=str, help="Instance hostname this address is added to")
-        parser.add_argument('--address', dest='address', required=True, type=str, help="Receptor address")
-        parser.add_argument('--port', dest='port', type=int, help="Receptor listener port")
-        parser.add_argument('--websocket_path', dest='websocket_path', type=str, default="", help="Path for websockets")
-        parser.add_argument('--is_internal', action='store_true', help="If true, address only resolvable within the Kubernetes cluster")
-        parser.add_argument('--protocol', type=str, default='tcp', choices=['tcp', 'ws', 'wss'], help="Protocol to use for the Receptor listener")
-        parser.add_argument('--canonical', action='store_true', help="If true, address is the canonical address for the instance")
-        parser.add_argument('--peers_from_control_nodes', action='store_true', help="If true, control nodes will peer to this address")
-
-    def handle(self, **options):
-        address_options = {
-            k: options[k]
-            for k in ('instance', 'address', 'port', 'websocket_path', 'is_internal', 'protocol', 'peers_from_control_nodes', 'canonical')
-            if options[k]
-        }
-        changed = add_address(**address_options)
-        if changed:
-            print("(changed: True)")

awx/main/management/commands/list_instances.py

@@ -55,7 +55,7 @@ class Command(BaseCommand):
 
                 capacity = f' capacity={x.capacity}' if x.node_type != 'hop' else ''
                 version = f" version={x.version or '?'}" if x.node_type != 'hop' else ''
-                heartbeat = f' heartbeat="{x.last_seen:%Y-%m-%d %H:%M:%S}"' if x.last_seen else ''
+                heartbeat = f' heartbeat="{x.last_seen:%Y-%m-%d %H:%M:%S}"' if x.capacity or x.node_type == 'hop' else ''
                 print(f'\t{color}{x.hostname}{capacity} node_type={x.node_type}{version}{heartbeat}{end_color}')
 
             print()

awx/main/management/commands/provision_instance.py

@@ -25,17 +25,20 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('--hostname', dest='hostname', type=str, help="Hostname used during provisioning")
+        parser.add_argument('--listener_port', dest='listener_port', type=int, help="Receptor listener port")
         parser.add_argument('--node_type', type=str, default='hybrid', choices=['control', 'execution', 'hop', 'hybrid'], help="Instance Node type")
         parser.add_argument('--uuid', type=str, help="Instance UUID")
 
-    def _register_hostname(self, hostname, node_type, uuid):
+    def _register_hostname(self, hostname, node_type, uuid, listener_port):
         if not hostname:
             if not settings.AWX_AUTO_DEPROVISION_INSTANCES:
                 raise CommandError('Registering with values from settings only intended for use in K8s installs')
 
             from awx.main.management.commands.register_queue import RegisterQueue
 
-            (changed, instance) = Instance.objects.register(ip_address=os.environ.get('MY_POD_IP'), node_type='control', node_uuid=settings.SYSTEM_UUID)
+            (changed, instance) = Instance.objects.register(
+                ip_address=os.environ.get('MY_POD_IP'), listener_port=listener_port, node_type='control', node_uuid=settings.SYSTEM_UUID
+            )
             RegisterQueue(settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME, 100, 0, [], is_container_group=False).register()
             RegisterQueue(
                 settings.DEFAULT_EXECUTION_QUEUE_NAME,
@@ -48,17 +51,16 @@ class Command(BaseCommand):
                 max_concurrent_jobs=settings.DEFAULT_EXECUTION_QUEUE_MAX_CONCURRENT_JOBS,
             ).register()
         else:
-            (changed, instance) = Instance.objects.register(hostname=hostname, node_type=node_type, node_uuid=uuid)
+            (changed, instance) = Instance.objects.register(hostname=hostname, node_type=node_type, node_uuid=uuid, listener_port=listener_port)
         if changed:
             print("Successfully registered instance {}".format(hostname))
         else:
             print("Instance already registered {}".format(instance.hostname))
-
         self.changed = changed
 
     @transaction.atomic
     def handle(self, **options):
         self.changed = False
-        self._register_hostname(options.get('hostname'), options.get('node_type'), options.get('uuid'))
+        self._register_hostname(options.get('hostname'), options.get('node_type'), options.get('uuid'), options.get('listener_port'))
         if self.changed:
             print("(changed: True)")

awx/main/management/commands/register_peers.py

@@ -1,7 +1,9 @@
+import warnings
+
 from django.core.management.base import BaseCommand, CommandError
 from django.db import transaction
 
-from awx.main.models import Instance, InstanceLink, ReceptorAddress
+from awx.main.models import Instance, InstanceLink
 
 
 class Command(BaseCommand):
@@ -26,9 +28,7 @@ class Command(BaseCommand):
 
     def handle(self, **options):
         # provides a mapping of hostname to Instance objects
-        nodes = Instance.objects.all().in_bulk(field_name='hostname')
-        # provides a mapping of address to ReceptorAddress objects
-        addresses = ReceptorAddress.objects.all().in_bulk(field_name='address')
+        nodes = Instance.objects.in_bulk(field_name='hostname')
 
         if options['source'] not in nodes:
             raise CommandError(f"Host {options['source']} is not a registered instance.")
@@ -39,14 +39,6 @@ class Command(BaseCommand):
         if options['exact'] is not None and options['disconnect']:
             raise CommandError("The option --disconnect may not be used with --exact.")
 
-        # make sure each target has a receptor address
-        peers = options['peers'] or []
-        disconnect = options['disconnect'] or []
-        exact = options['exact'] or []
-        for peer in peers + disconnect + exact:
-            if peer not in addresses:
-                raise CommandError(f"Peer {peer} does not have a receptor address.")
-
         # No 1-cycles
         for collection in ('peers', 'disconnect', 'exact'):
             if options[collection] is not None and options['source'] in options[collection]:
@@ -55,12 +47,9 @@ class Command(BaseCommand):
         # No 2-cycles
         if options['peers'] or options['exact'] is not None:
             peers = set(options['peers'] or options['exact'])
-            if options['source'] in addresses:
-                incoming = set(InstanceLink.objects.filter(target=addresses[options['source']]).values_list('source__hostname', flat=True))
-            else:
-                incoming = set()
+            incoming = set(InstanceLink.objects.filter(target=nodes[options['source']]).values_list('source__hostname', flat=True))
             if peers & incoming:
-                raise CommandError(f"Source node {options['source']} should not link to nodes already peering to it: {peers & incoming}.")
+                warnings.warn(f"Source node {options['source']} should not link to nodes already peering to it: {peers & incoming}.")
 
         if options['peers']:
             missing_peers = set(options['peers']) - set(nodes)
@@ -71,7 +60,7 @@ class Command(BaseCommand):
             results = 0
             for target in options['peers']:
                 _, created = InstanceLink.objects.update_or_create(
-                    source=nodes[options['source']], target=addresses[target], defaults={'link_state': InstanceLink.States.ESTABLISHED}
+                    source=nodes[options['source']], target=nodes[target], defaults={'link_state': InstanceLink.States.ESTABLISHED}
                 )
                 if created:
                     results += 1
@@ -81,9 +70,9 @@ class Command(BaseCommand):
         if options['disconnect']:
             results = 0
             for target in options['disconnect']:
-                if target not in addresses:  # Be permissive, the node might have already been de-registered.
+                if target not in nodes:  # Be permissive, the node might have already been de-registered.
                     continue
-                n, _ = InstanceLink.objects.filter(source=nodes[options['source']], target=addresses[target]).delete()
+                n, _ = InstanceLink.objects.filter(source=nodes[options['source']], target=nodes[target]).delete()
                 results += n
 
             print(f"{results} peer links removed from the database.")
@@ -92,11 +81,11 @@ class Command(BaseCommand):
             additions = 0
             with transaction.atomic():
                 peers = set(options['exact'])
-                links = set(InstanceLink.objects.filter(source=nodes[options['source']]).values_list('target__address', flat=True))
-                removals, _ = InstanceLink.objects.filter(source=nodes[options['source']], target__instance__hostname__in=links - peers).delete()
+                links = set(InstanceLink.objects.filter(source=nodes[options['source']]).values_list('target__hostname', flat=True))
+                removals, _ = InstanceLink.objects.filter(source=nodes[options['source']], target__hostname__in=links - peers).delete()
                 for target in peers - links:
                     _, created = InstanceLink.objects.update_or_create(
-                        source=nodes[options['source']], target=addresses[target], defaults={'link_state': InstanceLink.States.ESTABLISHED}
+                        source=nodes[options['source']], target=nodes[target], defaults={'link_state': InstanceLink.States.ESTABLISHED}
                     )
                     if created:
                         additions += 1

awx/main/management/commands/remove_receptor_address.py

@@ -1,26 +0,0 @@
-# Copyright (c) 2015 Ansible, Inc.
-# All Rights Reserved
-
-from django.core.management.base import BaseCommand
-
-from awx.main.models import ReceptorAddress
-
-
-class Command(BaseCommand):
-    """
-    Internal controller command.
-    Delete a receptor address.
-    """
-
-    help = "Add receptor address to an instance."
-
-    def add_arguments(self, parser):
-        parser.add_argument('--address', dest='address', type=str, help="Receptor address to remove")
-
-    def handle(self, **options):
-        deleted = ReceptorAddress.objects.filter(address=options['address']).delete()
-        if deleted[0]:
-            print(f"Successfully removed {options['address']}")
-            print("(changed: True)")
-        else:
-            print(f"Did not remove {options['address']}, not found")

awx/main/management/commands/run_callback_receiver.py

@@ -3,7 +3,6 @@
 
 from django.conf import settings
 from django.core.management.base import BaseCommand
-from awx.main.analytics.subsystem_metrics import CallbackReceiverMetricsServer
 
 from awx.main.dispatch.control import Control
 from awx.main.dispatch.worker import AWXConsumerRedis, CallbackBrokerWorker
@@ -26,9 +25,6 @@ class Command(BaseCommand):
             print(Control('callback_receiver').status())
             return
         consumer = None
-
-        CallbackReceiverMetricsServer().start()
-
         try:
             consumer = AWXConsumerRedis(
                 'callback_receiver',

awx/main/management/commands/run_dispatcher.py

@@ -10,7 +10,6 @@ from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.control import Control
 from awx.main.dispatch.pool import AutoscalePool
 from awx.main.dispatch.worker import AWXConsumerPG, TaskWorker
-from awx.main.analytics.subsystem_metrics import DispatcherMetricsServer
 
 logger = logging.getLogger('awx.main.dispatch')
 
@@ -63,8 +62,6 @@ class Command(BaseCommand):
 
         consumer = None
 
-        DispatcherMetricsServer().start()
-
         try:
             queues = ['tower_broadcast_all', 'tower_settings_change', get_task_queuename()]
             consumer = AWXConsumerPG('dispatcher', TaskWorker(), queues, AutoscalePool(min_workers=4), schedule=settings.CELERYBEAT_SCHEDULE)

awx/main/management/commands/run_wsrelay.py

@@ -16,7 +16,6 @@ from awx.main.analytics.broadcast_websocket import (
     RelayWebsocketStatsManager,
     safe_name,
 )
-from awx.main.analytics.subsystem_metrics import WebsocketsMetricsServer
 from awx.main.wsrelay import WebSocketRelayManager
 
 
@@ -92,8 +91,6 @@ class Command(BaseCommand):
         return host_stats
 
     def handle(self, *arg, **options):
-        WebsocketsMetricsServer().start()
-
         # it's necessary to delay this import in case
         # database migrations are still running
         from awx.main.models.ha import Instance

awx/main/managers.py

@@ -115,14 +115,7 @@ class InstanceManager(models.Manager):
             return node[0]
         raise RuntimeError("No instance found with the current cluster host id")
 
-    def register(
-        self,
-        node_uuid=None,
-        hostname=None,
-        ip_address="",
-        node_type='hybrid',
-        defaults=None,
-    ):
+    def register(self, node_uuid=None, hostname=None, ip_address="", listener_port=None, node_type='hybrid', defaults=None):
         if not hostname:
             hostname = settings.CLUSTER_HOST_ID
 
@@ -168,6 +161,9 @@ class InstanceManager(models.Manager):
                 if instance.node_type != node_type:
                     instance.node_type = node_type
                     update_fields.append('node_type')
+                if instance.listener_port != listener_port:
+                    instance.listener_port = listener_port
+                    update_fields.append('listener_port')
                 if update_fields:
                     instance.save(update_fields=update_fields)
                     return (True, instance)
@@ -178,13 +174,11 @@ class InstanceManager(models.Manager):
             create_defaults = {
                 'node_state': Instance.States.INSTALLED,
                 'capacity': 0,
-                'managed': True,
             }
             if defaults is not None:
                 create_defaults.update(defaults)
             uuid_option = {'uuid': node_uuid if node_uuid is not None else uuid.uuid4()}
             if node_type == 'execution' and 'version' not in create_defaults:
                 create_defaults['version'] = RECEPTOR_PENDING
-            instance = self.create(hostname=hostname, ip_address=ip_address, node_type=node_type, **create_defaults, **uuid_option)
-
+            instance = self.create(hostname=hostname, ip_address=ip_address, listener_port=listener_port, node_type=node_type, **create_defaults, **uuid_option)
         return (True, instance)

awx/main/middleware.py

@@ -5,12 +5,11 @@ import logging
 import threading
 import time
 import urllib.parse
-from pathlib import Path
 
 from django.conf import settings
 from django.contrib.auth import logout
 from django.contrib.auth.models import User
-from django.db.migrations.recorder import MigrationRecorder
+from django.db.migrations.executor import MigrationExecutor
 from django.db import connection
 from django.shortcuts import redirect
 from django.apps import apps
@@ -18,11 +17,9 @@ from django.utils.deprecation import MiddlewareMixin
 from django.utils.translation import gettext_lazy as _
 from django.urls import reverse, resolve
 
-from awx.main import migrations
 from awx.main.utils.named_url_graph import generate_graph, GraphNode
 from awx.conf import fields, register
 from awx.main.utils.profiling import AWXProfiler
-from awx.main.utils.common import memoize
 
 
 logger = logging.getLogger('awx.main.middleware')
@@ -201,22 +198,9 @@ class URLModificationMiddleware(MiddlewareMixin):
             request.path_info = new_path
 
 
-@memoize(ttl=20)
-def is_migrating():
-    latest_number = 0
-    latest_name = ''
-    for migration_path in Path(migrations.__path__[0]).glob('[0-9]*.py'):
-        try:
-            migration_number = int(migration_path.name.split('_', 1)[0])
-        except ValueError:
-            continue
-        if migration_number > latest_number:
-            latest_number = migration_number
-            latest_name = migration_path.name[: -len('.py')]
-    return not MigrationRecorder(connection).migration_qs.filter(app='main', name=latest_name).exists()
-
-
 class MigrationRanCheckMiddleware(MiddlewareMixin):
     def process_request(self, request):
-        if is_migrating() and getattr(resolve(request.path), 'url_name', '') != 'migrations_notran':
+        executor = MigrationExecutor(connection)
+        plan = executor.migration_plan(executor.loader.graph.leaf_nodes())
+        if bool(plan) and getattr(resolve(request.path), 'url_name', '') != 'migrations_notran':
             return redirect(reverse("ui:migrations_notran"))

awx/main/migrations/0188_add_bitbucket_dc_webhook.py

@@ -1,52 +0,0 @@
-# Generated by Django 4.2.6 on 2023-11-16 21:00
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ('main', '0187_hop_nodes'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='job',
-            name='webhook_service',
-            field=models.CharField(
-                blank=True,
-                choices=[('github', 'GitHub'), ('gitlab', 'GitLab'), ('bitbucket_dc', 'BitBucket DataCenter')],
-                help_text='Service that webhook requests will be accepted from',
-                max_length=16,
-            ),
-        ),
-        migrations.AlterField(
-            model_name='jobtemplate',
-            name='webhook_service',
-            field=models.CharField(
-                blank=True,
-                choices=[('github', 'GitHub'), ('gitlab', 'GitLab'), ('bitbucket_dc', 'BitBucket DataCenter')],
-                help_text='Service that webhook requests will be accepted from',
-                max_length=16,
-            ),
-        ),
-        migrations.AlterField(
-            model_name='workflowjob',
-            name='webhook_service',
-            field=models.CharField(
-                blank=True,
-                choices=[('github', 'GitHub'), ('gitlab', 'GitLab'), ('bitbucket_dc', 'BitBucket DataCenter')],
-                help_text='Service that webhook requests will be accepted from',
-                max_length=16,
-            ),
-        ),
-        migrations.AlterField(
-            model_name='workflowjobtemplate',
-            name='webhook_service',
-            field=models.CharField(
-                blank=True,
-                choices=[('github', 'GitHub'), ('gitlab', 'GitLab'), ('bitbucket_dc', 'BitBucket DataCenter')],
-                help_text='Service that webhook requests will be accepted from',
-                max_length=16,
-            ),
-        ),
-    ]

awx/main/migrations/0189_inbound_hop_nodes.py

@@ -1,150 +0,0 @@
-# Generated by Django 4.2.6 on 2024-01-19 19:24
-
-import django.core.validators
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-def create_receptor_addresses(apps, schema_editor):
-    """
-    If listener_port was defined on an instance, create a receptor address for it
-    """
-    Instance = apps.get_model('main', 'Instance')
-    ReceptorAddress = apps.get_model('main', 'ReceptorAddress')
-    for instance in Instance.objects.exclude(listener_port=None):
-        ReceptorAddress.objects.create(
-            instance=instance,
-            address=instance.hostname,
-            port=instance.listener_port,
-            peers_from_control_nodes=instance.peers_from_control_nodes,
-            protocol='tcp',
-            is_internal=False,
-            canonical=True,
-        )
-
-
-def link_to_receptor_addresses(apps, schema_editor):
-    """
-    Modify each InstanceLink to point to the newly created
-    ReceptorAddresses, using the new target field
-    """
-    InstanceLink = apps.get_model('main', 'InstanceLink')
-    for link in InstanceLink.objects.all():
-        link.target = link.target_old.receptor_addresses.get()
-        link.save()
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ('main', '0188_add_bitbucket_dc_webhook'),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='ReceptorAddress',
-            fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('address', models.CharField(help_text='Routable address for this instance.', max_length=255)),
-                (
-                    'port',
-                    models.IntegerField(
-                        default=27199,
-                        help_text='Port for the address.',
-                        validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(65535)],
-                    ),
-                ),
-                ('websocket_path', models.CharField(blank=True, default='', help_text='Websocket path.', max_length=255)),
-                (
-                    'protocol',
-                    models.CharField(
-                        choices=[('tcp', 'TCP'), ('ws', 'WS'), ('wss', 'WSS')],
-                        default='tcp',
-                        help_text="Protocol to use for the Receptor listener, 'tcp', 'wss', or 'ws'.",
-                        max_length=10,
-                    ),
-                ),
-                ('is_internal', models.BooleanField(default=False, help_text='If True, only routable within the Kubernetes cluster.')),
-                ('canonical', models.BooleanField(default=False, help_text='If True, this address is the canonical address for the instance.')),
-                (
-                    'peers_from_control_nodes',
-                    models.BooleanField(default=False, help_text='If True, control plane cluster nodes should automatically peer to it.'),
-                ),
-            ],
-        ),
-        migrations.RemoveConstraint(
-            model_name='instancelink',
-            name='source_and_target_can_not_be_equal',
-        ),
-        migrations.RenameField(
-            model_name='instancelink',
-            old_name='target',
-            new_name='target_old',
-        ),
-        migrations.AlterUniqueTogether(
-            name='instancelink',
-            unique_together=set(),
-        ),
-        migrations.AddField(
-            model_name='instance',
-            name='managed',
-            field=models.BooleanField(default=False, editable=False, help_text='If True, this instance is managed by the control plane.'),
-        ),
-        migrations.AlterField(
-            model_name='instancelink',
-            name='source',
-            field=models.ForeignKey(help_text='The source instance of this peer link.', on_delete=django.db.models.deletion.CASCADE, to='main.instance'),
-        ),
-        migrations.AddField(
-            model_name='receptoraddress',
-            name='instance',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='receptor_addresses', to='main.instance'),
-        ),
-        migrations.AddField(
-            model_name='activitystream',
-            name='receptor_address',
-            field=models.ManyToManyField(blank=True, to='main.receptoraddress'),
-        ),
-        migrations.AddConstraint(
-            model_name='receptoraddress',
-            constraint=models.UniqueConstraint(fields=('address',), name='unique_receptor_address', violation_error_message='Receptor address must be unique.'),
-        ),
-        migrations.AddField(
-            model_name='instancelink',
-            name='target',
-            field=models.ForeignKey(
-                help_text='The target receptor address of this peer link.', null=True, on_delete=django.db.models.deletion.CASCADE, to='main.receptoraddress'
-            ),
-        ),
-        migrations.RunPython(create_receptor_addresses),
-        migrations.RunPython(link_to_receptor_addresses),
-        migrations.RemoveField(
-            model_name='instance',
-            name='peers_from_control_nodes',
-        ),
-        migrations.RemoveField(
-            model_name='instance',
-            name='listener_port',
-        ),
-        migrations.RemoveField(
-            model_name='instancelink',
-            name='target_old',
-        ),
-        migrations.AlterField(
-            model_name='instance',
-            name='peers',
-            field=models.ManyToManyField(related_name='peers_from', through='main.InstanceLink', to='main.receptoraddress'),
-        ),
-        migrations.AlterField(
-            model_name='instancelink',
-            name='target',
-            field=models.ForeignKey(
-                help_text='The target receptor address of this peer link.', on_delete=django.db.models.deletion.CASCADE, to='main.receptoraddress'
-            ),
-        ),
-        migrations.AddConstraint(
-            model_name='instancelink',
-            constraint=models.UniqueConstraint(
-                fields=('source', 'target'), name='unique_source_target', violation_error_message='Field source and target must be unique together.'
-            ),
-        ),
-    ]

awx/main/migrations/_inventory_source_vars.py

@@ -76,7 +76,7 @@ class azure_rm(PluginFileInjector):
         user_filters = []
         old_filterables = [
             ('resource_groups', 'resource_group'),
-            ('tags', 'tags'),
+            ('tags', 'tags')
             # locations / location would be an entry
             # but this would conflict with source_regions
         ]

awx/main/models/__init__.py

@@ -6,15 +6,12 @@ from django.conf import settings  # noqa
 from django.db import connection
 from django.db.models.signals import pre_delete  # noqa
 
-from ansible_base.lib.utils.models import prevent_search
-
 # AWX
-from awx.main.models.base import BaseModel, PrimordialModel, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES  # noqa
+from awx.main.models.base import BaseModel, PrimordialModel, prevent_search, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES  # noqa
 from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutMaxBytesExceeded  # noqa
 from awx.main.models.organization import Organization, Profile, Team, UserSessionMembership  # noqa
 from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env  # noqa
 from awx.main.models.projects import Project, ProjectUpdate  # noqa
-from awx.main.models.receptor_address import ReceptorAddress  # noqa
 from awx.main.models.inventory import (  # noqa
     CustomInventoryScript,
     Group,

awx/main/models/activity_stream.py

@@ -77,7 +77,6 @@ class ActivityStream(models.Model):
     notification_template = models.ManyToManyField("NotificationTemplate", blank=True)
     notification = models.ManyToManyField("Notification", blank=True)
     label = models.ManyToManyField("Label", blank=True)
-    receptor_address = models.ManyToManyField("ReceptorAddress", blank=True)
     role = models.ManyToManyField("Role", blank=True)
     instance = models.ManyToManyField("Instance", blank=True)
     instance_group = models.ManyToManyField("InstanceGroup", blank=True)

awx/main/models/ad_hoc_commands.py

@@ -12,11 +12,9 @@ from django.utils.text import Truncator
 from django.utils.translation import gettext_lazy as _
 from django.core.exceptions import ValidationError
 
-from ansible_base.lib.utils.models import prevent_search
-
 # AWX
 from awx.api.versioning import reverse
-from awx.main.models.base import AD_HOC_JOB_TYPE_CHOICES, VERBOSITY_CHOICES, VarsDictProperty
+from awx.main.models.base import prevent_search, AD_HOC_JOB_TYPE_CHOICES, VERBOSITY_CHOICES, VarsDictProperty
 from awx.main.models.events import AdHocCommandEvent, UnpartitionedAdHocCommandEvent
 from awx.main.models.unified_jobs import UnifiedJob
 from awx.main.models.notifications import JobNotificationMixin, NotificationTemplate

awx/main/models/base.py

@@ -15,6 +15,7 @@ from awx.main.utils import encrypt_field, parse_yaml_or_json
 from awx.main.constants import CLOUD_PROVIDERS
 
 __all__ = [
+    'prevent_search',
     'VarsDictProperty',
     'BaseModel',
     'CreatedModifiedModel',
@@ -383,6 +384,23 @@ class NotificationFieldsModel(BaseModel):
     notification_templates_started = models.ManyToManyField("NotificationTemplate", blank=True, related_name='%(class)s_notification_templates_for_started')
 
 
+def prevent_search(relation):
+    """
+    Used to mark a model field or relation as "restricted from filtering"
+    e.g.,
+
+    class AuthToken(BaseModel):
+        user = prevent_search(models.ForeignKey(...))
+        sensitive_data = prevent_search(models.CharField(...))
+
+    The flag set by this function is used by
+    `awx.api.filters.FieldLookupBackend` to block fields and relations that
+    should not be searchable/filterable via search query params
+    """
+    setattr(relation, '__prevent_search__', True)
+    return relation
+
+
 def accepts_json(relation):
     """
     Used to mark a model field as allowing JSON e.g,. JobTemplate.extra_vars

awx/main/models/credential/__init__.py

@@ -953,25 +953,6 @@ ManagedCredentialType(
     },
 )
 
-ManagedCredentialType(
-    namespace='bitbucket_dc_token',
-    kind='token',
-    name=gettext_noop('Bitbucket Data Center HTTP Access Token'),
-    managed=True,
-    inputs={
-        'fields': [
-            {
-                'id': 'token',
-                'label': gettext_noop('Token'),
-                'type': 'string',
-                'secret': True,
-                'help_text': gettext_noop('This token needs to come from your user settings in Bitbucket'),
-            }
-        ],
-        'required': ['token'],
-    },
-)
-
 ManagedCredentialType(
     namespace='insights',
     kind='insights',
@@ -1216,26 +1197,6 @@ ManagedCredentialType(
     },
 )
 
-ManagedCredentialType(
-    namespace='terraform',
-    kind='cloud',
-    name=gettext_noop('Terraform backend configuration'),
-    managed=True,
-    inputs={
-        'fields': [
-            {
-                'id': 'configuration',
-                'label': gettext_noop('Backend configuration'),
-                'type': 'string',
-                'secret': True,
-                'multiline': True,
-                'help_text': gettext_noop('Terraform backend config as Hashicorp configuration language.'),
-            },
-        ],
-        'required': ['configuration'],
-    },
-)
-
 
 class CredentialInputSource(PrimordialModel):
     class Meta:

awx/main/models/credential/injectors.py

@@ -122,11 +122,3 @@ def kubernetes_bearer_token(cred, env, private_data_dir):
         env['K8S_AUTH_SSL_CA_CERT'] = to_container_path(path, private_data_dir)
     else:
         env['K8S_AUTH_VERIFY_SSL'] = 'False'
-
-
-def terraform(cred, env, private_data_dir):
-    handle, path = tempfile.mkstemp(dir=os.path.join(private_data_dir, 'env'))
-    with os.fdopen(handle, 'w') as f:
-        os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
-        f.write(cred.get_input('configuration'))
-    env['TF_BACKEND_CONFIG_FILE'] = to_container_path(path, private_data_dir)

awx/main/models/events.py

@@ -124,6 +124,8 @@ class BasePlaybookEvent(CreatedModifiedModel):
         'parent_uuid',
         'start_line',
         'end_line',
+        'host_id',
+        'host_name',
         'verbosity',
     ]
     WRAPUP_EVENT = 'playbook_on_stats'
@@ -471,7 +473,7 @@ class JobEvent(BasePlaybookEvent):
     An event/message logged from the callback when running a job.
     """
 
-    VALID_KEYS = BasePlaybookEvent.VALID_KEYS + ['job_id', 'workflow_job_id', 'job_created', 'host_id', 'host_name']
+    VALID_KEYS = BasePlaybookEvent.VALID_KEYS + ['job_id', 'workflow_job_id', 'job_created']
     JOB_REFERENCE = 'job_id'
 
     objects = DeferJobCreatedManager()

awx/main/models/ha.py

@@ -5,7 +5,7 @@ from decimal import Decimal
 import logging
 import os
 
-from django.core.validators import MinValueValidator
+from django.core.validators import MinValueValidator, MaxValueValidator
 from django.db import models, connection
 from django.db.models.signals import post_save, post_delete
 from django.dispatch import receiver
@@ -17,8 +17,6 @@ from django.db.models import Sum, Q
 import redis
 from solo.models import SingletonModel
 
-from ansible_base.lib.utils.models import prevent_search
-
 # AWX
 from awx import __version__ as awx_application_version
 from awx.main.utils import is_testing
@@ -26,7 +24,7 @@ from awx.api.versioning import reverse
 from awx.main.fields import ImplicitRoleField
 from awx.main.managers import InstanceManager, UUID_DEFAULT
 from awx.main.constants import JOB_FOLDER_PREFIX
-from awx.main.models.base import BaseModel, HasEditsMixin
+from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search
 from awx.main.models.rbac import (
     ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
     ROLE_SINGLETON_SYSTEM_AUDITOR,
@@ -34,7 +32,6 @@ from awx.main.models.rbac import (
 from awx.main.models.unified_jobs import UnifiedJob
 from awx.main.utils.common import get_corrected_cpu, get_cpu_effective_capacity, get_corrected_memory, get_mem_effective_capacity
 from awx.main.models.mixins import RelatedJobsMixin, ResourceMixin
-from awx.main.models.receptor_address import ReceptorAddress
 
 # ansible-runner
 from ansible_runner.utils.capacity import get_cpu_count, get_mem_in_bytes
@@ -65,19 +62,8 @@ class HasPolicyEditsMixin(HasEditsMixin):
 
 
 class InstanceLink(BaseModel):
-    class Meta:
-        ordering = ("id",)
-        # add constraint for source and target to be unique together
-        constraints = [
-            models.UniqueConstraint(
-                fields=["source", "target"],
-                name="unique_source_target",
-                violation_error_message=_("Field source and target must be unique together."),
-            )
-        ]
-
-    source = models.ForeignKey('Instance', on_delete=models.CASCADE, help_text=_("The source instance of this peer link."))
-    target = models.ForeignKey('ReceptorAddress', on_delete=models.CASCADE, help_text=_("The target receptor address of this peer link."))
+    source = models.ForeignKey('Instance', on_delete=models.CASCADE, related_name='+')
+    target = models.ForeignKey('Instance', on_delete=models.CASCADE, related_name='reverse_peers')
 
     class States(models.TextChoices):
         ADDING = 'adding', _('Adding')
@@ -88,6 +74,11 @@ class InstanceLink(BaseModel):
         choices=States.choices, default=States.ADDING, max_length=16, help_text=_("Indicates the current life cycle stage of this peer link.")
     )
 
+    class Meta:
+        unique_together = ('source', 'target')
+        ordering = ("id",)
+        constraints = [models.CheckConstraint(check=~models.Q(source=models.F('target')), name='source_and_target_can_not_be_equal')]
+
 
 class Instance(HasPolicyEditsMixin, BaseModel):
     """A model representing an AWX instance running against this database."""
@@ -117,7 +108,6 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         default="",
         max_length=50,
     )
-
     # Auto-fields, implementation is different from BaseModel
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
@@ -193,9 +183,16 @@ class Instance(HasPolicyEditsMixin, BaseModel):
     node_state = models.CharField(
         choices=States.choices, default=States.READY, max_length=16, help_text=_("Indicates the current life cycle stage of this instance.")
     )
+    listener_port = models.PositiveIntegerField(
+        blank=True,
+        null=True,
+        default=None,
+        validators=[MinValueValidator(1024), MaxValueValidator(65535)],
+        help_text=_("Port that Receptor will listen for incoming connections on."),
+    )
 
-    managed = models.BooleanField(help_text=_("If True, this instance is managed by the control plane."), default=False, editable=False)
-    peers = models.ManyToManyField('ReceptorAddress', through=InstanceLink, through_fields=('source', 'target'), related_name='peers_from')
+    peers = models.ManyToManyField('self', symmetrical=False, through=InstanceLink, through_fields=('source', 'target'), related_name='peers_from')
+    peers_from_control_nodes = models.BooleanField(default=False, help_text=_("If True, control plane cluster nodes should automatically peer to it."))
 
     POLICY_FIELDS = frozenset(('managed_by_policy', 'hostname', 'capacity_adjustment'))
 
@@ -242,26 +239,6 @@ class Instance(HasPolicyEditsMixin, BaseModel):
             return True
         return self.health_check_started > self.last_health_check
 
-    @property
-    def canonical_address(self):
-        return self.receptor_addresses.filter(canonical=True).first()
-
-    @property
-    def canonical_address_port(self):
-        # note: don't create a different query for receptor addresses, as this is prefetched on the View for optimization
-        for addr in self.receptor_addresses.all():
-            if addr.canonical:
-                return addr.port
-        return None
-
-    @property
-    def canonical_address_peers_from_control_nodes(self):
-        # note: don't create a different query for receptor addresses, as this is prefetched on the View for optimization
-        for addr in self.receptor_addresses.all():
-            if addr.canonical:
-                return addr.peers_from_control_nodes
-        return False
-
     def get_cleanup_task_kwargs(self, **kwargs):
         """
         Produce options to use for the command: ansible-runner worker cleanup
@@ -522,35 +499,6 @@ def schedule_write_receptor_config(broadcast=True):
             write_receptor_config()  # just run locally
 
 
-@receiver(post_save, sender=ReceptorAddress)
-def receptor_address_saved(sender, instance, **kwargs):
-    from awx.main.signals import disable_activity_stream
-
-    address = instance
-
-    control_instances = set(Instance.objects.filter(node_type__in=[Instance.Types.CONTROL, Instance.Types.HYBRID]))
-    if address.peers_from_control_nodes:
-        # if control_instances is not a subset of current peers of address, then
-        # that means we need to add some InstanceLinks
-        if not control_instances <= set(address.peers_from.all()):
-            with disable_activity_stream():
-                for control_instance in control_instances:
-                    InstanceLink.objects.update_or_create(source=control_instance, target=address)
-                schedule_write_receptor_config()
-    else:
-        if address.peers_from.exists():
-            with disable_activity_stream():
-                address.peers_from.remove(*control_instances)
-                schedule_write_receptor_config()
-
-
-@receiver(post_delete, sender=ReceptorAddress)
-def receptor_address_deleted(sender, instance, **kwargs):
-    address = instance
-    if address.peers_from_control_nodes:
-        schedule_write_receptor_config()
-
-
 @receiver(post_save, sender=Instance)
 def on_instance_saved(sender, instance, created=False, raw=False, **kwargs):
     '''
@@ -561,14 +509,11 @@ def on_instance_saved(sender, instance, created=False, raw=False, **kwargs):
     2. a node changes its value of peers_from_control_nodes
     3. a new control node comes online and has instances to peer to
     '''
-    from awx.main.signals import disable_activity_stream
-
     if created and settings.IS_K8S and instance.node_type in [Instance.Types.CONTROL, Instance.Types.HYBRID]:
-        peers_addresses = ReceptorAddress.objects.filter(peers_from_control_nodes=True)
-        if peers_addresses.exists():
-            with disable_activity_stream():
-                instance.peers.add(*peers_addresses)
-                schedule_write_receptor_config(broadcast=False)
+        inst = Instance.objects.filter(peers_from_control_nodes=True)
+        if set(instance.peers.all()) != set(inst):
+            instance.peers.set(inst)
+            schedule_write_receptor_config(broadcast=False)
 
     if settings.IS_K8S and instance.node_type in [Instance.Types.HOP, Instance.Types.EXECUTION]:
         if instance.node_state == Instance.States.DEPROVISIONING:
@@ -577,6 +522,16 @@ def on_instance_saved(sender, instance, created=False, raw=False, **kwargs):
             # wait for jobs on the node to complete, then delete the
             # node and kick off write_receptor_config
             connection.on_commit(lambda: remove_deprovisioned_node.apply_async([instance.hostname]))
+        else:
+            control_instances = set(Instance.objects.filter(node_type__in=[Instance.Types.CONTROL, Instance.Types.HYBRID]))
+            if instance.peers_from_control_nodes:
+                if (control_instances & set(instance.peers_from.all())) != set(control_instances):
+                    instance.peers_from.add(*control_instances)
+                    schedule_write_receptor_config()  # keep method separate to make pytest mocking easier
+            else:
+                if set(control_instances) & set(instance.peers_from.all()):
+                    instance.peers_from.remove(*control_instances)
+                    schedule_write_receptor_config()
 
     if created or instance.has_policy_changes():
         schedule_policy_task()
@@ -591,6 +546,8 @@ def on_instance_group_deleted(sender, instance, using, **kwargs):
 @receiver(post_delete, sender=Instance)
 def on_instance_deleted(sender, instance, using, **kwargs):
     schedule_policy_task()
+    if settings.IS_K8S and instance.node_type in (Instance.Types.EXECUTION, Instance.Types.HOP) and instance.peers_from_control_nodes:
+        schedule_write_receptor_config()
 
 
 class UnifiedJobTemplateInstanceGroupMembership(models.Model):

awx/main/models/inventory.py

@@ -25,8 +25,6 @@ from django.db.models import Q
 # REST Framework
 from rest_framework.exceptions import ParseError
 
-from ansible_base.lib.utils.models import prevent_search
-
 # AWX
 from awx.api.versioning import reverse
 from awx.main.constants import CLOUD_PROVIDERS
@@ -37,7 +35,7 @@ from awx.main.fields import (
     OrderedManyToManyField,
 )
 from awx.main.managers import HostManager, HostMetricActiveManager
-from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, accepts_json
+from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, prevent_search, accepts_json
 from awx.main.models.events import InventoryUpdateEvent, UnpartitionedInventoryUpdateEvent
 from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate
 from awx.main.models.mixins import (

awx/main/models/jobs.py

@@ -20,14 +20,13 @@ from django.core.exceptions import FieldDoesNotExist
 # REST Framework
 from rest_framework.exceptions import ParseError
 
-from ansible_base.lib.utils.models import prevent_search
-
 # AWX
 from awx.api.versioning import reverse
 from awx.main.constants import HOST_FACTS_FIELDS
 from awx.main.models.base import (
     BaseModel,
     CreatedModifiedModel,
+    prevent_search,
     accepts_json,
     JOB_TYPE_CHOICES,
     NEW_JOB_TYPE_CHOICES,

awx/main/models/mixins.py

@@ -9,6 +9,7 @@ import requests
 # Django
 from django.apps import apps
 from django.conf import settings
+from django.contrib.auth.models import User  # noqa
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError
 from django.db import models
@@ -16,9 +17,8 @@ from django.db.models.query import QuerySet
 from django.utils.crypto import get_random_string
 from django.utils.translation import gettext_lazy as _
 
-from ansible_base.lib.utils.models import prevent_search
-
 # AWX
+from awx.main.models.base import prevent_search
 from awx.main.models.rbac import Role, RoleAncestorEntry
 from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser, polymorphic
 from awx.main.utils.execution_environments import get_default_execution_environment
@@ -64,12 +64,13 @@ class ResourceMixin(models.Model):
 
     @staticmethod
     def _accessible_pk_qs(cls, accessor, role_field, content_types=None):
-        if accessor._meta.model_name == 'user':
+        if type(accessor) == User:
             ancestor_roles = accessor.roles.all()
         elif type(accessor) == Role:
             ancestor_roles = [accessor]
         else:
-            raise RuntimeError(f'Role filters only valid for users and ancestor role, received {accessor}')
+            accessor_type = ContentType.objects.get_for_model(accessor)
+            ancestor_roles = Role.objects.filter(content_type__pk=accessor_type.id, object_id=accessor.id)
 
         if content_types is None:
             ct_kwarg = dict(content_type_id=ContentType.objects.get_for_model(cls).id)
@@ -527,6 +528,7 @@ class CustomVirtualEnvMixin(models.Model):
 
 
 class RelatedJobsMixin(object):
+
     """
     This method is intended to be overwritten.
     Called by get_active_jobs()
@@ -561,7 +563,6 @@ class WebhookTemplateMixin(models.Model):
     SERVICES = [
         ('github', "GitHub"),
         ('gitlab', "GitLab"),
-        ('bitbucket_dc', "BitBucket DataCenter"),
     ]
 
     webhook_service = models.CharField(max_length=16, choices=SERVICES, blank=True, help_text=_('Service that webhook requests will be accepted from'))
@@ -622,7 +623,6 @@ class WebhookMixin(models.Model):
         service_header = {
             'github': ('Authorization', 'token {}'),
             'gitlab': ('PRIVATE-TOKEN', '{}'),
-            'bitbucket_dc': ('Authorization', 'Bearer {}'),
         }
         service_statuses = {
             'github': {
@@ -640,14 +640,6 @@ class WebhookMixin(models.Model):
                 'error': 'failed',  # GitLab doesn't have an 'error' status distinct from 'failed' :(
                 'canceled': 'canceled',
             },
-            'bitbucket_dc': {
-                'pending': 'INPROGRESS',  # Bitbucket DC doesn't have any other statuses distinct from INPROGRESS, SUCCESSFUL, FAILED :(
-                'running': 'INPROGRESS',
-                'successful': 'SUCCESSFUL',
-                'failed': 'FAILED',
-                'error': 'FAILED',
-                'canceled': 'FAILED',
-            },
         }
 
         statuses = service_statuses[self.webhook_service]
@@ -656,18 +648,11 @@ class WebhookMixin(models.Model):
             return
         try:
             license_type = get_licenser().validate().get('license_type')
-            if self.webhook_service == 'bitbucket_dc':
-                data = {
-                    'state': statuses[status],
-                    'key': 'ansible/awx' if license_type == 'open' else 'ansible/tower',
-                    'url': self.get_ui_url(),
-                }
-            else:
-                data = {
-                    'state': statuses[status],
-                    'context': 'ansible/awx' if license_type == 'open' else 'ansible/tower',
-                    'target_url': self.get_ui_url(),
-                }
+            data = {
+                'state': statuses[status],
+                'context': 'ansible/awx' if license_type == 'open' else 'ansible/tower',
+                'target_url': self.get_ui_url(),
+            }
             k, v = service_header[self.webhook_service]
             headers = {k: v.format(self.webhook_credential.get_input('token')), 'Content-Type': 'application/json'}
             response = requests.post(status_api, data=json.dumps(data), headers=headers, timeout=30)

awx/main/models/notifications.py

@@ -5,7 +5,6 @@ from copy import deepcopy
 import datetime
 import logging
 import json
-import traceback
 
 from django.db import models
 from django.conf import settings
@@ -16,11 +15,9 @@ from django.utils.encoding import smart_str, force_str
 from jinja2 import sandbox, ChainableUndefined
 from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
 
-from ansible_base.lib.utils.models import prevent_search
-
 # AWX
 from awx.api.versioning import reverse
-from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel
+from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel, prevent_search
 from awx.main.utils import encrypt_field, decrypt_field, set_environ
 from awx.main.notifications.email_backend import CustomEmailBackend
 from awx.main.notifications.slack_backend import SlackBackend
@@ -485,29 +482,14 @@ class JobNotificationMixin(object):
         if msg_template:
             try:
                 msg = env.from_string(msg_template).render(**context)
-            except (TemplateSyntaxError, UndefinedError, SecurityError) as e:
-                msg = '\r\n'.join([e.message, ''.join(traceback.format_exception(None, e, e.__traceback__).replace('\n', '\r\n'))])
+            except (TemplateSyntaxError, UndefinedError, SecurityError):
+                msg = ''
 
         if body_template:
             try:
                 body = env.from_string(body_template).render(**context)
-            except (TemplateSyntaxError, UndefinedError, SecurityError) as e:
-                body = '\r\n'.join([e.message, ''.join(traceback.format_exception(None, e, e.__traceback__).replace('\n', '\r\n'))])
-
-        # https://datatracker.ietf.org/doc/html/rfc2822#section-2.2
-        # Body should have at least 2 CRLF, some clients will interpret
-        # the email incorrectly with blank body.  So we will check that
-
-        if len(body.strip().splitlines()) <= 2:
-            # blank body
-            body = '\r\n'.join(
-                [
-                    "The template rendering return a blank body.",
-                    "Please check the template.",
-                    "Refer to https://github.com/ansible/awx/issues/13983",
-                    "for further information.",
-                ]
-            )
+            except (TemplateSyntaxError, UndefinedError, SecurityError):
+                body = ''
 
         return (msg, body)
 

awx/main/models/rbac.py

@@ -15,10 +15,12 @@ from django.utils.translation import gettext_lazy as _
 
 # AWX
 from awx.api.versioning import reverse
+from django.contrib.auth.models import User  # noqa
 
 __all__ = [
     'Role',
     'batch_role_ancestor_rebuilding',
+    'get_roles_on_resource',
     'ROLE_SINGLETON_SYSTEM_ADMINISTRATOR',
     'ROLE_SINGLETON_SYSTEM_AUDITOR',
     'role_summary_fields_generator',
@@ -168,10 +170,16 @@ class Role(models.Model):
         return reverse('api:role_detail', kwargs={'pk': self.pk}, request=request)
 
     def __contains__(self, accessor):
-        if accessor._meta.model_name == 'user':
+        if type(accessor) == User:
             return self.ancestors.filter(members=accessor).exists()
+        elif accessor.__class__.__name__ == 'Team':
+            return self.ancestors.filter(pk=accessor.member_role.id).exists()
+        elif type(accessor) == Role:
+            return self.ancestors.filter(pk=accessor.pk).exists()
         else:
-            raise RuntimeError(f'Role evaluations only valid for users, received {accessor}')
+            accessor_type = ContentType.objects.get_for_model(accessor)
+            roles = Role.objects.filter(content_type__pk=accessor_type.id, object_id=accessor.id)
+            return self.ancestors.filter(pk__in=roles).exists()
 
     @property
     def name(self):
@@ -452,6 +460,31 @@ class RoleAncestorEntry(models.Model):
     object_id = models.PositiveIntegerField(null=False)
 
 
+def get_roles_on_resource(resource, accessor):
+    """
+    Returns a string list of the roles a accessor has for a given resource.
+    An accessor can be either a User, Role, or an arbitrary resource that
+    contains one or more Roles associated with it.
+    """
+
+    if type(accessor) == User:
+        roles = accessor.roles.all()
+    elif type(accessor) == Role:
+        roles = [accessor]
+    else:
+        accessor_type = ContentType.objects.get_for_model(accessor)
+        roles = Role.objects.filter(content_type__pk=accessor_type.id, object_id=accessor.id)
+
+    return [
+        role_field
+        for role_field in RoleAncestorEntry.objects.filter(
+            ancestor__in=roles, content_type_id=ContentType.objects.get_for_model(resource).id, object_id=resource.id
+        )
+        .values_list('role_field', flat=True)
+        .distinct()
+    ]
+
+
 def role_summary_fields_generator(content_object, role_field):
     global role_descriptions
     global role_names

awx/main/models/receptor_address.py

@@ -1,67 +0,0 @@
-from django.db import models
-from django.core.validators import MinValueValidator, MaxValueValidator
-from django.utils.translation import gettext_lazy as _
-from awx.api.versioning import reverse
-
-
-class Protocols(models.TextChoices):
-    TCP = 'tcp', 'TCP'
-    WS = 'ws', 'WS'
-    WSS = 'wss', 'WSS'
-
-
-class ReceptorAddress(models.Model):
-    class Meta:
-        app_label = 'main'
-        constraints = [
-            models.UniqueConstraint(
-                fields=["address"],
-                name="unique_receptor_address",
-                violation_error_message=_("Receptor address must be unique."),
-            )
-        ]
-
-    address = models.CharField(help_text=_("Routable address for this instance."), max_length=255)
-    port = models.IntegerField(help_text=_("Port for the address."), default=27199, validators=[MinValueValidator(0), MaxValueValidator(65535)])
-    websocket_path = models.CharField(help_text=_("Websocket path."), max_length=255, default="", blank=True)
-    protocol = models.CharField(
-        help_text=_("Protocol to use for the Receptor listener, 'tcp', 'wss', or 'ws'."), max_length=10, default=Protocols.TCP, choices=Protocols.choices
-    )
-    is_internal = models.BooleanField(help_text=_("If True, only routable within the Kubernetes cluster."), default=False)
-    canonical = models.BooleanField(help_text=_("If True, this address is the canonical address for the instance."), default=False)
-    peers_from_control_nodes = models.BooleanField(help_text=_("If True, control plane cluster nodes should automatically peer to it."), default=False)
-    instance = models.ForeignKey(
-        'Instance',
-        related_name='receptor_addresses',
-        on_delete=models.CASCADE,
-        null=False,
-    )
-
-    def __str__(self):
-        return self.get_full_address()
-
-    def get_full_address(self):
-        scheme = ""
-        path = ""
-        port = ""
-        if self.protocol == "ws":
-            scheme = "wss://"
-
-        if self.protocol == "ws" and self.websocket_path:
-            path = f"/{self.websocket_path}"
-
-        if self.port:
-            port = f":{self.port}"
-
-        return f"{scheme}{self.address}{port}{path}"
-
-    def get_peer_type(self):
-        if self.protocol == 'tcp':
-            return 'tcp-peer'
-        elif self.protocol in ['ws', 'wss']:
-            return 'ws-peer'
-        else:
-            return None
-
-    def get_absolute_url(self, request=None):
-        return reverse('api:receptor_address_detail', kwargs={'pk': self.pk}, request=request)

awx/main/models/unified_jobs.py

@@ -30,10 +30,8 @@ from rest_framework.exceptions import ParseError
 # Django-Polymorphic
 from polymorphic.models import PolymorphicModel
 
-from ansible_base.lib.utils.models import prevent_search, get_type_for_model
-
 # AWX
-from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel, NotificationFieldsModel
+from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel, NotificationFieldsModel, prevent_search
 from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.control import Control as ControlDispatcher
 from awx.main.registrar import activity_stream_registrar
@@ -44,6 +42,7 @@ from awx.main.utils.common import (
     _inventory_updates,
     copy_model_by_class,
     copy_m2m_relationships,
+    get_type_for_model,
     parse_yaml_or_json,
     getattr_dne,
     ScheduleDependencyManager,

awx/main/models/workflow.py

@@ -23,11 +23,9 @@ from crum import get_current_user
 from jinja2 import sandbox
 from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
 
-from ansible_base.lib.utils.models import prevent_search
-
 # AWX
 from awx.api.versioning import reverse
-from awx.main.models import accepts_json, UnifiedJobTemplate, UnifiedJob
+from awx.main.models import prevent_search, accepts_json, UnifiedJobTemplate, UnifiedJob
 from awx.main.models.notifications import NotificationTemplate, JobNotificationMixin
 from awx.main.models.base import CreatedModifiedModel, VarsDictProperty
 from awx.main.models.rbac import ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR

awx/main/notifications/custom_notification_base.py

@@ -1,6 +1,5 @@
 # Copyright (c) 2019 Ansible, Inc.
 # All Rights Reserved.
-# -*-coding:utf-8-*-
 
 
 class CustomNotificationBase(object):

awx/main/notifications/twilio_backend.py

@@ -39,15 +39,11 @@ class TwilioBackend(AWXBaseEmailBackend, CustomNotificationBase):
             logger.error(smart_str(_("Exception connecting to Twilio: {}").format(e)))
 
         for m in messages:
-            failed = False
-            for dest in m.to:
-                try:
-                    logger.debug(smart_str(_("FROM: {} / TO: {}").format(m.from_email, dest)))
-                    connection.messages.create(to=dest, from_=m.from_email, body=m.subject)
-                    sent_messages += 1
-                except Exception as e:
-                    logger.error(smart_str(_("Exception sending messages: {}").format(e)))
-                    failed = True
-            if not self.fail_silently and failed:
-                raise
+            try:
+                connection.messages.create(to=m.to, from_=m.from_email, body=m.subject)
+                sent_messages += 1
+            except Exception as e:
+                logger.error(smart_str(_("Exception sending messages: {}").format(e)))
+                if not self.fail_silently:
+                    raise
         return sent_messages

awx/main/routing.py

@@ -4,15 +4,13 @@ import logging
 from django.conf import settings
 from django.urls import re_path
 
+from channels.auth import AuthMiddlewareStack
 from channels.routing import ProtocolTypeRouter, URLRouter
 
-from ansible_base.lib.channels.middleware import DrfAuthMiddlewareStack
-
 from . import consumers
 
 
 logger = logging.getLogger('awx.main.routing')
-_application = None
 
 
 class AWXProtocolTypeRouter(ProtocolTypeRouter):
@@ -28,91 +26,13 @@ class AWXProtocolTypeRouter(ProtocolTypeRouter):
         super().__init__(*args, **kwargs)
 
 
-class MultipleURLRouterAdapter:
-    """
-    Django channels doesn't nicely support Auth_1(urls_1), Auth_2(urls_2), ..., Auth_n(urls_n)
-    This class allows assocating a websocket url with an auth
-    Ordering matters. The first matching url will be used.
-    """
-
-    def __init__(self, *auths):
-        self._auths = [a for a in auths]
-
-    async def __call__(self, scope, receive, send):
-        """
-        Loop through the list of passed in URLRouter's (they may or may not be wrapped by auth).
-        We know we have exhausted the list of URLRouter patterns when we get a
-        ValueError('No route found for path %s'). When that happens, move onto the next
-        URLRouter.
-        If the final URLRouter raises an error, re-raise it in the end.
-
-        We know that we found a match when no error is raised, end the loop.
-        """
-        last_index = len(self._auths) - 1
-        for i, auth in enumerate(self._auths):
-            try:
-                return await auth.__call__(scope, receive, send)
-            except ValueError as e:
-                if str(e).startswith('No route found for path'):
-                    # Only surface the error if on the last URLRouter
-                    if i == last_index:
-                        raise
-
-
 websocket_urlpatterns = [
-    re_path(r'api/websocket/$', consumers.EventConsumer.as_asgi()),
     re_path(r'websocket/$', consumers.EventConsumer.as_asgi()),
-]
-websocket_relay_urlpatterns = [
     re_path(r'websocket/relay/$', consumers.RelayConsumer.as_asgi()),
 ]
 
-
-def application_func(cls=AWXProtocolTypeRouter) -> ProtocolTypeRouter:
-    return cls(
-        {
-            'websocket': MultipleURLRouterAdapter(
-                URLRouter(websocket_relay_urlpatterns),
-                DrfAuthMiddlewareStack(URLRouter(websocket_urlpatterns)),
-            )
-        }
-    )
-
-
-def __getattr__(name: str) -> ProtocolTypeRouter:
-    """
-    Defer instantiating application.
-    For testing, we just need it to NOT run on import.
-
-    https://peps.python.org/pep-0562/#specification
-
-    Normally, someone would get application from this module via:
-        from awx.main.routing import application
-
-    and do something with the application:
-        application.do_something()
-
-    What does the callstack look like when the import runs?
-    ...
-        awx.main.routing.__getattribute__(...)                              # <-- we don't define this so NOOP as far as we are concerned
-        if '__getattr__' in awx.main.routing.__dict__:                      # <-- this triggers the function we are in
-            return awx.main.routing.__dict__.__getattr__("application")
-
-    Why isn't this function simply implemented as:
-        def __getattr__(name):
-            if not _application:
-              _application = application_func()
-            return _application
-
-    It could. I manually tested it and it passes test_routing.py.
-
-    But my understanding after reading the PEP-0562 specification link above is that
-    performance would be a bit worse due to the extra __getattribute__ calls when
-    we reference non-global variables.
-    """
-    if name == "application":
-        globs = globals()
-        if not globs['_application']:
-            globs['_application'] = application_func()
-        return globs['_application']
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+application = AWXProtocolTypeRouter(
+    {
+        'websocket': AuthMiddlewareStack(URLRouter(websocket_urlpatterns)),
+    }
+)

awx/main/scheduler/task_manager.py

@@ -17,8 +17,6 @@ from django.utils.timezone import now as tz_now
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 
-from ansible_base.lib.utils.models import get_type_for_model
-
 # AWX
 from awx.main.dispatch.reaper import reap_job
 from awx.main.models import (
@@ -36,6 +34,7 @@ from awx.main.models import (
 from awx.main.scheduler.dag_workflow import WorkflowDAG
 from awx.main.utils.pglock import advisory_lock
 from awx.main.utils import (
+    get_type_for_model,
     ScheduleTaskManager,
     ScheduleWorkflowManager,
 )
@@ -68,7 +67,7 @@ class TaskBase:
         # initialize each metric to 0 and force metric_has_changed to true. This
         # ensures each task manager metric will be overridden when pipe_execute
         # is called later.
-        self.subsystem_metrics = s_metrics.DispatcherMetrics(auto_pipe_execute=False)
+        self.subsystem_metrics = s_metrics.Metrics(auto_pipe_execute=False)
         self.start_time = time.time()
 
         # We want to avoid calling settings in loops, so cache these settings at init time
@@ -105,7 +104,7 @@ class TaskBase:
             try:
                 # increment task_manager_schedule_calls regardless if the other
                 # metrics are recorded
-                s_metrics.DispatcherMetrics(auto_pipe_execute=True).inc(f"{self.prefix}__schedule_calls", 1)
+                s_metrics.Metrics(auto_pipe_execute=True).inc(f"{self.prefix}__schedule_calls", 1)
                 # Only record metrics if the last time recording was more
                 # than SUBSYSTEM_METRICS_TASK_MANAGER_RECORD_INTERVAL ago.
                 # Prevents a short-duration task manager that runs directly after a

awx/main/tasks/callback.py

@@ -29,7 +29,7 @@ class RunnerCallback:
         self.safe_env = {}
         self.event_ct = 0
         self.model = model
-        self.update_attempts = int(getattr(settings, 'DISPATCHER_DB_DOWNTOWN_TOLLERANCE', settings.DISPATCHER_DB_DOWNTIME_TOLERANCE) / 5)
+        self.update_attempts = int(settings.DISPATCHER_DB_DOWNTIME_TOLERANCE / 5)
         self.wrapup_event_dispatched = False
         self.artifacts_processed = False
         self.extra_update_fields = {}
@@ -95,17 +95,17 @@ class RunnerCallback:
         if self.parent_workflow_job_id:
             event_data['workflow_job_id'] = self.parent_workflow_job_id
         event_data['job_created'] = self.job_created
-
-        host = event_data.get('event_data', {}).get('host', '').strip()
-        if host:
-            event_data['host_name'] = host
-            if host in self.host_map:
-                event_data['host_id'] = self.host_map[host]
-        else:
-            event_data['host_name'] = ''
-            event_data['host_id'] = ''
-        if event_data.get('event') == 'playbook_on_stats':
-            event_data['host_map'] = self.host_map
+        if self.host_map:
+            host = event_data.get('event_data', {}).get('host', '').strip()
+            if host:
+                event_data['host_name'] = host
+                if host in self.host_map:
+                    event_data['host_id'] = self.host_map[host]
+            else:
+                event_data['host_name'] = ''
+                event_data['host_id'] = ''
+            if event_data.get('event') == 'playbook_on_stats':
+                event_data['host_map'] = self.host_map
 
         if isinstance(self, RunnerCallbackForProjectUpdate):
             # need a better way to have this check.

awx/main/tasks/jobs.py

@@ -114,7 +114,7 @@ class BaseTask(object):
 
     def __init__(self):
         self.cleanup_paths = []
-        self.update_attempts = int(getattr(settings, 'DISPATCHER_DB_DOWNTOWN_TOLLERANCE', settings.DISPATCHER_DB_DOWNTIME_TOLERANCE) / 5)
+        self.update_attempts = int(settings.DISPATCHER_DB_DOWNTIME_TOLERANCE / 5)
         self.runner_callback = self.callback_class(model=self.model)
 
     def update_model(self, pk, _attempt=0, **updates):

awx/main/tasks/receptor.py

@@ -27,7 +27,7 @@ from awx.main.utils.common import (
 )
 from awx.main.constants import MAX_ISOLATED_PATH_COLON_DELIMITER
 from awx.main.tasks.signals import signal_state, signal_callback, SignalExit
-from awx.main.models import Instance, InstanceLink, UnifiedJob, ReceptorAddress
+from awx.main.models import Instance, InstanceLink, UnifiedJob
 from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.publish import task
 from awx.main.utils.pglock import advisory_lock
@@ -676,44 +676,36 @@ RECEPTOR_CONFIG_STARTER = (
 )
 
 
-def should_update_config(new_config):
+def should_update_config(instances):
     '''
     checks that the list of instances matches the list of
     tcp-peers in the config
     '''
-
     current_config = read_receptor_config()  # this gets receptor conf lock
+    current_peers = []
     for config_entry in current_config:
-        if config_entry not in new_config:
-            logger.warning(f"{config_entry} should not be in receptor config. Updating.")
-            return True
-    for config_entry in new_config:
-        if config_entry not in current_config:
-            logger.warning(f"{config_entry} missing from receptor config. Updating.")
-            return True
+        for key, value in config_entry.items():
+            if key.endswith('-peer'):
+                current_peers.append(value['address'])
+    intended_peers = [f"{i.hostname}:{i.listener_port}" for i in instances]
+    logger.debug(f"Peers current {current_peers} intended {intended_peers}")
+    if set(current_peers) == set(intended_peers):
+        return False  # config file is already update to date
 
-    return False
+    return True
 
 
 def generate_config_data():
     # returns two values
     #   receptor config - based on current database peers
     #   should_update   - If True, receptor_config differs from the receptor conf file on disk
-    addresses = ReceptorAddress.objects.filter(peers_from_control_nodes=True)
+    instances = Instance.objects.filter(node_type__in=(Instance.Types.EXECUTION, Instance.Types.HOP), peers_from_control_nodes=True)
 
     receptor_config = list(RECEPTOR_CONFIG_STARTER)
-    for address in addresses:
-        if address.get_peer_type():
-            peer = {
-                f'{address.get_peer_type()}': {
-                    'address': f'{address.get_full_address()}',
-                    'tls': 'tlsclient',
-                }
-            }
-            receptor_config.append(peer)
-        else:
-            logger.warning(f"Receptor address {address} has unsupported peer type, skipping.")
-    should_update = should_update_config(receptor_config)
+    for instance in instances:
+        peer = {'tcp-peer': {'address': f'{instance.hostname}:{instance.listener_port}', 'tls': 'tlsclient'}}
+        receptor_config.append(peer)
+    should_update = should_update_config(instances)
     return receptor_config, should_update
 
 
@@ -755,13 +747,14 @@ def write_receptor_config():
             with lock:
                 with open(__RECEPTOR_CONF, 'w') as file:
                     yaml.dump(receptor_config, file, default_flow_style=False)
+
             reload_receptor()
 
 
 @task(queue=get_task_queuename)
 def remove_deprovisioned_node(hostname):
     InstanceLink.objects.filter(source__hostname=hostname).update(link_state=InstanceLink.States.REMOVING)
-    InstanceLink.objects.filter(target__instance__hostname=hostname).update(link_state=InstanceLink.States.REMOVING)
+    InstanceLink.objects.filter(target__hostname=hostname).update(link_state=InstanceLink.States.REMOVING)
 
     node_jobs = UnifiedJob.objects.filter(
         execution_node=hostname,

awx/main/tasks/system.py

@@ -62,7 +62,7 @@ from awx.main.tasks.receptor import get_receptor_ctl, worker_info, worker_cleanu
 from awx.main.consumers import emit_channel_notification
 from awx.main import analytics
 from awx.conf import settings_registry
-from awx.main.analytics.subsystem_metrics import DispatcherMetrics
+from awx.main.analytics.subsystem_metrics import Metrics
 
 from rest_framework.exceptions import PermissionDenied
 
@@ -113,7 +113,7 @@ def dispatch_startup():
     cluster_node_heartbeat()
     reaper.startup_reaping()
     reaper.reap_waiting(grace_period=0)
-    m = DispatcherMetrics()
+    m = Metrics()
     m.reset_values()
 
 
@@ -495,7 +495,7 @@ def inspect_established_receptor_connections(mesh_status):
     update_links = []
     for link in all_links:
         if link.link_state != InstanceLink.States.REMOVING:
-            if link.target.instance.hostname in active_receptor_conns.get(link.source.hostname, {}):
+            if link.target.hostname in active_receptor_conns.get(link.source.hostname, {}):
                 if link.link_state is not InstanceLink.States.ESTABLISHED:
                     link.link_state = InstanceLink.States.ESTABLISHED
                     update_links.append(link)

awx/main/tests/functional/api/test_instance_peers.py

@@ -1,16 +1,19 @@
 import pytest
 import yaml
+import itertools
 from unittest import mock
 
+from django.db.utils import IntegrityError
+
 from awx.api.versioning import reverse
-from awx.main.models import Instance, ReceptorAddress
+from awx.main.models import Instance
 from awx.api.views.instance_install_bundle import generate_group_vars_all_yml
 
 
 def has_peer(group_vars, peer):
     peers = group_vars.get('receptor_peers', [])
     for p in peers:
-        if p['address'] == peer:
+        if f"{p['host']}:{p['port']}" == peer:
             return True
     return False
 
@@ -21,314 +24,119 @@ class TestPeers:
     def configure_settings(self, settings):
         settings.IS_K8S = True
 
-    @pytest.mark.parametrize('node_type', ['hop', 'execution'])
-    def test_peering_to_self(self, node_type, admin_user, patch):
+    @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
+    def test_prevent_peering_to_self(self, node_type):
         """
         cannot peer to self
         """
-        instance = Instance.objects.create(hostname='abc', node_type=node_type)
-        addr = ReceptorAddress.objects.create(instance=instance, address='abc', canonical=True)
-        resp = patch(
-            url=reverse('api:instance_detail', kwargs={'pk': instance.pk}),
-            data={"hostname": "abc", "node_type": node_type, "peers": [addr.id]},
-            user=admin_user,
-            expect=400,
-        )
-        assert 'Instance cannot peer to its own address.' in str(resp.data)
+        control_instance = Instance.objects.create(hostname='abc', node_type=node_type)
+        with pytest.raises(IntegrityError):
+            control_instance.peers.add(control_instance)
 
     @pytest.mark.parametrize('node_type', ['control', 'hybrid', 'hop', 'execution'])
     def test_creating_node(self, node_type, admin_user, post):
         """
         can only add hop and execution nodes via API
         """
-        resp = post(
+        post(
             url=reverse('api:instance_list'),
             data={"hostname": "abc", "node_type": node_type},
             user=admin_user,
             expect=400 if node_type in ['control', 'hybrid'] else 201,
         )
-        if resp.status_code == 400:
-            assert 'Can only create execution or hop nodes.' in str(resp.data)
 
     def test_changing_node_type(self, admin_user, patch):
         """
         cannot change node type
         """
         hop = Instance.objects.create(hostname='abc', node_type="hop")
-        resp = patch(
+        patch(
             url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
             data={"node_type": "execution"},
             user=admin_user,
             expect=400,
         )
-        assert 'Cannot change node type.' in str(resp.data)
 
-    @pytest.mark.parametrize(
-        'payload_port, payload_peers_from, initial_port, initial_peers_from',
-        [
-            (-1, -1, None, None),
-            (-1, -1, 27199, False),
-            (-1, -1, 27199, True),
-            (None, -1, None, None),
-            (None, False, None, None),
-            (-1, False, None, None),
-            (27199, True, 27199, True),
-            (27199, False, 27199, False),
-            (27199, -1, 27199, True),
-            (27199, -1, 27199, False),
-            (-1, True, 27199, True),
-            (-1, False, 27199, False),
-        ],
-    )
-    def test_no_op(self, payload_port, payload_peers_from, initial_port, initial_peers_from, admin_user, patch):
-        node = Instance.objects.create(hostname='abc', node_type='hop')
-        if initial_port is not None:
-            ReceptorAddress.objects.create(address=node.hostname, port=initial_port, canonical=True, peers_from_control_nodes=initial_peers_from, instance=node)
-
-            assert ReceptorAddress.objects.filter(instance=node).count() == 1
-        else:
-            assert ReceptorAddress.objects.filter(instance=node).count() == 0
-
-        data = {'enabled': True}  # Just to have something to post.
-        if payload_port != -1:
-            data['listener_port'] = payload_port
-        if payload_peers_from != -1:
-            data['peers_from_control_nodes'] = payload_peers_from
-
-        patch(
-            url=reverse('api:instance_detail', kwargs={'pk': node.pk}),
-            data=data,
+    @pytest.mark.parametrize('node_type', ['hop', 'execution'])
+    def test_listener_port_null(self, node_type, admin_user, post):
+        """
+        listener_port can be None
+        """
+        post(
+            url=reverse('api:instance_list'),
+            data={"hostname": "abc", "node_type": node_type, "listener_port": None},
             user=admin_user,
-            expect=200,
+            expect=201,
         )
 
-        assert ReceptorAddress.objects.filter(instance=node).count() == (0 if initial_port is None else 1)
-        if initial_port is not None:
-            ra = ReceptorAddress.objects.get(instance=node, canonical=True)
-            assert ra.port == initial_port
-            assert ra.peers_from_control_nodes == initial_peers_from
-
-    @pytest.mark.parametrize(
-        'payload_port, payload_peers_from',
-        [
-            (27199, True),
-            (27199, False),
-            (27199, -1),
-        ],
-    )
-    def test_creates_canonical_address(self, payload_port, payload_peers_from, admin_user, patch):
-        node = Instance.objects.create(hostname='abc', node_type='hop')
-        assert ReceptorAddress.objects.filter(instance=node).count() == 0
-
-        data = {'enabled': True}  # Just to have something to post.
-        if payload_port != -1:
-            data['listener_port'] = payload_port
-        if payload_peers_from != -1:
-            data['peers_from_control_nodes'] = payload_peers_from
-
-        patch(
-            url=reverse('api:instance_detail', kwargs={'pk': node.pk}),
-            data=data,
+    @pytest.mark.parametrize('node_type, allowed', [('control', False), ('hybrid', False), ('hop', True), ('execution', True)])
+    def test_peers_from_control_nodes_allowed(self, node_type, allowed, post, admin_user):
+        """
+        only hop and execution nodes can have peers_from_control_nodes set to True
+        """
+        post(
+            url=reverse('api:instance_list'),
+            data={"hostname": "abc", "peers_from_control_nodes": True, "node_type": node_type, "listener_port": 6789},
             user=admin_user,
-            expect=200,
+            expect=201 if allowed else 400,
         )
 
-        assert ReceptorAddress.objects.filter(instance=node).count() == 1
-        ra = ReceptorAddress.objects.get(instance=node, canonical=True)
-        assert ra.port == payload_port
-        assert ra.peers_from_control_nodes == (payload_peers_from if payload_peers_from != -1 else False)
-
-    @pytest.mark.parametrize(
-        'payload_port, payload_peers_from, initial_port, initial_peers_from',
-        [
-            (None, False, 27199, True),
-            (None, -1, 27199, True),
-            (None, False, 27199, False),
-            (None, -1, 27199, False),
-        ],
-    )
-    def test_deletes_canonical_address(self, payload_port, payload_peers_from, initial_port, initial_peers_from, admin_user, patch):
-        node = Instance.objects.create(hostname='abc', node_type='hop')
-        ReceptorAddress.objects.create(address=node.hostname, port=initial_port, canonical=True, peers_from_control_nodes=initial_peers_from, instance=node)
-
-        assert ReceptorAddress.objects.filter(instance=node).count() == 1
-
-        data = {'enabled': True}  # Just to have something to post.
-        if payload_port != -1:
-            data['listener_port'] = payload_port
-        if payload_peers_from != -1:
-            data['peers_from_control_nodes'] = payload_peers_from
-
-        patch(
-            url=reverse('api:instance_detail', kwargs={'pk': node.pk}),
-            data=data,
-            user=admin_user,
-            expect=200,
-        )
-
-        assert ReceptorAddress.objects.filter(instance=node).count() == 0
-
-    @pytest.mark.parametrize(
-        'payload_port, payload_peers_from, initial_port, initial_peers_from',
-        [
-            (27199, True, 27199, False),
-            (27199, False, 27199, True),
-            (-1, True, 27199, False),
-            (-1, False, 27199, True),
-        ],
-    )
-    def test_updates_canonical_address(self, payload_port, payload_peers_from, initial_port, initial_peers_from, admin_user, patch):
-        node = Instance.objects.create(hostname='abc', node_type='hop')
-        ReceptorAddress.objects.create(address=node.hostname, port=initial_port, canonical=True, peers_from_control_nodes=initial_peers_from, instance=node)
-
-        assert ReceptorAddress.objects.filter(instance=node).count() == 1
-
-        data = {'enabled': True}  # Just to have something to post.
-        if payload_port != -1:
-            data['listener_port'] = payload_port
-        if payload_peers_from != -1:
-            data['peers_from_control_nodes'] = payload_peers_from
-
-        patch(
-            url=reverse('api:instance_detail', kwargs={'pk': node.pk}),
-            data=data,
-            user=admin_user,
-            expect=200,
-        )
-
-        assert ReceptorAddress.objects.filter(instance=node).count() == 1
-        ra = ReceptorAddress.objects.get(instance=node, canonical=True)
-        assert ra.port == initial_port  # At the present time, changing ports is not allowed
-        assert ra.peers_from_control_nodes == payload_peers_from
-
-    @pytest.mark.parametrize(
-        'payload_port, payload_peers_from, initial_port, initial_peers_from, error_msg',
-        [
-            (-1, True, None, None, "Cannot enable peers_from_control_nodes"),
-            (None, True, None, None, "Cannot enable peers_from_control_nodes"),
-            (None, True, 21799, True, "Cannot enable peers_from_control_nodes"),
-            (None, True, 21799, False, "Cannot enable peers_from_control_nodes"),
-            (21800, -1, 21799, True, "Cannot change listener port"),
-            (21800, True, 21799, True, "Cannot change listener port"),
-            (21800, False, 21799, True, "Cannot change listener port"),
-            (21800, -1, 21799, False, "Cannot change listener port"),
-            (21800, True, 21799, False, "Cannot change listener port"),
-            (21800, False, 21799, False, "Cannot change listener port"),
-        ],
-    )
-    def test_canonical_address_validation_error(self, payload_port, payload_peers_from, initial_port, initial_peers_from, error_msg, admin_user, patch):
-        node = Instance.objects.create(hostname='abc', node_type='hop')
-        if initial_port is not None:
-            ReceptorAddress.objects.create(address=node.hostname, port=initial_port, canonical=True, peers_from_control_nodes=initial_peers_from, instance=node)
-
-            assert ReceptorAddress.objects.filter(instance=node).count() == 1
-        else:
-            assert ReceptorAddress.objects.filter(instance=node).count() == 0
-
-        data = {'enabled': True}  # Just to have something to post.
-        if payload_port != -1:
-            data['listener_port'] = payload_port
-        if payload_peers_from != -1:
-            data['peers_from_control_nodes'] = payload_peers_from
-
-        resp = patch(
-            url=reverse('api:instance_detail', kwargs={'pk': node.pk}),
-            data=data,
+    def test_listener_port_is_required(self, admin_user, post):
+        """
+        if adding instance to peers list, that instance must have listener_port set
+        """
+        Instance.objects.create(hostname='abc', node_type="hop", listener_port=None)
+        post(
+            url=reverse('api:instance_list'),
+            data={"hostname": "ex", "peers_from_control_nodes": False, "node_type": "execution", "listener_port": None, "peers": ["abc"]},
             user=admin_user,
             expect=400,
         )
 
-        assert error_msg in str(resp.data)
-
-    def test_changing_managed_listener_port(self, admin_user, patch):
+    def test_peers_from_control_nodes_listener_port_enabled(self, admin_user, post):
         """
-        if instance is managed, cannot change listener port at all
+        if peers_from_control_nodes is True, listener_port must an integer
+        Assert that all other combinations are allowed
         """
-        hop = Instance.objects.create(hostname='abc', node_type="hop", managed=True)
-        resp = patch(
-            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
-            data={"listener_port": 5678},
-            user=admin_user,
-            expect=400,  # cannot set port
-        )
-        assert 'Cannot change listener port for managed nodes.' in str(resp.data)
-        ReceptorAddress.objects.create(instance=hop, address='hop', port=27199, canonical=True)
-        resp = patch(
-            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
-            data={"listener_port": None},
-            user=admin_user,
-            expect=400,  # cannot unset port
-        )
-        assert 'Cannot change listener port for managed nodes.' in str(resp.data)
-
-    def test_bidirectional_peering(self, admin_user, patch):
-        """
-        cannot peer to node that is already to peered to it
-        if A -> B, then disallow B -> A
-        """
-        hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
-        hop1addr = ReceptorAddress.objects.create(instance=hop1, address='hop1', canonical=True)
-        hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
-        hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', canonical=True)
-        hop1.peers.add(hop2addr)
-        resp = patch(
-            url=reverse('api:instance_detail', kwargs={'pk': hop2.pk}),
-            data={"peers": [hop1addr.id]},
-            user=admin_user,
-            expect=400,
-        )
-        assert 'Instance hop1 is already peered to this instance.' in str(resp.data)
-
-    def test_multiple_peers_same_instance(self, admin_user, patch):
-        """
-        cannot peer to more than one address of the same instance
-        """
-        hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
-        hop1addr1 = ReceptorAddress.objects.create(instance=hop1, address='hop1', canonical=True)
-        hop1addr2 = ReceptorAddress.objects.create(instance=hop1, address='hop1alternate')
-        hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
-        resp = patch(
-            url=reverse('api:instance_detail', kwargs={'pk': hop2.pk}),
-            data={"peers": [hop1addr1.id, hop1addr2.id]},
-            user=admin_user,
-            expect=400,
-        )
-        assert 'Cannot peer to the same instance more than once.' in str(resp.data)
+        for index, item in enumerate(itertools.product(['hop', 'execution'], [True, False], [None, 6789])):
+            node_type, peers_from, listener_port = item
+            # only disallowed case is when peers_from is True and listener port is None
+            disallowed = peers_from and not listener_port
+            post(
+                url=reverse('api:instance_list'),
+                data={"hostname": f"abc{index}", "peers_from_control_nodes": peers_from, "node_type": node_type, "listener_port": listener_port},
+                user=admin_user,
+                expect=400 if disallowed else 201,
+            )
 
     @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
-    def test_changing_peers_control_nodes(self, node_type, admin_user, patch):
+    def test_disallow_modifying_peers_control_nodes(self, node_type, admin_user, patch):
         """
         for control nodes, peers field should not be
         modified directly via patch.
         """
-        control = Instance.objects.create(hostname='abc', node_type=node_type, managed=True)
-        hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
-        hop1addr = ReceptorAddress.objects.create(instance=hop1, address='hop1', peers_from_control_nodes=True, canonical=True)
-        hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
-        hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', canonical=True)
-        assert [hop1addr] == list(control.peers.all())  # only hop1addr should be peered
-        resp = patch(
-            url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
-            data={"peers": [hop2addr.id]},
-            user=admin_user,
-            expect=400,  # cannot add peers manually
-        )
-        assert 'Setting peers manually for managed nodes is not allowed.' in str(resp.data)
-
+        control = Instance.objects.create(hostname='abc', node_type=node_type)
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop', peers_from_control_nodes=True, listener_port=6789)
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop', peers_from_control_nodes=False, listener_port=6789)
+        assert [hop1] == list(control.peers.all())  # only hop1 should be peered
         patch(
             url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
-            data={"peers": [hop1addr.id]},
+            data={"peers": ["hop2"]},
+            user=admin_user,
+            expect=400,  # cannot add peers directly
+        )
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
+            data={"peers": ["hop1"]},
             user=admin_user,
             expect=200,  # patching with current peers list should be okay
         )
-        resp = patch(
+        patch(
             url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
             data={"peers": []},
             user=admin_user,
             expect=400,  # cannot remove peers directly
         )
-        assert 'Setting peers manually for managed nodes is not allowed.' in str(resp.data)
-
         patch(
             url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
             data={},
@@ -340,25 +148,23 @@ class TestPeers:
             url=reverse('api:instance_detail', kwargs={'pk': hop2.pk}),
             data={"peers_from_control_nodes": True},
             user=admin_user,
-            expect=200,
+            expect=200,  # patching without data should be fine too
         )
-        assert {hop1addr, hop2addr} == set(control.peers.all())  # hop1 and hop2 should now be peered from control node
+        assert {hop1, hop2} == set(control.peers.all())  # hop1 and hop2 should now be peered from control node
 
-    def test_changing_hostname(self, admin_user, patch):
+    def test_disallow_changing_hostname(self, admin_user, patch):
         """
         cannot change hostname
         """
         hop = Instance.objects.create(hostname='hop', node_type='hop')
-        resp = patch(
+        patch(
             url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
             data={"hostname": "hop2"},
             user=admin_user,
             expect=400,
         )
 
-        assert 'Cannot change hostname.' in str(resp.data)
-
-    def test_changing_node_state(self, admin_user, patch):
+    def test_disallow_changing_node_state(self, admin_user, patch):
         """
         only allow setting to deprovisioning
         """
@@ -369,54 +175,12 @@ class TestPeers:
             user=admin_user,
             expect=200,
         )
-        resp = patch(
+        patch(
             url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
             data={"node_state": "ready"},
             user=admin_user,
             expect=400,
         )
-        assert "Can only change instances to the 'deprovisioning' state." in str(resp.data)
-
-    def test_changing_managed_node_state(self, admin_user, patch):
-        """
-        cannot change node state of managed node
-        """
-        hop = Instance.objects.create(hostname='hop', node_type='hop', managed=True)
-        resp = patch(
-            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
-            data={"node_state": "deprovisioning"},
-            user=admin_user,
-            expect=400,
-        )
-
-        assert 'Cannot deprovision managed nodes.' in str(resp.data)
-
-    def test_changing_managed_peers_from_control_nodes(self, admin_user, patch):
-        """
-        cannot change peers_from_control_nodes of managed node
-        """
-        hop = Instance.objects.create(hostname='hop', node_type='hop', managed=True)
-        ReceptorAddress.objects.create(instance=hop, address='hop', peers_from_control_nodes=True, canonical=True)
-        resp = patch(
-            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
-            data={"peers_from_control_nodes": False},
-            user=admin_user,
-            expect=400,
-        )
-
-        assert 'Cannot change peers_from_control_nodes for managed nodes.' in str(resp.data)
-
-        hop.peers_from_control_nodes = False
-        hop.save()
-
-        resp = patch(
-            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
-            data={"peers_from_control_nodes": False},
-            user=admin_user,
-            expect=400,
-        )
-
-        assert 'Cannot change peers_from_control_nodes for managed nodes.' in str(resp.data)
 
     @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
     def test_control_node_automatically_peers(self, node_type):
@@ -427,10 +191,9 @@ class TestPeers:
         peer to hop should be removed if hop is deleted
         """
 
-        hop = Instance.objects.create(hostname='hop', node_type='hop')
-        hopaddr = ReceptorAddress.objects.create(instance=hop, address='hop', peers_from_control_nodes=True, canonical=True)
+        hop = Instance.objects.create(hostname='hop', node_type='hop', peers_from_control_nodes=True, listener_port=6789)
         control = Instance.objects.create(hostname='abc', node_type=node_type)
-        assert hopaddr in control.peers.all()
+        assert hop in control.peers.all()
         hop.delete()
         assert not control.peers.exists()
 
@@ -440,50 +203,26 @@ class TestPeers:
         if a new node comes online, other peer relationships should
         remain intact
         """
-        hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
-        hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
-        hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', canonical=True)
-        hop1.peers.add(hop2addr)
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=True)
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
+        hop1.peers.add(hop2)
 
         # a control node is added
-        Instance.objects.create(hostname='control', node_type=node_type)
+        Instance.objects.create(hostname='control', node_type=node_type, listener_port=None)
 
         assert hop1.peers.exists()
 
-    def test_reverse_peers(self, admin_user, get):
-        """
-        if hop1 peers to hop2, hop1 should
-        be in hop2's reverse_peers list
-        """
-        hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
-        hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
-        hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', canonical=True)
-        hop1.peers.add(hop2addr)
-
-        resp = get(
-            url=reverse('api:instance_detail', kwargs={'pk': hop2.pk}),
-            user=admin_user,
-            expect=200,
-        )
-
-        assert hop1.pk in resp.data['reverse_peers']
-
-    def test_group_vars(self):
+    def test_group_vars(self, get, admin_user):
         """
         control > hop1 > hop2 < execution
         """
-        control = Instance.objects.create(hostname='control', node_type='control')
-        hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
-        ReceptorAddress.objects.create(instance=hop1, address='hop1', peers_from_control_nodes=True, port=6789, canonical=True)
+        control = Instance.objects.create(hostname='control', node_type='control', listener_port=None)
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=True)
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
+        execution = Instance.objects.create(hostname='execution', node_type='execution', listener_port=6789)
 
-        hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
-        hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', peers_from_control_nodes=False, port=6789, canonical=True)
-
-        execution = Instance.objects.create(hostname='execution', node_type='execution')
-        ReceptorAddress.objects.create(instance=execution, address='execution', peers_from_control_nodes=False, port=6789, canonical=True)
-
-        execution.peers.add(hop2addr)
-        hop1.peers.add(hop2addr)
+        execution.peers.add(hop2)
+        hop1.peers.add(hop2)
 
         control_vars = yaml.safe_load(generate_group_vars_all_yml(control))
         hop1_vars = yaml.safe_load(generate_group_vars_all_yml(hop1))
@@ -526,15 +265,13 @@ class TestPeers:
             control = Instance.objects.create(hostname='control1', node_type='control')
             write_method.assert_not_called()
 
-            # new address with peers_from_control_nodes False (no)
-            hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
-            hop1addr = ReceptorAddress.objects.create(instance=hop1, address='hop1', peers_from_control_nodes=False, canonical=True)
+            # new hop node with peers_from_control_nodes False (no)
+            hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
             hop1.delete()
             write_method.assert_not_called()
 
-            # new address with peers_from_control_nodes True (yes)
-            hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
-            hop1addr = ReceptorAddress.objects.create(instance=hop1, address='hop1', peers_from_control_nodes=True, canonical=True)
+            # new hop node with peers_from_control_nodes True (yes)
+            hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=True)
             write_method.assert_called()
             write_method.reset_mock()
 
@@ -543,21 +280,20 @@ class TestPeers:
             write_method.assert_called()
             write_method.reset_mock()
 
-            # new address with peers_from_control_nodes False and peered to another hop node (no)
-            hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
-            ReceptorAddress.objects.create(instance=hop2, address='hop2', peers_from_control_nodes=False, canonical=True)
-            hop2.peers.add(hop1addr)
+            # new hop node with peers_from_control_nodes False and peered to another hop node (no)
+            hop2 = Instance.objects.create(hostname='hop2', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
+            hop2.peers.add(hop1)
             hop2.delete()
             write_method.assert_not_called()
 
             # changing peers_from_control_nodes to False (yes)
-            hop1addr.peers_from_control_nodes = False
-            hop1addr.save()
+            hop1.peers_from_control_nodes = False
+            hop1.save()
             write_method.assert_called()
             write_method.reset_mock()
 
-            # deleting address that has peers_from_control_nodes to False (no)
-            hop1.delete()  # cascade deletes to hop1addr
+            # deleting hop node that has peers_from_control_nodes to False (no)
+            hop1.delete()
             write_method.assert_not_called()
 
             # deleting control nodes (no)
@@ -579,8 +315,8 @@ class TestPeers:
 
             # not peered, so config file should not be updated
             for i in range(3):
-                inst = Instance.objects.create(hostname=f"exNo-{i}", node_type='execution')
-                ReceptorAddress.objects.create(instance=inst, address=f"exNo-{i}", port=6789, peers_from_control_nodes=False, canonical=True)
+                Instance.objects.create(hostname=f"exNo-{i}", node_type='execution', listener_port=6789, peers_from_control_nodes=False)
+
             _, should_update = generate_config_data()
             assert not should_update
 
@@ -588,13 +324,11 @@ class TestPeers:
             expected_peers = []
             for i in range(3):
                 expected_peers.append(f"hop-{i}:6789")
-                inst = Instance.objects.create(hostname=f"hop-{i}", node_type='hop')
-                ReceptorAddress.objects.create(instance=inst, address=f"hop-{i}", port=6789, peers_from_control_nodes=True, canonical=True)
+                Instance.objects.create(hostname=f"hop-{i}", node_type='hop', listener_port=6789, peers_from_control_nodes=True)
 
             for i in range(3):
                 expected_peers.append(f"exYes-{i}:6789")
-                inst = Instance.objects.create(hostname=f"exYes-{i}", node_type='execution')
-                ReceptorAddress.objects.create(instance=inst, address=f"exYes-{i}", port=6789, peers_from_control_nodes=True, canonical=True)
+                Instance.objects.create(hostname=f"exYes-{i}", node_type='execution', listener_port=6789, peers_from_control_nodes=True)
 
             new_config, should_update = generate_config_data()
             assert should_update

awx/main/tests/functional/api/test_survey_spec.py

@@ -2,12 +2,12 @@ from unittest import mock
 import pytest
 import json
 
-from ansible_base.lib.utils.models import get_type_for_model
 
 from awx.api.versioning import reverse
 from awx.main.models.jobs import JobTemplate, Job
 from awx.main.models.activity_stream import ActivityStream
 from awx.main.access import JobTemplateAccess
+from awx.main.utils.common import get_type_for_model
 
 
 @pytest.fixture

awx/main/tests/functional/test_bulk.py

@@ -309,139 +309,3 @@ def test_bulk_job_set_all_prompt(job_template, organization, inventory, project,
     assert node[0].limit == 'kansas'
     assert node[0].skip_tags == 'foobar'
     assert node[0].job_tags == 'untagged'
-
-
-@pytest.mark.django_db
-@pytest.mark.parametrize('num_hosts, num_queries', [(1, 70), (10, 150), (25, 250)])
-def test_bulk_host_delete_num_queries(organization, inventory, post, get, user, num_hosts, num_queries, django_assert_max_num_queries):
-    '''
-    If I am a...
-      org admin
-      inventory admin at org level
-      admin of a particular inventory
-      superuser
-
-    Bulk Host delete should take under a certain number of queries
-    '''
-    users_list = setup_admin_users_list(organization, inventory, user)
-    for u in users_list:
-        hosts = [{'name': str(uuid4())} for i in range(num_hosts)]
-        with django_assert_max_num_queries(num_queries):
-            bulk_host_create_response = post(reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': hosts}, u, expect=201).data
-            assert len(bulk_host_create_response['hosts']) == len(hosts), f"unexpected number of hosts created for user {u}"
-            hosts_ids_created = get_inventory_hosts(get, inventory.id, u)
-            bulk_host_delete_response = post(reverse('api:bulk_host_delete'), {'hosts': hosts_ids_created}, u, expect=201).data
-            assert len(bulk_host_delete_response['hosts'].keys()) == len(hosts), f"unexpected number of hosts deleted for user {u}"
-
-
-@pytest.mark.django_db
-def test_bulk_host_delete_rbac(organization, inventory, post, get, user):
-    '''
-    If I am a...
-      org admin
-      inventory admin at org level
-      admin of a particular invenotry
-          ... I can bulk delete hosts
-
-    Everyone else cannot
-    '''
-    admin_users_list = setup_admin_users_list(organization, inventory, user)
-    users_list = setup_none_admin_uses_list(organization, inventory, user)
-
-    for indx, u in enumerate(admin_users_list):
-        bulk_host_create_response = post(
-            reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': [{'name': f'foobar-{indx}'}]}, u, expect=201
-        ).data
-        assert len(bulk_host_create_response['hosts']) == 1, f"unexpected number of hosts created for user {u}"
-        assert Host.objects.filter(inventory__id=inventory.id)[0].name == f'foobar-{indx}'
-        hosts_ids_created = get_inventory_hosts(get, inventory.id, u)
-        bulk_host_delete_response = post(reverse('api:bulk_host_delete'), {'hosts': hosts_ids_created}, u, expect=201).data
-        assert len(bulk_host_delete_response['hosts'].keys()) == 1, f"unexpected number of hosts deleted by user {u}"
-
-    for indx, create_u in enumerate(admin_users_list):
-        bulk_host_create_response = post(
-            reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': [{'name': f'foobar2-{indx}'}]}, create_u, expect=201
-        ).data
-        print(bulk_host_create_response)
-        assert bulk_host_create_response['hosts'][0]['name'] == f'foobar2-{indx}'
-        hosts_ids_created = get_inventory_hosts(get, inventory.id, create_u)
-        print(f"Try to delete {hosts_ids_created}")
-        for delete_u in users_list:
-            bulk_host_delete_response = post(reverse('api:bulk_host_delete'), {'hosts': hosts_ids_created}, delete_u, expect=403).data
-            assert "Lack permissions to delete hosts from this inventory." in bulk_host_delete_response['inventories'].values()
-
-
-@pytest.mark.django_db
-def test_bulk_host_delete_from_multiple_inv(organization, inventory, post, get, user):
-    '''
-    If I am inventory admin at org level
-
-    Bulk Host delete should be enabled only on my inventory
-    '''
-    num_hosts = 10
-    inventory.organization = organization
-
-    # Create second inventory
-    inv2 = organization.inventories.create(name="second-test-inv")
-    inv2.organization = organization
-    admin2_user = user('inventory2_admin', False)
-    inv2.admin_role.members.add(admin2_user)
-
-    admin_user = user('inventory_admin', False)
-    inventory.admin_role.members.add(admin_user)
-
-    organization.member_role.members.add(admin_user)
-    organization.member_role.members.add(admin2_user)
-
-    hosts = [{'name': str(uuid4())} for i in range(num_hosts)]
-    hosts2 = [{'name': str(uuid4())} for i in range(num_hosts)]
-
-    # create hosts in each of the inventories
-    bulk_host_create_response = post(reverse('api:bulk_host_create'), {'inventory': inventory.id, 'hosts': hosts}, admin_user, expect=201).data
-    assert len(bulk_host_create_response['hosts']) == len(hosts), f"unexpected number of hosts created for user {admin_user}"
-
-    bulk_host_create_response2 = post(reverse('api:bulk_host_create'), {'inventory': inv2.id, 'hosts': hosts2}, admin2_user, expect=201).data
-    assert len(bulk_host_create_response2['hosts']) == len(hosts), f"unexpected number of hosts created for user {admin2_user}"
-
-    # get all hosts ids - from both inventories
-    hosts_ids_created = get_inventory_hosts(get, inventory.id, admin_user)
-    hosts_ids_created += get_inventory_hosts(get, inv2.id, admin2_user)
-
-    expected_error = "Lack permissions to delete hosts from this inventory."
-    # try to delete ALL hosts with admin user of inventory 1.
-    for inv_name, invadmin in zip([inv2.name, inventory.name], [admin_user, admin2_user]):
-        bulk_host_delete_response = post(reverse('api:bulk_host_delete'), {'hosts': hosts_ids_created}, invadmin, expect=403).data
-        result_message = bulk_host_delete_response['inventories'][inv_name]
-        assert result_message == expected_error, f"deleted hosts without permission by user {invadmin}"
-
-
-def setup_admin_users_list(organization, inventory, user):
-    inventory.organization = organization
-    inventory_admin = user('inventory_admin', False)
-    org_admin = user('org_admin', False)
-    org_inv_admin = user('org_admin', False)
-    superuser = user('admin', True)
-    for u in [org_admin, org_inv_admin, inventory_admin]:
-        organization.member_role.members.add(u)
-    organization.admin_role.members.add(org_admin)
-    organization.inventory_admin_role.members.add(org_inv_admin)
-    inventory.admin_role.members.add(inventory_admin)
-    return [inventory_admin, org_inv_admin, superuser, org_admin]
-
-
-def setup_none_admin_uses_list(organization, inventory, user):
-    inventory.organization = organization
-    auditor = user('auditor', False)
-    member = user('member', False)
-    use_inv_member = user('member', False)
-    for u in [auditor, member, use_inv_member]:
-        organization.member_role.members.add(u)
-    inventory.use_role.members.add(use_inv_member)
-    organization.auditor_role.members.add(auditor)
-    return [auditor, member, use_inv_member]
-
-
-def get_inventory_hosts(get, inv_id, use_user):
-    data = get(reverse('api:inventory_hosts_list', kwargs={'pk': inv_id}), use_user, expect=200).data
-    results = [host['id'] for host in data['results']]
-    return results

awx/main/tests/functional/test_credential.py

@@ -81,7 +81,6 @@ def test_default_cred_types():
             'aws_secretsmanager_credential',
             'azure_kv',
             'azure_rm',
-            'bitbucket_dc_token',
             'centrify_vault_kv',
             'conjur',
             'controller',
@@ -101,7 +100,6 @@ def test_default_cred_types():
             'satellite6',
             'scm',
             'ssh',
-            'terraform',
             'thycotic_dsv',
             'thycotic_tss',
             'vault',

awx/main/tests/functional/test_credential_plugins.py

@@ -40,33 +40,6 @@ def test_hashivault_kubernetes_auth():
         assert res == expected_res
 
 
-def test_hashivault_client_cert_auth_explicit_role():
-    kwargs = {
-        'client_cert_role': 'test-cert-1',
-    }
-    expected_res = {
-        'name': 'test-cert-1',
-    }
-    res = hashivault.client_cert_auth(**kwargs)
-    assert res == expected_res
-
-
-def test_hashivault_client_cert_auth_no_role():
-    kwargs = {}
-    expected_res = {
-        'name': None,
-    }
-    res = hashivault.client_cert_auth(**kwargs)
-    assert res == expected_res
-
-
-def test_hashivault_userpass_auth():
-    kwargs = {'username': 'the_username', 'password': 'the_password'}
-    expected_res = {'username': 'the_username', 'password': 'the_password'}
-    res = hashivault.userpass_auth(**kwargs)
-    assert res == expected_res
-
-
 def test_hashivault_handle_auth_token():
     kwargs = {
         'token': 'the_token',
@@ -100,22 +73,6 @@ def test_hashivault_handle_auth_kubernetes():
             assert token == 'the_token'
 
 
-def test_hashivault_handle_auth_client_cert():
-    kwargs = {
-        'client_cert_public': "foo",
-        'client_cert_private': "bar",
-        'client_cert_role': 'test-cert-1',
-    }
-    auth_params = {
-        'name': 'test-cert-1',
-    }
-    with mock.patch.object(hashivault, 'method_auth') as method_mock:
-        method_mock.return_value = 'the_token'
-        token = hashivault.handle_auth(**kwargs)
-        method_mock.assert_called_with(**kwargs, auth_param=auth_params)
-        assert token == 'the_token'
-
-
 def test_hashivault_handle_auth_not_enough_args():
     with pytest.raises(Exception):
         hashivault.handle_auth()

awx/main/tests/functional/test_linkstate.py

@@ -1,30 +0,0 @@
-import pytest
-
-from awx.main.models import Instance, ReceptorAddress, InstanceLink
-from awx.main.tasks.system import inspect_established_receptor_connections
-
-
-@pytest.mark.django_db
-class TestLinkState:
-    @pytest.fixture(autouse=True)
-    def configure_settings(self, settings):
-        settings.IS_K8S = True
-
-    def test_inspect_established_receptor_connections(self):
-        '''
-        Change link state from ADDING to ESTABLISHED
-        if the receptor status KnownConnectionCosts field
-        has an entry for the source and target node.
-        '''
-        hop1 = Instance.objects.create(hostname='hop1')
-        hop2 = Instance.objects.create(hostname='hop2')
-        hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', port=5678)
-        InstanceLink.objects.create(source=hop1, target=hop2addr, link_state=InstanceLink.States.ADDING)
-
-        # calling with empty KnownConnectionCosts should not change the link state
-        inspect_established_receptor_connections({"KnownConnectionCosts": {}})
-        assert InstanceLink.objects.get(source=hop1, target=hop2addr).link_state == InstanceLink.States.ADDING
-
-        mesh_state = {"KnownConnectionCosts": {"hop1": {"hop2": 1}}}
-        inspect_established_receptor_connections(mesh_state)
-        assert InstanceLink.objects.get(source=hop1, target=hop2addr).link_state == InstanceLink.States.ESTABLISHED

awx/main/tests/functional/test_migrations.py

@@ -42,29 +42,3 @@ class TestMigrationSmoke:
         final_state = migrator.apply_tested_migration(final_migration)
         Instance = final_state.apps.get_model('main', 'Instance')
         assert Instance.objects.filter(hostname='foobar').count() == 1
-
-    def test_receptor_address(self, migrator):
-        old_state = migrator.apply_initial_migration(('main', '0188_add_bitbucket_dc_webhook'))
-        Instance = old_state.apps.get_model('main', 'Instance')
-        for i in range(3):
-            Instance.objects.create(hostname=f'foobar{i}', node_type='hop')
-        foo = Instance.objects.create(hostname='foo', node_type='execution', listener_port=1234)
-        bar = Instance.objects.create(hostname='bar', node_type='execution', listener_port=None)
-        bar.peers.add(foo)
-
-        new_state = migrator.apply_tested_migration(
-            ('main', '0189_inbound_hop_nodes'),
-        )
-        Instance = new_state.apps.get_model('main', 'Instance')
-        ReceptorAddress = new_state.apps.get_model('main', 'ReceptorAddress')
-
-        # We can now test how our migration worked, new field is there:
-        assert ReceptorAddress.objects.filter(address='foo', port=1234).count() == 1
-        assert not ReceptorAddress.objects.filter(address='bar').exists()
-
-        bar = Instance.objects.get(hostname='bar')
-        fooaddr = ReceptorAddress.objects.get(address='foo')
-
-        bar_peers = bar.peers.all()
-        assert len(bar_peers) == 1
-        assert fooaddr in bar_peers

awx/main/tests/functional/test_rbac_core.py

@@ -208,6 +208,6 @@ def test_auto_parenting():
 @pytest.mark.django_db
 def test_update_parents_keeps_teams(team, project):
     project.update_role.parents.add(team.member_role)
-    assert list(Project.accessible_objects(team.member_role, 'update_role')) == [project]  # test prep sanity check
+    assert team.member_role in project.update_role  # test prep sanity check
     update_role_parentage_for_instance(project)
-    assert list(Project.accessible_objects(team.member_role, 'update_role')) == [project]  # actual assertion
+    assert team.member_role in project.update_role  # actual assertion

awx/main/tests/functional/test_rbac_team.py

@@ -92,7 +92,7 @@ def test_team_accessible_by(team, user, project):
     u = user('team_member', False)
 
     team.member_role.children.add(project.use_role)
-    assert list(Project.accessible_objects(team.member_role, 'read_role')) == [project]
+    assert team in project.read_role
     assert u not in project.read_role
 
     team.member_role.members.add(u)
@@ -104,7 +104,7 @@ def test_team_accessible_objects(team, user, project):
     u = user('team_member', False)
 
     team.member_role.children.add(project.use_role)
-    assert len(Project.accessible_objects(team.member_role, 'read_role')) == 1
+    assert len(Project.accessible_objects(team, 'read_role')) == 1
     assert not Project.accessible_objects(u, 'read_role')
 
     team.member_role.members.add(u)

awx/main/tests/functional/test_routing.py

@@ -1,90 +0,0 @@
-import pytest
-
-from django.contrib.auth.models import AnonymousUser
-
-from channels.routing import ProtocolTypeRouter
-from channels.testing.websocket import WebsocketCommunicator
-
-
-from awx.main.consumers import WebsocketSecretAuthHelper
-
-
-@pytest.fixture
-def application():
-    # code in routing hits the db on import because .. settings cache
-    from awx.main.routing import application_func
-
-    yield application_func(ProtocolTypeRouter)
-
-
-@pytest.fixture
-def websocket_server_generator(application):
-    def fn(endpoint):
-        return WebsocketCommunicator(application, endpoint)
-
-    return fn
-
-
-@pytest.mark.asyncio
-@pytest.mark.django_db
-class TestWebsocketRelay:
-    @pytest.fixture
-    def websocket_relay_secret_generator(self, settings):
-        def fn(secret, set_broadcast_websocket_secret=False):
-            secret_backup = settings.BROADCAST_WEBSOCKET_SECRET
-            settings.BROADCAST_WEBSOCKET_SECRET = 'foobar'
-            res = ('secret'.encode('utf-8'), WebsocketSecretAuthHelper.construct_secret().encode('utf-8'))
-            if set_broadcast_websocket_secret is False:
-                settings.BROADCAST_WEBSOCKET_SECRET = secret_backup
-            return res
-
-        return fn
-
-    @pytest.fixture
-    def websocket_relay_secret(self, settings, websocket_relay_secret_generator):
-        return websocket_relay_secret_generator('foobar', set_broadcast_websocket_secret=True)
-
-    async def test_authorized(self, websocket_server_generator, websocket_relay_secret):
-        server = websocket_server_generator('/websocket/relay/')
-
-        server.scope['headers'] = (websocket_relay_secret,)
-        connected, _ = await server.connect()
-        assert connected is True
-
-    async def test_not_authorized(self, websocket_server_generator):
-        server = websocket_server_generator('/websocket/relay/')
-        connected, _ = await server.connect()
-        assert connected is False, "Connection to the relay websocket without auth. We expected the client to be denied."
-
-    async def test_wrong_secret(self, websocket_server_generator, websocket_relay_secret_generator):
-        server = websocket_server_generator('/websocket/relay/')
-
-        server.scope['headers'] = (websocket_relay_secret_generator('foobar', set_broadcast_websocket_secret=False),)
-        connected, _ = await server.connect()
-        assert connected is False
-
-
-@pytest.mark.asyncio
-@pytest.mark.django_db
-class TestWebsocketEventConsumer:
-    async def test_unauthorized_anonymous(self, websocket_server_generator):
-        server = websocket_server_generator('/websocket/')
-
-        server.scope['user'] = AnonymousUser()
-        connected, _ = await server.connect()
-        assert connected is False, "Anonymous user should NOT be allowed to login."
-
-    @pytest.mark.skip(reason="Ran out of coding time.")
-    async def test_authorized(self, websocket_server_generator, application, admin):
-        server = websocket_server_generator('/websocket/')
-
-        """
-        I ran out of time. Here is what I was thinking ...
-        Inject a valid session into the cookies in the header
-
-        server.scope['headers'] = (
-            (b'cookie', ...),
-        )
-        """
-        connected, _ = await server.connect()
-        assert connected is True, "User should be allowed in via cookies auth via a session key in the cookies"

awx/main/tests/unit/api/test_filters.py

@@ -3,13 +3,15 @@
 import pytest
 
 # Django
-from rest_framework.exceptions import PermissionDenied
+from django.core.exceptions import FieldDoesNotExist
 
-from ansible_base.rest_filters.rest_framework.field_lookup_backend import FieldLookupBackend
+from rest_framework.exceptions import PermissionDenied, ParseError
 
+from awx.api.filters import FieldLookupBackend, OrderByBackend, get_field_from_path
 from awx.main.models import (
     AdHocCommand,
     ActivityStream,
+    Credential,
     Job,
     JobTemplate,
     SystemJob,
@@ -18,11 +20,88 @@ from awx.main.models import (
     WorkflowJob,
     WorkflowJobTemplate,
     WorkflowJobOptions,
+    InventorySource,
+    JobEvent,
 )
 from awx.main.models.oauth import OAuth2Application
 from awx.main.models.jobs import JobOptions
 
 
+def test_related():
+    field_lookup = FieldLookupBackend()
+    lookup = '__'.join(['inventory', 'organization', 'pk'])
+    field, new_lookup = field_lookup.get_field_from_lookup(InventorySource, lookup)
+    print(field)
+    print(new_lookup)
+
+
+def test_invalid_filter_key():
+    field_lookup = FieldLookupBackend()
+    # FieldDoesNotExist is caught and converted to ParseError by filter_queryset
+    with pytest.raises(FieldDoesNotExist) as excinfo:
+        field_lookup.value_to_python(JobEvent, 'event_data.task_action', 'foo')
+    assert 'has no field named' in str(excinfo)
+
+
+def test_invalid_field_hop():
+    with pytest.raises(ParseError) as excinfo:
+        get_field_from_path(Credential, 'organization__description__user')
+    assert 'No related model for' in str(excinfo)
+
+
+def test_invalid_order_by_key():
+    field_order_by = OrderByBackend()
+    with pytest.raises(ParseError) as excinfo:
+        [f for f in field_order_by._validate_ordering_fields(JobEvent, ('event_data.task_action',))]
+    assert 'has no field named' in str(excinfo)
+
+
+@pytest.mark.parametrize(u"empty_value", [u'', ''])
+def test_empty_in(empty_value):
+    field_lookup = FieldLookupBackend()
+    with pytest.raises(ValueError) as excinfo:
+        field_lookup.value_to_python(JobTemplate, 'project__name__in', empty_value)
+    assert 'empty value for __in' in str(excinfo.value)
+
+
+@pytest.mark.parametrize(u"valid_value", [u'foo', u'foo,'])
+def test_valid_in(valid_value):
+    field_lookup = FieldLookupBackend()
+    value, new_lookup, _ = field_lookup.value_to_python(JobTemplate, 'project__name__in', valid_value)
+    assert 'foo' in value
+
+
+def test_invalid_field():
+    invalid_field = u"ヽヾ"
+    field_lookup = FieldLookupBackend()
+    with pytest.raises(ValueError) as excinfo:
+        field_lookup.value_to_python(WorkflowJobTemplate, invalid_field, 'foo')
+    assert 'is not an allowed field name. Must be ascii encodable.' in str(excinfo.value)
+
+
+def test_valid_iexact():
+    field_lookup = FieldLookupBackend()
+    value, new_lookup, _ = field_lookup.value_to_python(JobTemplate, 'project__name__iexact', 'foo')
+    assert 'foo' in value
+
+
+def test_invalid_iexact():
+    field_lookup = FieldLookupBackend()
+    with pytest.raises(ValueError) as excinfo:
+        field_lookup.value_to_python(Job, 'id__iexact', '1')
+    assert 'is not a text field and cannot be filtered by case-insensitive search' in str(excinfo.value)
+
+
+@pytest.mark.parametrize('lookup_suffix', ['', 'contains', 'startswith', 'in'])
+@pytest.mark.parametrize('password_field', Credential.PASSWORD_FIELDS)
+def test_filter_on_password_field(password_field, lookup_suffix):
+    field_lookup = FieldLookupBackend()
+    lookup = '__'.join(filter(None, [password_field, lookup_suffix]))
+    with pytest.raises(PermissionDenied) as excinfo:
+        field, new_lookup = field_lookup.get_field_from_lookup(Credential, lookup)
+    assert 'not allowed' in str(excinfo.value)
+
+
 @pytest.mark.parametrize(
     'model, query',
     [
@@ -49,3 +128,10 @@ def test_filter_sensitive_fields_and_relations(model, query):
     with pytest.raises(PermissionDenied) as excinfo:
         field, new_lookup = field_lookup.get_field_from_lookup(model, query)
     assert 'not allowed' in str(excinfo.value)
+
+
+def test_looping_filters_prohibited():
+    field_lookup = FieldLookupBackend()
+    with pytest.raises(ParseError) as loop_exc:
+        field_lookup.get_field_from_lookup(Job, 'job_events__job__job_events')
+    assert 'job_events' in str(loop_exc.value)

awx/main/tests/unit/models/test_receptor_address.py

@@ -1,32 +0,0 @@
-from awx.main.models import ReceptorAddress
-import pytest
-
-ReceptorAddress()
-
-
-@pytest.mark.parametrize(
-    'address, protocol, port, websocket_path, expected',
-    [
-        ('foo', 'tcp', 27199, '', 'foo:27199'),
-        ('bar', 'ws', 6789, '', 'wss://bar:6789'),
-        ('mal', 'ws', 6789, 'path', 'wss://mal:6789/path'),
-        ('example.com', 'ws', 443, 'path', 'wss://example.com:443/path'),
-    ],
-)
-def test_get_full_address(address, protocol, port, websocket_path, expected):
-    receptor_address = ReceptorAddress(address=address, protocol=protocol, port=port, websocket_path=websocket_path)
-    assert receptor_address.get_full_address() == expected
-
-
-@pytest.mark.parametrize(
-    'protocol, expected',
-    [
-        ('tcp', 'tcp-peer'),
-        ('ws', 'ws-peer'),
-        ('wss', 'ws-peer'),
-        ('foo', None),
-    ],
-)
-def test_get_peer_type(protocol, expected):
-    receptor_address = ReceptorAddress(protocol=protocol)
-    assert receptor_address.get_peer_type() == expected

awx/main/tests/unit/test_tasks.py

@@ -1085,27 +1085,6 @@ class TestJobCredentials(TestJobExecution):
         assert open(env['ANSIBLE_NET_SSH_KEYFILE'], 'r').read() == self.EXAMPLE_PRIVATE_KEY
         assert safe_env['ANSIBLE_NET_PASSWORD'] == HIDDEN_PASSWORD
 
-    def test_terraform_cloud_credentials(self, job, private_data_dir, mock_me):
-        terraform = CredentialType.defaults['terraform']()
-        hcl_config = '''
-        backend "s3" {
-            bucket = "s3_sample_bucket"
-            key    = "/tf_state/"
-            region = "us-east-1"
-        }
-        '''
-        credential = Credential(pk=1, credential_type=terraform, inputs={'configuration': hcl_config})
-        credential.inputs['configuration'] = encrypt_field(credential, 'configuration')
-        job.credentials.add(credential)
-
-        env = {}
-        safe_env = {}
-        credential.credential_type.inject_credential(credential, env, safe_env, [], private_data_dir)
-
-        local_path = to_host_path(env['TF_BACKEND_CONFIG_FILE'], private_data_dir)
-        config = open(local_path, 'r').read()
-        assert config == hcl_config
-
     def test_custom_environment_injectors_with_jinja_syntax_error(self, private_data_dir, mock_me):
         some_cloud = CredentialType(
             kind='cloud',

awx/main/tests/unit/utils/test_common.py

@@ -12,8 +12,6 @@ from unittest import mock
 
 from rest_framework.exceptions import ParseError
 
-from ansible_base.lib.utils.models import get_type_for_model
-
 from awx.main.utils import common
 from awx.api.validators import HostnameRegexValidator
 
@@ -108,7 +106,7 @@ TEST_MODELS = [
 # Cases relied on for scheduler dependent jobs list
 @pytest.mark.parametrize('model,name', TEST_MODELS)
 def test_get_type_for_model(model, name):
-    assert get_type_for_model(model) == name
+    assert common.get_type_for_model(model) == name
 
 
 def test_get_model_for_invalid_type():

awx/main/tests/unit/utils/test_filters.py

@@ -68,9 +68,7 @@ class mockHost:
 
 @mock.patch('awx.main.utils.filters.get_model', return_value=mockHost())
 class TestSmartFilterQueryFromString:
-    @mock.patch(
-        'ansible_base.rest_filters.rest_framework.field_lookup_backend.get_fields_from_path', lambda model, path: ([model], path)
-    )  # disable field filtering, because a__b isn't a real Host field
+    @mock.patch('awx.api.filters.get_fields_from_path', lambda model, path: ([model], path))  # disable field filtering, because a__b isn't a real Host field
     @pytest.mark.parametrize(
         "filter_string,q_expected",
         [

awx/main/utils/common.py

@@ -52,10 +52,12 @@ __all__ = [
     'get_awx_http_client_headers',
     'get_awx_version',
     'update_scm_url',
+    'get_type_for_model',
     'get_model_for_type',
     'copy_model_by_class',
     'copy_m2m_relationships',
     'prefetch_page_capabilities',
+    'to_python_boolean',
     'datetime_hook',
     'ignore_inventory_computed_fields',
     'ignore_inventory_group_removal',
@@ -108,6 +110,18 @@ def get_object_or_400(klass, *args, **kwargs):
         raise ParseError(*e.args)
 
 
+def to_python_boolean(value, allow_none=False):
+    value = str(value)
+    if value.lower() in ('true', '1', 't'):
+        return True
+    elif value.lower() in ('false', '0', 'f'):
+        return False
+    elif allow_none and value.lower() in ('none', 'null'):
+        return None
+    else:
+        raise ValueError(_(u'Unable to convert "%s" to boolean') % value)
+
+
 def datetime_hook(d):
     new_d = {}
     for key, value in d.items():
@@ -555,6 +569,14 @@ def copy_m2m_relationships(obj1, obj2, fields, kwargs=None):
                 dest_field.add(*list(src_field_value.all().values_list('id', flat=True)))
 
 
+def get_type_for_model(model):
+    """
+    Return type name for a given model class.
+    """
+    opts = model._meta.concrete_model._meta
+    return camelcase_to_underscore(opts.object_name)
+
+
 def get_model_for_type(type_name):
     """
     Return model class for a given type name.

awx/main/utils/db.py

@@ -1,10 +1,27 @@
 # Copyright (c) 2017 Ansible by Red Hat
 # All Rights Reserved.
 
+from itertools import chain
 
 from awx.settings.application_name import set_application_name
 from django.conf import settings
 
 
+def get_all_field_names(model):
+    # Implements compatibility with _meta.get_all_field_names
+    # See: https://docs.djangoproject.com/en/1.11/ref/models/meta/#migrating-from-the-old-api
+    return list(
+        set(
+            chain.from_iterable(
+                (field.name, field.attname) if hasattr(field, 'attname') else (field.name,)
+                for field in model._meta.get_fields()
+                # For complete backwards compatibility, you may want to exclude
+                # GenericForeignKey from the results.
+                if not (field.many_to_one and field.related_model is None)
+            )
+        )
+    )
+
+
 def set_connection_name(function):
     set_application_name(settings.DATABASES, settings.CLUSTER_HOST_ID, function=function)

awx/main/utils/filters.py

@@ -161,7 +161,7 @@ class SmartFilter(object):
             else:
                 # detect loops and restrict access to sensitive fields
                 # this import is intentional here to avoid a circular import
-                from ansible_base.rest_filters.rest_framework.field_lookup_backend import FieldLookupBackend
+                from awx.api.filters import FieldLookupBackend
 
                 FieldLookupBackend().get_field_from_lookup(Host, k)
                 kwargs[k] = v

awx/main/wsrelay.py

@@ -20,6 +20,7 @@ from awx.main.analytics.broadcast_websocket import (
     RelayWebsocketStats,
     RelayWebsocketStatsManager,
 )
+import awx.main.analytics.subsystem_metrics as s_metrics
 
 logger = logging.getLogger('awx.main.wsrelay')
 
@@ -53,6 +54,7 @@ class WebsocketRelayConnection:
         self.protocol = protocol
         self.verify_ssl = verify_ssl
         self.channel_layer = None
+        self.subsystem_metrics = s_metrics.Metrics(instance_name=name)
         self.producers = dict()
         self.connected = False
 
@@ -339,7 +341,7 @@ class WebSocketRelayManager(object):
 
             if deleted_remote_hosts:
                 logger.info(f"Removing {deleted_remote_hosts} from websocket broadcast list")
-                await asyncio.gather(*[self.cleanup_offline_host(h) for h in deleted_remote_hosts])
+                await asyncio.gather(self.cleanup_offline_host(h) for h in deleted_remote_hosts)
 
             if new_remote_hosts:
                 logger.info(f"Adding {new_remote_hosts} to websocket broadcast list")

awx/playbooks/project_update.yml

@@ -216,54 +216,42 @@
     - block:
         - name: Fetch galaxy roles from roles/requirements.(yml/yaml)
           ansible.builtin.command:
-            cmd: "ansible-galaxy role install -r {{ req_file }} {{ verbosity }}"
+            cmd: "ansible-galaxy role install -r {{ item }} {{ verbosity }}"
           register: galaxy_result
-          vars:
-            req_file: "{{ lookup('ansible.builtin.first_found', req_candidates, skip=True) }}"
-            req_candidates:
-              - "{{ project_path | quote }}/roles/requirements.yml"
-              - "{{ project_path | quote }}/roles/requirements.yaml"
+          with_fileglob:
+            - "{{ project_path | quote }}/roles/requirements.yaml"
+            - "{{ project_path | quote }}/roles/requirements.yml"
           changed_when: "'was installed successfully' in galaxy_result.stdout"
-          when:
-            - roles_enabled | bool
-            - req_file
+          when: roles_enabled | bool
           tags:
             - install_roles
 
         - name: Fetch galaxy collections from collections/requirements.(yml/yaml)
           ansible.builtin.command:
-            cmd: "ansible-galaxy collection install -r {{ req_file }} {{ verbosity }}"
+            cmd: "ansible-galaxy collection install -r {{ item }} {{ verbosity }}"
           register: galaxy_collection_result
-          vars:
-            req_file: "{{ lookup('ansible.builtin.first_found', req_candidates, skip=True) }}"
-            req_candidates:
-              - "{{ project_path | quote }}/collections/requirements.yml"
-              - "{{ project_path | quote }}/collections/requirements.yaml"
-              - "{{ project_path | quote }}/requirements.yml"
-              - "{{ project_path | quote }}/requirements.yaml"
+          with_fileglob:
+            - "{{ project_path | quote }}/collections/requirements.yaml"
+            - "{{ project_path | quote }}/collections/requirements.yml"
           changed_when: "'Nothing to do.' not in galaxy_collection_result.stdout"
           when:
             - "ansible_version.full is version_compare('2.9', '>=')"
             - collections_enabled | bool
-            - req_file
           tags:
             - install_collections
 
         - name: Fetch galaxy roles and collections from requirements.(yml/yaml)
           ansible.builtin.command:
-            cmd: "ansible-galaxy install -r {{ req_file }} {{ verbosity }}"
+            cmd: "ansible-galaxy install -r {{ item }} {{ verbosity }}"
           register: galaxy_combined_result
-          vars:
-            req_file: "{{ lookup('ansible.builtin.first_found', req_candidates, skip=True) }}"
-            req_candidates:
-              - "{{ project_path | quote }}/requirements.yaml"
-              - "{{ project_path | quote }}/requirements.yml"
+          with_fileglob:
+            - "{{ project_path | quote }}/requirements.yaml"
+            - "{{ project_path | quote }}/requirements.yml"
           changed_when: "'Nothing to do.' not in galaxy_combined_result.stdout"
           when:
             - "ansible_version.full is version_compare('2.10', '>=')"
             - collections_enabled | bool
             - roles_enabled | bool
-            - req_file
           tags:
             - install_collections
             - install_roles

awx/settings/defaults.py

@@ -11,7 +11,6 @@ from datetime import timedelta
 
 # python-ldap
 import ldap
-from split_settings.tools import include
 
 
 DEBUG = True
@@ -37,18 +36,6 @@ DATABASES = {
     }
 }
 
-# Special database overrides for dispatcher connections listening to pg_notify
-LISTENER_DATABASES = {
-    'default': {
-        'OPTIONS': {
-            'keepalives': 1,
-            'keepalives_idle': 5,
-            'keepalives_interval': 5,
-            'keepalives_count': 5,
-        },
-    }
-}
-
 # Whether or not the deployment is a K8S-based deployment
 # In K8S-based deployments, instances have zero capacity - all playbook
 # automation is intended to flow through defined Container Groups that
@@ -144,9 +131,6 @@ BULK_JOB_MAX_LAUNCH = 100
 # Maximum number of host that can be created in 1 bulk host create
 BULK_HOST_MAX_CREATE = 100
 
-# Maximum number of host that can be deleted in 1 bulk host delete
-BULK_HOST_MAX_DELETE = 250
-
 SITE_ID = 1
 
 # Make this unique, and don't share it with anybody.
@@ -352,7 +336,7 @@ INSTALLED_APPS = [
     'awx.ui',
     'awx.sso',
     'solo',
-    'ansible_base.rest_filters',
+    'ansible_base',
 ]
 
 INTERNAL_IPS = ('127.0.0.1',)
@@ -367,6 +351,12 @@ REST_FRAMEWORK = {
         'awx.api.authentication.LoggedBasicAuthentication',
     ),
     'DEFAULT_PERMISSION_CLASSES': ('awx.api.permissions.ModelAccessPermission',),
+    'DEFAULT_FILTER_BACKENDS': (
+        'awx.api.filters.TypeFilterBackend',
+        'awx.api.filters.FieldLookupBackend',
+        'rest_framework.filters.SearchFilter',
+        'awx.api.filters.OrderByBackend',
+    ),
     'DEFAULT_PARSER_CLASSES': ('awx.api.parsers.JSONParser',),
     'DEFAULT_RENDERER_CLASSES': ('awx.api.renderers.DefaultJSONRenderer', 'awx.api.renderers.BrowsableAPIRenderer'),
     'DEFAULT_METADATA_CLASS': 'awx.api.metadata.Metadata',
@@ -1074,42 +1064,3 @@ CLEANUP_HOST_METRICS_HARD_THRESHOLD = 36  # months
 # Host metric summary monthly task - last time of run
 HOST_METRIC_SUMMARY_TASK_LAST_TS = None
 HOST_METRIC_SUMMARY_TASK_INTERVAL = 7  # days
-
-
-# TODO: cmeyers, replace with with register pattern
-# The register pattern is particularly nice for this because we need
-# to know the process to start the thread that will be the server.
-# The registration location should be the same location as we would
-# call MetricsServer.start()
-# Note: if we don't get to this TODO, then at least create constants
-# for the services strings below.
-# TODO: cmeyers, break this out into a separate django app so other
-# projects can take advantage.
-
-METRICS_SERVICE_CALLBACK_RECEIVER = 'callback_receiver'
-METRICS_SERVICE_DISPATCHER = 'dispatcher'
-METRICS_SERVICE_WEBSOCKETS = 'websockets'
-
-METRICS_SUBSYSTEM_CONFIG = {
-    'server': {
-        METRICS_SERVICE_CALLBACK_RECEIVER: {
-            'port': 8014,
-        },
-        METRICS_SERVICE_DISPATCHER: {
-            'port': 8015,
-        },
-        METRICS_SERVICE_WEBSOCKETS: {
-            'port': 8016,
-        },
-    }
-}
-
-
-# django-ansible-base
-ANSIBLE_BASE_TEAM_MODEL = 'main.Team'
-ANSIBLE_BASE_ORGANIZATION_MODEL = 'main.Organization'
-
-from ansible_base.lib import dynamic_config  # noqa: E402
-
-settings_file = os.path.join(os.path.dirname(dynamic_config.__file__), 'dynamic_settings.py')
-include(settings_file)

awx/settings/development.py

@@ -21,7 +21,7 @@ from split_settings.tools import optional, include
 from .defaults import *  # NOQA
 
 # awx-manage shell_plus --notebook
-NOTEBOOK_ARGUMENTS = ['--NotebookApp.token=', '--ip', '0.0.0.0', '--port', '9888', '--allow-root', '--no-browser']
+NOTEBOOK_ARGUMENTS = ['--NotebookApp.token=', '--ip', '0.0.0.0', '--port', '8888', '--allow-root', '--no-browser']
 
 # print SQL queries in shell_plus
 SHELL_PLUS_PRINT_SQL = False