Switch to docker_compose_v2 in tools playbook (#14872)

Switch to docker_compose_v2

Fix
```
"Configuration error - kwargs_from_env() got an unexpected keyword argument 'ssl_version'"}
```

.github/actions/stale/stale.yml

@@ -1,31 +0,0 @@
-name: 'Close stale issues and PRs'
-on:
-  schedule:
-    - cron: '30 1 * * *'
-
-jobs:
-  stale:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/stale@v9
-        with:
-
-          debug-only: true
-          operations-per-run: 30
-
-          days-before-stale: 30
-          days-before-close: 5
-
-          days-before-issue-stale: 180
-          days-before-issue-close: 14
-          stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.'
-          close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.'
-          exempt-issue-labels: 'needs_triage'
-          stale-issue-label: 'no_issue_activity'
-
-          days-before-pr-stale: 90
-          days-before-pr-close: 14
-          stale-pr-message: 'This PR is stale because it has been open 45 days with no activity. Remove stale label or comment or this will be closed in 10 days.'
-          close-pr-message: 'This PR was closed because it has been stalled for 10 days with no activity.'
-          exempt-pr-labels: 'needs_triage'
-          stale-pr-label: 'no_pr_activity'

.gitignore

@@ -169,3 +169,6 @@ awx/ui_next/build
 # Docs build stuff
 docs/docsite/build/
 _readthedocs/
+
+# Pyenv
+.python-version

Makefile

@@ -538,7 +538,8 @@ docker-compose: awx/projects docker-compose-sources
 	ansible-galaxy install --ignore-certs -r tools/docker-compose/ansible/requirements.yml;
 	ansible-playbook -i tools/docker-compose/inventory tools/docker-compose/ansible/initialize_containers.yml \
 	    -e enable_vault=$(VAULT) \
-	    -e vault_tls=$(VAULT_TLS);
+	    -e vault_tls=$(VAULT_TLS) \
+	    -e enable_ldap=$(LDAP);
 	$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans
 
 docker-compose-credential-plugins: awx/projects docker-compose-sources

awx/api/serializers.py

@@ -6,7 +6,7 @@ import copy
 import json
 import logging
 import re
-from collections import OrderedDict
+from collections import Counter, OrderedDict
 from datetime import timedelta
 from uuid import uuid4
 
@@ -82,6 +82,7 @@ from awx.main.models import (
     Project,
     ProjectUpdate,
     ProjectUpdateEvent,
+    ReceptorAddress,
     RefreshToken,
     Role,
     Schedule,
@@ -636,7 +637,7 @@ class BaseSerializer(serializers.ModelSerializer, metaclass=BaseSerializerMetacl
             exclusions = self.get_validation_exclusions(self.instance)
             obj = self.instance or self.Meta.model()
             for k, v in attrs.items():
-                if k not in exclusions:
+                if k not in exclusions and k != 'canonical_address_port':
                     setattr(obj, k, v)
             obj.full_clean(exclude=exclusions)
             # full_clean may modify values on the instance; copy those changes
@@ -5176,16 +5177,21 @@ class NotificationTemplateSerializer(BaseSerializer):
                 body = messages[event].get('body', {})
                 if body:
                     try:
-                        rendered_body = (
-                            sandbox.ImmutableSandboxedEnvironment(undefined=DescriptiveUndefined).from_string(body).render(JobNotificationMixin.context_stub())
-                        )
-                        potential_body = json.loads(rendered_body)
-                        if not isinstance(potential_body, dict):
-                            error_list.append(
-                                _("Webhook body for '{}' should be a json dictionary. Found type '{}'.".format(event, type(potential_body).__name__))
-                            )
-                    except json.JSONDecodeError as exc:
-                        error_list.append(_("Webhook body for '{}' is not a valid json dictionary ({}).".format(event, exc)))
+                        sandbox.ImmutableSandboxedEnvironment(undefined=DescriptiveUndefined).from_string(body).render(JobNotificationMixin.context_stub())
+
+                        # https://github.com/ansible/awx/issues/14410
+
+                        # When rendering something such as "{{ job.id }}"
+                        # the return type is not a dict, unlike "{{ job_metadata }}" which is a dict
+
+                        # potential_body = json.loads(rendered_body)
+
+                        # if not isinstance(potential_body, dict):
+                        #     error_list.append(
+                        #         _("Webhook body for '{}' should be a json dictionary. Found type '{}'.".format(event, type(potential_body).__name__))
+                        #     )
+                    except Exception as exc:
+                        error_list.append(_("Webhook body for '{}' is not valid. The following gave an error ({}).".format(event, exc)))
 
         if error_list:
             raise serializers.ValidationError(error_list)
@@ -5458,17 +5464,25 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria
 class InstanceLinkSerializer(BaseSerializer):
     class Meta:
         model = InstanceLink
-        fields = ('id', 'url', 'related', 'source', 'target', 'link_state')
+        fields = ('id', 'related', 'source', 'target', 'target_full_address', 'link_state')
 
     source = serializers.SlugRelatedField(slug_field="hostname", queryset=Instance.objects.all())
-    target = serializers.SlugRelatedField(slug_field="hostname", queryset=Instance.objects.all())
+
+    target = serializers.SerializerMethodField()
+    target_full_address = serializers.SerializerMethodField()
 
     def get_related(self, obj):
         res = super(InstanceLinkSerializer, self).get_related(obj)
         res['source_instance'] = self.reverse('api:instance_detail', kwargs={'pk': obj.source.id})
-        res['target_instance'] = self.reverse('api:instance_detail', kwargs={'pk': obj.target.id})
+        res['target_address'] = self.reverse('api:receptor_address_detail', kwargs={'pk': obj.target.id})
         return res
 
+    def get_target(self, obj):
+        return obj.target.instance.hostname
+
+    def get_target_full_address(self, obj):
+        return obj.target.get_full_address()
+
 
 class InstanceNodeSerializer(BaseSerializer):
     class Meta:
@@ -5476,6 +5490,29 @@ class InstanceNodeSerializer(BaseSerializer):
         fields = ('id', 'hostname', 'node_type', 'node_state', 'enabled')
 
 
+class ReceptorAddressSerializer(BaseSerializer):
+    full_address = serializers.SerializerMethodField()
+
+    class Meta:
+        model = ReceptorAddress
+        fields = (
+            'id',
+            'url',
+            'address',
+            'port',
+            'protocol',
+            'websocket_path',
+            'is_internal',
+            'canonical',
+            'instance',
+            'peers_from_control_nodes',
+            'full_address',
+        )
+
+    def get_full_address(self, obj):
+        return obj.get_full_address()
+
+
 class InstanceSerializer(BaseSerializer):
     show_capabilities = ['edit']
 
@@ -5484,11 +5521,17 @@ class InstanceSerializer(BaseSerializer):
     jobs_running = serializers.IntegerField(help_text=_('Count of jobs in the running or waiting state that are targeted for this instance'), read_only=True)
     jobs_total = serializers.IntegerField(help_text=_('Count of all jobs that target this instance'), read_only=True)
     health_check_pending = serializers.SerializerMethodField()
-    peers = serializers.SlugRelatedField(many=True, required=False, slug_field="hostname", queryset=Instance.objects.all())
+    peers = serializers.PrimaryKeyRelatedField(
+        help_text=_('Primary keys of receptor addresses to peer to.'), many=True, required=False, queryset=ReceptorAddress.objects.all()
+    )
+    reverse_peers = serializers.SerializerMethodField()
+    listener_port = serializers.IntegerField(source='canonical_address_port', required=False, allow_null=True)
+    peers_from_control_nodes = serializers.BooleanField(source='canonical_address_peers_from_control_nodes', required=False)
+    protocol = serializers.SerializerMethodField()
 
     class Meta:
         model = Instance
-        read_only_fields = ('ip_address', 'uuid', 'version')
+        read_only_fields = ('ip_address', 'uuid', 'version', 'managed', 'reverse_peers')
         fields = (
             'id',
             'hostname',
@@ -5519,10 +5562,13 @@ class InstanceSerializer(BaseSerializer):
             'managed_by_policy',
             'node_type',
             'node_state',
+            'managed',
             'ip_address',
-            'listener_port',
             'peers',
+            'reverse_peers',
+            'listener_port',
             'peers_from_control_nodes',
+            'protocol',
         )
         extra_kwargs = {
             'node_type': {'initial': Instance.Types.EXECUTION, 'default': Instance.Types.EXECUTION},
@@ -5544,16 +5590,54 @@ class InstanceSerializer(BaseSerializer):
 
     def get_related(self, obj):
         res = super(InstanceSerializer, self).get_related(obj)
+        res['receptor_addresses'] = self.reverse('api:instance_receptor_addresses_list', kwargs={'pk': obj.pk})
         res['jobs'] = self.reverse('api:instance_unified_jobs_list', kwargs={'pk': obj.pk})
+        res['peers'] = self.reverse('api:instance_peers_list', kwargs={"pk": obj.pk})
         res['instance_groups'] = self.reverse('api:instance_instance_groups_list', kwargs={'pk': obj.pk})
         if obj.node_type in [Instance.Types.EXECUTION, Instance.Types.HOP]:
             res['install_bundle'] = self.reverse('api:instance_install_bundle', kwargs={'pk': obj.pk})
-        res['peers'] = self.reverse('api:instance_peers_list', kwargs={"pk": obj.pk})
         if self.context['request'].user.is_superuser or self.context['request'].user.is_system_auditor:
             if obj.node_type == 'execution':
                 res['health_check'] = self.reverse('api:instance_health_check', kwargs={'pk': obj.pk})
         return res
 
+    def create_or_update(self, validated_data, obj=None, create=True):
+        # create a managed receptor address if listener port is defined
+        port = validated_data.pop('listener_port', -1)
+        peers_from_control_nodes = validated_data.pop('peers_from_control_nodes', -1)
+
+        # delete the receptor address if the port is explicitly set to None
+        if obj and port == None:
+            obj.receptor_addresses.filter(address=obj.hostname).delete()
+
+        if create:
+            instance = super(InstanceSerializer, self).create(validated_data)
+        else:
+            instance = super(InstanceSerializer, self).update(obj, validated_data)
+            instance.refresh_from_db()  # instance canonical address lookup is deferred, so needs to be reloaded
+
+        # only create or update if port is defined in validated_data or already exists in the
+        # canonical address
+        # this prevents creating a receptor address if peers_from_control_nodes is in
+        # validated_data but a port is not set
+        if (port != None and port != -1) or instance.canonical_address_port:
+            kwargs = {}
+            if port != -1:
+                kwargs['port'] = port
+            if peers_from_control_nodes != -1:
+                kwargs['peers_from_control_nodes'] = peers_from_control_nodes
+            if kwargs:
+                kwargs['canonical'] = True
+                instance.receptor_addresses.update_or_create(address=instance.hostname, defaults=kwargs)
+
+        return instance
+
+    def create(self, validated_data):
+        return self.create_or_update(validated_data, create=True)
+
+    def update(self, obj, validated_data):
+        return self.create_or_update(validated_data, obj, create=False)
+
     def get_summary_fields(self, obj):
         summary = super().get_summary_fields(obj)
 
@@ -5563,6 +5647,16 @@ class InstanceSerializer(BaseSerializer):
 
         return summary
 
+    def get_reverse_peers(self, obj):
+        return Instance.objects.prefetch_related('peers').filter(peers__in=obj.receptor_addresses.all()).values_list('id', flat=True)
+
+    def get_protocol(self, obj):
+        # note: don't create a different query for receptor addresses, as this is prefetched on the View for optimization
+        for addr in obj.receptor_addresses.all():
+            if addr.canonical:
+                return addr.protocol
+        return ""
+
     def get_consumed_capacity(self, obj):
         return obj.consumed_capacity
 
@@ -5576,47 +5670,20 @@ class InstanceSerializer(BaseSerializer):
         return obj.health_check_pending
 
     def validate(self, attrs):
-        def get_field_from_model_or_attrs(fd):
-            return attrs.get(fd, self.instance and getattr(self.instance, fd) or None)
-
-        def check_peers_changed():
-            '''
-            return True if
-            - 'peers' in attrs
-            - instance peers matches peers in attrs
-            '''
-            return self.instance and 'peers' in attrs and set(self.instance.peers.all()) != set(attrs['peers'])
+        # Oddly, using 'source' on a DRF field populates attrs with the source name, so we should rename it back
+        if 'canonical_address_port' in attrs:
+            attrs['listener_port'] = attrs.pop('canonical_address_port')
+        if 'canonical_address_peers_from_control_nodes' in attrs:
+            attrs['peers_from_control_nodes'] = attrs.pop('canonical_address_peers_from_control_nodes')
 
         if not self.instance and not settings.IS_K8S:
             raise serializers.ValidationError(_("Can only create instances on Kubernetes or OpenShift."))
 
-        node_type = get_field_from_model_or_attrs("node_type")
-        peers_from_control_nodes = get_field_from_model_or_attrs("peers_from_control_nodes")
-        listener_port = get_field_from_model_or_attrs("listener_port")
-        peers = attrs.get('peers', [])
-
-        if peers_from_control_nodes and node_type not in (Instance.Types.EXECUTION, Instance.Types.HOP):
-            raise serializers.ValidationError(_("peers_from_control_nodes can only be enabled for execution or hop nodes."))
-
-        if node_type in [Instance.Types.CONTROL, Instance.Types.HYBRID]:
-            if check_peers_changed():
-                raise serializers.ValidationError(
-                    _("Setting peers manually for control nodes is not allowed. Enable peers_from_control_nodes on the hop and execution nodes instead.")
-                )
-
-        if not listener_port and peers_from_control_nodes:
-            raise serializers.ValidationError(_("Field listener_port must be a valid integer when peers_from_control_nodes is enabled."))
-
-        if not listener_port and self.instance and self.instance.peers_from.exists():
-            raise serializers.ValidationError(_("Field listener_port must be a valid integer when other nodes peer to it."))
-
-        for peer in peers:
-            if peer.listener_port is None:
-                raise serializers.ValidationError(_("Field listener_port must be set on peer ") + peer.hostname + ".")
-
-        if not settings.IS_K8S:
-            if check_peers_changed():
-                raise serializers.ValidationError(_("Cannot change peers."))
+        # cannot enable peers_from_control_nodes if listener_port is not set
+        if attrs.get('peers_from_control_nodes'):
+            port = attrs.get('listener_port', -1)  # -1 denotes missing, None denotes explicit null
+            if (port is None) or (port == -1 and self.instance and self.instance.canonical_address is None):
+                raise serializers.ValidationError(_("Cannot enable peers_from_control_nodes if listener_port is not set."))
 
         return super().validate(attrs)
 
@@ -5636,8 +5703,8 @@ class InstanceSerializer(BaseSerializer):
                     raise serializers.ValidationError(_("Can only change the state on Kubernetes or OpenShift."))
                 if value != Instance.States.DEPROVISIONING:
                     raise serializers.ValidationError(_("Can only change instances to the 'deprovisioning' state."))
-                if self.instance.node_type not in (Instance.Types.EXECUTION, Instance.Types.HOP):
-                    raise serializers.ValidationError(_("Can only deprovision execution or hop nodes."))
+                if self.instance.managed:
+                    raise serializers.ValidationError(_("Cannot deprovision managed nodes."))
         else:
             if value and value != Instance.States.INSTALLED:
                 raise serializers.ValidationError(_("Can only create instances in the 'installed' state."))
@@ -5656,18 +5723,48 @@ class InstanceSerializer(BaseSerializer):
     def validate_listener_port(self, value):
         """
         Cannot change listener port, unless going from none to integer, and vice versa
+        If instance is managed, cannot change listener port at all
         """
-        if value and self.instance and self.instance.listener_port and self.instance.listener_port != value:
-            raise serializers.ValidationError(_("Cannot change listener port."))
+        if self.instance:
+            canonical_address_port = self.instance.canonical_address_port
+            if value and canonical_address_port and canonical_address_port != value:
+                raise serializers.ValidationError(_("Cannot change listener port."))
+            if self.instance.managed and value != canonical_address_port:
+                raise serializers.ValidationError(_("Cannot change listener port for managed nodes."))
+        return value
+
+    def validate_peers(self, value):
+        # cannot peer to an instance more than once
+        peers_instances = Counter(p.instance_id for p in value)
+        if any(count > 1 for count in peers_instances.values()):
+            raise serializers.ValidationError(_("Cannot peer to the same instance more than once."))
+
+        if self.instance:
+            instance_addresses = set(self.instance.receptor_addresses.all())
+            setting_peers = set(value)
+            peers_changed = set(self.instance.peers.all()) != setting_peers
+
+            if not settings.IS_K8S and peers_changed:
+                raise serializers.ValidationError(_("Cannot change peers."))
+
+            if self.instance.managed and peers_changed:
+                raise serializers.ValidationError(_("Setting peers manually for managed nodes is not allowed."))
+
+            # cannot peer to self
+            if instance_addresses & setting_peers:
+                raise serializers.ValidationError(_("Instance cannot peer to its own address."))
+
+            # cannot peer to an instance that is already peered to this instance
+            if instance_addresses:
+                for p in setting_peers:
+                    if set(p.instance.peers.all()) & instance_addresses:
+                        raise serializers.ValidationError(_(f"Instance {p.instance.hostname} is already peered to this instance."))
 
         return value
 
     def validate_peers_from_control_nodes(self, value):
-        """
-        Can only enable for K8S based deployments
-        """
-        if value and not settings.IS_K8S:
-            raise serializers.ValidationError(_("Can only be enabled on Kubernetes or Openshift."))
+        if self.instance and self.instance.managed and self.instance.canonical_address_peers_from_control_nodes != value:
+            raise serializers.ValidationError(_("Cannot change peers_from_control_nodes for managed nodes."))
 
         return value
 

awx/api/templates/instance_install_bundle/group_vars/all.yml

@@ -17,19 +17,18 @@ custom_worksign_public_keyfile: receptor/work_public_key.pem
 custom_tls_certfile: receptor/tls/receptor.crt
 custom_tls_keyfile: receptor/tls/receptor.key
 custom_ca_certfile: receptor/tls/ca/mesh-CA.crt
-receptor_protocol: 'tcp'
-{% if instance.listener_port %}
+{% if listener_port %}
+receptor_protocol: {{ listener_protocol }}
 receptor_listener: true
-receptor_port: {{ instance.listener_port }}
+receptor_port: {{ listener_port }}
 {% else %}
 receptor_listener: false
 {% endif %}
 {% if peers %}
 receptor_peers:
 {% for peer in peers %}
-  - host: {{ peer.host }}
-    port: {{ peer.port }}
-    protocol: tcp
+  - address: {{ peer.address }}
+    protocol: {{ peer.protocol }}
 {% endfor %}
 {% endif %}
 {% verbatim %}

awx/api/templates/instance_install_bundle/requirements.yml

@@ -1,4 +1,4 @@
 ---
 collections:
   - name: ansible.receptor
-    version: 2.0.2
+    version: 2.0.3

awx/api/urls/instance.py

@@ -10,6 +10,7 @@ from awx.api.views import (
     InstanceInstanceGroupsList,
     InstanceHealthCheck,
     InstancePeersList,
+    InstanceReceptorAddressesList,
 )
 from awx.api.views.instance_install_bundle import InstanceInstallBundle
 
@@ -21,6 +22,7 @@ urls = [
     re_path(r'^(?P<pk>[0-9]+)/instance_groups/$', InstanceInstanceGroupsList.as_view(), name='instance_instance_groups_list'),
     re_path(r'^(?P<pk>[0-9]+)/health_check/$', InstanceHealthCheck.as_view(), name='instance_health_check'),
     re_path(r'^(?P<pk>[0-9]+)/peers/$', InstancePeersList.as_view(), name='instance_peers_list'),
+    re_path(r'^(?P<pk>[0-9]+)/receptor_addresses/$', InstanceReceptorAddressesList.as_view(), name='instance_receptor_addresses_list'),
     re_path(r'^(?P<pk>[0-9]+)/install_bundle/$', InstanceInstallBundle.as_view(), name='instance_install_bundle'),
 ]
 

awx/api/urls/receptor_address.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2017 Ansible, Inc.
+# All Rights Reserved.
+
+from django.urls import re_path
+
+from awx.api.views import (
+    ReceptorAddressesList,
+    ReceptorAddressDetail,
+)
+
+
+urls = [
+    re_path(r'^$', ReceptorAddressesList.as_view(), name='receptor_addresses_list'),
+    re_path(r'^(?P<pk>[0-9]+)/$', ReceptorAddressDetail.as_view(), name='receptor_address_detail'),
+]
+
+__all__ = ['urls']

awx/api/urls/urls.py

@@ -85,6 +85,7 @@ from .oauth2_root import urls as oauth2_root_urls
 from .workflow_approval_template import urls as workflow_approval_template_urls
 from .workflow_approval import urls as workflow_approval_urls
 from .analytics import urls as analytics_urls
+from .receptor_address import urls as receptor_address_urls
 
 v2_urls = [
     re_path(r'^$', ApiV2RootView.as_view(), name='api_v2_root_view'),
@@ -155,6 +156,7 @@ v2_urls = [
     re_path(r'^bulk/host_create/$', BulkHostCreateView.as_view(), name='bulk_host_create'),
     re_path(r'^bulk/host_delete/$', BulkHostDeleteView.as_view(), name='bulk_host_delete'),
     re_path(r'^bulk/job_launch/$', BulkJobLaunchView.as_view(), name='bulk_job_launch'),
+    re_path(r'^receptor_addresses/', include(receptor_address_urls)),
 ]
 
 

awx/api/views/__init__.py

@@ -337,12 +337,20 @@ class InstanceList(ListCreateAPIView):
     search_fields = ('hostname',)
     ordering = ('id',)
 
+    def get_queryset(self):
+        qs = super().get_queryset().prefetch_related('receptor_addresses')
+        return qs
+
 
 class InstanceDetail(RetrieveUpdateAPIView):
     name = _("Instance Detail")
     model = models.Instance
     serializer_class = serializers.InstanceSerializer
 
+    def get_queryset(self):
+        qs = super().get_queryset().prefetch_related('receptor_addresses')
+        return qs
+
     def update_raw_data(self, data):
         # these fields are only valid on creation of an instance, so they unwanted on detail view
         data.pop('node_type', None)
@@ -375,13 +383,37 @@ class InstanceUnifiedJobsList(SubListAPIView):
 
 
 class InstancePeersList(SubListAPIView):
-    name = _("Instance Peers")
+    name = _("Peers")
+    model = models.ReceptorAddress
+    serializer_class = serializers.ReceptorAddressSerializer
     parent_model = models.Instance
-    model = models.Instance
-    serializer_class = serializers.InstanceSerializer
     parent_access = 'read'
-    search_fields = {'hostname'}
     relationship = 'peers'
+    search_fields = ('address',)
+
+
+class InstanceReceptorAddressesList(SubListAPIView):
+    name = _("Receptor Addresses")
+    model = models.ReceptorAddress
+    parent_key = 'instance'
+    parent_model = models.Instance
+    serializer_class = serializers.ReceptorAddressSerializer
+    search_fields = ('address',)
+
+
+class ReceptorAddressesList(ListAPIView):
+    name = _("Receptor Addresses")
+    model = models.ReceptorAddress
+    serializer_class = serializers.ReceptorAddressSerializer
+    search_fields = ('address',)
+
+
+class ReceptorAddressDetail(RetrieveAPIView):
+    name = _("Receptor Address Detail")
+    model = models.ReceptorAddress
+    serializer_class = serializers.ReceptorAddressSerializer
+    parent_model = models.Instance
+    relationship = 'receptor_addresses'
 
 
 class InstanceInstanceGroupsList(InstanceGroupMembershipMixin, SubListCreateAttachDetachAPIView):

awx/api/views/instance_install_bundle.py

@@ -124,10 +124,19 @@ def generate_inventory_yml(instance_obj):
 
 
 def generate_group_vars_all_yml(instance_obj):
+    # get peers
     peers = []
-    for instance in instance_obj.peers.all():
-        peers.append(dict(host=instance.hostname, port=instance.listener_port))
-    all_yaml = render_to_string("instance_install_bundle/group_vars/all.yml", context=dict(instance=instance_obj, peers=peers))
+    for addr in instance_obj.peers.select_related('instance'):
+        peers.append(dict(address=addr.get_full_address(), protocol=addr.protocol))
+    context = dict(instance=instance_obj, peers=peers)
+
+    canonical_addr = instance_obj.canonical_address
+    if canonical_addr:
+        context['listener_port'] = canonical_addr.port
+        protocol = canonical_addr.protocol if canonical_addr.protocol != 'wss' else 'ws'
+        context['listener_protocol'] = protocol
+
+    all_yaml = render_to_string("instance_install_bundle/group_vars/all.yml", context=context)
     # convert consecutive newlines with a single newline
     return re.sub(r'\n+', '\n', all_yaml)
 

awx/api/views/mesh_visualizer.py

@@ -17,7 +17,7 @@ class MeshVisualizer(APIView):
     def get(self, request, format=None):
         data = {
             'nodes': InstanceNodeSerializer(Instance.objects.all(), many=True).data,
-            'links': InstanceLinkSerializer(InstanceLink.objects.select_related('target', 'source'), many=True).data,
+            'links': InstanceLinkSerializer(InstanceLink.objects.select_related('target__instance', 'source'), many=True).data,
         }
 
         return Response(data)

awx/api/views/root.py

@@ -84,6 +84,7 @@ class ApiVersionRootView(APIView):
         data['ping'] = reverse('api:api_v2_ping_view', request=request)
         data['instances'] = reverse('api:instance_list', request=request)
         data['instance_groups'] = reverse('api:instance_group_list', request=request)
+        data['receptor_addresses'] = reverse('api:receptor_addresses_list', request=request)
         data['config'] = reverse('api:api_v2_config_view', request=request)
         data['settings'] = reverse('api:setting_category_list', request=request)
         data['me'] = reverse('api:user_me_list', request=request)

awx/main/access.py

@@ -57,6 +57,7 @@ from awx.main.models import (
     Project,
     ProjectUpdate,
     ProjectUpdateEvent,
+    ReceptorAddress,
     Role,
     Schedule,
     SystemJob,
@@ -2430,6 +2431,29 @@ class InventoryUpdateEventAccess(BaseAccess):
         return False
 
 
+class ReceptorAddressAccess(BaseAccess):
+    """
+    I can see receptor address records whenever I can access the instance
+    """
+
+    model = ReceptorAddress
+
+    def filtered_queryset(self):
+        return self.model.objects.filter(Q(instance__in=Instance.accessible_pk_qs(self.user, 'read_role')))
+
+    @check_superuser
+    def can_add(self, data):
+        return False
+
+    @check_superuser
+    def can_change(self, obj, data):
+        return False
+
+    @check_superuser
+    def can_delete(self, obj):
+        return False
+
+
 class SystemJobEventAccess(BaseAccess):
     """
     I can only see manage System Jobs events if I'm a super user

awx/main/analytics/analytics_tasks.py

@@ -2,7 +2,7 @@
 import logging
 
 # AWX
-from awx.main.analytics.subsystem_metrics import Metrics
+from awx.main.analytics.subsystem_metrics import DispatcherMetrics, CallbackReceiverMetrics
 from awx.main.dispatch.publish import task
 from awx.main.dispatch import get_task_queuename
 
@@ -11,4 +11,5 @@ logger = logging.getLogger('awx.main.scheduler')
 
 @task(queue=get_task_queuename)
 def send_subsystem_metrics():
-    Metrics().send_metrics()
+    DispatcherMetrics().send_metrics()
+    CallbackReceiverMetrics().send_metrics()

awx/main/analytics/subsystem_metrics.py

@@ -1,10 +1,15 @@
+import itertools
 import redis
 import json
 import time
 import logging
 
+import prometheus_client
+from prometheus_client.core import GaugeMetricFamily, HistogramMetricFamily
+from prometheus_client.registry import CollectorRegistry
 from django.conf import settings
-from django.apps import apps
+from django.http import HttpRequest
+from rest_framework.request import Request
 
 from awx.main.consumers import emit_channel_notification
 from awx.main.utils import is_testing
@@ -13,6 +18,30 @@ root_key = settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX
 logger = logging.getLogger('awx.main.analytics')
 
 
+class MetricsNamespace:
+    def __init__(self, namespace):
+        self._namespace = namespace
+
+
+class MetricsServerSettings(MetricsNamespace):
+    def port(self):
+        return settings.METRICS_SUBSYSTEM_CONFIG['server'][self._namespace]['port']
+
+
+class MetricsServer(MetricsServerSettings):
+    def __init__(self, namespace, registry):
+        MetricsNamespace.__init__(self, namespace)
+        self._registry = registry
+
+    def start(self):
+        try:
+            # TODO: addr for ipv6 ?
+            prometheus_client.start_http_server(self.port(), addr='localhost', registry=self._registry)
+        except Exception:
+            logger.error(f"MetricsServer failed to start for service '{self._namespace}.")
+            raise
+
+
 class BaseM:
     def __init__(self, field, help_text):
         self.field = field
@@ -148,76 +177,40 @@ class HistogramM(BaseM):
         return output_text
 
 
-class Metrics:
-    def __init__(self, auto_pipe_execute=False, instance_name=None):
+class Metrics(MetricsNamespace):
+    # metric name, help_text
+    METRICSLIST = []
+    _METRICSLIST = [
+        FloatM('subsystem_metrics_pipe_execute_seconds', 'Time spent saving metrics to redis'),
+        IntM('subsystem_metrics_pipe_execute_calls', 'Number of calls to pipe_execute'),
+        FloatM('subsystem_metrics_send_metrics_seconds', 'Time spent sending metrics to other nodes'),
+    ]
+
+    def __init__(self, namespace, auto_pipe_execute=False, instance_name=None, metrics_have_changed=True, **kwargs):
+        MetricsNamespace.__init__(self, namespace)
+
         self.pipe = redis.Redis.from_url(settings.BROKER_URL).pipeline()
         self.conn = redis.Redis.from_url(settings.BROKER_URL)
         self.last_pipe_execute = time.time()
         # track if metrics have been modified since last saved to redis
         # start with True so that we get an initial save to redis
-        self.metrics_have_changed = True
+        self.metrics_have_changed = metrics_have_changed
         self.pipe_execute_interval = settings.SUBSYSTEM_METRICS_INTERVAL_SAVE_TO_REDIS
         self.send_metrics_interval = settings.SUBSYSTEM_METRICS_INTERVAL_SEND_METRICS
         # auto pipe execute will commit transaction of metric data to redis
         # at a regular interval (pipe_execute_interval). If set to False,
         # the calling function should call .pipe_execute() explicitly
         self.auto_pipe_execute = auto_pipe_execute
-        Instance = apps.get_model('main', 'Instance')
         if instance_name:
             self.instance_name = instance_name
         elif is_testing():
             self.instance_name = "awx_testing"
         else:
-            self.instance_name = Instance.objects.my_hostname()
+            self.instance_name = settings.CLUSTER_HOST_ID  # Same as Instance.objects.my_hostname() BUT we do not need to import Instance
 
-        # metric name, help_text
-        METRICSLIST = [
-            SetIntM('callback_receiver_events_queue_size_redis', 'Current number of events in redis queue'),
-            IntM('callback_receiver_events_popped_redis', 'Number of events popped from redis'),
-            IntM('callback_receiver_events_in_memory', 'Current number of events in memory (in transfer from redis to db)'),
-            IntM('callback_receiver_batch_events_errors', 'Number of times batch insertion failed'),
-            FloatM('callback_receiver_events_insert_db_seconds', 'Total time spent saving events to database'),
-            IntM('callback_receiver_events_insert_db', 'Number of events batch inserted into database'),
-            IntM('callback_receiver_events_broadcast', 'Number of events broadcast to other control plane nodes'),
-            HistogramM(
-                'callback_receiver_batch_events_insert_db', 'Number of events batch inserted into database', settings.SUBSYSTEM_METRICS_BATCH_INSERT_BUCKETS
-            ),
-            SetFloatM('callback_receiver_event_processing_avg_seconds', 'Average processing time per event per callback receiver batch'),
-            FloatM('subsystem_metrics_pipe_execute_seconds', 'Time spent saving metrics to redis'),
-            IntM('subsystem_metrics_pipe_execute_calls', 'Number of calls to pipe_execute'),
-            FloatM('subsystem_metrics_send_metrics_seconds', 'Time spent sending metrics to other nodes'),
-            SetFloatM('task_manager_get_tasks_seconds', 'Time spent in loading tasks from db'),
-            SetFloatM('task_manager_start_task_seconds', 'Time spent starting task'),
-            SetFloatM('task_manager_process_running_tasks_seconds', 'Time spent processing running tasks'),
-            SetFloatM('task_manager_process_pending_tasks_seconds', 'Time spent processing pending tasks'),
-            SetFloatM('task_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
-            IntM('task_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
-            SetFloatM('task_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
-            SetIntM('task_manager_tasks_started', 'Number of tasks started'),
-            SetIntM('task_manager_running_processed', 'Number of running tasks processed'),
-            SetIntM('task_manager_pending_processed', 'Number of pending tasks processed'),
-            SetIntM('task_manager_tasks_blocked', 'Number of tasks blocked from running'),
-            SetFloatM('task_manager_commit_seconds', 'Time spent in db transaction, including on_commit calls'),
-            SetFloatM('dependency_manager_get_tasks_seconds', 'Time spent loading pending tasks from db'),
-            SetFloatM('dependency_manager_generate_dependencies_seconds', 'Time spent generating dependencies for pending tasks'),
-            SetFloatM('dependency_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
-            IntM('dependency_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
-            SetFloatM('dependency_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
-            SetIntM('dependency_manager_pending_processed', 'Number of pending tasks processed'),
-            SetFloatM('workflow_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
-            IntM('workflow_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
-            SetFloatM('workflow_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
-            SetFloatM('workflow_manager_spawn_workflow_graph_jobs_seconds', 'Time spent spawning workflow tasks'),
-            SetFloatM('workflow_manager_get_tasks_seconds', 'Time spent loading workflow tasks from db'),
-            # dispatcher subsystem metrics
-            SetIntM('dispatcher_pool_scale_up_events', 'Number of times local dispatcher scaled up a worker since startup'),
-            SetIntM('dispatcher_pool_active_task_count', 'Number of active tasks in the worker pool when last task was submitted'),
-            SetIntM('dispatcher_pool_max_worker_count', 'Highest number of workers in worker pool in last collection interval, about 20s'),
-            SetFloatM('dispatcher_availability', 'Fraction of time (in last collection interval) dispatcher was able to receive messages'),
-        ]
         # turn metric list into dictionary with the metric name as a key
         self.METRICS = {}
-        for m in METRICSLIST:
+        for m in itertools.chain(self.METRICSLIST, self._METRICSLIST):
             self.METRICS[m.field] = m
 
         # track last time metrics were sent to other nodes
@@ -230,7 +223,7 @@ class Metrics:
             m.reset_value(self.conn)
         self.metrics_have_changed = True
         self.conn.delete(root_key + "_lock")
-        for m in self.conn.scan_iter(root_key + '_instance_*'):
+        for m in self.conn.scan_iter(root_key + '-' + self._namespace + '_instance_*'):
             self.conn.delete(m)
 
     def inc(self, field, value):
@@ -297,7 +290,7 @@ class Metrics:
     def send_metrics(self):
         # more than one thread could be calling this at the same time, so should
         # acquire redis lock before sending metrics
-        lock = self.conn.lock(root_key + '_lock')
+        lock = self.conn.lock(root_key + '-' + self._namespace + '_lock')
         if not lock.acquire(blocking=False):
             return
         try:
@@ -307,9 +300,10 @@ class Metrics:
                 payload = {
                     'instance': self.instance_name,
                     'metrics': serialized_metrics,
+                    'metrics_namespace': self._namespace,
                 }
                 # store the serialized data locally as well, so that load_other_metrics will read it
-                self.conn.set(root_key + '_instance_' + self.instance_name, serialized_metrics)
+                self.conn.set(root_key + '-' + self._namespace + '_instance_' + self.instance_name, serialized_metrics)
                 emit_channel_notification("metrics", payload)
 
                 self.previous_send_metrics.set(current_time)
@@ -331,14 +325,14 @@ class Metrics:
         instances_filter = request.query_params.getlist("node")
         # get a sorted list of instance names
         instance_names = [self.instance_name]
-        for m in self.conn.scan_iter(root_key + '_instance_*'):
+        for m in self.conn.scan_iter(root_key + '-' + self._namespace + '_instance_*'):
             instance_names.append(m.decode('UTF-8').split('_instance_')[1])
         instance_names.sort()
         # load data, including data from the this local instance
         instance_data = {}
         for instance in instance_names:
             if len(instances_filter) == 0 or instance in instances_filter:
-                instance_data_from_redis = self.conn.get(root_key + '_instance_' + instance)
+                instance_data_from_redis = self.conn.get(root_key + '-' + self._namespace + '_instance_' + instance)
                 # data from other instances may not be available. That is OK.
                 if instance_data_from_redis:
                     instance_data[instance] = json.loads(instance_data_from_redis.decode('UTF-8'))
@@ -357,6 +351,120 @@ class Metrics:
         return output_text
 
 
+class DispatcherMetrics(Metrics):
+    METRICSLIST = [
+        SetFloatM('task_manager_get_tasks_seconds', 'Time spent in loading tasks from db'),
+        SetFloatM('task_manager_start_task_seconds', 'Time spent starting task'),
+        SetFloatM('task_manager_process_running_tasks_seconds', 'Time spent processing running tasks'),
+        SetFloatM('task_manager_process_pending_tasks_seconds', 'Time spent processing pending tasks'),
+        SetFloatM('task_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
+        IntM('task_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
+        SetFloatM('task_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
+        SetIntM('task_manager_tasks_started', 'Number of tasks started'),
+        SetIntM('task_manager_running_processed', 'Number of running tasks processed'),
+        SetIntM('task_manager_pending_processed', 'Number of pending tasks processed'),
+        SetIntM('task_manager_tasks_blocked', 'Number of tasks blocked from running'),
+        SetFloatM('task_manager_commit_seconds', 'Time spent in db transaction, including on_commit calls'),
+        SetFloatM('dependency_manager_get_tasks_seconds', 'Time spent loading pending tasks from db'),
+        SetFloatM('dependency_manager_generate_dependencies_seconds', 'Time spent generating dependencies for pending tasks'),
+        SetFloatM('dependency_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
+        IntM('dependency_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
+        SetFloatM('dependency_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
+        SetIntM('dependency_manager_pending_processed', 'Number of pending tasks processed'),
+        SetFloatM('workflow_manager__schedule_seconds', 'Time spent in running the entire _schedule'),
+        IntM('workflow_manager__schedule_calls', 'Number of calls to _schedule, after lock is acquired'),
+        SetFloatM('workflow_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
+        SetFloatM('workflow_manager_spawn_workflow_graph_jobs_seconds', 'Time spent spawning workflow tasks'),
+        SetFloatM('workflow_manager_get_tasks_seconds', 'Time spent loading workflow tasks from db'),
+        # dispatcher subsystem metrics
+        SetIntM('dispatcher_pool_scale_up_events', 'Number of times local dispatcher scaled up a worker since startup'),
+        SetIntM('dispatcher_pool_active_task_count', 'Number of active tasks in the worker pool when last task was submitted'),
+        SetIntM('dispatcher_pool_max_worker_count', 'Highest number of workers in worker pool in last collection interval, about 20s'),
+        SetFloatM('dispatcher_availability', 'Fraction of time (in last collection interval) dispatcher was able to receive messages'),
+    ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(settings.METRICS_SERVICE_DISPATCHER, *args, **kwargs)
+
+
+class CallbackReceiverMetrics(Metrics):
+    METRICSLIST = [
+        SetIntM('callback_receiver_events_queue_size_redis', 'Current number of events in redis queue'),
+        IntM('callback_receiver_events_popped_redis', 'Number of events popped from redis'),
+        IntM('callback_receiver_events_in_memory', 'Current number of events in memory (in transfer from redis to db)'),
+        IntM('callback_receiver_batch_events_errors', 'Number of times batch insertion failed'),
+        FloatM('callback_receiver_events_insert_db_seconds', 'Total time spent saving events to database'),
+        IntM('callback_receiver_events_insert_db', 'Number of events batch inserted into database'),
+        IntM('callback_receiver_events_broadcast', 'Number of events broadcast to other control plane nodes'),
+        HistogramM(
+            'callback_receiver_batch_events_insert_db', 'Number of events batch inserted into database', settings.SUBSYSTEM_METRICS_BATCH_INSERT_BUCKETS
+        ),
+        SetFloatM('callback_receiver_event_processing_avg_seconds', 'Average processing time per event per callback receiver batch'),
+    ]
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(settings.METRICS_SERVICE_CALLBACK_RECEIVER, *args, **kwargs)
+
+
 def metrics(request):
-    m = Metrics()
-    return m.generate_metrics(request)
+    output_text = ''
+    for m in [DispatcherMetrics(), CallbackReceiverMetrics()]:
+        output_text += m.generate_metrics(request)
+    return output_text
+
+
+class CustomToPrometheusMetricsCollector(prometheus_client.registry.Collector):
+    """
+    Takes the metric data from redis -> our custom metric fields -> prometheus
+    library metric fields.
+
+    The plan is to get rid of the use of redis, our custom metric fields, and
+    to switch fully to the prometheus library. At that point, this translation
+    code will be deleted.
+    """
+
+    def __init__(self, metrics_obj, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._metrics = metrics_obj
+
+    def collect(self):
+        my_hostname = settings.CLUSTER_HOST_ID
+
+        instance_data = self._metrics.load_other_metrics(Request(HttpRequest()))
+        if not instance_data:
+            logger.debug(f"No metric data not found in redis for metric namespace '{self._metrics._namespace}'")
+            return None
+
+        host_metrics = instance_data.get(my_hostname)
+        for _, metric in self._metrics.METRICS.items():
+            entry = host_metrics.get(metric.field)
+            if not entry:
+                logger.debug(f"{self._metrics._namespace} metric '{metric.field}' not found in redis data payload {json.dumps(instance_data, indent=2)}")
+                continue
+            if isinstance(metric, HistogramM):
+                buckets = list(zip(metric.buckets, entry['counts']))
+                buckets = [[str(i[0]), str(i[1])] for i in buckets]
+                yield HistogramMetricFamily(metric.field, metric.help_text, buckets=buckets, sum_value=entry['sum'])
+            else:
+                yield GaugeMetricFamily(metric.field, metric.help_text, value=entry)
+
+
+class CallbackReceiverMetricsServer(MetricsServer):
+    def __init__(self):
+        registry = CollectorRegistry(auto_describe=True)
+        registry.register(CustomToPrometheusMetricsCollector(DispatcherMetrics(metrics_have_changed=False)))
+        super().__init__(settings.METRICS_SERVICE_CALLBACK_RECEIVER, registry)
+
+
+class DispatcherMetricsServer(MetricsServer):
+    def __init__(self):
+        registry = CollectorRegistry(auto_describe=True)
+        registry.register(CustomToPrometheusMetricsCollector(CallbackReceiverMetrics(metrics_have_changed=False)))
+        super().__init__(settings.METRICS_SERVICE_DISPATCHER, registry)
+
+
+class WebsocketsMetricsServer(MetricsServer):
+    def __init__(self):
+        registry = CollectorRegistry(auto_describe=True)
+        # registry.register()
+        super().__init__(settings.METRICS_SERVICE_WEBSOCKETS, registry)

awx/main/consumers.py

@@ -106,7 +106,7 @@ class RelayConsumer(AsyncJsonWebsocketConsumer):
         if group == "metrics":
             message = json.loads(message['text'])
             conn = redis.Redis.from_url(settings.BROKER_URL)
-            conn.set(settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX + "_instance_" + message['instance'], message['metrics'])
+            conn.set(settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX + "-" + message['metrics_namespace'] + "_instance_" + message['instance'], message['metrics'])
         else:
             await self.channel_layer.group_send(group, message)
 

awx/main/dispatch/__init__.py

@@ -105,7 +105,11 @@ def create_listener_connection():
     for k, v in settings.LISTENER_DATABASES.get('default', {}).get('OPTIONS', {}).items():
         conf['OPTIONS'][k] = v
 
-    connection_data = f"dbname={conf['NAME']} host={conf['HOST']} user={conf['USER']} password={conf['PASSWORD']} port={conf['PORT']}"
+    # Allow password-less authentication
+    if 'PASSWORD' in conf:
+        conf['OPTIONS']['password'] = conf.pop('PASSWORD')
+
+    connection_data = f"dbname={conf['NAME']} host={conf['HOST']} user={conf['USER']} port={conf['PORT']}"
     return psycopg.connect(connection_data, autocommit=True, **conf['OPTIONS'])
 
 

awx/main/dispatch/worker/base.py

@@ -168,7 +168,7 @@ class AWXConsumerPG(AWXConsumerBase):
         init_time = time.time()
         self.pg_down_time = init_time - self.pg_max_wait  # allow no grace period
         self.last_cleanup = init_time
-        self.subsystem_metrics = s_metrics.Metrics(auto_pipe_execute=False)
+        self.subsystem_metrics = s_metrics.DispatcherMetrics(auto_pipe_execute=False)
         self.last_metrics_gather = init_time
         self.listen_cumulative_time = 0.0
         if schedule:

awx/main/dispatch/worker/callback.py

@@ -72,7 +72,7 @@ class CallbackBrokerWorker(BaseWorker):
     def __init__(self):
         self.buff = {}
         self.redis = redis.Redis.from_url(settings.BROKER_URL)
-        self.subsystem_metrics = s_metrics.Metrics(auto_pipe_execute=False)
+        self.subsystem_metrics = s_metrics.CallbackReceiverMetrics(auto_pipe_execute=False)
         self.queue_pop = 0
         self.queue_name = settings.CALLBACK_QUEUE
         self.prof = AWXProfiler("CallbackBrokerWorker")

awx/main/fields.py

@@ -5,6 +5,7 @@
 import copy
 import json
 import re
+import sys
 import urllib.parse
 
 from jinja2 import sandbox, StrictUndefined
@@ -406,11 +407,13 @@ class SmartFilterField(models.TextField):
         # https://docs.python.org/2/library/stdtypes.html#truth-value-testing
         if not value:
             return None
-        value = urllib.parse.unquote(value)
-        try:
-            SmartFilter().query_from_string(value)
-        except RuntimeError as e:
-            raise models.base.ValidationError(e)
+        # avoid doing too much during migrations
+        if 'migrate' not in sys.argv:
+            value = urllib.parse.unquote(value)
+            try:
+                SmartFilter().query_from_string(value)
+            except RuntimeError as e:
+                raise models.base.ValidationError(e)
         return super(SmartFilterField, self).get_prep_value(value)
 
 

awx/main/management/commands/add_receptor_address.py

@@ -0,0 +1,53 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+from django.core.management.base import BaseCommand
+
+from awx.main.models import Instance, ReceptorAddress
+
+
+def add_address(**kwargs):
+    try:
+        instance = Instance.objects.get(hostname=kwargs.pop('instance'))
+        kwargs['instance'] = instance
+
+        if kwargs.get('canonical') and instance.receptor_addresses.filter(canonical=True).exclude(address=kwargs['address']).exists():
+            print(f"Instance {instance.hostname} already has a canonical address, skipping")
+            return False
+        # if ReceptorAddress already exists with address, just update
+        # otherwise, create new ReceptorAddress
+        addr, _ = ReceptorAddress.objects.update_or_create(address=kwargs.pop('address'), defaults=kwargs)
+        print(f"Successfully added receptor address {addr.get_full_address()}")
+        return True
+    except Exception as e:
+        print(f"Error adding receptor address: {e}")
+        return False
+
+
+class Command(BaseCommand):
+    """
+    Internal controller command.
+    Register receptor address to an already-registered instance.
+    """
+
+    help = "Add receptor address to an instance."
+
+    def add_arguments(self, parser):
+        parser.add_argument('--instance', dest='instance', required=True, type=str, help="Instance hostname this address is added to")
+        parser.add_argument('--address', dest='address', required=True, type=str, help="Receptor address")
+        parser.add_argument('--port', dest='port', type=int, help="Receptor listener port")
+        parser.add_argument('--websocket_path', dest='websocket_path', type=str, default="", help="Path for websockets")
+        parser.add_argument('--is_internal', action='store_true', help="If true, address only resolvable within the Kubernetes cluster")
+        parser.add_argument('--protocol', type=str, default='tcp', choices=['tcp', 'ws', 'wss'], help="Protocol to use for the Receptor listener")
+        parser.add_argument('--canonical', action='store_true', help="If true, address is the canonical address for the instance")
+        parser.add_argument('--peers_from_control_nodes', action='store_true', help="If true, control nodes will peer to this address")
+
+    def handle(self, **options):
+        address_options = {
+            k: options[k]
+            for k in ('instance', 'address', 'port', 'websocket_path', 'is_internal', 'protocol', 'peers_from_control_nodes', 'canonical')
+            if options[k]
+        }
+        changed = add_address(**address_options)
+        if changed:
+            print("(changed: True)")

awx/main/management/commands/list_instances.py

@@ -55,7 +55,7 @@ class Command(BaseCommand):
 
                 capacity = f' capacity={x.capacity}' if x.node_type != 'hop' else ''
                 version = f" version={x.version or '?'}" if x.node_type != 'hop' else ''
-                heartbeat = f' heartbeat="{x.last_seen:%Y-%m-%d %H:%M:%S}"' if x.capacity or x.node_type == 'hop' else ''
+                heartbeat = f' heartbeat="{x.last_seen:%Y-%m-%d %H:%M:%S}"' if x.last_seen else ''
                 print(f'\t{color}{x.hostname}{capacity} node_type={x.node_type}{version}{heartbeat}{end_color}')
 
             print()

awx/main/management/commands/provision_instance.py

@@ -25,20 +25,17 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('--hostname', dest='hostname', type=str, help="Hostname used during provisioning")
-        parser.add_argument('--listener_port', dest='listener_port', type=int, help="Receptor listener port")
         parser.add_argument('--node_type', type=str, default='hybrid', choices=['control', 'execution', 'hop', 'hybrid'], help="Instance Node type")
         parser.add_argument('--uuid', type=str, help="Instance UUID")
 
-    def _register_hostname(self, hostname, node_type, uuid, listener_port):
+    def _register_hostname(self, hostname, node_type, uuid):
         if not hostname:
             if not settings.AWX_AUTO_DEPROVISION_INSTANCES:
                 raise CommandError('Registering with values from settings only intended for use in K8s installs')
 
             from awx.main.management.commands.register_queue import RegisterQueue
 
-            (changed, instance) = Instance.objects.register(
-                ip_address=os.environ.get('MY_POD_IP'), listener_port=listener_port, node_type='control', node_uuid=settings.SYSTEM_UUID
-            )
+            (changed, instance) = Instance.objects.register(ip_address=os.environ.get('MY_POD_IP'), node_type='control', node_uuid=settings.SYSTEM_UUID)
             RegisterQueue(settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME, 100, 0, [], is_container_group=False).register()
             RegisterQueue(
                 settings.DEFAULT_EXECUTION_QUEUE_NAME,
@@ -51,16 +48,17 @@ class Command(BaseCommand):
                 max_concurrent_jobs=settings.DEFAULT_EXECUTION_QUEUE_MAX_CONCURRENT_JOBS,
             ).register()
         else:
-            (changed, instance) = Instance.objects.register(hostname=hostname, node_type=node_type, node_uuid=uuid, listener_port=listener_port)
+            (changed, instance) = Instance.objects.register(hostname=hostname, node_type=node_type, node_uuid=uuid)
         if changed:
             print("Successfully registered instance {}".format(hostname))
         else:
             print("Instance already registered {}".format(instance.hostname))
+
         self.changed = changed
 
     @transaction.atomic
     def handle(self, **options):
         self.changed = False
-        self._register_hostname(options.get('hostname'), options.get('node_type'), options.get('uuid'), options.get('listener_port'))
+        self._register_hostname(options.get('hostname'), options.get('node_type'), options.get('uuid'))
         if self.changed:
             print("(changed: True)")

awx/main/management/commands/register_peers.py

@@ -1,9 +1,7 @@
-import warnings
-
 from django.core.management.base import BaseCommand, CommandError
 from django.db import transaction
 
-from awx.main.models import Instance, InstanceLink
+from awx.main.models import Instance, InstanceLink, ReceptorAddress
 
 
 class Command(BaseCommand):
@@ -28,7 +26,9 @@ class Command(BaseCommand):
 
     def handle(self, **options):
         # provides a mapping of hostname to Instance objects
-        nodes = Instance.objects.in_bulk(field_name='hostname')
+        nodes = Instance.objects.all().in_bulk(field_name='hostname')
+        # provides a mapping of address to ReceptorAddress objects
+        addresses = ReceptorAddress.objects.all().in_bulk(field_name='address')
 
         if options['source'] not in nodes:
             raise CommandError(f"Host {options['source']} is not a registered instance.")
@@ -39,6 +39,14 @@ class Command(BaseCommand):
         if options['exact'] is not None and options['disconnect']:
             raise CommandError("The option --disconnect may not be used with --exact.")
 
+        # make sure each target has a receptor address
+        peers = options['peers'] or []
+        disconnect = options['disconnect'] or []
+        exact = options['exact'] or []
+        for peer in peers + disconnect + exact:
+            if peer not in addresses:
+                raise CommandError(f"Peer {peer} does not have a receptor address.")
+
         # No 1-cycles
         for collection in ('peers', 'disconnect', 'exact'):
             if options[collection] is not None and options['source'] in options[collection]:
@@ -47,9 +55,12 @@ class Command(BaseCommand):
         # No 2-cycles
         if options['peers'] or options['exact'] is not None:
             peers = set(options['peers'] or options['exact'])
-            incoming = set(InstanceLink.objects.filter(target=nodes[options['source']]).values_list('source__hostname', flat=True))
+            if options['source'] in addresses:
+                incoming = set(InstanceLink.objects.filter(target=addresses[options['source']]).values_list('source__hostname', flat=True))
+            else:
+                incoming = set()
             if peers & incoming:
-                warnings.warn(f"Source node {options['source']} should not link to nodes already peering to it: {peers & incoming}.")
+                raise CommandError(f"Source node {options['source']} should not link to nodes already peering to it: {peers & incoming}.")
 
         if options['peers']:
             missing_peers = set(options['peers']) - set(nodes)
@@ -60,7 +71,7 @@ class Command(BaseCommand):
             results = 0
             for target in options['peers']:
                 _, created = InstanceLink.objects.update_or_create(
-                    source=nodes[options['source']], target=nodes[target], defaults={'link_state': InstanceLink.States.ESTABLISHED}
+                    source=nodes[options['source']], target=addresses[target], defaults={'link_state': InstanceLink.States.ESTABLISHED}
                 )
                 if created:
                     results += 1
@@ -70,9 +81,9 @@ class Command(BaseCommand):
         if options['disconnect']:
             results = 0
             for target in options['disconnect']:
-                if target not in nodes:  # Be permissive, the node might have already been de-registered.
+                if target not in addresses:  # Be permissive, the node might have already been de-registered.
                     continue
-                n, _ = InstanceLink.objects.filter(source=nodes[options['source']], target=nodes[target]).delete()
+                n, _ = InstanceLink.objects.filter(source=nodes[options['source']], target=addresses[target]).delete()
                 results += n
 
             print(f"{results} peer links removed from the database.")
@@ -81,11 +92,11 @@ class Command(BaseCommand):
             additions = 0
             with transaction.atomic():
                 peers = set(options['exact'])
-                links = set(InstanceLink.objects.filter(source=nodes[options['source']]).values_list('target__hostname', flat=True))
-                removals, _ = InstanceLink.objects.filter(source=nodes[options['source']], target__hostname__in=links - peers).delete()
+                links = set(InstanceLink.objects.filter(source=nodes[options['source']]).values_list('target__address', flat=True))
+                removals, _ = InstanceLink.objects.filter(source=nodes[options['source']], target__instance__hostname__in=links - peers).delete()
                 for target in peers - links:
                     _, created = InstanceLink.objects.update_or_create(
-                        source=nodes[options['source']], target=nodes[target], defaults={'link_state': InstanceLink.States.ESTABLISHED}
+                        source=nodes[options['source']], target=addresses[target], defaults={'link_state': InstanceLink.States.ESTABLISHED}
                     )
                     if created:
                         additions += 1

awx/main/management/commands/remove_receptor_address.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+
+from django.core.management.base import BaseCommand
+
+from awx.main.models import ReceptorAddress
+
+
+class Command(BaseCommand):
+    """
+    Internal controller command.
+    Delete a receptor address.
+    """
+
+    help = "Add receptor address to an instance."
+
+    def add_arguments(self, parser):
+        parser.add_argument('--address', dest='address', type=str, help="Receptor address to remove")
+
+    def handle(self, **options):
+        deleted = ReceptorAddress.objects.filter(address=options['address']).delete()
+        if deleted[0]:
+            print(f"Successfully removed {options['address']}")
+            print("(changed: True)")
+        else:
+            print(f"Did not remove {options['address']}, not found")

awx/main/management/commands/run_callback_receiver.py

@@ -3,6 +3,7 @@
 
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from awx.main.analytics.subsystem_metrics import CallbackReceiverMetricsServer
 
 from awx.main.dispatch.control import Control
 from awx.main.dispatch.worker import AWXConsumerRedis, CallbackBrokerWorker
@@ -25,6 +26,9 @@ class Command(BaseCommand):
             print(Control('callback_receiver').status())
             return
         consumer = None
+
+        CallbackReceiverMetricsServer().start()
+
         try:
             consumer = AWXConsumerRedis(
                 'callback_receiver',

awx/main/management/commands/run_dispatcher.py

@@ -10,6 +10,7 @@ from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.control import Control
 from awx.main.dispatch.pool import AutoscalePool
 from awx.main.dispatch.worker import AWXConsumerPG, TaskWorker
+from awx.main.analytics.subsystem_metrics import DispatcherMetricsServer
 
 logger = logging.getLogger('awx.main.dispatch')
 
@@ -62,6 +63,8 @@ class Command(BaseCommand):
 
         consumer = None
 
+        DispatcherMetricsServer().start()
+
         try:
             queues = ['tower_broadcast_all', 'tower_settings_change', get_task_queuename()]
             consumer = AWXConsumerPG('dispatcher', TaskWorker(), queues, AutoscalePool(min_workers=4), schedule=settings.CELERYBEAT_SCHEDULE)

awx/main/management/commands/run_wsrelay.py

@@ -16,6 +16,7 @@ from awx.main.analytics.broadcast_websocket import (
     RelayWebsocketStatsManager,
     safe_name,
 )
+from awx.main.analytics.subsystem_metrics import WebsocketsMetricsServer
 from awx.main.wsrelay import WebSocketRelayManager
 
 
@@ -91,6 +92,8 @@ class Command(BaseCommand):
         return host_stats
 
     def handle(self, *arg, **options):
+        WebsocketsMetricsServer().start()
+
         # it's necessary to delay this import in case
         # database migrations are still running
         from awx.main.models.ha import Instance

awx/main/managers.py

@@ -115,7 +115,14 @@ class InstanceManager(models.Manager):
             return node[0]
         raise RuntimeError("No instance found with the current cluster host id")
 
-    def register(self, node_uuid=None, hostname=None, ip_address="", listener_port=None, node_type='hybrid', defaults=None):
+    def register(
+        self,
+        node_uuid=None,
+        hostname=None,
+        ip_address="",
+        node_type='hybrid',
+        defaults=None,
+    ):
         if not hostname:
             hostname = settings.CLUSTER_HOST_ID
 
@@ -161,9 +168,6 @@ class InstanceManager(models.Manager):
                 if instance.node_type != node_type:
                     instance.node_type = node_type
                     update_fields.append('node_type')
-                if instance.listener_port != listener_port:
-                    instance.listener_port = listener_port
-                    update_fields.append('listener_port')
                 if update_fields:
                     instance.save(update_fields=update_fields)
                     return (True, instance)
@@ -174,11 +178,13 @@ class InstanceManager(models.Manager):
             create_defaults = {
                 'node_state': Instance.States.INSTALLED,
                 'capacity': 0,
+                'managed': True,
             }
             if defaults is not None:
                 create_defaults.update(defaults)
             uuid_option = {'uuid': node_uuid if node_uuid is not None else uuid.uuid4()}
             if node_type == 'execution' and 'version' not in create_defaults:
                 create_defaults['version'] = RECEPTOR_PENDING
-            instance = self.create(hostname=hostname, ip_address=ip_address, listener_port=listener_port, node_type=node_type, **create_defaults, **uuid_option)
+            instance = self.create(hostname=hostname, ip_address=ip_address, node_type=node_type, **create_defaults, **uuid_option)
+
         return (True, instance)

awx/main/migrations/0189_inbound_hop_nodes.py

@@ -0,0 +1,150 @@
+# Generated by Django 4.2.6 on 2024-01-19 19:24
+
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def create_receptor_addresses(apps, schema_editor):
+    """
+    If listener_port was defined on an instance, create a receptor address for it
+    """
+    Instance = apps.get_model('main', 'Instance')
+    ReceptorAddress = apps.get_model('main', 'ReceptorAddress')
+    for instance in Instance.objects.exclude(listener_port=None):
+        ReceptorAddress.objects.create(
+            instance=instance,
+            address=instance.hostname,
+            port=instance.listener_port,
+            peers_from_control_nodes=instance.peers_from_control_nodes,
+            protocol='tcp',
+            is_internal=False,
+            canonical=True,
+        )
+
+
+def link_to_receptor_addresses(apps, schema_editor):
+    """
+    Modify each InstanceLink to point to the newly created
+    ReceptorAddresses, using the new target field
+    """
+    InstanceLink = apps.get_model('main', 'InstanceLink')
+    for link in InstanceLink.objects.all():
+        link.target = link.target_old.receptor_addresses.get()
+        link.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('main', '0188_add_bitbucket_dc_webhook'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ReceptorAddress',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('address', models.CharField(help_text='Routable address for this instance.', max_length=255)),
+                (
+                    'port',
+                    models.IntegerField(
+                        default=27199,
+                        help_text='Port for the address.',
+                        validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(65535)],
+                    ),
+                ),
+                ('websocket_path', models.CharField(blank=True, default='', help_text='Websocket path.', max_length=255)),
+                (
+                    'protocol',
+                    models.CharField(
+                        choices=[('tcp', 'TCP'), ('ws', 'WS'), ('wss', 'WSS')],
+                        default='tcp',
+                        help_text="Protocol to use for the Receptor listener, 'tcp', 'wss', or 'ws'.",
+                        max_length=10,
+                    ),
+                ),
+                ('is_internal', models.BooleanField(default=False, help_text='If True, only routable within the Kubernetes cluster.')),
+                ('canonical', models.BooleanField(default=False, help_text='If True, this address is the canonical address for the instance.')),
+                (
+                    'peers_from_control_nodes',
+                    models.BooleanField(default=False, help_text='If True, control plane cluster nodes should automatically peer to it.'),
+                ),
+            ],
+        ),
+        migrations.RemoveConstraint(
+            model_name='instancelink',
+            name='source_and_target_can_not_be_equal',
+        ),
+        migrations.RenameField(
+            model_name='instancelink',
+            old_name='target',
+            new_name='target_old',
+        ),
+        migrations.AlterUniqueTogether(
+            name='instancelink',
+            unique_together=set(),
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='managed',
+            field=models.BooleanField(default=False, editable=False, help_text='If True, this instance is managed by the control plane.'),
+        ),
+        migrations.AlterField(
+            model_name='instancelink',
+            name='source',
+            field=models.ForeignKey(help_text='The source instance of this peer link.', on_delete=django.db.models.deletion.CASCADE, to='main.instance'),
+        ),
+        migrations.AddField(
+            model_name='receptoraddress',
+            name='instance',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='receptor_addresses', to='main.instance'),
+        ),
+        migrations.AddField(
+            model_name='activitystream',
+            name='receptor_address',
+            field=models.ManyToManyField(blank=True, to='main.receptoraddress'),
+        ),
+        migrations.AddConstraint(
+            model_name='receptoraddress',
+            constraint=models.UniqueConstraint(fields=('address',), name='unique_receptor_address', violation_error_message='Receptor address must be unique.'),
+        ),
+        migrations.AddField(
+            model_name='instancelink',
+            name='target',
+            field=models.ForeignKey(
+                help_text='The target receptor address of this peer link.', null=True, on_delete=django.db.models.deletion.CASCADE, to='main.receptoraddress'
+            ),
+        ),
+        migrations.RunPython(create_receptor_addresses),
+        migrations.RunPython(link_to_receptor_addresses),
+        migrations.RemoveField(
+            model_name='instance',
+            name='peers_from_control_nodes',
+        ),
+        migrations.RemoveField(
+            model_name='instance',
+            name='listener_port',
+        ),
+        migrations.RemoveField(
+            model_name='instancelink',
+            name='target_old',
+        ),
+        migrations.AlterField(
+            model_name='instance',
+            name='peers',
+            field=models.ManyToManyField(related_name='peers_from', through='main.InstanceLink', to='main.receptoraddress'),
+        ),
+        migrations.AlterField(
+            model_name='instancelink',
+            name='target',
+            field=models.ForeignKey(
+                help_text='The target receptor address of this peer link.', on_delete=django.db.models.deletion.CASCADE, to='main.receptoraddress'
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name='instancelink',
+            constraint=models.UniqueConstraint(
+                fields=('source', 'target'), name='unique_source_target', violation_error_message='Field source and target must be unique together.'
+            ),
+        ),
+    ]

awx/main/models/__init__.py

@@ -14,6 +14,7 @@ from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutM
 from awx.main.models.organization import Organization, Profile, Team, UserSessionMembership  # noqa
 from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env  # noqa
 from awx.main.models.projects import Project, ProjectUpdate  # noqa
+from awx.main.models.receptor_address import ReceptorAddress  # noqa
 from awx.main.models.inventory import (  # noqa
     CustomInventoryScript,
     Group,

awx/main/models/activity_stream.py

@@ -77,6 +77,7 @@ class ActivityStream(models.Model):
     notification_template = models.ManyToManyField("NotificationTemplate", blank=True)
     notification = models.ManyToManyField("Notification", blank=True)
     label = models.ManyToManyField("Label", blank=True)
+    receptor_address = models.ManyToManyField("ReceptorAddress", blank=True)
     role = models.ManyToManyField("Role", blank=True)
     instance = models.ManyToManyField("Instance", blank=True)
     instance_group = models.ManyToManyField("InstanceGroup", blank=True)

awx/main/models/credential/__init__.py

@@ -1216,6 +1216,26 @@ ManagedCredentialType(
     },
 )
 
+ManagedCredentialType(
+    namespace='terraform',
+    kind='cloud',
+    name=gettext_noop('Terraform backend configuration'),
+    managed=True,
+    inputs={
+        'fields': [
+            {
+                'id': 'configuration',
+                'label': gettext_noop('Backend configuration'),
+                'type': 'string',
+                'secret': True,
+                'multiline': True,
+                'help_text': gettext_noop('Terraform backend config as Hashicorp configuration language.'),
+            },
+        ],
+        'required': ['configuration'],
+    },
+)
+
 
 class CredentialInputSource(PrimordialModel):
     class Meta:

awx/main/models/credential/injectors.py

@@ -122,3 +122,11 @@ def kubernetes_bearer_token(cred, env, private_data_dir):
         env['K8S_AUTH_SSL_CA_CERT'] = to_container_path(path, private_data_dir)
     else:
         env['K8S_AUTH_VERIFY_SSL'] = 'False'
+
+
+def terraform(cred, env, private_data_dir):
+    handle, path = tempfile.mkstemp(dir=os.path.join(private_data_dir, 'env'))
+    with os.fdopen(handle, 'w') as f:
+        os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
+        f.write(cred.get_input('configuration'))
+    env['TF_BACKEND_CONFIG_FILE'] = to_container_path(path, private_data_dir)

awx/main/models/events.py

@@ -124,8 +124,6 @@ class BasePlaybookEvent(CreatedModifiedModel):
         'parent_uuid',
         'start_line',
         'end_line',
-        'host_id',
-        'host_name',
         'verbosity',
     ]
     WRAPUP_EVENT = 'playbook_on_stats'
@@ -473,7 +471,7 @@ class JobEvent(BasePlaybookEvent):
     An event/message logged from the callback when running a job.
     """
 
-    VALID_KEYS = BasePlaybookEvent.VALID_KEYS + ['job_id', 'workflow_job_id', 'job_created']
+    VALID_KEYS = BasePlaybookEvent.VALID_KEYS + ['job_id', 'workflow_job_id', 'job_created', 'host_id', 'host_name']
     JOB_REFERENCE = 'job_id'
 
     objects = DeferJobCreatedManager()

awx/main/models/ha.py

@@ -5,7 +5,7 @@ from decimal import Decimal
 import logging
 import os
 
-from django.core.validators import MinValueValidator, MaxValueValidator
+from django.core.validators import MinValueValidator
 from django.db import models, connection
 from django.db.models.signals import post_save, post_delete
 from django.dispatch import receiver
@@ -34,6 +34,7 @@ from awx.main.models.rbac import (
 from awx.main.models.unified_jobs import UnifiedJob
 from awx.main.utils.common import get_corrected_cpu, get_cpu_effective_capacity, get_corrected_memory, get_mem_effective_capacity
 from awx.main.models.mixins import RelatedJobsMixin, ResourceMixin
+from awx.main.models.receptor_address import ReceptorAddress
 
 # ansible-runner
 from ansible_runner.utils.capacity import get_cpu_count, get_mem_in_bytes
@@ -64,8 +65,19 @@ class HasPolicyEditsMixin(HasEditsMixin):
 
 
 class InstanceLink(BaseModel):
-    source = models.ForeignKey('Instance', on_delete=models.CASCADE, related_name='+')
-    target = models.ForeignKey('Instance', on_delete=models.CASCADE, related_name='reverse_peers')
+    class Meta:
+        ordering = ("id",)
+        # add constraint for source and target to be unique together
+        constraints = [
+            models.UniqueConstraint(
+                fields=["source", "target"],
+                name="unique_source_target",
+                violation_error_message=_("Field source and target must be unique together."),
+            )
+        ]
+
+    source = models.ForeignKey('Instance', on_delete=models.CASCADE, help_text=_("The source instance of this peer link."))
+    target = models.ForeignKey('ReceptorAddress', on_delete=models.CASCADE, help_text=_("The target receptor address of this peer link."))
 
     class States(models.TextChoices):
         ADDING = 'adding', _('Adding')
@@ -76,11 +88,6 @@ class InstanceLink(BaseModel):
         choices=States.choices, default=States.ADDING, max_length=16, help_text=_("Indicates the current life cycle stage of this peer link.")
     )
 
-    class Meta:
-        unique_together = ('source', 'target')
-        ordering = ("id",)
-        constraints = [models.CheckConstraint(check=~models.Q(source=models.F('target')), name='source_and_target_can_not_be_equal')]
-
 
 class Instance(HasPolicyEditsMixin, BaseModel):
     """A model representing an AWX instance running against this database."""
@@ -110,6 +117,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         default="",
         max_length=50,
     )
+
     # Auto-fields, implementation is different from BaseModel
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
@@ -185,16 +193,9 @@ class Instance(HasPolicyEditsMixin, BaseModel):
     node_state = models.CharField(
         choices=States.choices, default=States.READY, max_length=16, help_text=_("Indicates the current life cycle stage of this instance.")
     )
-    listener_port = models.PositiveIntegerField(
-        blank=True,
-        null=True,
-        default=None,
-        validators=[MinValueValidator(1024), MaxValueValidator(65535)],
-        help_text=_("Port that Receptor will listen for incoming connections on."),
-    )
 
-    peers = models.ManyToManyField('self', symmetrical=False, through=InstanceLink, through_fields=('source', 'target'), related_name='peers_from')
-    peers_from_control_nodes = models.BooleanField(default=False, help_text=_("If True, control plane cluster nodes should automatically peer to it."))
+    managed = models.BooleanField(help_text=_("If True, this instance is managed by the control plane."), default=False, editable=False)
+    peers = models.ManyToManyField('ReceptorAddress', through=InstanceLink, through_fields=('source', 'target'), related_name='peers_from')
 
     POLICY_FIELDS = frozenset(('managed_by_policy', 'hostname', 'capacity_adjustment'))
 
@@ -241,6 +242,26 @@ class Instance(HasPolicyEditsMixin, BaseModel):
             return True
         return self.health_check_started > self.last_health_check
 
+    @property
+    def canonical_address(self):
+        return self.receptor_addresses.filter(canonical=True).first()
+
+    @property
+    def canonical_address_port(self):
+        # note: don't create a different query for receptor addresses, as this is prefetched on the View for optimization
+        for addr in self.receptor_addresses.all():
+            if addr.canonical:
+                return addr.port
+        return None
+
+    @property
+    def canonical_address_peers_from_control_nodes(self):
+        # note: don't create a different query for receptor addresses, as this is prefetched on the View for optimization
+        for addr in self.receptor_addresses.all():
+            if addr.canonical:
+                return addr.peers_from_control_nodes
+        return False
+
     def get_cleanup_task_kwargs(self, **kwargs):
         """
         Produce options to use for the command: ansible-runner worker cleanup
@@ -501,6 +522,35 @@ def schedule_write_receptor_config(broadcast=True):
             write_receptor_config()  # just run locally
 
 
+@receiver(post_save, sender=ReceptorAddress)
+def receptor_address_saved(sender, instance, **kwargs):
+    from awx.main.signals import disable_activity_stream
+
+    address = instance
+
+    control_instances = set(Instance.objects.filter(node_type__in=[Instance.Types.CONTROL, Instance.Types.HYBRID]))
+    if address.peers_from_control_nodes:
+        # if control_instances is not a subset of current peers of address, then
+        # that means we need to add some InstanceLinks
+        if not control_instances <= set(address.peers_from.all()):
+            with disable_activity_stream():
+                for control_instance in control_instances:
+                    InstanceLink.objects.update_or_create(source=control_instance, target=address)
+                schedule_write_receptor_config()
+    else:
+        if address.peers_from.exists():
+            with disable_activity_stream():
+                address.peers_from.remove(*control_instances)
+                schedule_write_receptor_config()
+
+
+@receiver(post_delete, sender=ReceptorAddress)
+def receptor_address_deleted(sender, instance, **kwargs):
+    address = instance
+    if address.peers_from_control_nodes:
+        schedule_write_receptor_config()
+
+
 @receiver(post_save, sender=Instance)
 def on_instance_saved(sender, instance, created=False, raw=False, **kwargs):
     '''
@@ -511,11 +561,14 @@ def on_instance_saved(sender, instance, created=False, raw=False, **kwargs):
     2. a node changes its value of peers_from_control_nodes
     3. a new control node comes online and has instances to peer to
     '''
+    from awx.main.signals import disable_activity_stream
+
     if created and settings.IS_K8S and instance.node_type in [Instance.Types.CONTROL, Instance.Types.HYBRID]:
-        inst = Instance.objects.filter(peers_from_control_nodes=True)
-        if set(instance.peers.all()) != set(inst):
-            instance.peers.set(inst)
-            schedule_write_receptor_config(broadcast=False)
+        peers_addresses = ReceptorAddress.objects.filter(peers_from_control_nodes=True)
+        if peers_addresses.exists():
+            with disable_activity_stream():
+                instance.peers.add(*peers_addresses)
+                schedule_write_receptor_config(broadcast=False)
 
     if settings.IS_K8S and instance.node_type in [Instance.Types.HOP, Instance.Types.EXECUTION]:
         if instance.node_state == Instance.States.DEPROVISIONING:
@@ -524,16 +577,6 @@ def on_instance_saved(sender, instance, created=False, raw=False, **kwargs):
             # wait for jobs on the node to complete, then delete the
             # node and kick off write_receptor_config
             connection.on_commit(lambda: remove_deprovisioned_node.apply_async([instance.hostname]))
-        else:
-            control_instances = set(Instance.objects.filter(node_type__in=[Instance.Types.CONTROL, Instance.Types.HYBRID]))
-            if instance.peers_from_control_nodes:
-                if (control_instances & set(instance.peers_from.all())) != set(control_instances):
-                    instance.peers_from.add(*control_instances)
-                    schedule_write_receptor_config()  # keep method separate to make pytest mocking easier
-            else:
-                if set(control_instances) & set(instance.peers_from.all()):
-                    instance.peers_from.remove(*control_instances)
-                    schedule_write_receptor_config()
 
     if created or instance.has_policy_changes():
         schedule_policy_task()
@@ -548,8 +591,6 @@ def on_instance_group_deleted(sender, instance, using, **kwargs):
 @receiver(post_delete, sender=Instance)
 def on_instance_deleted(sender, instance, using, **kwargs):
     schedule_policy_task()
-    if settings.IS_K8S and instance.node_type in (Instance.Types.EXECUTION, Instance.Types.HOP) and instance.peers_from_control_nodes:
-        schedule_write_receptor_config()
 
 
 class UnifiedJobTemplateInstanceGroupMembership(models.Model):

awx/main/models/receptor_address.py

@@ -0,0 +1,67 @@
+from django.db import models
+from django.core.validators import MinValueValidator, MaxValueValidator
+from django.utils.translation import gettext_lazy as _
+from awx.api.versioning import reverse
+
+
+class Protocols(models.TextChoices):
+    TCP = 'tcp', 'TCP'
+    WS = 'ws', 'WS'
+    WSS = 'wss', 'WSS'
+
+
+class ReceptorAddress(models.Model):
+    class Meta:
+        app_label = 'main'
+        constraints = [
+            models.UniqueConstraint(
+                fields=["address"],
+                name="unique_receptor_address",
+                violation_error_message=_("Receptor address must be unique."),
+            )
+        ]
+
+    address = models.CharField(help_text=_("Routable address for this instance."), max_length=255)
+    port = models.IntegerField(help_text=_("Port for the address."), default=27199, validators=[MinValueValidator(0), MaxValueValidator(65535)])
+    websocket_path = models.CharField(help_text=_("Websocket path."), max_length=255, default="", blank=True)
+    protocol = models.CharField(
+        help_text=_("Protocol to use for the Receptor listener, 'tcp', 'wss', or 'ws'."), max_length=10, default=Protocols.TCP, choices=Protocols.choices
+    )
+    is_internal = models.BooleanField(help_text=_("If True, only routable within the Kubernetes cluster."), default=False)
+    canonical = models.BooleanField(help_text=_("If True, this address is the canonical address for the instance."), default=False)
+    peers_from_control_nodes = models.BooleanField(help_text=_("If True, control plane cluster nodes should automatically peer to it."), default=False)
+    instance = models.ForeignKey(
+        'Instance',
+        related_name='receptor_addresses',
+        on_delete=models.CASCADE,
+        null=False,
+    )
+
+    def __str__(self):
+        return self.get_full_address()
+
+    def get_full_address(self):
+        scheme = ""
+        path = ""
+        port = ""
+        if self.protocol == "ws":
+            scheme = "wss://"
+
+        if self.protocol == "ws" and self.websocket_path:
+            path = f"/{self.websocket_path}"
+
+        if self.port:
+            port = f":{self.port}"
+
+        return f"{scheme}{self.address}{port}{path}"
+
+    def get_peer_type(self):
+        if self.protocol == 'tcp':
+            return 'tcp-peer'
+        elif self.protocol in ['ws', 'wss']:
+            return 'ws-peer'
+        else:
+            return None
+
+    def get_absolute_url(self, request=None):
+        return reverse('api:receptor_address_detail', kwargs={'pk': self.pk}, request=request)

awx/main/routing.py

@@ -4,9 +4,10 @@ import logging
 from django.conf import settings
 from django.urls import re_path
 
-from channels.auth import AuthMiddlewareStack
 from channels.routing import ProtocolTypeRouter, URLRouter
 
+from ansible_base.lib.channels.middleware import DrfAuthMiddlewareStack
+
 from . import consumers
 
 
@@ -27,12 +28,13 @@ class AWXProtocolTypeRouter(ProtocolTypeRouter):
 
 
 websocket_urlpatterns = [
+    re_path(r'api/websocket/$', consumers.EventConsumer.as_asgi()),
     re_path(r'websocket/$', consumers.EventConsumer.as_asgi()),
     re_path(r'websocket/relay/$', consumers.RelayConsumer.as_asgi()),
 ]
 
 application = AWXProtocolTypeRouter(
     {
-        'websocket': AuthMiddlewareStack(URLRouter(websocket_urlpatterns)),
+        'websocket': DrfAuthMiddlewareStack(URLRouter(websocket_urlpatterns)),
     }
 )

awx/main/scheduler/task_manager.py

@@ -68,7 +68,7 @@ class TaskBase:
         # initialize each metric to 0 and force metric_has_changed to true. This
         # ensures each task manager metric will be overridden when pipe_execute
         # is called later.
-        self.subsystem_metrics = s_metrics.Metrics(auto_pipe_execute=False)
+        self.subsystem_metrics = s_metrics.DispatcherMetrics(auto_pipe_execute=False)
         self.start_time = time.time()
 
         # We want to avoid calling settings in loops, so cache these settings at init time
@@ -105,7 +105,7 @@ class TaskBase:
             try:
                 # increment task_manager_schedule_calls regardless if the other
                 # metrics are recorded
-                s_metrics.Metrics(auto_pipe_execute=True).inc(f"{self.prefix}__schedule_calls", 1)
+                s_metrics.DispatcherMetrics(auto_pipe_execute=True).inc(f"{self.prefix}__schedule_calls", 1)
                 # Only record metrics if the last time recording was more
                 # than SUBSYSTEM_METRICS_TASK_MANAGER_RECORD_INTERVAL ago.
                 # Prevents a short-duration task manager that runs directly after a

awx/main/tasks/callback.py

@@ -95,17 +95,17 @@ class RunnerCallback:
         if self.parent_workflow_job_id:
             event_data['workflow_job_id'] = self.parent_workflow_job_id
         event_data['job_created'] = self.job_created
-        if self.host_map:
-            host = event_data.get('event_data', {}).get('host', '').strip()
-            if host:
-                event_data['host_name'] = host
-                if host in self.host_map:
-                    event_data['host_id'] = self.host_map[host]
-            else:
-                event_data['host_name'] = ''
-                event_data['host_id'] = ''
-            if event_data.get('event') == 'playbook_on_stats':
-                event_data['host_map'] = self.host_map
+
+        host = event_data.get('event_data', {}).get('host', '').strip()
+        if host:
+            event_data['host_name'] = host
+            if host in self.host_map:
+                event_data['host_id'] = self.host_map[host]
+        else:
+            event_data['host_name'] = ''
+            event_data['host_id'] = ''
+        if event_data.get('event') == 'playbook_on_stats':
+            event_data['host_map'] = self.host_map
 
         if isinstance(self, RunnerCallbackForProjectUpdate):
             # need a better way to have this check.

awx/main/tasks/receptor.py

@@ -27,7 +27,7 @@ from awx.main.utils.common import (
 )
 from awx.main.constants import MAX_ISOLATED_PATH_COLON_DELIMITER
 from awx.main.tasks.signals import signal_state, signal_callback, SignalExit
-from awx.main.models import Instance, InstanceLink, UnifiedJob
+from awx.main.models import Instance, InstanceLink, UnifiedJob, ReceptorAddress
 from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.publish import task
 from awx.main.utils.pglock import advisory_lock
@@ -676,36 +676,44 @@ RECEPTOR_CONFIG_STARTER = (
 )
 
 
-def should_update_config(instances):
+def should_update_config(new_config):
     '''
     checks that the list of instances matches the list of
     tcp-peers in the config
     '''
-    current_config = read_receptor_config()  # this gets receptor conf lock
-    current_peers = []
-    for config_entry in current_config:
-        for key, value in config_entry.items():
-            if key.endswith('-peer'):
-                current_peers.append(value['address'])
-    intended_peers = [f"{i.hostname}:{i.listener_port}" for i in instances]
-    logger.debug(f"Peers current {current_peers} intended {intended_peers}")
-    if set(current_peers) == set(intended_peers):
-        return False  # config file is already update to date
 
-    return True
+    current_config = read_receptor_config()  # this gets receptor conf lock
+    for config_entry in current_config:
+        if config_entry not in new_config:
+            logger.warning(f"{config_entry} should not be in receptor config. Updating.")
+            return True
+    for config_entry in new_config:
+        if config_entry not in current_config:
+            logger.warning(f"{config_entry} missing from receptor config. Updating.")
+            return True
+
+    return False
 
 
 def generate_config_data():
     # returns two values
     #   receptor config - based on current database peers
     #   should_update   - If True, receptor_config differs from the receptor conf file on disk
-    instances = Instance.objects.filter(node_type__in=(Instance.Types.EXECUTION, Instance.Types.HOP), peers_from_control_nodes=True)
+    addresses = ReceptorAddress.objects.filter(peers_from_control_nodes=True)
 
     receptor_config = list(RECEPTOR_CONFIG_STARTER)
-    for instance in instances:
-        peer = {'tcp-peer': {'address': f'{instance.hostname}:{instance.listener_port}', 'tls': 'tlsclient'}}
-        receptor_config.append(peer)
-    should_update = should_update_config(instances)
+    for address in addresses:
+        if address.get_peer_type():
+            peer = {
+                f'{address.get_peer_type()}': {
+                    'address': f'{address.get_full_address()}',
+                    'tls': 'tlsclient',
+                }
+            }
+            receptor_config.append(peer)
+        else:
+            logger.warning(f"Receptor address {address} has unsupported peer type, skipping.")
+    should_update = should_update_config(receptor_config)
     return receptor_config, should_update
 
 
@@ -747,14 +755,13 @@ def write_receptor_config():
             with lock:
                 with open(__RECEPTOR_CONF, 'w') as file:
                     yaml.dump(receptor_config, file, default_flow_style=False)
-
             reload_receptor()
 
 
 @task(queue=get_task_queuename)
 def remove_deprovisioned_node(hostname):
     InstanceLink.objects.filter(source__hostname=hostname).update(link_state=InstanceLink.States.REMOVING)
-    InstanceLink.objects.filter(target__hostname=hostname).update(link_state=InstanceLink.States.REMOVING)
+    InstanceLink.objects.filter(target__instance__hostname=hostname).update(link_state=InstanceLink.States.REMOVING)
 
     node_jobs = UnifiedJob.objects.filter(
         execution_node=hostname,

awx/main/tasks/system.py

@@ -62,7 +62,7 @@ from awx.main.tasks.receptor import get_receptor_ctl, worker_info, worker_cleanu
 from awx.main.consumers import emit_channel_notification
 from awx.main import analytics
 from awx.conf import settings_registry
-from awx.main.analytics.subsystem_metrics import Metrics
+from awx.main.analytics.subsystem_metrics import DispatcherMetrics
 
 from rest_framework.exceptions import PermissionDenied
 
@@ -113,7 +113,7 @@ def dispatch_startup():
     cluster_node_heartbeat()
     reaper.startup_reaping()
     reaper.reap_waiting(grace_period=0)
-    m = Metrics()
+    m = DispatcherMetrics()
     m.reset_values()
 
 
@@ -495,7 +495,7 @@ def inspect_established_receptor_connections(mesh_status):
     update_links = []
     for link in all_links:
         if link.link_state != InstanceLink.States.REMOVING:
-            if link.target.hostname in active_receptor_conns.get(link.source.hostname, {}):
+            if link.target.instance.hostname in active_receptor_conns.get(link.source.hostname, {}):
                 if link.link_state is not InstanceLink.States.ESTABLISHED:
                     link.link_state = InstanceLink.States.ESTABLISHED
                     update_links.append(link)

awx/main/tests/functional/api/test_instance_peers.py

@@ -1,19 +1,16 @@
 import pytest
 import yaml
-import itertools
 from unittest import mock
 
-from django.db.utils import IntegrityError
-
 from awx.api.versioning import reverse
-from awx.main.models import Instance
+from awx.main.models import Instance, ReceptorAddress
 from awx.api.views.instance_install_bundle import generate_group_vars_all_yml
 
 
 def has_peer(group_vars, peer):
     peers = group_vars.get('receptor_peers', [])
     for p in peers:
-        if f"{p['host']}:{p['port']}" == peer:
+        if p['address'] == peer:
             return True
     return False
 
@@ -24,119 +21,314 @@ class TestPeers:
     def configure_settings(self, settings):
         settings.IS_K8S = True
 
-    @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
-    def test_prevent_peering_to_self(self, node_type):
+    @pytest.mark.parametrize('node_type', ['hop', 'execution'])
+    def test_peering_to_self(self, node_type, admin_user, patch):
         """
         cannot peer to self
         """
-        control_instance = Instance.objects.create(hostname='abc', node_type=node_type)
-        with pytest.raises(IntegrityError):
-            control_instance.peers.add(control_instance)
+        instance = Instance.objects.create(hostname='abc', node_type=node_type)
+        addr = ReceptorAddress.objects.create(instance=instance, address='abc', canonical=True)
+        resp = patch(
+            url=reverse('api:instance_detail', kwargs={'pk': instance.pk}),
+            data={"hostname": "abc", "node_type": node_type, "peers": [addr.id]},
+            user=admin_user,
+            expect=400,
+        )
+        assert 'Instance cannot peer to its own address.' in str(resp.data)
 
     @pytest.mark.parametrize('node_type', ['control', 'hybrid', 'hop', 'execution'])
     def test_creating_node(self, node_type, admin_user, post):
         """
         can only add hop and execution nodes via API
         """
-        post(
+        resp = post(
             url=reverse('api:instance_list'),
             data={"hostname": "abc", "node_type": node_type},
             user=admin_user,
             expect=400 if node_type in ['control', 'hybrid'] else 201,
         )
+        if resp.status_code == 400:
+            assert 'Can only create execution or hop nodes.' in str(resp.data)
 
     def test_changing_node_type(self, admin_user, patch):
         """
         cannot change node type
         """
         hop = Instance.objects.create(hostname='abc', node_type="hop")
-        patch(
+        resp = patch(
             url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
             data={"node_type": "execution"},
             user=admin_user,
             expect=400,
         )
+        assert 'Cannot change node type.' in str(resp.data)
 
-    @pytest.mark.parametrize('node_type', ['hop', 'execution'])
-    def test_listener_port_null(self, node_type, admin_user, post):
-        """
-        listener_port can be None
-        """
-        post(
-            url=reverse('api:instance_list'),
-            data={"hostname": "abc", "node_type": node_type, "listener_port": None},
+    @pytest.mark.parametrize(
+        'payload_port, payload_peers_from, initial_port, initial_peers_from',
+        [
+            (-1, -1, None, None),
+            (-1, -1, 27199, False),
+            (-1, -1, 27199, True),
+            (None, -1, None, None),
+            (None, False, None, None),
+            (-1, False, None, None),
+            (27199, True, 27199, True),
+            (27199, False, 27199, False),
+            (27199, -1, 27199, True),
+            (27199, -1, 27199, False),
+            (-1, True, 27199, True),
+            (-1, False, 27199, False),
+        ],
+    )
+    def test_no_op(self, payload_port, payload_peers_from, initial_port, initial_peers_from, admin_user, patch):
+        node = Instance.objects.create(hostname='abc', node_type='hop')
+        if initial_port is not None:
+            ReceptorAddress.objects.create(address=node.hostname, port=initial_port, canonical=True, peers_from_control_nodes=initial_peers_from, instance=node)
+
+            assert ReceptorAddress.objects.filter(instance=node).count() == 1
+        else:
+            assert ReceptorAddress.objects.filter(instance=node).count() == 0
+
+        data = {'enabled': True}  # Just to have something to post.
+        if payload_port != -1:
+            data['listener_port'] = payload_port
+        if payload_peers_from != -1:
+            data['peers_from_control_nodes'] = payload_peers_from
+
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': node.pk}),
+            data=data,
             user=admin_user,
-            expect=201,
+            expect=200,
         )
 
-    @pytest.mark.parametrize('node_type, allowed', [('control', False), ('hybrid', False), ('hop', True), ('execution', True)])
-    def test_peers_from_control_nodes_allowed(self, node_type, allowed, post, admin_user):
-        """
-        only hop and execution nodes can have peers_from_control_nodes set to True
-        """
-        post(
-            url=reverse('api:instance_list'),
-            data={"hostname": "abc", "peers_from_control_nodes": True, "node_type": node_type, "listener_port": 6789},
+        assert ReceptorAddress.objects.filter(instance=node).count() == (0 if initial_port is None else 1)
+        if initial_port is not None:
+            ra = ReceptorAddress.objects.get(instance=node, canonical=True)
+            assert ra.port == initial_port
+            assert ra.peers_from_control_nodes == initial_peers_from
+
+    @pytest.mark.parametrize(
+        'payload_port, payload_peers_from',
+        [
+            (27199, True),
+            (27199, False),
+            (27199, -1),
+        ],
+    )
+    def test_creates_canonical_address(self, payload_port, payload_peers_from, admin_user, patch):
+        node = Instance.objects.create(hostname='abc', node_type='hop')
+        assert ReceptorAddress.objects.filter(instance=node).count() == 0
+
+        data = {'enabled': True}  # Just to have something to post.
+        if payload_port != -1:
+            data['listener_port'] = payload_port
+        if payload_peers_from != -1:
+            data['peers_from_control_nodes'] = payload_peers_from
+
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': node.pk}),
+            data=data,
             user=admin_user,
-            expect=201 if allowed else 400,
+            expect=200,
         )
 
-    def test_listener_port_is_required(self, admin_user, post):
-        """
-        if adding instance to peers list, that instance must have listener_port set
-        """
-        Instance.objects.create(hostname='abc', node_type="hop", listener_port=None)
-        post(
-            url=reverse('api:instance_list'),
-            data={"hostname": "ex", "peers_from_control_nodes": False, "node_type": "execution", "listener_port": None, "peers": ["abc"]},
+        assert ReceptorAddress.objects.filter(instance=node).count() == 1
+        ra = ReceptorAddress.objects.get(instance=node, canonical=True)
+        assert ra.port == payload_port
+        assert ra.peers_from_control_nodes == (payload_peers_from if payload_peers_from != -1 else False)
+
+    @pytest.mark.parametrize(
+        'payload_port, payload_peers_from, initial_port, initial_peers_from',
+        [
+            (None, False, 27199, True),
+            (None, -1, 27199, True),
+            (None, False, 27199, False),
+            (None, -1, 27199, False),
+        ],
+    )
+    def test_deletes_canonical_address(self, payload_port, payload_peers_from, initial_port, initial_peers_from, admin_user, patch):
+        node = Instance.objects.create(hostname='abc', node_type='hop')
+        ReceptorAddress.objects.create(address=node.hostname, port=initial_port, canonical=True, peers_from_control_nodes=initial_peers_from, instance=node)
+
+        assert ReceptorAddress.objects.filter(instance=node).count() == 1
+
+        data = {'enabled': True}  # Just to have something to post.
+        if payload_port != -1:
+            data['listener_port'] = payload_port
+        if payload_peers_from != -1:
+            data['peers_from_control_nodes'] = payload_peers_from
+
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': node.pk}),
+            data=data,
+            user=admin_user,
+            expect=200,
+        )
+
+        assert ReceptorAddress.objects.filter(instance=node).count() == 0
+
+    @pytest.mark.parametrize(
+        'payload_port, payload_peers_from, initial_port, initial_peers_from',
+        [
+            (27199, True, 27199, False),
+            (27199, False, 27199, True),
+            (-1, True, 27199, False),
+            (-1, False, 27199, True),
+        ],
+    )
+    def test_updates_canonical_address(self, payload_port, payload_peers_from, initial_port, initial_peers_from, admin_user, patch):
+        node = Instance.objects.create(hostname='abc', node_type='hop')
+        ReceptorAddress.objects.create(address=node.hostname, port=initial_port, canonical=True, peers_from_control_nodes=initial_peers_from, instance=node)
+
+        assert ReceptorAddress.objects.filter(instance=node).count() == 1
+
+        data = {'enabled': True}  # Just to have something to post.
+        if payload_port != -1:
+            data['listener_port'] = payload_port
+        if payload_peers_from != -1:
+            data['peers_from_control_nodes'] = payload_peers_from
+
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': node.pk}),
+            data=data,
+            user=admin_user,
+            expect=200,
+        )
+
+        assert ReceptorAddress.objects.filter(instance=node).count() == 1
+        ra = ReceptorAddress.objects.get(instance=node, canonical=True)
+        assert ra.port == initial_port  # At the present time, changing ports is not allowed
+        assert ra.peers_from_control_nodes == payload_peers_from
+
+    @pytest.mark.parametrize(
+        'payload_port, payload_peers_from, initial_port, initial_peers_from, error_msg',
+        [
+            (-1, True, None, None, "Cannot enable peers_from_control_nodes"),
+            (None, True, None, None, "Cannot enable peers_from_control_nodes"),
+            (None, True, 21799, True, "Cannot enable peers_from_control_nodes"),
+            (None, True, 21799, False, "Cannot enable peers_from_control_nodes"),
+            (21800, -1, 21799, True, "Cannot change listener port"),
+            (21800, True, 21799, True, "Cannot change listener port"),
+            (21800, False, 21799, True, "Cannot change listener port"),
+            (21800, -1, 21799, False, "Cannot change listener port"),
+            (21800, True, 21799, False, "Cannot change listener port"),
+            (21800, False, 21799, False, "Cannot change listener port"),
+        ],
+    )
+    def test_canonical_address_validation_error(self, payload_port, payload_peers_from, initial_port, initial_peers_from, error_msg, admin_user, patch):
+        node = Instance.objects.create(hostname='abc', node_type='hop')
+        if initial_port is not None:
+            ReceptorAddress.objects.create(address=node.hostname, port=initial_port, canonical=True, peers_from_control_nodes=initial_peers_from, instance=node)
+
+            assert ReceptorAddress.objects.filter(instance=node).count() == 1
+        else:
+            assert ReceptorAddress.objects.filter(instance=node).count() == 0
+
+        data = {'enabled': True}  # Just to have something to post.
+        if payload_port != -1:
+            data['listener_port'] = payload_port
+        if payload_peers_from != -1:
+            data['peers_from_control_nodes'] = payload_peers_from
+
+        resp = patch(
+            url=reverse('api:instance_detail', kwargs={'pk': node.pk}),
+            data=data,
             user=admin_user,
             expect=400,
         )
 
-    def test_peers_from_control_nodes_listener_port_enabled(self, admin_user, post):
+        assert error_msg in str(resp.data)
+
+    def test_changing_managed_listener_port(self, admin_user, patch):
         """
-        if peers_from_control_nodes is True, listener_port must an integer
-        Assert that all other combinations are allowed
+        if instance is managed, cannot change listener port at all
         """
-        for index, item in enumerate(itertools.product(['hop', 'execution'], [True, False], [None, 6789])):
-            node_type, peers_from, listener_port = item
-            # only disallowed case is when peers_from is True and listener port is None
-            disallowed = peers_from and not listener_port
-            post(
-                url=reverse('api:instance_list'),
-                data={"hostname": f"abc{index}", "peers_from_control_nodes": peers_from, "node_type": node_type, "listener_port": listener_port},
-                user=admin_user,
-                expect=400 if disallowed else 201,
-            )
+        hop = Instance.objects.create(hostname='abc', node_type="hop", managed=True)
+        resp = patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
+            data={"listener_port": 5678},
+            user=admin_user,
+            expect=400,  # cannot set port
+        )
+        assert 'Cannot change listener port for managed nodes.' in str(resp.data)
+        ReceptorAddress.objects.create(instance=hop, address='hop', port=27199, canonical=True)
+        resp = patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
+            data={"listener_port": None},
+            user=admin_user,
+            expect=400,  # cannot unset port
+        )
+        assert 'Cannot change listener port for managed nodes.' in str(resp.data)
+
+    def test_bidirectional_peering(self, admin_user, patch):
+        """
+        cannot peer to node that is already to peered to it
+        if A -> B, then disallow B -> A
+        """
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
+        hop1addr = ReceptorAddress.objects.create(instance=hop1, address='hop1', canonical=True)
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
+        hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', canonical=True)
+        hop1.peers.add(hop2addr)
+        resp = patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop2.pk}),
+            data={"peers": [hop1addr.id]},
+            user=admin_user,
+            expect=400,
+        )
+        assert 'Instance hop1 is already peered to this instance.' in str(resp.data)
+
+    def test_multiple_peers_same_instance(self, admin_user, patch):
+        """
+        cannot peer to more than one address of the same instance
+        """
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
+        hop1addr1 = ReceptorAddress.objects.create(instance=hop1, address='hop1', canonical=True)
+        hop1addr2 = ReceptorAddress.objects.create(instance=hop1, address='hop1alternate')
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
+        resp = patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop2.pk}),
+            data={"peers": [hop1addr1.id, hop1addr2.id]},
+            user=admin_user,
+            expect=400,
+        )
+        assert 'Cannot peer to the same instance more than once.' in str(resp.data)
 
     @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
-    def test_disallow_modifying_peers_control_nodes(self, node_type, admin_user, patch):
+    def test_changing_peers_control_nodes(self, node_type, admin_user, patch):
         """
         for control nodes, peers field should not be
         modified directly via patch.
         """
-        control = Instance.objects.create(hostname='abc', node_type=node_type)
-        hop1 = Instance.objects.create(hostname='hop1', node_type='hop', peers_from_control_nodes=True, listener_port=6789)
-        hop2 = Instance.objects.create(hostname='hop2', node_type='hop', peers_from_control_nodes=False, listener_port=6789)
-        assert [hop1] == list(control.peers.all())  # only hop1 should be peered
-        patch(
+        control = Instance.objects.create(hostname='abc', node_type=node_type, managed=True)
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
+        hop1addr = ReceptorAddress.objects.create(instance=hop1, address='hop1', peers_from_control_nodes=True, canonical=True)
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
+        hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', canonical=True)
+        assert [hop1addr] == list(control.peers.all())  # only hop1addr should be peered
+        resp = patch(
             url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
-            data={"peers": ["hop2"]},
+            data={"peers": [hop2addr.id]},
             user=admin_user,
-            expect=400,  # cannot add peers directly
+            expect=400,  # cannot add peers manually
         )
+        assert 'Setting peers manually for managed nodes is not allowed.' in str(resp.data)
+
         patch(
             url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
-            data={"peers": ["hop1"]},
+            data={"peers": [hop1addr.id]},
             user=admin_user,
             expect=200,  # patching with current peers list should be okay
         )
-        patch(
+        resp = patch(
             url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
             data={"peers": []},
             user=admin_user,
             expect=400,  # cannot remove peers directly
         )
+        assert 'Setting peers manually for managed nodes is not allowed.' in str(resp.data)
+
         patch(
             url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
             data={},
@@ -148,23 +340,25 @@ class TestPeers:
             url=reverse('api:instance_detail', kwargs={'pk': hop2.pk}),
             data={"peers_from_control_nodes": True},
             user=admin_user,
-            expect=200,  # patching without data should be fine too
+            expect=200,
         )
-        assert {hop1, hop2} == set(control.peers.all())  # hop1 and hop2 should now be peered from control node
+        assert {hop1addr, hop2addr} == set(control.peers.all())  # hop1 and hop2 should now be peered from control node
 
-    def test_disallow_changing_hostname(self, admin_user, patch):
+    def test_changing_hostname(self, admin_user, patch):
         """
         cannot change hostname
         """
         hop = Instance.objects.create(hostname='hop', node_type='hop')
-        patch(
+        resp = patch(
             url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
             data={"hostname": "hop2"},
             user=admin_user,
             expect=400,
         )
 
-    def test_disallow_changing_node_state(self, admin_user, patch):
+        assert 'Cannot change hostname.' in str(resp.data)
+
+    def test_changing_node_state(self, admin_user, patch):
         """
         only allow setting to deprovisioning
         """
@@ -175,12 +369,54 @@ class TestPeers:
             user=admin_user,
             expect=200,
         )
-        patch(
+        resp = patch(
             url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
             data={"node_state": "ready"},
             user=admin_user,
             expect=400,
         )
+        assert "Can only change instances to the 'deprovisioning' state." in str(resp.data)
+
+    def test_changing_managed_node_state(self, admin_user, patch):
+        """
+        cannot change node state of managed node
+        """
+        hop = Instance.objects.create(hostname='hop', node_type='hop', managed=True)
+        resp = patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
+            data={"node_state": "deprovisioning"},
+            user=admin_user,
+            expect=400,
+        )
+
+        assert 'Cannot deprovision managed nodes.' in str(resp.data)
+
+    def test_changing_managed_peers_from_control_nodes(self, admin_user, patch):
+        """
+        cannot change peers_from_control_nodes of managed node
+        """
+        hop = Instance.objects.create(hostname='hop', node_type='hop', managed=True)
+        ReceptorAddress.objects.create(instance=hop, address='hop', peers_from_control_nodes=True, canonical=True)
+        resp = patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
+            data={"peers_from_control_nodes": False},
+            user=admin_user,
+            expect=400,
+        )
+
+        assert 'Cannot change peers_from_control_nodes for managed nodes.' in str(resp.data)
+
+        hop.peers_from_control_nodes = False
+        hop.save()
+
+        resp = patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
+            data={"peers_from_control_nodes": False},
+            user=admin_user,
+            expect=400,
+        )
+
+        assert 'Cannot change peers_from_control_nodes for managed nodes.' in str(resp.data)
 
     @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
     def test_control_node_automatically_peers(self, node_type):
@@ -191,9 +427,10 @@ class TestPeers:
         peer to hop should be removed if hop is deleted
         """
 
-        hop = Instance.objects.create(hostname='hop', node_type='hop', peers_from_control_nodes=True, listener_port=6789)
+        hop = Instance.objects.create(hostname='hop', node_type='hop')
+        hopaddr = ReceptorAddress.objects.create(instance=hop, address='hop', peers_from_control_nodes=True, canonical=True)
         control = Instance.objects.create(hostname='abc', node_type=node_type)
-        assert hop in control.peers.all()
+        assert hopaddr in control.peers.all()
         hop.delete()
         assert not control.peers.exists()
 
@@ -203,26 +440,50 @@ class TestPeers:
         if a new node comes online, other peer relationships should
         remain intact
         """
-        hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=True)
-        hop2 = Instance.objects.create(hostname='hop2', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
-        hop1.peers.add(hop2)
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
+        hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', canonical=True)
+        hop1.peers.add(hop2addr)
 
         # a control node is added
-        Instance.objects.create(hostname='control', node_type=node_type, listener_port=None)
+        Instance.objects.create(hostname='control', node_type=node_type)
 
         assert hop1.peers.exists()
 
-    def test_group_vars(self, get, admin_user):
+    def test_reverse_peers(self, admin_user, get):
+        """
+        if hop1 peers to hop2, hop1 should
+        be in hop2's reverse_peers list
+        """
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
+        hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', canonical=True)
+        hop1.peers.add(hop2addr)
+
+        resp = get(
+            url=reverse('api:instance_detail', kwargs={'pk': hop2.pk}),
+            user=admin_user,
+            expect=200,
+        )
+
+        assert hop1.pk in resp.data['reverse_peers']
+
+    def test_group_vars(self):
         """
         control > hop1 > hop2 < execution
         """
-        control = Instance.objects.create(hostname='control', node_type='control', listener_port=None)
-        hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=True)
-        hop2 = Instance.objects.create(hostname='hop2', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
-        execution = Instance.objects.create(hostname='execution', node_type='execution', listener_port=6789)
+        control = Instance.objects.create(hostname='control', node_type='control')
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
+        ReceptorAddress.objects.create(instance=hop1, address='hop1', peers_from_control_nodes=True, port=6789, canonical=True)
 
-        execution.peers.add(hop2)
-        hop1.peers.add(hop2)
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
+        hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', peers_from_control_nodes=False, port=6789, canonical=True)
+
+        execution = Instance.objects.create(hostname='execution', node_type='execution')
+        ReceptorAddress.objects.create(instance=execution, address='execution', peers_from_control_nodes=False, port=6789, canonical=True)
+
+        execution.peers.add(hop2addr)
+        hop1.peers.add(hop2addr)
 
         control_vars = yaml.safe_load(generate_group_vars_all_yml(control))
         hop1_vars = yaml.safe_load(generate_group_vars_all_yml(hop1))
@@ -265,13 +526,15 @@ class TestPeers:
             control = Instance.objects.create(hostname='control1', node_type='control')
             write_method.assert_not_called()
 
-            # new hop node with peers_from_control_nodes False (no)
-            hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
+            # new address with peers_from_control_nodes False (no)
+            hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
+            hop1addr = ReceptorAddress.objects.create(instance=hop1, address='hop1', peers_from_control_nodes=False, canonical=True)
             hop1.delete()
             write_method.assert_not_called()
 
-            # new hop node with peers_from_control_nodes True (yes)
-            hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=True)
+            # new address with peers_from_control_nodes True (yes)
+            hop1 = Instance.objects.create(hostname='hop1', node_type='hop')
+            hop1addr = ReceptorAddress.objects.create(instance=hop1, address='hop1', peers_from_control_nodes=True, canonical=True)
             write_method.assert_called()
             write_method.reset_mock()
 
@@ -280,20 +543,21 @@ class TestPeers:
             write_method.assert_called()
             write_method.reset_mock()
 
-            # new hop node with peers_from_control_nodes False and peered to another hop node (no)
-            hop2 = Instance.objects.create(hostname='hop2', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
-            hop2.peers.add(hop1)
+            # new address with peers_from_control_nodes False and peered to another hop node (no)
+            hop2 = Instance.objects.create(hostname='hop2', node_type='hop')
+            ReceptorAddress.objects.create(instance=hop2, address='hop2', peers_from_control_nodes=False, canonical=True)
+            hop2.peers.add(hop1addr)
             hop2.delete()
             write_method.assert_not_called()
 
             # changing peers_from_control_nodes to False (yes)
-            hop1.peers_from_control_nodes = False
-            hop1.save()
+            hop1addr.peers_from_control_nodes = False
+            hop1addr.save()
             write_method.assert_called()
             write_method.reset_mock()
 
-            # deleting hop node that has peers_from_control_nodes to False (no)
-            hop1.delete()
+            # deleting address that has peers_from_control_nodes to False (no)
+            hop1.delete()  # cascade deletes to hop1addr
             write_method.assert_not_called()
 
             # deleting control nodes (no)
@@ -315,8 +579,8 @@ class TestPeers:
 
             # not peered, so config file should not be updated
             for i in range(3):
-                Instance.objects.create(hostname=f"exNo-{i}", node_type='execution', listener_port=6789, peers_from_control_nodes=False)
-
+                inst = Instance.objects.create(hostname=f"exNo-{i}", node_type='execution')
+                ReceptorAddress.objects.create(instance=inst, address=f"exNo-{i}", port=6789, peers_from_control_nodes=False, canonical=True)
             _, should_update = generate_config_data()
             assert not should_update
 
@@ -324,11 +588,13 @@ class TestPeers:
             expected_peers = []
             for i in range(3):
                 expected_peers.append(f"hop-{i}:6789")
-                Instance.objects.create(hostname=f"hop-{i}", node_type='hop', listener_port=6789, peers_from_control_nodes=True)
+                inst = Instance.objects.create(hostname=f"hop-{i}", node_type='hop')
+                ReceptorAddress.objects.create(instance=inst, address=f"hop-{i}", port=6789, peers_from_control_nodes=True, canonical=True)
 
             for i in range(3):
                 expected_peers.append(f"exYes-{i}:6789")
-                Instance.objects.create(hostname=f"exYes-{i}", node_type='execution', listener_port=6789, peers_from_control_nodes=True)
+                inst = Instance.objects.create(hostname=f"exYes-{i}", node_type='execution')
+                ReceptorAddress.objects.create(instance=inst, address=f"exYes-{i}", port=6789, peers_from_control_nodes=True, canonical=True)
 
             new_config, should_update = generate_config_data()
             assert should_update

awx/main/tests/functional/test_credential.py

@@ -101,6 +101,7 @@ def test_default_cred_types():
             'satellite6',
             'scm',
             'ssh',
+            'terraform',
             'thycotic_dsv',
             'thycotic_tss',
             'vault',

awx/main/tests/functional/test_linkstate.py

@@ -0,0 +1,30 @@
+import pytest
+
+from awx.main.models import Instance, ReceptorAddress, InstanceLink
+from awx.main.tasks.system import inspect_established_receptor_connections
+
+
+@pytest.mark.django_db
+class TestLinkState:
+    @pytest.fixture(autouse=True)
+    def configure_settings(self, settings):
+        settings.IS_K8S = True
+
+    def test_inspect_established_receptor_connections(self):
+        '''
+        Change link state from ADDING to ESTABLISHED
+        if the receptor status KnownConnectionCosts field
+        has an entry for the source and target node.
+        '''
+        hop1 = Instance.objects.create(hostname='hop1')
+        hop2 = Instance.objects.create(hostname='hop2')
+        hop2addr = ReceptorAddress.objects.create(instance=hop2, address='hop2', port=5678)
+        InstanceLink.objects.create(source=hop1, target=hop2addr, link_state=InstanceLink.States.ADDING)
+
+        # calling with empty KnownConnectionCosts should not change the link state
+        inspect_established_receptor_connections({"KnownConnectionCosts": {}})
+        assert InstanceLink.objects.get(source=hop1, target=hop2addr).link_state == InstanceLink.States.ADDING
+
+        mesh_state = {"KnownConnectionCosts": {"hop1": {"hop2": 1}}}
+        inspect_established_receptor_connections(mesh_state)
+        assert InstanceLink.objects.get(source=hop1, target=hop2addr).link_state == InstanceLink.States.ESTABLISHED

awx/main/tests/functional/test_migrations.py

@@ -42,3 +42,29 @@ class TestMigrationSmoke:
         final_state = migrator.apply_tested_migration(final_migration)
         Instance = final_state.apps.get_model('main', 'Instance')
         assert Instance.objects.filter(hostname='foobar').count() == 1
+
+    def test_receptor_address(self, migrator):
+        old_state = migrator.apply_initial_migration(('main', '0188_add_bitbucket_dc_webhook'))
+        Instance = old_state.apps.get_model('main', 'Instance')
+        for i in range(3):
+            Instance.objects.create(hostname=f'foobar{i}', node_type='hop')
+        foo = Instance.objects.create(hostname='foo', node_type='execution', listener_port=1234)
+        bar = Instance.objects.create(hostname='bar', node_type='execution', listener_port=None)
+        bar.peers.add(foo)
+
+        new_state = migrator.apply_tested_migration(
+            ('main', '0189_inbound_hop_nodes'),
+        )
+        Instance = new_state.apps.get_model('main', 'Instance')
+        ReceptorAddress = new_state.apps.get_model('main', 'ReceptorAddress')
+
+        # We can now test how our migration worked, new field is there:
+        assert ReceptorAddress.objects.filter(address='foo', port=1234).count() == 1
+        assert not ReceptorAddress.objects.filter(address='bar').exists()
+
+        bar = Instance.objects.get(hostname='bar')
+        fooaddr = ReceptorAddress.objects.get(address='foo')
+
+        bar_peers = bar.peers.all()
+        assert len(bar_peers) == 1
+        assert fooaddr in bar_peers

awx/main/tests/unit/models/test_receptor_address.py

@@ -0,0 +1,32 @@
+from awx.main.models import ReceptorAddress
+import pytest
+
+ReceptorAddress()
+
+
+@pytest.mark.parametrize(
+    'address, protocol, port, websocket_path, expected',
+    [
+        ('foo', 'tcp', 27199, '', 'foo:27199'),
+        ('bar', 'ws', 6789, '', 'wss://bar:6789'),
+        ('mal', 'ws', 6789, 'path', 'wss://mal:6789/path'),
+        ('example.com', 'ws', 443, 'path', 'wss://example.com:443/path'),
+    ],
+)
+def test_get_full_address(address, protocol, port, websocket_path, expected):
+    receptor_address = ReceptorAddress(address=address, protocol=protocol, port=port, websocket_path=websocket_path)
+    assert receptor_address.get_full_address() == expected
+
+
+@pytest.mark.parametrize(
+    'protocol, expected',
+    [
+        ('tcp', 'tcp-peer'),
+        ('ws', 'ws-peer'),
+        ('wss', 'ws-peer'),
+        ('foo', None),
+    ],
+)
+def test_get_peer_type(protocol, expected):
+    receptor_address = ReceptorAddress(protocol=protocol)
+    assert receptor_address.get_peer_type() == expected

awx/main/tests/unit/test_tasks.py

@@ -1085,6 +1085,27 @@ class TestJobCredentials(TestJobExecution):
         assert open(env['ANSIBLE_NET_SSH_KEYFILE'], 'r').read() == self.EXAMPLE_PRIVATE_KEY
         assert safe_env['ANSIBLE_NET_PASSWORD'] == HIDDEN_PASSWORD
 
+    def test_terraform_cloud_credentials(self, job, private_data_dir, mock_me):
+        terraform = CredentialType.defaults['terraform']()
+        hcl_config = '''
+        backend "s3" {
+            bucket = "s3_sample_bucket"
+            key    = "/tf_state/"
+            region = "us-east-1"
+        }
+        '''
+        credential = Credential(pk=1, credential_type=terraform, inputs={'configuration': hcl_config})
+        credential.inputs['configuration'] = encrypt_field(credential, 'configuration')
+        job.credentials.add(credential)
+
+        env = {}
+        safe_env = {}
+        credential.credential_type.inject_credential(credential, env, safe_env, [], private_data_dir)
+
+        local_path = to_host_path(env['TF_BACKEND_CONFIG_FILE'], private_data_dir)
+        config = open(local_path, 'r').read()
+        assert config == hcl_config
+
     def test_custom_environment_injectors_with_jinja_syntax_error(self, private_data_dir, mock_me):
         some_cloud = CredentialType(
             kind='cloud',

awx/main/wsrelay.py

@@ -20,7 +20,6 @@ from awx.main.analytics.broadcast_websocket import (
     RelayWebsocketStats,
     RelayWebsocketStatsManager,
 )
-import awx.main.analytics.subsystem_metrics as s_metrics
 
 logger = logging.getLogger('awx.main.wsrelay')
 
@@ -54,7 +53,6 @@ class WebsocketRelayConnection:
         self.protocol = protocol
         self.verify_ssl = verify_ssl
         self.channel_layer = None
-        self.subsystem_metrics = s_metrics.Metrics(instance_name=name)
         self.producers = dict()
         self.connected = False
 

awx/settings/defaults.py

@@ -1076,6 +1076,35 @@ HOST_METRIC_SUMMARY_TASK_LAST_TS = None
 HOST_METRIC_SUMMARY_TASK_INTERVAL = 7  # days
 
 
+# TODO: cmeyers, replace with with register pattern
+# The register pattern is particularly nice for this because we need
+# to know the process to start the thread that will be the server.
+# The registration location should be the same location as we would
+# call MetricsServer.start()
+# Note: if we don't get to this TODO, then at least create constants
+# for the services strings below.
+# TODO: cmeyers, break this out into a separate django app so other
+# projects can take advantage.
+
+METRICS_SERVICE_CALLBACK_RECEIVER = 'callback_receiver'
+METRICS_SERVICE_DISPATCHER = 'dispatcher'
+METRICS_SERVICE_WEBSOCKETS = 'websockets'
+
+METRICS_SUBSYSTEM_CONFIG = {
+    'server': {
+        METRICS_SERVICE_CALLBACK_RECEIVER: {
+            'port': 8014,
+        },
+        METRICS_SERVICE_DISPATCHER: {
+            'port': 8015,
+        },
+        METRICS_SERVICE_WEBSOCKETS: {
+            'port': 8016,
+        },
+    }
+}
+
+
 # django-ansible-base
 ANSIBLE_BASE_TEAM_MODEL = 'main.Team'
 ANSIBLE_BASE_ORGANIZATION_MODEL = 'main.Organization'

awx/settings/development.py

@@ -21,7 +21,7 @@ from split_settings.tools import optional, include
 from .defaults import *  # NOQA
 
 # awx-manage shell_plus --notebook
-NOTEBOOK_ARGUMENTS = ['--NotebookApp.token=', '--ip', '0.0.0.0', '--port', '8888', '--allow-root', '--no-browser']
+NOTEBOOK_ARGUMENTS = ['--NotebookApp.token=', '--ip', '0.0.0.0', '--port', '9888', '--allow-root', '--no-browser']
 
 # print SQL queries in shell_plus
 SHELL_PLUS_PRINT_SQL = False

awx/ui/src/api/index.js

@@ -29,6 +29,7 @@ import Notifications from './models/Notifications';
 import Organizations from './models/Organizations';
 import ProjectUpdates from './models/ProjectUpdates';
 import Projects from './models/Projects';
+import ReceptorAddresses from './models/Receptor';
 import Roles from './models/Roles';
 import Root from './models/Root';
 import Schedules from './models/Schedules';
@@ -79,6 +80,7 @@ const NotificationsAPI = new Notifications();
 const OrganizationsAPI = new Organizations();
 const ProjectUpdatesAPI = new ProjectUpdates();
 const ProjectsAPI = new Projects();
+const ReceptorAPI = new ReceptorAddresses();
 const RolesAPI = new Roles();
 const RootAPI = new Root();
 const SchedulesAPI = new Schedules();
@@ -130,6 +132,7 @@ export {
   OrganizationsAPI,
   ProjectUpdatesAPI,
   ProjectsAPI,
+  ReceptorAPI,
   RolesAPI,
   RootAPI,
   SchedulesAPI,

awx/ui/src/api/models/Instances.js

@@ -8,6 +8,7 @@ class Instances extends Base {
     this.readHealthCheckDetail = this.readHealthCheckDetail.bind(this);
     this.healthCheck = this.healthCheck.bind(this);
     this.readInstanceGroup = this.readInstanceGroup.bind(this);
+    this.readReceptorAddresses = this.readReceptorAddresses.bind(this);
     this.deprovisionInstance = this.deprovisionInstance.bind(this);
   }
 
@@ -27,6 +28,17 @@ class Instances extends Base {
     return this.http.get(`${this.baseUrl}${instanceId}/instance_groups/`);
   }
 
+  readReceptorAddresses(instanceId) {
+    return this.http.get(`${this.baseUrl}${instanceId}/receptor_addresses/`);
+  }
+
+  updateReceptorAddresses(instanceId, data) {
+    return this.http.post(
+      `${this.baseUrl}${instanceId}/receptor_addresses/`,
+      data
+    );
+  }
+
   deprovisionInstance(instanceId) {
     return this.http.patch(`${this.baseUrl}${instanceId}/`, {
       node_state: 'deprovisioning',

awx/ui/src/api/models/Receptor.js

@@ -0,0 +1,14 @@
+import Base from '../Base';
+
+class ReceptorAddresses extends Base {
+  constructor(http) {
+    super(http);
+    this.baseUrl = 'api/v2/receptor_addresses/';
+  }
+
+  updateReceptorAddresses(instanceId, data) {
+    return this.http.post(`${this.baseUrl}`, data);
+  }
+}
+
+export default ReceptorAddresses;

awx/ui/src/screens/Instances/Instance.js

@@ -12,6 +12,7 @@ import { SettingsAPI } from 'api';
 import ContentLoading from 'components/ContentLoading';
 import InstanceDetail from './InstanceDetail';
 import InstancePeerList from './InstancePeers';
+import InstanceListenerAddressList from './InstanceListenerAddressList';
 
 function Instance({ setBreadcrumb }) {
   const { me } = useConfig();
@@ -54,7 +55,12 @@ function Instance({ setBreadcrumb }) {
   }, [request]);
 
   if (isK8s) {
-    tabsArray.push({ name: t`Peers`, link: `${match.url}/peers`, id: 1 });
+    tabsArray.push({
+      name: t`Listener Addresses`,
+      link: `${match.url}/listener_addresses`,
+      id: 1,
+    });
+    tabsArray.push({ name: t`Peers`, link: `${match.url}/peers`, id: 2 });
   }
   if (isLoading) {
     return <ContentLoading />;
@@ -72,6 +78,14 @@ function Instance({ setBreadcrumb }) {
           <Route path="/instances/:id/details" key="details">
             <InstanceDetail isK8s={isK8s} setBreadcrumb={setBreadcrumb} />
           </Route>
+          {isK8s && (
+            <Route
+              path="/instances/:id/listener_addresses"
+              key="listener_addresses"
+            >
+              <InstanceListenerAddressList setBreadcrumb={setBreadcrumb} />
+            </Route>
+          )}
           {isK8s && (
             <Route path="/instances/:id/peers" key="peers">
               <InstancePeerList setBreadcrumb={setBreadcrumb} />

awx/ui/src/screens/Instances/InstanceAdd/InstanceAdd.js

@@ -9,6 +9,10 @@ function InstanceAdd() {
   const [formError, setFormError] = useState();
   const handleSubmit = async (values) => {
     try {
+      if (values.listener_port === undefined) {
+        values.listener_port = null;
+      }
+
       const {
         data: { id },
       } = await InstancesAPI.create(values);

awx/ui/src/screens/Instances/InstanceAdd/InstanceAdd.test.js

@@ -36,6 +36,7 @@ describe('<InstanceAdd />', () => {
       });
     });
     expect(InstancesAPI.create).toHaveBeenCalledWith({
+      listener_port: null, // injected if listener_port is not set
       node_type: 'hop',
     });
     expect(history.location.pathname).toBe('/instances/13/details');

awx/ui/src/screens/Instances/InstanceDetail/InstanceDetail.js

@@ -183,6 +183,7 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
   }
   const isHopNode = instance.node_type === 'hop';
   const isExecutionNode = instance.node_type === 'execution';
+  const isManaged = instance.managed;
 
   return (
     <>
@@ -208,33 +209,31 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
           <Detail label={t`Node Type`} value={instance.node_type} />
           <Detail label={t`Host`} value={instance.ip_address} />
           <Detail label={t`Listener Port`} value={instance.listener_port} />
+          {!isManaged && instance.related?.install_bundle && (
+            <Detail
+              label={t`Install Bundle`}
+              value={
+                <Tooltip content={t`Click to download bundle`}>
+                  <Button
+                    component="a"
+                    isSmall
+                    href={`${instance.related?.install_bundle}`}
+                    target="_blank"
+                    variant="secondary"
+                    dataCy="install-bundle-download-button"
+                    rel="noopener noreferrer"
+                  >
+                    <DownloadIcon />
+                  </Button>
+                </Tooltip>
+              }
+            />
+          )}
           {(isExecutionNode || isHopNode) && (
-            <>
-              {instance.related?.install_bundle && (
-                <Detail
-                  label={t`Install Bundle`}
-                  value={
-                    <Tooltip content={t`Click to download bundle`}>
-                      <Button
-                        component="a"
-                        isSmall
-                        href={`${instance.related?.install_bundle}`}
-                        target="_blank"
-                        variant="secondary"
-                        dataCy="install-bundle-download-button"
-                        rel="noopener noreferrer"
-                      >
-                        <DownloadIcon />
-                      </Button>
-                    </Tooltip>
-                  }
-                />
-              )}
-              <Detail
-                label={t`Peers from control nodes`}
-                value={instance.peers_from_control_nodes ? t`On` : t`Off`}
-              />
-            </>
+            <Detail
+              label={t`Peers from control nodes`}
+              value={instance.peers_from_control_nodes ? t`On` : t`Off`}
+            />
           )}
           {!isHopNode && (
             <>
@@ -294,7 +293,9 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
                         value={instance.capacity_adjustment}
                         onChange={handleChangeValue}
                         isDisabled={
-                          !config?.me?.is_superuser || !instance.enabled
+                          !config?.me?.is_superuser ||
+                          !instance.enabled ||
+                          !isManaged
                         }
                         data-cy="slider"
                       />
@@ -338,31 +339,31 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
           )}
         </DetailList>
         <CardActionsRow>
-          {config?.me?.is_superuser && isK8s && (isExecutionNode || isHopNode) && (
-            <Button
-              ouiaId="instance-detail-edit-button"
-              aria-label={t`edit`}
-              component={Link}
-              to={`/instances/${id}/edit`}
-            >
-              {t`Edit`}
-            </Button>
-          )}
-          {config?.me?.is_superuser &&
-            isK8s &&
-            (isExecutionNode || isHopNode) && (
+          {config?.me?.is_superuser && isK8s && !isManaged && (
+            <>
+              <Button
+                ouiaId="instance-detail-edit-button"
+                aria-label={t`edit`}
+                component={Link}
+                to={`/instances/${id}/edit`}
+              >
+                {t`Edit`}
+              </Button>
               <RemoveInstanceButton
                 dataCy="remove-instance-button"
                 itemsToRemove={[instance]}
                 isK8s={isK8s}
                 onRemove={removeInstances}
               />
-            )}
+            </>
+          )}
           {isExecutionNode && (
             <Tooltip content={t`Run a health check on the instance`}>
               <Button
                 isDisabled={
-                  !config?.me?.is_superuser || instance.health_check_pending
+                  !config?.me?.is_superuser ||
+                  instance.health_check_pending ||
+                  instance.managed
                 }
                 variant="primary"
                 ouiaId="health-check-button"
@@ -376,12 +377,14 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
               </Button>
             </Tooltip>
           )}
-          <InstanceToggle
-            css="display: inline-flex;"
-            fetchInstances={fetchDetails}
-            instance={instance}
-            dataCy="enable-instance"
-          />
+          {!isHopNode && (
+            <InstanceToggle
+              css="display: inline-flex;"
+              fetchInstances={fetchDetails}
+              instance={instance}
+              dataCy="enable-instance"
+            />
+          )}
         </CardActionsRow>
 
         {error && (

awx/ui/src/screens/Instances/InstanceDetail/InstanceDetail.test.js

@@ -48,6 +48,7 @@ describe('<InstanceDetail/>', () => {
         cpu_capacity: 32,
         mem_capacity: 38,
         enabled: true,
+        managed: false,
         managed_by_policy: true,
         node_type: 'execution',
         node_state: 'ready',

awx/ui/src/screens/Instances/InstanceList/InstanceListItem.js

@@ -114,7 +114,8 @@ function InstanceListItem({
   );
 
   const isHopNode = instance.node_type === 'hop';
-  const isExecutionNode = instance.node_type === 'execution';
+  const isManaged = instance.managed;
+
   return (
     <>
       <Tr
@@ -138,7 +139,7 @@ function InstanceListItem({
             rowIndex,
             isSelected,
             onSelect,
-            disable: !(isExecutionNode || isHopNode),
+            disable: isManaged,
           }}
           dataLabel={t`Selected`}
         />

awx/ui/src/screens/Instances/InstanceListenerAddressList/InstanceListenerAddressList.js

@@ -0,0 +1,154 @@
+import React, { useCallback, useEffect } from 'react';
+import { t } from '@lingui/macro';
+import { CardBody } from 'components/Card';
+import PaginatedTable, {
+  getSearchableKeys,
+  HeaderCell,
+  HeaderRow,
+  // ToolbarAddButton,
+} from 'components/PaginatedTable';
+import useToast from 'hooks/useToast';
+import { getQSConfig } from 'util/qs';
+import { useParams } from 'react-router-dom';
+import useRequest from 'hooks/useRequest';
+import DataListToolbar from 'components/DataListToolbar';
+import { InstancesAPI, ReceptorAPI } from 'api';
+import useSelected from 'hooks/useSelected';
+import InstanceListenerAddressListItem from './InstanceListenerAddressListItem';
+
+const QS_CONFIG = getQSConfig('peer', {
+  page: 1,
+  page_size: 20,
+  order_by: 'pk',
+});
+
+function InstanceListenerAddressList({ setBreadcrumb }) {
+  const { id } = useParams();
+  const { Toast, toastProps } = useToast();
+  const {
+    isLoading,
+    error: contentError,
+    request: fetchListenerAddresses,
+    result: {
+      instance,
+      listenerAddresses,
+      count,
+      relatedSearchableKeys,
+      searchableKeys,
+    },
+  } = useRequest(
+    useCallback(async () => {
+      const [
+        { data: detail },
+        {
+          data: { results },
+        },
+        actions,
+      ] = await Promise.all([
+        InstancesAPI.readDetail(id),
+        ReceptorAPI.read(),
+        InstancesAPI.readOptions(),
+      ]);
+
+      const listenerAddress_list = [];
+
+      for (let q = 0; q < results.length; q++) {
+        const receptor = results[q];
+        if (receptor.managed === true) continue;
+        if (id.toString() === receptor.instance.toString()) {
+          receptor.name = detail.hostname;
+          listenerAddress_list.push(receptor);
+        }
+      }
+
+      return {
+        instance: detail,
+        listenerAddresses: listenerAddress_list,
+        count: listenerAddress_list.length,
+        relatedSearchableKeys: (actions?.data?.related_search_fields || []).map(
+          (val) => val.slice(0, -8)
+        ),
+        searchableKeys: getSearchableKeys(actions.data.actions?.GET),
+      };
+    }, [id]),
+    {
+      instance: {},
+      listenerAddresses: [],
+      count: 0,
+      relatedSearchableKeys: [],
+      searchableKeys: [],
+    }
+  );
+
+  useEffect(() => {
+    fetchListenerAddresses();
+  }, [fetchListenerAddresses]);
+
+  useEffect(() => {
+    if (instance) {
+      setBreadcrumb(instance);
+    }
+  }, [instance, setBreadcrumb]);
+
+  const { selected, isAllSelected, handleSelect, clearSelected, selectAll } =
+    useSelected(listenerAddresses);
+
+  return (
+    <CardBody>
+      <PaginatedTable
+        contentError={contentError}
+        hasContentLoading={isLoading}
+        items={listenerAddresses}
+        itemCount={count}
+        pluralizedItemName={t`Listener Addresses`}
+        qsConfig={QS_CONFIG}
+        onRowClick={handleSelect}
+        clearSelected={clearSelected}
+        toolbarSearchableKeys={searchableKeys}
+        toolbarRelatedSearchableKeys={relatedSearchableKeys}
+        toolbarSearchColumns={[
+          {
+            name: t`Name`,
+            key: 'hostname__icontains',
+            isDefault: true,
+          },
+        ]}
+        toolbarSortColumns={[
+          {
+            name: t`Name`,
+            key: 'hostname',
+          },
+        ]}
+        headerRow={
+          <HeaderRow qsConfig={QS_CONFIG}>
+            <HeaderCell sortKey="address">{t`Address`}</HeaderCell>
+            <HeaderCell sortKey="port">{t`Port`}</HeaderCell>
+            <HeaderCell sortKey="protocol">{t`Protocol`}</HeaderCell>
+            <HeaderCell sortKey="canonical">{t`Canonical`}</HeaderCell>
+          </HeaderRow>
+        }
+        renderToolbar={(props) => (
+          <DataListToolbar
+            {...props}
+            isAllSelected={isAllSelected}
+            onSelectAll={selectAll}
+            qsConfig={QS_CONFIG}
+            additionalControls={[]}
+          />
+        )}
+        renderRow={(listenerAddress, index) => (
+          <InstanceListenerAddressListItem
+            isSelected={selected.some((row) => row.id === listenerAddress.id)}
+            onSelect={() => handleSelect(listenerAddress)}
+            key={listenerAddress.id}
+            peerListenerAddress={listenerAddress}
+            rowIndex={index}
+          />
+        )}
+      />
+      <Toast {...toastProps} />
+    </CardBody>
+  );
+}
+
+export default InstanceListenerAddressList;

awx/ui/src/screens/Instances/InstanceListenerAddressList/InstanceListenerAddressListItem.js

@@ -0,0 +1,46 @@
+import React from 'react';
+import { t } from '@lingui/macro';
+import 'styled-components/macro';
+import { Tr, Td } from '@patternfly/react-table';
+
+function InstanceListenerAddressListItem({
+  peerListenerAddress,
+  isSelected,
+  onSelect,
+  rowIndex,
+}) {
+  const labelId = `check-action-${peerListenerAddress.id}`;
+  return (
+    <Tr
+      id={`peerListenerAddress-row-${peerListenerAddress.id}`}
+      ouiaId={`peerListenerAddress-row-${peerListenerAddress.id}`}
+    >
+      <Td
+        select={{
+          rowIndex,
+          isSelected,
+          onSelect,
+        }}
+        dataLabel={t`Selected`}
+      />
+
+      <Td id={labelId} dataLabel={t`Address`}>
+        {peerListenerAddress.address}
+      </Td>
+
+      <Td id={labelId} dataLabel={t`Port`}>
+        {peerListenerAddress.port}
+      </Td>
+
+      <Td id={labelId} dataLabel={t`Protocol`}>
+        {peerListenerAddress.protocol}
+      </Td>
+
+      <Td id={labelId} dataLabel={t`Canonical`}>
+        {peerListenerAddress.canonical.toString()}
+      </Td>
+    </Tr>
+  );
+}
+
+export default InstanceListenerAddressListItem;

awx/ui/src/screens/Instances/InstanceListenerAddressList/index.js

@@ -0,0 +1 @@
+export { default } from './InstanceListenerAddressList';

awx/ui/src/screens/Instances/InstancePeers/InstancePeerList.js

@@ -16,7 +16,7 @@ import { getQSConfig, parseQueryString, mergeParams } from 'util/qs';
 import { useLocation, useParams } from 'react-router-dom';
 import useRequest, { useDismissableError } from 'hooks/useRequest';
 import DataListToolbar from 'components/DataListToolbar';
-import { InstancesAPI } from 'api';
+import { InstancesAPI, ReceptorAPI } from 'api';
 import useExpanded from 'hooks/useExpanded';
 import useSelected from 'hooks/useSelected';
 import InstancePeerListItem from './InstancePeerListItem';
@@ -24,7 +24,7 @@ import InstancePeerListItem from './InstancePeerListItem';
 const QS_CONFIG = getQSConfig('peer', {
   page: 1,
   page_size: 20,
-  order_by: 'hostname',
+  order_by: 'pk',
 });
 
 function InstancePeerList({ setBreadcrumb }) {
@@ -47,18 +47,35 @@ function InstancePeerList({ setBreadcrumb }) {
       const [
         { data: detail },
         {
-          data: { results, count: itemNumber },
+          data: { results },
         },
         actions,
+        instances,
       ] = await Promise.all([
         InstancesAPI.readDetail(id),
         InstancesAPI.readPeers(id, params),
         InstancesAPI.readOptions(),
+        InstancesAPI.read(),
       ]);
+
+      const address_list = [];
+
+      for (let q = 0; q < results.length; q++) {
+        const receptor = results[q];
+        if (receptor.managed === true) continue;
+        const host = instances.data.results.filter(
+          (obj) => obj.id === receptor.instance
+        )[0];
+        const copy = receptor;
+        copy.hostname = host.hostname;
+        copy.node_type = host.node_type;
+        address_list.push(copy);
+      }
+
       return {
         instance: detail,
-        peers: results,
-        count: itemNumber,
+        peers: address_list,
+        count: address_list.length,
         relatedSearchableKeys: (actions?.data?.related_search_fields || []).map(
           (val) => val.slice(0, -8)
         ),
@@ -90,15 +107,73 @@ function InstancePeerList({ setBreadcrumb }) {
     useSelected(peers);
 
   const fetchInstancesToAssociate = useCallback(
-    (params) =>
-      InstancesAPI.read(
+    async (params) => {
+      const address_list = [];
+
+      const instances = await InstancesAPI.read(
         mergeParams(params, {
-          ...{ not__id: id },
           ...{ not__node_type: ['control', 'hybrid'] },
-          ...{ not__hostname: instance.peers },
         })
-      ),
-    [id, instance]
+      );
+      const receptors = (await ReceptorAPI.read()).data.results;
+
+      // get instance ids of the current peered receptor ids
+      const already_peered_instance_ids = [];
+      for (let h = 0; h < instance.peers.length; h++) {
+        const matched = receptors.filter((obj) => obj.id === instance.peers[h]);
+        matched.forEach((element) => {
+          already_peered_instance_ids.push(element.instance);
+        });
+      }
+
+      for (let q = 0; q < receptors.length; q++) {
+        const receptor = receptors[q];
+
+        if (already_peered_instance_ids.includes(receptor.instance)) {
+          // ignore reverse peers
+          continue;
+        }
+
+        if (instance.peers.includes(receptor.id)) {
+          // no links to existing links
+          continue;
+        }
+
+        if (instance.id === receptor.instance) {
+          // no links to thy self
+          continue;
+        }
+
+        if (instance.managed) {
+          // no managed nodes
+          continue;
+        }
+
+        const host = instances.data.results.filter(
+          (obj) => obj.id === receptor.instance
+        )[0];
+
+        if (host === undefined) {
+          // no hosts
+          continue;
+        }
+
+        if (receptor.is_internal) {
+          continue;
+        }
+
+        const copy = receptor;
+        copy.hostname = host.hostname;
+        copy.node_type = host.node_type;
+        copy.canonical = copy.canonical.toString();
+        address_list.push(copy);
+      }
+
+      instances.data.results = address_list;
+
+      return instances;
+    },
+    [instance]
   );
 
   const {
@@ -108,17 +183,15 @@ function InstancePeerList({ setBreadcrumb }) {
   } = useRequest(
     useCallback(
       async (instancesPeerToAssociate) => {
-        const selected_hostname = instancesPeerToAssociate.map(
-          (obj) => obj.hostname
-        );
-        const new_peers = [
-          ...new Set([...instance.peers, ...selected_hostname]),
-        ];
+        const selected_peers = instancesPeerToAssociate.map((obj) => obj.id);
+
+        const new_peers = [...new Set([...instance.peers, ...selected_peers])];
         await InstancesAPI.update(instance.id, { peers: new_peers });
+
         fetchPeers();
         addToast({
           id: instancesPeerToAssociate,
-          title: t`${selected_hostname} added as a peer. Please be sure to run the install bundle for ${instance.hostname} again in order to see changes take effect.`,
+          title: t`Peers update on ${instance.hostname}.  Please be sure to run the install bundle for ${instance.hostname} again in order to see changes take effect.`,
           variant: AlertVariant.success,
           hasTimeout: true,
         });
@@ -133,17 +206,18 @@ function InstancePeerList({ setBreadcrumb }) {
     error: disassociateError,
   } = useRequest(
     useCallback(async () => {
-      const new_peers = [];
-      const selected_hostname = selected.map((obj) => obj.hostname);
-      for (let i = 0; i < instance.peers.length; i++) {
-        if (!selected_hostname.includes(instance.peers[i])) {
-          new_peers.push(instance.peers[i]);
-        }
+      let new_peers = instance.peers;
+
+      const selected_ids = selected.map((obj) => obj.id);
+
+      for (let i = 0; i < selected_ids.length; i++) {
+        new_peers = new_peers.filter((s_id) => s_id !== selected_ids[i]);
       }
       await InstancesAPI.update(instance.id, { peers: new_peers });
+
       fetchPeers();
       addToast({
-        title: t`${selected_hostname} removed. Please be sure to run the install bundle for ${instance.hostname} again in order to see changes take effect.`,
+        title: t`Peer removed. Please be sure to run the install bundle for ${instance.hostname} again in order to see changes take effect.`,
         variant: AlertVariant.success,
         hasTimeout: true,
       });
@@ -190,9 +264,11 @@ function InstancePeerList({ setBreadcrumb }) {
             <HeaderCell
               tooltip={t`Cannot run health check on hop nodes.`}
               sortKey="hostname"
-            >{t`Name`}</HeaderCell>
-            <HeaderCell sortKey="errors">{t`Status`}</HeaderCell>
+            >{t`Instance Name`}</HeaderCell>
+            <HeaderCell sortKey="address">{t`Address`}</HeaderCell>
+            <HeaderCell sortKey="port">{t`Port`}</HeaderCell>
             <HeaderCell sortKey="node_type">{t`Node Type`}</HeaderCell>
+            <HeaderCell sortKey="canonical">{t`Canonical`}</HeaderCell>
           </HeaderRow>
         }
         renderToolbar={(props) => (
@@ -218,7 +294,7 @@ function InstancePeerList({ setBreadcrumb }) {
                   key="disassociate"
                   onDisassociate={handlePeersDiassociate}
                   itemsToDisassociate={selected}
-                  modalTitle={t`Remove instance from peers?`}
+                  modalTitle={t`Remove peers?`}
                 />
               ),
             ]}
@@ -243,12 +319,15 @@ function InstancePeerList({ setBreadcrumb }) {
           isModalOpen={isModalOpen}
           onAssociate={handlePeerAssociate}
           onClose={() => setIsModalOpen(false)}
-          title={t`Select Instances`}
+          title={t`Select Peer Addresses`}
           optionsRequest={readInstancesOptions}
           displayKey="hostname"
           columns={[
             { key: 'hostname', name: t`Name` },
+            { key: 'address', name: t`Address` },
+            { key: 'port', name: t`Port` },
             { key: 'node_type', name: t`Node Type` },
+            { key: 'protocol', name: t`Protocol` },
           ]}
         />
       )}

awx/ui/src/screens/Instances/InstancePeers/InstancePeerListItem.js

@@ -2,10 +2,8 @@ import React from 'react';
 import { Link } from 'react-router-dom';
 import { t } from '@lingui/macro';
 import 'styled-components/macro';
-import { Tooltip } from '@patternfly/react-core';
 import { Tr, Td, ExpandableRowContent } from '@patternfly/react-table';
 import { formatDateString } from 'util/dates';
-import StatusLabel from 'components/StatusLabel';
 import { Detail, DetailList } from 'components/DetailList';
 
 function InstancePeerListItem({
@@ -43,29 +41,26 @@ function InstancePeerListItem({
           }}
           dataLabel={t`Selected`}
         />
+
         <Td id={labelId} dataLabel={t`Name`}>
-          <Link to={`/instances/${peerInstance.id}/details`}>
+          <Link to={`/instances/${peerInstance.instance}/details`}>
             <b>{peerInstance.hostname}</b>
           </Link>
         </Td>
 
-        <Td dataLabel={t`Status`}>
-          <Tooltip
-            content={
-              <div>
-                {t`Last Health Check`}
-                &nbsp;
-                {formatDateString(
-                  peerInstance.last_health_check ?? peerInstance.last_seen
-                )}
-              </div>
-            }
-          >
-            <StatusLabel status={peerInstance.node_state} />
-          </Tooltip>
+        <Td id={labelId} dataLabel={t`Address`}>
+          {peerInstance.address}
+        </Td>
+
+        <Td id={labelId} dataLabel={t`Port`}>
+          {peerInstance.port}
         </Td>
 
         <Td dataLabel={t`Node Type`}>{peerInstance.node_type}</Td>
+
+        <Td id={labelId} dataLabel={t`Canonical`}>
+          {peerInstance.canonical.toString()}
+        </Td>
       </Tr>
       {!isHopNode && (
         <Tr

awx/ui/src/screens/Instances/Instances.js

@@ -25,6 +25,7 @@ function Instances() {
       [`/instances/${instance.id}`]: `${instance.hostname}`,
       [`/instances/${instance.id}/details`]: t`Details`,
       [`/instances/${instance.id}/peers`]: t`Peers`,
+      [`/instances/${instance.id}/listener_addresses`]: t`Listener Addresses`,
       [`/instances/${instance.id}/edit`]: t`Edit Instance`,
     });
   }, []);

awx/ui/src/screens/Instances/Shared/InstanceForm.js

@@ -1,6 +1,6 @@
-import React, { useCallback } from 'react';
+import React from 'react';
 import { t } from '@lingui/macro';
-import { Formik, useField, useFormikContext } from 'formik';
+import { Formik, useField } from 'formik';
 import { Form, FormGroup, CardBody } from '@patternfly/react-core';
 import { FormColumnLayout } from 'components/FormLayout';
 import FormField, {
@@ -9,7 +9,6 @@ import FormField, {
 } from 'components/FormField';
 import FormActionGroup from 'components/FormActionGroup';
 import AnsibleSelect from 'components/AnsibleSelect';
-import { PeersLookup } from 'components/Lookup';
 import { required } from 'util/validators';
 
 const INSTANCE_TYPES = [
@@ -23,16 +22,6 @@ function InstanceFormFields({ isEdit }) {
     validate: required(t`Set a value for this field`),
   });
 
-  const { setFieldValue } = useFormikContext();
-
-  const [peersField, peersMeta, peersHelpers] = useField('peers');
-
-  const handlePeersUpdate = useCallback(
-    (value) => {
-      setFieldValue('peers', value);
-    },
-    [setFieldValue]
-  );
   return (
     <>
       <FormField
@@ -91,20 +80,6 @@ function InstanceFormFields({ isEdit }) {
           isDisabled={isEdit}
         />
       </FormGroup>
-      <PeersLookup
-        helperTextInvalid={peersMeta.error}
-        isValid={!peersMeta.touched || !peersMeta.error}
-        onBlur={() => peersHelpers.setTouched()}
-        onChange={handlePeersUpdate}
-        value={peersField.value}
-        tooltip={t`Select the Peers Instances.`}
-        fieldName="peers"
-        formLabel={t`Peers`}
-        multiple
-        typePeers
-        id="peers"
-        isRequired
-      />
       <FormGroup fieldId="instance-option-checkboxes" label={t`Options`}>
         <CheckboxField
           id="enabled"

awx_collection/plugins/modules/instance.py

@@ -67,6 +67,8 @@ options:
       description:
         - List of peers to connect outbound to. Only configurable for hop and execution nodes.
         - To remove all current peers, set value to an empty list, [].
+        - Each item is an ID or address of a receptor address.
+        - If item is address, it must be unique across all receptor addresses.
       required: False
       type: list
       elements: str
@@ -83,12 +85,24 @@ EXAMPLES = '''
   awx.awx.instance:
     hostname: my-instance.prod.example.com
     capacity_adjustment: 0.4
-    listener_port: 31337
 
 - name: Deprovision the instance
   awx.awx.instance:
     hostname: my-instance.prod.example.com
     node_state: deprovisioning
+
+- name: Create execution node
+  awx.awx.instance:
+    hostname: execution.example.com
+    node_type: execution
+    peers:
+      - 12
+      - route.to.hop.example.com
+
+- name: Remove peers
+  awx.awx.instance:
+    hostname: execution.example.com
+    peers:
 '''
 
 from ..module_utils.controller_api import ControllerAPIModule
@@ -124,6 +138,17 @@ def main():
     # Attempt to look up an existing item based on the provided data
     existing_item = module.get_one('instances', name_or_id=hostname)
 
+    # peer item can be an id or address
+    # if address, get the id
+    peers_ids = []
+    if peers:
+        for p in peers:
+            if not p.isdigit():
+                p_id = module.get_one('receptor_addresses', allow_none=False, data={'address': p})
+                peers_ids.append(p_id['id'])
+            else:
+                peers_ids.append(p)
+
     # Create the data that gets sent for create and update
     new_fields = {'hostname': hostname}
     if capacity_adjustment is not None:
@@ -139,7 +164,7 @@ def main():
     if listener_port is not None:
         new_fields['listener_port'] = listener_port
     if peers is not None:
-        new_fields['peers'] = peers
+        new_fields['peers'] = peers_ids
     if peers_from_control_nodes is not None:
         new_fields['peers_from_control_nodes'] = peers_from_control_nodes
 

awx_collection/test/awx/test_completeness.py

@@ -54,7 +54,7 @@ extra_endpoints = {
 }
 
 # Global module parameters we can ignore
-ignore_parameters = ['state', 'new_name', 'update_secrets', 'copy_from']
+ignore_parameters = ['state', 'new_name', 'update_secrets', 'copy_from', 'is_internal']
 
 # Some modules take additional parameters that do not appear in the API
 # Add the module name as the key with the value being the list of params to ignore
@@ -248,7 +248,7 @@ def test_completeness(collection_import, request, admin_user, job_template, exec
         singular_endpoint = '{0}'.format(endpoint)
         if singular_endpoint.endswith('ies'):
             singular_endpoint = singular_endpoint[:-3]
-        if singular_endpoint != 'settings' and singular_endpoint.endswith('s'):
+        elif singular_endpoint != 'settings' and singular_endpoint.endswith('s'):
             singular_endpoint = singular_endpoint[:-1]
         module_name = '{0}'.format(singular_endpoint)
 

awx_collection/test/awx/test_export.py

@@ -134,21 +134,17 @@ def test_export_simple(
 
 
 @pytest.mark.django_db
-def test_export_system_auditor(run_module, schedule, system_auditor):  # noqa: F811
+def test_export_system_auditor(run_module, organization, system_auditor):  # noqa: F811
     """
-    This test illustrates that deficiency of export when ran as non-root user (i.e. system auditor).
-    The OPTIONS endpoint does NOT return POST for a system auditor. This is bad for the export code
-    because it relies on crawling the OPTIONS POST response to determine the fields to export.
+    This test illustrates that export of resources can now happen
+    when ran as non-root user (i.e. system auditor). The OPTIONS
+    endpoint does NOT return POST for a system auditor, but now we
+    make a best-effort to parse the description string, which will
+    often have the fields.
     """
     result = run_module('export', dict(all=True), system_auditor)
-    assert result.get('failed', False), result.get('msg', result)
+    assert not result.get('failed', False), result.get('msg', result)
+    assert 'msg' not in result
+    assert 'assets' in result
 
-    assert 'Failed to export assets substring not found' in result['msg'], (
-        'If you found this error then you have probably fixed a feature! The export code attempts to assertain the POST fields from the `description` field,'
-        ' but both the API side and the client inference code are lacking.'
-    )
-
-    # r = result['assets']['schedules'][0]
-    # assert r['natural_key']['name'] == 'test-sched'
-
-    # assert 'rrule' not in r, 'If you found this error then you have probably fixed a feature! We WANT rrule to be found in the export schedule payload.'
+    find_by(result['assets'], 'organizations', 'name', 'Default')

awx_collection/test/awx/test_instance.py

@@ -13,39 +13,32 @@ def test_peers_adding_and_removing(run_module, admin_user):
     with override_settings(IS_K8S=True):
         result = run_module(
             'instance',
-            {'hostname': 'hopnode1', 'node_type': 'hop', 'peers_from_control_nodes': True, 'node_state': 'installed', 'listener_port': 27199},
+            {'hostname': 'hopnode', 'node_type': 'hop', 'node_state': 'installed', 'listener_port': 6789},
             admin_user,
         )
         assert result['changed']
 
-        hop_node_1 = Instance.objects.get(pk=result.get('id'))
+        hop_node = Instance.objects.get(pk=result.get('id'))
 
-        assert hop_node_1.peers_from_control_nodes is True
-        assert hop_node_1.node_type == 'hop'
+        assert hop_node.node_type == 'hop'
+
+        address = hop_node.receptor_addresses.get(pk=result.get('id'))
+        assert address.port == 6789
 
         result = run_module(
             'instance',
-            {'hostname': 'hopnode2', 'node_type': 'hop', 'peers_from_control_nodes': True, 'node_state': 'installed', 'listener_port': 27199},
-            admin_user,
-        )
-        assert result['changed']
-
-        hop_node_2 = Instance.objects.get(pk=result.get('id'))
-
-        result = run_module(
-            'instance',
-            {'hostname': 'executionnode', 'node_type': 'execution', 'node_state': 'installed', 'listener_port': 27199, 'peers': ['hopnode1', 'hopnode2']},
+            {'hostname': 'executionnode', 'node_type': 'execution', 'node_state': 'installed', 'peers': ['hopnode']},
             admin_user,
         )
         assert result['changed']
 
         execution_node = Instance.objects.get(pk=result.get('id'))
 
-        assert set(execution_node.peers.all()) == {hop_node_1, hop_node_2}
+        assert set(execution_node.peers.all()) == {address}
 
         result = run_module(
             'instance',
-            {'hostname': 'executionnode', 'node_type': 'execution', 'node_state': 'installed', 'listener_port': 27199, 'peers': []},
+            {'hostname': 'executionnode', 'node_type': 'execution', 'node_state': 'installed', 'peers': []},
             admin_user,
         )
 

awx_collection/tests/integration/targets/instance/tasks/main.yml

@@ -42,7 +42,6 @@
         node_type: execution
         node_state: installed
         capacity_adjustment: 0.4
-        listener_port: 31337
       register: result
 
     - assert:
@@ -74,11 +73,9 @@
 - block:
     - name: Create hop node 1
       awx.awx.instance:
-        hostname: hopnode1
+        hostname: "{{ hostname1 }}"
         node_type: hop
         node_state: installed
-        listener_port: 27199
-        peers_from_control_nodes: True
       register: result
 
     - assert:
@@ -87,11 +84,9 @@
 
     - name: Create hop node 2
       awx.awx.instance:
-        hostname: hopnode2
+        hostname: "{{ hostname2 }}"
         node_type: hop
         node_state: installed
-        listener_port: 27199
-        peers_from_control_nodes: True
       register: result
 
     - assert:
@@ -100,13 +95,12 @@
 
     - name: Create execution node
       awx.awx.instance:
-        hostname: executionnode
+        hostname: "{{ hostname3 }}"
         node_type: execution
         node_state: installed
-        listener_port: 27199
         peers:
-          - "hopnode1"
-          - "hopnode2"
+          - "{{ hostname1 }}"
+          - "{{ hostname2 }}"
       register: result
 
     - assert:
@@ -115,10 +109,9 @@
 
     - name: Remove execution node peers
       awx.awx.instance:
-        hostname: executionnode
+        hostname: "{{ hostname3 }}"
         node_type: execution
         node_state: installed
-        listener_port: 27199
         peers: []
       register: result
 
@@ -126,4 +119,15 @@
         that:
           - result is changed
 
+  always:
+    - name: Deprovision the instances
+      awx.awx.instance:
+        hostname: "{{ item }}"
+        node_state: deprovisioning
+      with_items:
+        - "{{ hostname1 }}"
+        - "{{ hostname2 }}"
+        - "{{ hostname3 }}"
+
+
   when: IS_K8S

awxkit/awxkit/api/client.py

@@ -43,8 +43,8 @@ class Connection(object):
         self.session = requests.Session()
         self.uses_session_cookie = False
 
-    def get_session_requirements(self, next='/api/'):
-        self.get('/api/')  # this causes a cookie w/ the CSRF token to be set
+    def get_session_requirements(self, next=config.api_base_path):
+        self.get(config.api_base_path)  # this causes a cookie w/ the CSRF token to be set
         return dict(next=next)
 
     def login(self, username=None, password=None, token=None, **kwargs):
@@ -52,7 +52,7 @@ class Connection(object):
             _next = kwargs.get('next')
             if _next:
                 headers = self.session.headers.copy()
-                response = self.post('/api/login/', headers=headers, data=dict(username=username, password=password, next=_next))
+                response = self.post(f"{config.api_base_path}login/", headers=headers, data=dict(username=username, password=password, next=_next))
                 # The login causes a redirect so we need to search the history of the request to find the header
                 for historical_response in response.history:
                     if 'X-API-Session-Cookie-Name' in historical_response.headers:

awxkit/awxkit/api/mixins/has_status.py

@@ -3,6 +3,7 @@ import json
 
 from awxkit.utils import poll_until
 from awxkit.exceptions import WaitUntilTimeout
+from awxkit.config import config
 
 
 def bytes_to_str(obj):
@@ -83,7 +84,7 @@ class HasStatus(object):
         if getattr(self, 'job_explanation', '').startswith('Previous Task Failed'):
             try:
                 data = json.loads(self.job_explanation.replace('Previous Task Failed: ', ''))
-                dependency = self.walk('/api/v2/{0}s/{1}/'.format(data['job_type'], data['job_id']))
+                dependency = self.walk('/{0}v2/{1}s/{2}/'.format(config.api_base_path, data['job_type'], data['job_id']))
                 if hasattr(dependency, 'failure_output_details'):
                     msg += '\nDependency output:\n{}'.format(dependency.failure_output_details())
                 else:

awxkit/awxkit/api/pages/api.py

@@ -199,7 +199,7 @@ class ApiV2(base.Base):
                 return None
             fields['natural_key'] = natural_key
 
-        return utils.remove_encrypted(fields)
+        return fields
 
     def _export_list(self, endpoint):
         post_fields = utils.get_post_fields(endpoint, self._cache)
@@ -280,7 +280,7 @@ class ApiV2(base.Base):
                     _page = self._cache.get_by_natural_key(value)
                     post_data[field] = _page['id'] if _page is not None else None
                 else:
-                    post_data[field] = value
+                    post_data[field] = utils.remove_encrypted(value)
 
             _page = self._cache.get_by_natural_key(asset['natural_key'])
             try:

awxkit/awxkit/api/pages/base.py

@@ -150,19 +150,21 @@ class Base(Page):
             HTTPBasicAuth(client_id, client_secret)(req)
             req.headers['Content-Type'] = 'application/x-www-form-urlencoded'
             resp = self.connection.post(
-                '/api/o/token/', data={"grant_type": "password", "username": username, "password": password, "scope": scope}, headers=req.headers
+                f"{config.api_base_path}o/token/",
+                data={"grant_type": "password", "username": username, "password": password, "scope": scope},
+                headers=req.headers,
             )
         elif client_id:
             req.headers['Content-Type'] = 'application/x-www-form-urlencoded'
             resp = self.connection.post(
-                '/api/o/token/',
+                f"{config.api_base_path}o/token/",
                 data={"grant_type": "password", "username": username, "password": password, "client_id": client_id, "scope": scope},
                 headers=req.headers,
             )
         else:
             HTTPBasicAuth(username, password)(req)
             resp = self.connection.post(
-                '/api/v2/users/{}/personal_tokens/'.format(username),
+                '{0}v2/users/{1}/personal_tokens/'.format(config.api_base_path, username),
                 json={"description": description, "application": None, "scope": scope},
                 headers=req.headers,
             )
@@ -207,7 +209,7 @@ class Base(Page):
                 jobs = []
                 for active_job in active_jobs:
                     job_type = active_job['type']
-                    endpoint = '/api/v2/{}s/{}/'.format(job_type, active_job['id'])
+                    endpoint = '{}v2/{}s/{}/'.format(config.api_base_path, job_type, active_job['id'])
                     job = self.walk(endpoint)
                     jobs.append(job)
                     job.cancel()

awxkit/awxkit/api/resources.py

@@ -1,3 +1,6 @@
+from awxkit.config import config
+
+
 class Resources(object):
     _activity = r'activity_stream/\d+/'
     _activity_stream = 'activity_stream/'
@@ -281,7 +284,7 @@ class Resources(object):
     _workflow_job_workflow_nodes = r'workflow_jobs/\d+/workflow_nodes/'
     _subscriptions = 'config/subscriptions/'
     _workflow_jobs = 'workflow_jobs/'
-    api = '/api/'
+    api = str(config.api_base_path)
     common = api + r'v\d+/'
     v2 = api + 'v2/'
 

awxkit/awxkit/api/utils.py

@@ -15,7 +15,12 @@ def freeze(key):
 
 def parse_description(desc):
     options = {}
-    for line in desc[desc.index('POST') :].splitlines():
+    desc_lines = []
+    if 'POST' in desc:
+        desc_lines = desc[desc.index('POST') :].splitlines()
+    else:
+        desc_lines = desc.splitlines()
+    for line in desc_lines:
         match = descRE.match(line)
         if not match:
             continue

awxkit/awxkit/awx/utils.py

@@ -122,5 +122,5 @@ def as_user(v, username, password=None):
 
 
 def uses_sessions(connection):
-    session_login = connection.get('/api/login/')
+    session_login = connection.get(f"{config.api_base_path}login/")
     return session_login.status_code == 200

awxkit/awxkit/cli/custom.py

@@ -4,6 +4,7 @@ import json
 from .stdout import monitor, monitor_workflow
 from .utils import CustomRegistryMeta, color_enabled
 from awxkit import api
+from awxkit.config import config
 from awxkit.exceptions import NoContent
 
 
@@ -479,7 +480,7 @@ class RoleMixin(object):
                     options = ', '.join(RoleMixin.roles[flag])
                     raise ValueError("invalid choice: '{}' must be one of {}".format(role, options))
                 value = kwargs[flag]
-                target = '/api/v2/{}/{}'.format(resource, value)
+                target = '{}v2/{}/{}'.format(config.api_base_path, resource, value)
                 detail = self.page.__class__(target, self.page.connection).get()
                 object_roles = detail['summary_fields']['object_roles']
                 actual_role = object_roles[role + '_role']

awxkit/awxkit/cli/stdout.py

@@ -6,6 +6,7 @@ import sys
 import time
 
 from .utils import cprint, color_enabled, STATUS_COLORS
+from awxkit.config import config
 from awxkit.utils import to_str
 
 
@@ -17,7 +18,7 @@ def monitor_workflow(response, session, print_stdout=True, action_timeout=None,
     }
 
     def fetch(seen):
-        results = response.connection.get('/api/v2/unified_jobs', payload).json()['results']
+        results = response.connection.get(f"{config.api_base_path}v2/unified_jobs", payload).json()['results']
 
         # erase lines we've previously printed
         if print_stdout and sys.stdout.isatty():

awxkit/awxkit/config.py

@@ -32,3 +32,4 @@ config.assume_untrusted = config.get('assume_untrusted', True)
 config.client_connection_attempts = int(os.getenv('AWXKIT_CLIENT_CONNECTION_ATTEMPTS', 5))
 config.prevent_teardown = to_bool(os.getenv('AWXKIT_PREVENT_TEARDOWN', False))
 config.use_sessions = to_bool(os.getenv('AWXKIT_SESSIONS', False))
+config.api_base_path = os.getenv('AWXKIT_API_BASE_PATH', '/api/')

awxkit/awxkit/utils/__init__.py

@@ -14,10 +14,8 @@ import yaml
 from awxkit.words import words
 from awxkit.exceptions import WaitUntilTimeout
 
-
 log = logging.getLogger(__name__)
 
-
 cloud_types = (
     'aws',
     'azure',

docs/docsite/conf.py

@@ -16,11 +16,11 @@ pubdate = datetime.strptime(pubdateshort, '%Y-%m-%d').strftime('%B %d, %Y')
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 html_title = 'Ansible AWX community documentation'
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 html_short_title = 'AWX community documentation'
 
 htmlhelp_basename = 'AWX_docs'
@@ -54,8 +54,8 @@ release = 'AWX latest'
 
 language = 'en'
 
-locale_dirs = ['locale/']   # path is example but recommended.
-gettext_compact = False     # optional.
+locale_dirs = ['locale/']  # path is example but recommended.
+gettext_compact = False  # optional.
 
 rst_epilog = """
 .. |atqi| replace:: *AWX Quick Installation Guide*
@@ -88,4 +88,8 @@ rst_epilog = """
 .. |rhaap| replace:: Red Hat Ansible Automation Platform
 .. |RHAT| replace:: Red Hat Ansible Automation Platform controller
 
-""" % (version, pubdateshort, pubdate)
+""" % (
+    version,
+    pubdateshort,
+    pubdate,
+)

docs/docsite/rst/rest_api/_swagger/swagger.py

@@ -35,12 +35,8 @@ def assets(app, exception):
         _, extension = os.path.splitext(asset)
         if extension in ('py', 'pyc'):
             continue
-        if not exception and os.path.exists(
-            os.path.join(app.outdir, '_static')
-        ):
-            copyfile(
-                os.path.join(here, asset),
-                os.path.join(app.outdir, '_static', asset))
+        if not exception and os.path.exists(os.path.join(app.outdir, '_static')):
+            copyfile(os.path.join(here, asset), os.path.join(app.outdir, '_static', asset))
 
 
 def setup(app):

docs/docsite/rst/rest_api/authentication.rst

@@ -89,8 +89,6 @@ Example with curl:
 
 .. code-block:: text
 
-   curl -X GET -H 'Authorization: Basic dXNlcjpwYXNzd29yZA==’ https://<awx-host>/api/v2/credentials -k -L
-
    # the --user flag adds this Authorization header for us
    curl -X GET --user 'user:password' https://<awx-host>/api/v2/credentials -k -L
 

docs/docsite/rst/userguide/credential_plugins.rst

@@ -263,30 +263,40 @@ When **HashiCorp Vault Secret Lookup** is selected for **Credential Type**, prov
 - **Server URL** (required): provide the URL used for communicating with HashiCorp Vault's secret management system
 - **Token**: specify the access token used to authenticate HashiCorp's server
 - **CA Certificate**: specify the CA certificate used to verify HashiCorp's server
-- **Approle Role_ID**: specify the ID for Approle authentication
+- **Approle Role_ID**: specify the ID if using Approle for authentication
 - **Approle Secret_ID**: specify the corresponding secret ID for Approle authentication
 - **Client Certificate**: specify a PEM-encoded client certificate when using the TLS auth method including any required intermediate certificates expected by Vault
 - **Client Certificate Key**: specify a PEM-encoded certificate private key when using the TLS auth method
 - **TLS Authentication Role**: specify the role or certificate name in Vault that corresponds to your client certificate when using the TLS auth method. If it is not provided, Vault will attempt to match the certificate automatically
 - **Namespace name** specify the namespace name (Vault Enterprise only)
 - **Kubernetes role** specify the role name when using Kubernetes authentication
+- **Username**: enter the username of the user to be used to authenticate this service
+- **Password**: enter the password associated with the user to authenticate this service
 - **Path to Auth**: specify a path if other than the default path of ``/approle``
 - **API Version** (required): select v1 for static lookups and v2 for versioned lookups
-- **Username and Password**: specify the username and password for the user account
 
-For more detail about the Approle auth method and its fields, refer to the `Vault documentation for Approle Auth Method <https://www.vaultproject.io/docs/auth/approle>`_.
 
-For more detail about the Userpass auth method and its fields, refer to the `Vault documentation for LDAP auth method <https://www.vaultproject.io/docs/auth/userpass>`_.
+For more detail about the Approle auth method and its fields, refer to the `Vault documentation for Approle Auth Method <https://developer.hashicorp.com/vault/docs/auth/approle>`_.
 
-For more detail about the Kubernetes auth method and its fields, refer to the `Vault documentation for Kubernetes auth method <https://developer.hashicorp.com/vault/docs/auth/kubernetes>` _.
+LDAP authentication requires LDAP to be configured in HashiCorp's Vault UI. A policy may be added to the user if they want access to a specific engine created. As long as the bind is set properly, the user should be able to successfully authenticate. Cubbyhole is the name of the default secret mount. If you have proper permissions, you can create other mounts and write key values to those. For more detail about the LDAP auth method and its fields, refer to the `Vault documentation for LDAP auth method <https://developer.hashicorp.com/vault/docs/auth/ldap>`_.
 
-For more detail about the TLS certificate auth method and its fields, refer to the `Vault documentation for TLS certificates auth method <https://developer.hashicorp.com/vault/docs/auth/cert>` _.
+For more detail about the userpass auth method and its fields, refer to the `Vault documentation for userpass auth method <https://developer.hashicorp.com/vault/docs/auth/userpass>`_.
 
-Below shows an example of a configured HashiCorp Vault Secret Lookup credential.
+For more detail about the Kubernetes auth method and its fields, refer to the `Vault documentation for Kubernetes auth method <https://developer.hashicorp.com/vault/docs/auth/kubernetes>`_.
+
+For more detail about the TLS certificate auth method and its fields, refer to the `Vault documentation for TLS certificates auth method <https://developer.hashicorp.com/vault/docs/auth/cert>`_.
+
+Below shows an example of a configured HashiCorp Vault Secret Lookup  credential for LDAP.
 
 .. image:: ../common/images/credentials-create-hashicorp-kv-credential.png 
    :alt: Example new HashiCorp Vault Secret lookup dialog
 
+To test the lookup, create another credential that uses the HashiCorp Vault lookup. The example below shows the metadata for a machine credential configured to look up HashiCorp Vault secret credentials:
+
+.. image:: ../common/images/credentials-machine-test-hashicorp-metadata.png 
+   :alt: Example machine credential lookup metadata for HashiCorp Vault.
+
+
 .. _ug_credentials_hashivaultssh:
 
 HashiCorp Vault Signed SSH
@@ -307,13 +317,15 @@ When **HashiCorp Vault Signed SSH** is selected for **Credential Type**, provide
 - **TLS Authentication Role**: specify the role or certificate name in Vault that corresponds to your client certificate when using the TLS auth method. If it is not provided, Vault will attempt to match the certificate automatically
 - **Namespace name** specify the namespace name (Vault Enterprise only)
 - **Kubernetes role** specify the role name when using Kubernetes authentication
+- **Username**: enter the username of the user to be used to authenticate this service
+- **Password**: enter the password associated with the user to authenticate this service
 - **Path to Auth**: specify a path if other than the default path of ``/approle``
 
-For more detail about the Approle auth method and its fields, refer to the `Vault documentation for Approle Auth Method <https://www.vaultproject.io/docs/auth/approle>`_.
+For more detail about the Approle auth method and its fields, refer to the `Vault documentation for Approle Auth Method <https://developer.hashicorp.com/vault/docs/auth/approle>`_.
 
-For more detail about the Kubernetes auth method and its fields, refer to the `Vault documentation for Kubernetes auth method <https://developer.hashicorp.com/vault/docs/auth/kubernetes>` _.
+For more detail about the Kubernetes auth method and its fields, refer to the `Vault documentation for Kubernetes auth method <https://developer.hashicorp.com/vault/docs/auth/kubernetes>`_.
 
-For more detail about the TLS certificate auth method and its fields, refer to the `Vault documentation for TLS certificates auth method <https://developer.hashicorp.com/vault/docs/auth/cert>` _.
+For more detail about the TLS certificate auth method and its fields, refer to the `Vault documentation for TLS certificates auth method <https://developer.hashicorp.com/vault/docs/auth/cert>`_.
 
 Below shows an example of a configured HashiCorp SSH Secrets Engine credential.
 

docs/release_process.md

@@ -146,10 +146,10 @@ This workflow will take the generated images and promote them to quay.io.
 
 ## Send notifications
 Send notifications to the following groups:
-  * AWX Mailing List
+  * [Ansible Community forum](https://forum.ansible.com/)
   * #social:ansible.com IRC (@newsbot for inclusion in bullhorn)
   * #awx:ansible.com (no @newsbot in this room)
-  * #ansible-controller slack channel
+  * #aap-controller slack channel
 
 These messages are templated out for you in the output of `get_next_release.yml`.
 

licenses/Cython.txt

@@ -1,176 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS

manage.py

@@ -5,4 +5,5 @@
 
 if __name__ == '__main__':
     from awx import manage
+
     manage()

requirements/requirements.in

@@ -9,7 +9,6 @@ botocore
 channels
 channels-redis==3.4.1  # see UPGRADE BLOCKERs
 cryptography>=41.0.2  # CVE-2023-38325
-Cython<3 # Since the bump to PyYAML 5.4.1 this is now a mandatory dep
 daphne
 distro
 django==4.2.6  # CVE-2023-43665

requirements/requirements.txt

@@ -88,8 +88,6 @@ cryptography==41.0.3
     #   pyopenssl
     #   service-identity
     #   social-auth-core
-cython==0.29.32
-    # via -r /awx_devel/requirements/requirements.in
 daphne==3.0.2
     # via
     #   -r /awx_devel/requirements/requirements.in

requirements/requirements_dev.txt

@@ -10,7 +10,6 @@ black
 pytest!=7.0.0
 pytest-cov
 pytest-django
-pytest-pythonpath
 pytest-mock==1.11.1
 pytest-timeout
 pytest-xdist==1.34.0 # 2.0.0 broke zuul for some reason

requirements/updater.sh

@@ -4,7 +4,9 @@ set -ue
 requirements_in="$(readlink -f ./requirements.in)"
 requirements="$(readlink -f ./requirements.txt)"
 requirements_git="$(readlink -f ./requirements_git.txt)"
+requirements_dev="$(readlink -f ./requirements_dev.txt)"
 pip_compile="pip-compile --no-header --quiet -r --allow-unsafe"
+sanitize_git="1"
 
 _cleanup() {
   cd /
@@ -21,18 +23,22 @@ generate_requirements() {
   # FIXME: https://github.com/jazzband/pip-tools/issues/1558
   ${venv}/bin/python3 -m pip install -U 'pip<22.0' pip-tools
 
-  ${pip_compile} "${requirements_in}" "${requirements_git}" --output-file requirements.txt
+  ${pip_compile} "$1" --output-file requirements.txt
   # consider the git requirements for purposes of resolving deps
   # Then remove any git+ lines from requirements.txt
-  while IFS= read -r line; do
-    if [[ $line != \#* ]]; then  # ignore comments
-      sed -i "\!${line%#*}!d" requirements.txt
-    fi
-  done < "${requirements_git}"
+  if [[ "$sanitize_git" == "1" ]] ; then
+    while IFS= read -r line; do
+      if [[ $line != \#* ]]; then  # ignore comments
+        sed -i "\!${line%#*}!d" requirements.txt
+      fi
+    done < "${requirements_git}"
+  fi;
 }
 
 main() {
   base_dir=$(pwd)
+  dest_requirements="${requirements}"
+  input_requirements="${requirements_in} ${requirements_git}"
 
   _tmp=$(python -c "import tempfile; print(tempfile.mkdtemp(suffix='.awx-requirements', dir='/tmp'))")
 
@@ -42,6 +48,12 @@ main() {
     "run")
       NEEDS_HELP=0
     ;;
+    "dev")
+      dest_requirements="${requirements_dev}"
+      input_requirements="${requirements_dev}"
+      sanitize_git=0
+      NEEDS_HELP=0
+    ;;
     "upgrade")
       NEEDS_HELP=0
       pip_compile="${pip_compile} --upgrade"
@@ -61,12 +73,13 @@ main() {
     echo "This script generates requirements.txt from requirements.in and requirements_git.in"
     echo "It should be run from within the awx container"
     echo ""
-    echo "Usage: $0 [run|upgrade]"
+    echo "Usage: $0 [run|upgrade|dev]"
     echo ""
     echo "Commands:"
     echo "help      Print this message"
     echo "run       Run the process only upgrading pinned libraries from requirements.in"
     echo "upgrade   Upgrade all libraries to latest while respecting pinnings"
+    echo "dev       Pin the development requirements file"
     echo ""
     exit
   fi
@@ -85,10 +98,10 @@ main() {
   cp -vf requirements.txt "${_tmp}"
   cd "${_tmp}"
 
-  generate_requirements
+  generate_requirements "${input_requirements}"
 
   echo "Changing $base_dir to /awx_devel/requirements"
-  cat requirements.txt | sed "s:$base_dir:/awx_devel/requirements:" > "${requirements}"
+  cat requirements.txt | sed "s:$base_dir:/awx_devel/requirements:" > "${dest_requirements}"
 
   _cleanup
 }

tools/data_generators/rbac_dummy_data_generator.py

@@ -53,7 +53,7 @@ from awx.main.models import (  # noqa
     WorkflowJobTemplateNode,
     batch_role_ancestor_rebuilding,
 )
-from awx.main.models.schedules import Schedule #noqa
+from awx.main.models.schedules import Schedule  # noqa
 
 from awx.main.signals import disable_activity_stream, disable_computed_fields  # noqa
 
@@ -595,8 +595,6 @@ def make_the_data():
                     schedule._is_new = _
                     schedules.append(schedule)
 
-
-
             print('# Creating %d Labels' % n_labels)
             org_idx = 0
             for n in spread(n_labels, n_organizations):

tools/docker-compose/README.md

@@ -538,13 +538,15 @@ To create a secret connected to this vault in AWX you can run the following play
 ```bash
 export CONTROLLER_USERNAME=<your username>
 export CONTROLLER_PASSWORD=<your password>
-ansible-playbook tools/docker-compose/ansible/plumb_vault.yml
+ansible-playbook tools/docker-compose/ansible/plumb_vault.yml -e enable_ldap=false
 ```
 
 This will create the following items in your AWX instance:
 * A credential called `Vault Lookup Cred` tied to the vault instance.
+* A credential called `Vault UserPass Lookup Cred` tied to the vault instance.
 * A custom credential type called `Vault Custom Cred Type`.
-* A credential called `Credential From Vault` which is of the created type using the `Vault Lookup Cred` to get the password.
+* A credential called `Credential From HashiCorp Vault via Token Auth` which is of the created type using the `Vault Lookup Cred` to get the secret.
+* A credential called `Credential From HashiCorp Vault via UserPass Auth` which is of the created type using the `Vault Userpass Lookup Cred` to get the secret.
 
 The custom credential type adds a variable when used in a playbook called `the_secret_from_vault`.
 If you have a playbook like:
@@ -559,7 +561,46 @@ If you have a playbook like:
         var: the_secret_from_vault
 ```
 
-And run it through AWX with the credential `Credential From Vault` tied to it, the debug should result in `this_is_the_secret_value`
+And run it through AWX with the credential `Credential From Vault via Token Auth` tied to it, the debug should result in `this_is_the_secret_value`. If you run it through AWX with the credential `Credential From Vault via Userpass Auth`, the debug should result in `this_is_the_userpass_secret_value`. 
+
+### HashiVault with LDAP
+
+If you wish to have your OpenLDAP container connected to the Vault container, you will first need to have the OpenLDAP container running alongside AWX and Vault. 
+
+
+```bash
+
+VAULT=true LDAP=true make docker-compose
+
+```
+
+Similar to the above, you will need to unseal the vault before we can run the other needed playbooks. 
+
+```bash
+
+ansible-playbook tools/docker-compose/ansible/unseal_vault.yml
+
+```
+
+Now that the vault is unsealed, we can plumb the vault container now while passing true to enable_ldap extra var. 
+
+
+```bash
+
+export CONTROLLER_USERNAME=<your username>
+
+export CONTROLLER_PASSWORD=<your password>
+
+ansible-playbook tools/docker-compose/ansible/plumb_vault.yml -e enable_ldap=true
+
+```
+
+This will populate your AWX instance with LDAP specific items. 
+
+- A vault LDAP Lookup Cred tied to the LDAP `awx_ldap_vault` user called `Vault LDAP Lookup Cred`
+- A credential called `Credential From HashiCorp Vault via LDAP Auth`  which is of the created type using the `Vault LDAP Lookup Cred` to get the secret.
+
+And run it through AWX with the credential `Credential From HashiCorp Vault via LDAP Auth` tied to it, the debug should result in `this_is_the_ldap_secret_value`.
 
 The extremely non-obvious input is the fact that the fact prefixes "data/" unexpectedly.
 This was discovered by inspecting the secret with the vault CLI, which may help with future troubleshooting.