docs: update execution_nodes.md to follow changes for receptor_collection (#14247)

receptor_host_identifier can be left out
of group_vars and will default to the
'ansible_host' variable

.github/workflows/pr_body_check.yml

@@ -7,6 +7,7 @@ on:
     types: [opened, edited, reopened, synchronize]
 jobs:
   pr-check:
+    if: github.repository_owner == 'ansible' && endsWith(github.repository, 'awx')
     name: Scan PR description for semantic versioning keywords
     runs-on: ubuntu-latest
     permissions:

.github/workflows/pr_body_check_jira.yml

@@ -1,35 +0,0 @@
----
-name: Check body for reference to jira
-on:
-  pull_request:
-    branches:
-      - release_**
-jobs:
-  pr-check:
-    if: github.repository_owner == 'ansible' && github.repository != 'awx'
-    name: Scan PR description for JIRA links
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-    steps:
-      - name: Check for JIRA lines
-        env:
-          PR_BODY: ${{ github.event.pull_request.body }}
-        run: |
-          echo "$PR_BODY" | grep "JIRA: None" > no_jira
-          echo "$PR_BODY" | grep "JIRA: https://.*[0-9]+"> jira
-          exit 0
-        # We exit 0 and set the shell to prevent the returns from the greps from failing this step
-        # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
-        shell: bash {0}
-
-      - name: Check for exactly one item
-        run: |
-          if [ $(cat no_jira jira | wc -l) != 1 ] ; then
-            echo "The PR body must contain exactly one of [ 'JIRA: None' or 'JIRA: <one or more links>' ]"
-            echo "We counted $(cat no_jira jira | wc -l)"
-            exit 255;
-          else
-            exit 0;
-          fi

awx/api/serializers.py

@@ -5356,10 +5356,16 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria
 class InstanceLinkSerializer(BaseSerializer):
     class Meta:
         model = InstanceLink
-        fields = ('source', 'target', 'link_state')
+        fields = ('id', 'url', 'related', 'source', 'target', 'link_state')
 
-    source = serializers.SlugRelatedField(slug_field="hostname", read_only=True)
+    source = serializers.SlugRelatedField(slug_field="hostname", queryset=Instance.objects.all())
-    target = serializers.SlugRelatedField(slug_field="hostname", read_only=True)
+    target = serializers.SlugRelatedField(slug_field="hostname", queryset=Instance.objects.all())
+
+    def get_related(self, obj):
+        res = super(InstanceLinkSerializer, self).get_related(obj)
+        res['source_instance'] = self.reverse('api:instance_detail', kwargs={'pk': obj.source.id})
+        res['target_instance'] = self.reverse('api:instance_detail', kwargs={'pk': obj.target.id})
+        return res
 
 
 class InstanceNodeSerializer(BaseSerializer):
@@ -5376,6 +5382,7 @@ class InstanceSerializer(BaseSerializer):
     jobs_running = serializers.IntegerField(help_text=_('Count of jobs in the running or waiting state that are targeted for this instance'), read_only=True)
     jobs_total = serializers.IntegerField(help_text=_('Count of all jobs that target this instance'), read_only=True)
     health_check_pending = serializers.SerializerMethodField()
+    peers = serializers.SlugRelatedField(many=True, required=False, slug_field="hostname", queryset=Instance.objects.all())
 
     class Meta:
         model = Instance
@@ -5412,6 +5419,8 @@ class InstanceSerializer(BaseSerializer):
             'node_state',
             'ip_address',
             'listener_port',
+            'peers',
+            'peers_from_control_nodes',
         )
         extra_kwargs = {
             'node_type': {'initial': Instance.Types.EXECUTION, 'default': Instance.Types.EXECUTION},
@@ -5464,22 +5473,57 @@ class InstanceSerializer(BaseSerializer):
     def get_health_check_pending(self, obj):
         return obj.health_check_pending
 
-    def validate(self, data):
+    def validate(self, attrs):
-        if self.instance:
+        def get_field_from_model_or_attrs(fd):
-            if self.instance.node_type == Instance.Types.HOP:
+            return attrs.get(fd, self.instance and getattr(self.instance, fd) or None)
-                raise serializers.ValidationError("Hop node instances may not be changed.")
+
-        else:
+        def check_peers_changed():
-            if not settings.IS_K8S:
+            '''
-                raise serializers.ValidationError("Can only create instances on Kubernetes or OpenShift.")
+            return True if
-        return data
+            - 'peers' in attrs
+            - instance peers matches peers in attrs
+            '''
+            return self.instance and 'peers' in attrs and set(self.instance.peers.all()) != set(attrs['peers'])
+
+        if not self.instance and not settings.IS_K8S:
+            raise serializers.ValidationError(_("Can only create instances on Kubernetes or OpenShift."))
+
+        node_type = get_field_from_model_or_attrs("node_type")
+        peers_from_control_nodes = get_field_from_model_or_attrs("peers_from_control_nodes")
+        listener_port = get_field_from_model_or_attrs("listener_port")
+        peers = attrs.get('peers', [])
+
+        if peers_from_control_nodes and node_type not in (Instance.Types.EXECUTION, Instance.Types.HOP):
+            raise serializers.ValidationError(_("peers_from_control_nodes can only be enabled for execution or hop nodes."))
+
+        if node_type in [Instance.Types.CONTROL, Instance.Types.HYBRID]:
+            if check_peers_changed():
+                raise serializers.ValidationError(
+                    _("Setting peers manually for control nodes is not allowed. Enable peers_from_control_nodes on the hop and execution nodes instead.")
+                )
+
+        if not listener_port and peers_from_control_nodes:
+            raise serializers.ValidationError(_("Field listener_port must be a valid integer when peers_from_control_nodes is enabled."))
+
+        if not listener_port and self.instance and self.instance.peers_from.exists():
+            raise serializers.ValidationError(_("Field listener_port must be a valid integer when other nodes peer to it."))
+
+        for peer in peers:
+            if peer.listener_port is None:
+                raise serializers.ValidationError(_("Field listener_port must be set on peer ") + peer.hostname + ".")
+
+        if not settings.IS_K8S:
+            if check_peers_changed():
+                raise serializers.ValidationError(_("Cannot change peers."))
+
+        return super().validate(attrs)
 
     def validate_node_type(self, value):
-        if not self.instance:
+        if not self.instance and value not in [Instance.Types.HOP, Instance.Types.EXECUTION]:
-            if value not in (Instance.Types.EXECUTION,):
+            raise serializers.ValidationError(_("Can only create execution or hop nodes."))
-                raise serializers.ValidationError("Can only create execution nodes.")
+
-        else:
+        if self.instance and self.instance.node_type != value:
-            if self.instance.node_type != value:
+            raise serializers.ValidationError(_("Cannot change node type."))
-                raise serializers.ValidationError("Cannot change node type.")
 
         return value
 
@@ -5487,30 +5531,41 @@ class InstanceSerializer(BaseSerializer):
         if self.instance:
             if value != self.instance.node_state:
                 if not settings.IS_K8S:
-                    raise serializers.ValidationError("Can only change the state on Kubernetes or OpenShift.")
+                    raise serializers.ValidationError(_("Can only change the state on Kubernetes or OpenShift."))
                 if value != Instance.States.DEPROVISIONING:
-                    raise serializers.ValidationError("Can only change instances to the 'deprovisioning' state.")
+                    raise serializers.ValidationError(_("Can only change instances to the 'deprovisioning' state."))
-                if self.instance.node_type not in (Instance.Types.EXECUTION,):
+                if self.instance.node_type not in (Instance.Types.EXECUTION, Instance.Types.HOP):
-                    raise serializers.ValidationError("Can only deprovision execution nodes.")
+                    raise serializers.ValidationError(_("Can only deprovision execution or hop nodes."))
         else:
             if value and value != Instance.States.INSTALLED:
-                raise serializers.ValidationError("Can only create instances in the 'installed' state.")
+                raise serializers.ValidationError(_("Can only create instances in the 'installed' state."))
 
         return value
 
     def validate_hostname(self, value):
         """
-        - Hostname cannot be "localhost" - but can be something like localhost.domain
+        Cannot change the hostname
-        - Cannot change the hostname of an-already instantiated & initialized Instance object
         """
         if self.instance and self.instance.hostname != value:
-            raise serializers.ValidationError("Cannot change hostname.")
+            raise serializers.ValidationError(_("Cannot change hostname."))
 
         return value
 
     def validate_listener_port(self, value):
-        if self.instance and self.instance.listener_port != value:
+        """
-            raise serializers.ValidationError("Cannot change listener port.")
+        Cannot change listener port, unless going from none to integer, and vice versa
+        """
+        if value and self.instance and self.instance.listener_port and self.instance.listener_port != value:
+            raise serializers.ValidationError(_("Cannot change listener port."))
+
+        return value
+
+    def validate_peers_from_control_nodes(self, value):
+        """
+        Can only enable for K8S based deployments
+        """
+        if value and not settings.IS_K8S:
+            raise serializers.ValidationError(_("Can only be enabled on Kubernetes or Openshift."))
 
         return value
 
@@ -5518,7 +5573,19 @@ class InstanceSerializer(BaseSerializer):
 class InstanceHealthCheckSerializer(BaseSerializer):
     class Meta:
         model = Instance
-        read_only_fields = ('uuid', 'hostname', 'version', 'last_health_check', 'errors', 'cpu', 'memory', 'cpu_capacity', 'mem_capacity', 'capacity')
+        read_only_fields = (
+            'uuid',
+            'hostname',
+            'ip_address',
+            'version',
+            'last_health_check',
+            'errors',
+            'cpu',
+            'memory',
+            'cpu_capacity',
+            'mem_capacity',
+            'capacity',
+        )
         fields = read_only_fields
 
 

awx/api/templates/instance_install_bundle/group_vars/all.yml

@@ -3,21 +3,35 @@ receptor_group: awx
 receptor_verify: true
 receptor_tls: true
 receptor_mintls13: false
+{% if instance.node_type == "execution" %}
 receptor_work_commands:
   ansible-runner:
     command: ansible-runner
     params: worker
     allowruntimeparams: true
     verifysignature: true
+additional_python_packages:
+  - ansible-runner
+{% endif %}
 custom_worksign_public_keyfile: receptor/work_public_key.pem
 custom_tls_certfile: receptor/tls/receptor.crt
 custom_tls_keyfile: receptor/tls/receptor.key
 custom_ca_certfile: receptor/tls/ca/mesh-CA.crt
 receptor_protocol: 'tcp'
+{% if instance.listener_port %}
 receptor_listener: true
 receptor_port: {{ instance.listener_port }}
-receptor_dependencies:
+{% else %}
-  - python39-pip
+receptor_listener: false
+{% endif %}
+{% if peers %}
+receptor_peers:
+{% for peer in peers %}
+  - host: {{ peer.host }}
+    port: {{ peer.port }}
+    protocol: tcp
+{% endfor %}
+{% endif %}
 {% verbatim %}
 podman_user: "{{ receptor_user }}"
 podman_group: "{{ receptor_group }}"

awx/api/templates/instance_install_bundle/install_receptor.yml

@@ -1,20 +1,16 @@
-{% verbatim %}
 ---
 - hosts: all
   become: yes
   tasks:
     - name: Create the receptor user
       user:
+{% verbatim %}
         name: "{{ receptor_user }}"
+{% endverbatim %}
         shell: /bin/bash
-    - name: Enable Copr repo for Receptor
+{% if instance.node_type == "execution" %}
-      command: dnf copr enable ansible-awx/receptor -y
     - import_role:
         name: ansible.receptor.podman
+{% endif %}
     - import_role:
         name: ansible.receptor.setup
-    - name: Install ansible-runner
-      pip:
-        name: ansible-runner
-        executable: pip3.9
-{% endverbatim %}

awx/api/templates/instance_install_bundle/requirements.yml

@@ -1,4 +1,4 @@
 ---
 collections:
   - name: ansible.receptor
-    version: 1.1.0
+    version: 2.0.0

awx/api/views/__init__.py

@@ -341,17 +341,18 @@ class InstanceDetail(RetrieveUpdateAPIView):
 
     def update_raw_data(self, data):
         # these fields are only valid on creation of an instance, so they unwanted on detail view
-        data.pop('listener_port', None)
         data.pop('node_type', None)
         data.pop('hostname', None)
+        data.pop('ip_address', None)
         return super(InstanceDetail, self).update_raw_data(data)
 
     def update(self, request, *args, **kwargs):
         r = super(InstanceDetail, self).update(request, *args, **kwargs)
         if status.is_success(r.status_code):
             obj = self.get_object()
-            obj.set_capacity_value()
+            capacity_changed = obj.set_capacity_value()
-            obj.save(update_fields=['capacity'])
+            if capacity_changed:
+                obj.save(update_fields=['capacity'])
             r.data = serializers.InstanceSerializer(obj, context=self.get_serializer_context()).to_representation(obj)
         return r
 

awx/api/views/instance_install_bundle.py

@@ -6,6 +6,8 @@ import io
 import ipaddress
 import os
 import tarfile
+import time
+import re
 
 import asn1
 from awx.api import serializers
@@ -40,6 +42,8 @@ RECEPTOR_OID = "1.3.6.1.4.1.2312.19.1"
 # │   │   └── receptor.key
 # │   └── work-public-key.pem
 # └── requirements.yml
+
+
 class InstanceInstallBundle(GenericAPIView):
     name = _('Install Bundle')
     model = models.Instance
@@ -49,9 +53,9 @@ class InstanceInstallBundle(GenericAPIView):
     def get(self, request, *args, **kwargs):
         instance_obj = self.get_object()
 
-        if instance_obj.node_type not in ('execution',):
+        if instance_obj.node_type not in ('execution', 'hop'):
             return Response(
-                data=dict(msg=_('Install bundle can only be generated for execution nodes.')),
+                data=dict(msg=_('Install bundle can only be generated for execution or hop nodes.')),
                 status=status.HTTP_400_BAD_REQUEST,
             )
 
@@ -66,37 +70,37 @@ class InstanceInstallBundle(GenericAPIView):
                 # generate and write the receptor key to receptor/tls/receptor.key in the tar file
                 key, cert = generate_receptor_tls(instance_obj)
 
+                def tar_addfile(tarinfo, filecontent):
+                    tarinfo.mtime = time.time()
+                    tarinfo.size = len(filecontent)
+                    tar.addfile(tarinfo, io.BytesIO(filecontent))
+
                 key_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/receptor/tls/receptor.key")
-                key_tarinfo.size = len(key)
+                tar_addfile(key_tarinfo, key)
-                tar.addfile(key_tarinfo, io.BytesIO(key))
 
                 cert_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/receptor/tls/receptor.crt")
                 cert_tarinfo.size = len(cert)
-                tar.addfile(cert_tarinfo, io.BytesIO(cert))
+                tar_addfile(cert_tarinfo, cert)
 
                 # generate and write install_receptor.yml to the tar file
-                playbook = generate_playbook().encode('utf-8')
+                playbook = generate_playbook(instance_obj).encode('utf-8')
                 playbook_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/install_receptor.yml")
-                playbook_tarinfo.size = len(playbook)
+                tar_addfile(playbook_tarinfo, playbook)
-                tar.addfile(playbook_tarinfo, io.BytesIO(playbook))
 
                 # generate and write inventory.yml to the tar file
                 inventory_yml = generate_inventory_yml(instance_obj).encode('utf-8')
                 inventory_yml_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/inventory.yml")
-                inventory_yml_tarinfo.size = len(inventory_yml)
+                tar_addfile(inventory_yml_tarinfo, inventory_yml)
-                tar.addfile(inventory_yml_tarinfo, io.BytesIO(inventory_yml))
 
                 # generate and write group_vars/all.yml to the tar file
                 group_vars = generate_group_vars_all_yml(instance_obj).encode('utf-8')
                 group_vars_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/group_vars/all.yml")
-                group_vars_tarinfo.size = len(group_vars)
+                tar_addfile(group_vars_tarinfo, group_vars)
-                tar.addfile(group_vars_tarinfo, io.BytesIO(group_vars))
 
                 # generate and write requirements.yml to the tar file
                 requirements_yml = generate_requirements_yml().encode('utf-8')
                 requirements_yml_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/requirements.yml")
-                requirements_yml_tarinfo.size = len(requirements_yml)
+                tar_addfile(requirements_yml_tarinfo, requirements_yml)
-                tar.addfile(requirements_yml_tarinfo, io.BytesIO(requirements_yml))
 
             # respond with the tarfile
             f.seek(0)
@@ -105,8 +109,10 @@ class InstanceInstallBundle(GenericAPIView):
             return response
 
 
-def generate_playbook():
+def generate_playbook(instance_obj):
-    return render_to_string("instance_install_bundle/install_receptor.yml")
+    playbook_yaml = render_to_string("instance_install_bundle/install_receptor.yml", context=dict(instance=instance_obj))
+    # convert consecutive newlines with a single newline
+    return re.sub(r'\n+', '\n', playbook_yaml)
 
 
 def generate_requirements_yml():
@@ -118,7 +124,12 @@ def generate_inventory_yml(instance_obj):
 
 
 def generate_group_vars_all_yml(instance_obj):
-    return render_to_string("instance_install_bundle/group_vars/all.yml", context=dict(instance=instance_obj))
+    peers = []
+    for instance in instance_obj.peers.all():
+        peers.append(dict(host=instance.hostname, port=instance.listener_port))
+    all_yaml = render_to_string("instance_install_bundle/group_vars/all.yml", context=dict(instance=instance_obj, peers=peers))
+    # convert consecutive newlines with a single newline
+    return re.sub(r'\n+', '\n', all_yaml)
 
 
 def generate_receptor_tls(instance_obj):

awx/main/management/commands/provision_instance.py

@@ -25,17 +25,20 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('--hostname', dest='hostname', type=str, help="Hostname used during provisioning")
+        parser.add_argument('--listener_port', dest='listener_port', type=int, help="Receptor listener port")
         parser.add_argument('--node_type', type=str, default='hybrid', choices=['control', 'execution', 'hop', 'hybrid'], help="Instance Node type")
         parser.add_argument('--uuid', type=str, help="Instance UUID")
 
-    def _register_hostname(self, hostname, node_type, uuid):
+    def _register_hostname(self, hostname, node_type, uuid, listener_port):
         if not hostname:
             if not settings.AWX_AUTO_DEPROVISION_INSTANCES:
                 raise CommandError('Registering with values from settings only intended for use in K8s installs')
 
             from awx.main.management.commands.register_queue import RegisterQueue
 
-            (changed, instance) = Instance.objects.register(ip_address=os.environ.get('MY_POD_IP'), node_type='control', node_uuid=settings.SYSTEM_UUID)
+            (changed, instance) = Instance.objects.register(
+                ip_address=os.environ.get('MY_POD_IP'), listener_port=listener_port, node_type='control', node_uuid=settings.SYSTEM_UUID
+            )
             RegisterQueue(settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME, 100, 0, [], is_container_group=False).register()
             RegisterQueue(
                 settings.DEFAULT_EXECUTION_QUEUE_NAME,
@@ -48,7 +51,7 @@ class Command(BaseCommand):
                 max_concurrent_jobs=settings.DEFAULT_EXECUTION_QUEUE_MAX_CONCURRENT_JOBS,
             ).register()
         else:
-            (changed, instance) = Instance.objects.register(hostname=hostname, node_type=node_type, node_uuid=uuid)
+            (changed, instance) = Instance.objects.register(hostname=hostname, node_type=node_type, node_uuid=uuid, listener_port=listener_port)
         if changed:
             print("Successfully registered instance {}".format(hostname))
         else:
@@ -58,6 +61,6 @@ class Command(BaseCommand):
     @transaction.atomic
     def handle(self, **options):
         self.changed = False
-        self._register_hostname(options.get('hostname'), options.get('node_type'), options.get('uuid'))
+        self._register_hostname(options.get('hostname'), options.get('node_type'), options.get('uuid'), options.get('listener_port'))
         if self.changed:
             print("(changed: True)")

awx/main/managers.py

@@ -115,10 +115,13 @@ class InstanceManager(models.Manager):
             return node[0]
         raise RuntimeError("No instance found with the current cluster host id")
 
-    def register(self, node_uuid=None, hostname=None, ip_address=None, node_type='hybrid', defaults=None):
+    def register(self, node_uuid=None, hostname=None, ip_address="", listener_port=None, node_type='hybrid', defaults=None):
         if not hostname:
             hostname = settings.CLUSTER_HOST_ID
 
+        if not ip_address:
+            ip_address = ""
+
         with advisory_lock('instance_registration_%s' % hostname):
             if settings.AWX_AUTO_DEPROVISION_INSTANCES:
                 # detect any instances with the same IP address.
@@ -157,6 +160,9 @@ class InstanceManager(models.Manager):
                 if instance.node_type != node_type:
                     instance.node_type = node_type
                     update_fields.append('node_type')
+                if instance.listener_port != listener_port:
+                    instance.listener_port = listener_port
+                    update_fields.append('listener_port')
                 if update_fields:
                     instance.save(update_fields=update_fields)
                     return (True, instance)
@@ -167,12 +173,11 @@ class InstanceManager(models.Manager):
             create_defaults = {
                 'node_state': Instance.States.INSTALLED,
                 'capacity': 0,
-                'listener_port': 27199,
             }
             if defaults is not None:
                 create_defaults.update(defaults)
             uuid_option = {'uuid': node_uuid if node_uuid is not None else uuid.uuid4()}
             if node_type == 'execution' and 'version' not in create_defaults:
                 create_defaults['version'] = RECEPTOR_PENDING
-            instance = self.create(hostname=hostname, ip_address=ip_address, node_type=node_type, **create_defaults, **uuid_option)
+            instance = self.create(hostname=hostname, ip_address=ip_address, listener_port=listener_port, node_type=node_type, **create_defaults, **uuid_option)
         return (True, instance)

awx/main/migrations/0187_hop_nodes.py

@@ -0,0 +1,75 @@
+# Generated by Django 4.2.3 on 2023-08-04 20:50
+
+import django.core.validators
+from django.db import migrations, models
+from django.conf import settings
+
+
+def automatically_peer_from_control_plane(apps, schema_editor):
+    if settings.IS_K8S:
+        Instance = apps.get_model('main', 'Instance')
+        Instance.objects.filter(node_type='execution').update(peers_from_control_nodes=True)
+        Instance.objects.filter(node_type='control').update(listener_port=None)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('main', '0186_drop_django_taggit'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='instancelink',
+            options={'ordering': ('id',)},
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='peers_from_control_nodes',
+            field=models.BooleanField(default=False, help_text='If True, control plane cluster nodes should automatically peer to it.'),
+        ),
+        migrations.AlterField(
+            model_name='instance',
+            name='ip_address',
+            field=models.CharField(blank=True, default='', max_length=50),
+        ),
+        migrations.AlterField(
+            model_name='instance',
+            name='listener_port',
+            field=models.PositiveIntegerField(
+                blank=True,
+                default=None,
+                help_text='Port that Receptor will listen for incoming connections on.',
+                null=True,
+                validators=[django.core.validators.MinValueValidator(1024), django.core.validators.MaxValueValidator(65535)],
+            ),
+        ),
+        migrations.AlterField(
+            model_name='instance',
+            name='peers',
+            field=models.ManyToManyField(related_name='peers_from', through='main.InstanceLink', to='main.instance'),
+        ),
+        migrations.AlterField(
+            model_name='instancelink',
+            name='link_state',
+            field=models.CharField(
+                choices=[('adding', 'Adding'), ('established', 'Established'), ('removing', 'Removing')],
+                default='adding',
+                help_text='Indicates the current life cycle stage of this peer link.',
+                max_length=16,
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name='instance',
+            constraint=models.UniqueConstraint(
+                condition=models.Q(('ip_address', ''), _negated=True),
+                fields=('ip_address',),
+                name='unique_ip_address_not_empty',
+                violation_error_message='Field ip_address must be unique.',
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name='instancelink',
+            constraint=models.CheckConstraint(check=models.Q(('source', models.F('target')), _negated=True), name='source_and_target_can_not_be_equal'),
+        ),
+        migrations.RunPython(automatically_peer_from_control_plane),
+    ]

awx/main/models/credential/__init__.py

@@ -17,6 +17,7 @@ from jinja2 import sandbox
 from django.db import models
 from django.utils.translation import gettext_lazy as _, gettext_noop
 from django.core.exceptions import ValidationError
+from django.conf import settings
 from django.utils.encoding import force_str
 from django.utils.functional import cached_property
 from django.utils.timezone import now
@@ -30,7 +31,7 @@ from awx.main.fields import (
     CredentialTypeInjectorField,
     DynamicCredentialInputField,
 )
-from awx.main.utils import decrypt_field, classproperty
+from awx.main.utils import decrypt_field, classproperty, set_environ
 from awx.main.utils.safe_yaml import safe_dump
 from awx.main.utils.execution_environments import to_container_path
 from awx.main.validators import validate_ssh_private_key
@@ -1252,7 +1253,9 @@ class CredentialInputSource(PrimordialModel):
                 backend_kwargs[field_name] = value
 
         backend_kwargs.update(self.metadata)
-        return backend(**backend_kwargs)
+
+        with set_environ(**settings.AWX_TASK_ENV):
+            return backend(**backend_kwargs)
 
     def get_absolute_url(self, request=None):
         view_name = 'api:credential_input_source_detail'

awx/main/models/ha.py

@@ -12,13 +12,14 @@ from django.dispatch import receiver
 from django.utils.translation import gettext_lazy as _
 from django.conf import settings
 from django.utils.timezone import now, timedelta
-from django.db.models import Sum
+from django.db.models import Sum, Q
 
 import redis
 from solo.models import SingletonModel
 
 # AWX
 from awx import __version__ as awx_application_version
+from awx.main.utils import is_testing
 from awx.api.versioning import reverse
 from awx.main.fields import ImplicitRoleField
 from awx.main.managers import InstanceManager, UUID_DEFAULT
@@ -70,16 +71,33 @@ class InstanceLink(BaseModel):
         REMOVING = 'removing', _('Removing')
 
     link_state = models.CharField(
-        choices=States.choices, default=States.ESTABLISHED, max_length=16, help_text=_("Indicates the current life cycle stage of this peer link.")
+        choices=States.choices, default=States.ADDING, max_length=16, help_text=_("Indicates the current life cycle stage of this peer link.")
     )
 
     class Meta:
         unique_together = ('source', 'target')
+        ordering = ("id",)
+        constraints = [models.CheckConstraint(check=~models.Q(source=models.F('target')), name='source_and_target_can_not_be_equal')]
 
 
 class Instance(HasPolicyEditsMixin, BaseModel):
     """A model representing an AWX instance running against this database."""
 
+    class Meta:
+        app_label = 'main'
+        ordering = ("hostname",)
+        constraints = [
+            models.UniqueConstraint(
+                fields=["ip_address"],
+                condition=~Q(ip_address=""),  # don't apply to constraint to empty entries
+                name="unique_ip_address_not_empty",
+                violation_error_message=_("Field ip_address must be unique."),
+            )
+        ]
+
+    def __str__(self):
+        return self.hostname
+
     objects = InstanceManager()
 
     # Fields set in instance registration
@@ -87,10 +105,8 @@ class Instance(HasPolicyEditsMixin, BaseModel):
     hostname = models.CharField(max_length=250, unique=True)
     ip_address = models.CharField(
         blank=True,
-        null=True,
+        default="",
-        default=None,
         max_length=50,
-        unique=True,
     )
     # Auto-fields, implementation is different from BaseModel
     created = models.DateTimeField(auto_now_add=True)
@@ -169,16 +185,14 @@ class Instance(HasPolicyEditsMixin, BaseModel):
     )
     listener_port = models.PositiveIntegerField(
         blank=True,
-        default=27199,
+        null=True,
-        validators=[MinValueValidator(1), MaxValueValidator(65535)],
+        default=None,
+        validators=[MinValueValidator(1024), MaxValueValidator(65535)],
         help_text=_("Port that Receptor will listen for incoming connections on."),
     )
 
-    peers = models.ManyToManyField('self', symmetrical=False, through=InstanceLink, through_fields=('source', 'target'))
+    peers = models.ManyToManyField('self', symmetrical=False, through=InstanceLink, through_fields=('source', 'target'), related_name='peers_from')
-
+    peers_from_control_nodes = models.BooleanField(default=False, help_text=_("If True, control plane cluster nodes should automatically peer to it."))
-    class Meta:
-        app_label = 'main'
-        ordering = ("hostname",)
 
     POLICY_FIELDS = frozenset(('managed_by_policy', 'hostname', 'capacity_adjustment'))
 
@@ -279,6 +293,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         return update_fields
 
     def set_capacity_value(self):
+        old_val = self.capacity
         """Sets capacity according to capacity adjustment rule (no save)"""
         if self.enabled and self.node_type != 'hop':
             lower_cap = min(self.mem_capacity, self.cpu_capacity)
@@ -286,6 +301,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
             self.capacity = lower_cap + (higher_cap - lower_cap) * self.capacity_adjustment
         else:
             self.capacity = 0
+        return int(self.capacity) != int(old_val)  # return True if value changed
 
     def refresh_capacity_fields(self):
         """Update derived capacity fields from cpu and memory (no save)"""
@@ -464,21 +480,50 @@ def on_instance_group_saved(sender, instance, created=False, raw=False, **kwargs
         instance.set_default_policy_fields()
 
 
+def schedule_write_receptor_config(broadcast=True):
+    from awx.main.tasks.receptor import write_receptor_config  # prevents circular import
+
+    # broadcast to all control instances to update their receptor configs
+    if broadcast:
+        connection.on_commit(lambda: write_receptor_config.apply_async(queue='tower_broadcast_all'))
+    else:
+        if not is_testing():
+            write_receptor_config()  # just run locally
+
+
 @receiver(post_save, sender=Instance)
 def on_instance_saved(sender, instance, created=False, raw=False, **kwargs):
-    if settings.IS_K8S and instance.node_type in (Instance.Types.EXECUTION,):
+    '''
+    Here we link control nodes to hop or execution nodes based on the
+    peers_from_control_nodes field.
+    write_receptor_config should be called on each control node when:
+    1. new node is created with peers_from_control_nodes enabled
+    2. a node changes its value of peers_from_control_nodes
+    3. a new control node comes online and has instances to peer to
+    '''
+    if created and settings.IS_K8S and instance.node_type in [Instance.Types.CONTROL, Instance.Types.HYBRID]:
+        inst = Instance.objects.filter(peers_from_control_nodes=True)
+        if set(instance.peers.all()) != set(inst):
+            instance.peers.set(inst)
+            schedule_write_receptor_config(broadcast=False)
+
+    if settings.IS_K8S and instance.node_type in [Instance.Types.HOP, Instance.Types.EXECUTION]:
         if instance.node_state == Instance.States.DEPROVISIONING:
             from awx.main.tasks.receptor import remove_deprovisioned_node  # prevents circular import
 
             # wait for jobs on the node to complete, then delete the
             # node and kick off write_receptor_config
             connection.on_commit(lambda: remove_deprovisioned_node.apply_async([instance.hostname]))
-
+        else:
-        if instance.node_state == Instance.States.INSTALLED:
+            control_instances = set(Instance.objects.filter(node_type__in=[Instance.Types.CONTROL, Instance.Types.HYBRID]))
-            from awx.main.tasks.receptor import write_receptor_config  # prevents circular import
+            if instance.peers_from_control_nodes:
-
+                if (control_instances & set(instance.peers_from.all())) != set(control_instances):
-            # broadcast to all control instances to update their receptor configs
+                    instance.peers_from.add(*control_instances)
-            connection.on_commit(lambda: write_receptor_config.apply_async(queue='tower_broadcast_all'))
+                    schedule_write_receptor_config()  # keep method separate to make pytest mocking easier
+            else:
+                if set(control_instances) & set(instance.peers_from.all()):
+                    instance.peers_from.remove(*control_instances)
+                    schedule_write_receptor_config()
 
     if created or instance.has_policy_changes():
         schedule_policy_task()
@@ -493,6 +538,8 @@ def on_instance_group_deleted(sender, instance, using, **kwargs):
 @receiver(post_delete, sender=Instance)
 def on_instance_deleted(sender, instance, using, **kwargs):
     schedule_policy_task()
+    if settings.IS_K8S and instance.node_type in (Instance.Types.EXECUTION, Instance.Types.HOP) and instance.peers_from_control_nodes:
+        schedule_write_receptor_config()
 
 
 class UnifiedJobTemplateInstanceGroupMembership(models.Model):

awx/main/tasks/receptor.py

@@ -30,6 +30,7 @@ from awx.main.tasks.signals import signal_state, signal_callback, SignalExit
 from awx.main.models import Instance, InstanceLink, UnifiedJob
 from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.publish import task
+from awx.main.utils.pglock import advisory_lock
 
 # Receptorctl
 from receptorctl.socket_interface import ReceptorControl
@@ -675,26 +676,41 @@ RECEPTOR_CONFIG_STARTER = (
 )
 
 
-@task()
+def should_update_config(instances):
-def write_receptor_config():
+    '''
-    lock = FileLock(__RECEPTOR_CONF_LOCKFILE)
+    checks that the list of instances matches the list of
-    with lock:
+    tcp-peers in the config
-        receptor_config = list(RECEPTOR_CONFIG_STARTER)
+    '''
+    current_config = read_receptor_config()  # this gets receptor conf lock
+    current_peers = []
+    for config_entry in current_config:
+        for key, value in config_entry.items():
+            if key.endswith('-peer'):
+                current_peers.append(value['address'])
+    intended_peers = [f"{i.hostname}:{i.listener_port}" for i in instances]
+    logger.debug(f"Peers current {current_peers} intended {intended_peers}")
+    if set(current_peers) == set(intended_peers):
+        return False  # config file is already update to date
 
-        this_inst = Instance.objects.me()
+    return True
-        instances = Instance.objects.filter(node_type=Instance.Types.EXECUTION)
-        existing_peers = {link.target_id for link in InstanceLink.objects.filter(source=this_inst)}
-        new_links = []
-        for instance in instances:
-            peer = {'tcp-peer': {'address': f'{instance.hostname}:{instance.listener_port}', 'tls': 'tlsclient'}}
-            receptor_config.append(peer)
-            if instance.id not in existing_peers:
-                new_links.append(InstanceLink(source=this_inst, target=instance, link_state=InstanceLink.States.ADDING))
 
-        InstanceLink.objects.bulk_create(new_links)
 
-        with open(__RECEPTOR_CONF, 'w') as file:
+def generate_config_data():
-            yaml.dump(receptor_config, file, default_flow_style=False)
+    # returns two values
+    #   receptor config - based on current database peers
+    #   should_update   - If True, receptor_config differs from the receptor conf file on disk
+    instances = Instance.objects.filter(node_type__in=(Instance.Types.EXECUTION, Instance.Types.HOP), peers_from_control_nodes=True)
+
+    receptor_config = list(RECEPTOR_CONFIG_STARTER)
+    for instance in instances:
+        peer = {'tcp-peer': {'address': f'{instance.hostname}:{instance.listener_port}', 'tls': 'tlsclient'}}
+        receptor_config.append(peer)
+    should_update = should_update_config(instances)
+    return receptor_config, should_update
+
+
+def reload_receptor():
+    logger.warning("Receptor config changed, reloading receptor")
 
     # This needs to be outside of the lock because this function itself will acquire the lock.
     receptor_ctl = get_receptor_ctl()
@@ -710,8 +726,29 @@ def write_receptor_config():
     else:
         raise RuntimeError("Receptor reload failed")
 
-    links = InstanceLink.objects.filter(source=this_inst, target__in=instances, link_state=InstanceLink.States.ADDING)
+
-    links.update(link_state=InstanceLink.States.ESTABLISHED)
+@task()
+def write_receptor_config():
+    """
+    This task runs async on each control node, K8S only.
+    It is triggered whenever remote is added or removed, or if peers_from_control_nodes
+    is flipped.
+    It is possible for write_receptor_config to be called multiple times.
+    For example, if new instances are added in quick succession.
+    To prevent that case, each control node first grabs a DB advisory lock, specific
+    to just that control node (i.e. multiple control nodes can run this function
+    at the same time, since it only writes the local receptor config file)
+    """
+    with advisory_lock(f"{settings.CLUSTER_HOST_ID}_write_receptor_config", wait=True):
+        # Config file needs to be updated
+        receptor_config, should_update = generate_config_data()
+        if should_update:
+            lock = FileLock(__RECEPTOR_CONF_LOCKFILE)
+            with lock:
+                with open(__RECEPTOR_CONF, 'w') as file:
+                    yaml.dump(receptor_config, file, default_flow_style=False)
+
+            reload_receptor()
 
 
 @task(queue=get_task_queuename)
@@ -731,6 +768,3 @@ def remove_deprovisioned_node(hostname):
 
     # This will as a side effect also delete the InstanceLinks that are tied to it.
     Instance.objects.filter(hostname=hostname).delete()
-
-    # Update the receptor configs for all of the control-plane.
-    write_receptor_config.apply_async(queue='tower_broadcast_all')

awx/main/tasks/system.py

@@ -512,13 +512,37 @@ def execution_node_health_check(node):
     return data
 
 
-def inspect_execution_nodes(instance_list):
+def inspect_established_receptor_connections(mesh_status):
-    with advisory_lock('inspect_execution_nodes_lock', wait=False):
+    '''
-        node_lookup = {inst.hostname: inst for inst in instance_list}
+    Flips link state from ADDING to ESTABLISHED
+    If the InstanceLink source and target match the entries
+    in Known Connection Costs, flip to Established.
+    '''
+    from awx.main.models import InstanceLink
 
+    all_links = InstanceLink.objects.filter(link_state=InstanceLink.States.ADDING)
+    if not all_links.exists():
+        return
+    active_receptor_conns = mesh_status['KnownConnectionCosts']
+    update_links = []
+    for link in all_links:
+        if link.link_state != InstanceLink.States.REMOVING:
+            if link.target.hostname in active_receptor_conns.get(link.source.hostname, {}):
+                if link.link_state is not InstanceLink.States.ESTABLISHED:
+                    link.link_state = InstanceLink.States.ESTABLISHED
+                    update_links.append(link)
+
+    InstanceLink.objects.bulk_update(update_links, ['link_state'])
+
+
+def inspect_execution_and_hop_nodes(instance_list):
+    with advisory_lock('inspect_execution_and_hop_nodes_lock', wait=False):
+        node_lookup = {inst.hostname: inst for inst in instance_list}
         ctl = get_receptor_ctl()
         mesh_status = ctl.simple_command('status')
 
+        inspect_established_receptor_connections(mesh_status)
+
         nowtime = now()
         workers = mesh_status['Advertisements']
 
@@ -576,7 +600,7 @@ def cluster_node_heartbeat(dispatch_time=None, worker_tasks=None):
             this_inst = inst
             break
 
-    inspect_execution_nodes(instance_list)
+    inspect_execution_and_hop_nodes(instance_list)
 
     for inst in list(instance_list):
         if inst == this_inst:

awx/main/tests/functional/api/test_instance.py

@@ -84,5 +84,6 @@ def test_custom_hostname_regex(post, admin_user):
                 "hostname": value[0],
                 "node_type": "execution",
                 "node_state": "installed",
+                "peers": [],
             }
             post(url=url, user=admin_user, data=data, expect=value[1])

awx/main/tests/functional/api/test_instance_peers.py

@@ -0,0 +1,342 @@
+import pytest
+import yaml
+import itertools
+from unittest import mock
+
+from django.db.utils import IntegrityError
+
+from awx.api.versioning import reverse
+from awx.main.models import Instance
+from awx.api.views.instance_install_bundle import generate_group_vars_all_yml
+
+
+def has_peer(group_vars, peer):
+    peers = group_vars.get('receptor_peers', [])
+    for p in peers:
+        if f"{p['host']}:{p['port']}" == peer:
+            return True
+    return False
+
+
+@pytest.mark.django_db
+class TestPeers:
+    @pytest.fixture(autouse=True)
+    def configure_settings(self, settings):
+        settings.IS_K8S = True
+
+    @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
+    def test_prevent_peering_to_self(self, node_type):
+        """
+        cannot peer to self
+        """
+        control_instance = Instance.objects.create(hostname='abc', node_type=node_type)
+        with pytest.raises(IntegrityError):
+            control_instance.peers.add(control_instance)
+
+    @pytest.mark.parametrize('node_type', ['control', 'hybrid', 'hop', 'execution'])
+    def test_creating_node(self, node_type, admin_user, post):
+        """
+        can only add hop and execution nodes via API
+        """
+        post(
+            url=reverse('api:instance_list'),
+            data={"hostname": "abc", "node_type": node_type},
+            user=admin_user,
+            expect=400 if node_type in ['control', 'hybrid'] else 201,
+        )
+
+    def test_changing_node_type(self, admin_user, patch):
+        """
+        cannot change node type
+        """
+        hop = Instance.objects.create(hostname='abc', node_type="hop")
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
+            data={"node_type": "execution"},
+            user=admin_user,
+            expect=400,
+        )
+
+    @pytest.mark.parametrize('node_type', ['hop', 'execution'])
+    def test_listener_port_null(self, node_type, admin_user, post):
+        """
+        listener_port can be None
+        """
+        post(
+            url=reverse('api:instance_list'),
+            data={"hostname": "abc", "node_type": node_type, "listener_port": None},
+            user=admin_user,
+            expect=201,
+        )
+
+    @pytest.mark.parametrize('node_type, allowed', [('control', False), ('hybrid', False), ('hop', True), ('execution', True)])
+    def test_peers_from_control_nodes_allowed(self, node_type, allowed, post, admin_user):
+        """
+        only hop and execution nodes can have peers_from_control_nodes set to True
+        """
+        post(
+            url=reverse('api:instance_list'),
+            data={"hostname": "abc", "peers_from_control_nodes": True, "node_type": node_type, "listener_port": 6789},
+            user=admin_user,
+            expect=201 if allowed else 400,
+        )
+
+    def test_listener_port_is_required(self, admin_user, post):
+        """
+        if adding instance to peers list, that instance must have listener_port set
+        """
+        Instance.objects.create(hostname='abc', node_type="hop", listener_port=None)
+        post(
+            url=reverse('api:instance_list'),
+            data={"hostname": "ex", "peers_from_control_nodes": False, "node_type": "execution", "listener_port": None, "peers": ["abc"]},
+            user=admin_user,
+            expect=400,
+        )
+
+    def test_peers_from_control_nodes_listener_port_enabled(self, admin_user, post):
+        """
+        if peers_from_control_nodes is True, listener_port must an integer
+        Assert that all other combinations are allowed
+        """
+        for index, item in enumerate(itertools.product(['hop', 'execution'], [True, False], [None, 6789])):
+            node_type, peers_from, listener_port = item
+            # only disallowed case is when peers_from is True and listener port is None
+            disallowed = peers_from and not listener_port
+            post(
+                url=reverse('api:instance_list'),
+                data={"hostname": f"abc{index}", "peers_from_control_nodes": peers_from, "node_type": node_type, "listener_port": listener_port},
+                user=admin_user,
+                expect=400 if disallowed else 201,
+            )
+
+    @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
+    def test_disallow_modifying_peers_control_nodes(self, node_type, admin_user, patch):
+        """
+        for control nodes, peers field should not be
+        modified directly via patch.
+        """
+        control = Instance.objects.create(hostname='abc', node_type=node_type)
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop', peers_from_control_nodes=True, listener_port=6789)
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop', peers_from_control_nodes=False, listener_port=6789)
+        assert [hop1] == list(control.peers.all())  # only hop1 should be peered
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
+            data={"peers": ["hop2"]},
+            user=admin_user,
+            expect=400,  # cannot add peers directly
+        )
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
+            data={"peers": ["hop1"]},
+            user=admin_user,
+            expect=200,  # patching with current peers list should be okay
+        )
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
+            data={"peers": []},
+            user=admin_user,
+            expect=400,  # cannot remove peers directly
+        )
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
+            data={},
+            user=admin_user,
+            expect=200,  # patching without data should be fine too
+        )
+        # patch hop2
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop2.pk}),
+            data={"peers_from_control_nodes": True},
+            user=admin_user,
+            expect=200,  # patching without data should be fine too
+        )
+        assert {hop1, hop2} == set(control.peers.all())  # hop1 and hop2 should now be peered from control node
+
+    def test_disallow_changing_hostname(self, admin_user, patch):
+        """
+        cannot change hostname
+        """
+        hop = Instance.objects.create(hostname='hop', node_type='hop')
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
+            data={"hostname": "hop2"},
+            user=admin_user,
+            expect=400,
+        )
+
+    def test_disallow_changing_node_state(self, admin_user, patch):
+        """
+        only allow setting to deprovisioning
+        """
+        hop = Instance.objects.create(hostname='hop', node_type='hop', node_state='installed')
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
+            data={"node_state": "deprovisioning"},
+            user=admin_user,
+            expect=200,
+        )
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
+            data={"node_state": "ready"},
+            user=admin_user,
+            expect=400,
+        )
+
+    @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
+    def test_control_node_automatically_peers(self, node_type):
+        """
+        a new control node should automatically
+        peer to hop
+
+        peer to hop should be removed if hop is deleted
+        """
+
+        hop = Instance.objects.create(hostname='hop', node_type='hop', peers_from_control_nodes=True, listener_port=6789)
+        control = Instance.objects.create(hostname='abc', node_type=node_type)
+        assert hop in control.peers.all()
+        hop.delete()
+        assert not control.peers.exists()
+
+    @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
+    def test_control_node_retains_other_peers(self, node_type):
+        """
+        if a new node comes online, other peer relationships should
+        remain intact
+        """
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=True)
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
+        hop1.peers.add(hop2)
+
+        # a control node is added
+        Instance.objects.create(hostname='control', node_type=node_type, listener_port=None)
+
+        assert hop1.peers.exists()
+
+    def test_group_vars(self, get, admin_user):
+        """
+        control > hop1 > hop2 < execution
+        """
+        control = Instance.objects.create(hostname='control', node_type='control', listener_port=None)
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=True)
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
+        execution = Instance.objects.create(hostname='execution', node_type='execution', listener_port=6789)
+
+        execution.peers.add(hop2)
+        hop1.peers.add(hop2)
+
+        control_vars = yaml.safe_load(generate_group_vars_all_yml(control))
+        hop1_vars = yaml.safe_load(generate_group_vars_all_yml(hop1))
+        hop2_vars = yaml.safe_load(generate_group_vars_all_yml(hop2))
+        execution_vars = yaml.safe_load(generate_group_vars_all_yml(execution))
+
+        # control group vars assertions
+        assert has_peer(control_vars, 'hop1:6789')
+        assert not has_peer(control_vars, 'hop2:6789')
+        assert not has_peer(control_vars, 'execution:6789')
+        assert not control_vars.get('receptor_listener', False)
+
+        # hop1 group vars assertions
+        assert has_peer(hop1_vars, 'hop2:6789')
+        assert not has_peer(hop1_vars, 'execution:6789')
+        assert hop1_vars.get('receptor_listener', False)
+
+        # hop2 group vars assertions
+        assert not has_peer(hop2_vars, 'hop1:6789')
+        assert not has_peer(hop2_vars, 'execution:6789')
+        assert hop2_vars.get('receptor_listener', False)
+        assert hop2_vars.get('receptor_peers', []) == []
+
+        # execution group vars assertions
+        assert has_peer(execution_vars, 'hop2:6789')
+        assert not has_peer(execution_vars, 'hop1:6789')
+        assert execution_vars.get('receptor_listener', False)
+
+    def test_write_receptor_config_called(self):
+        """
+        Assert that write_receptor_config is called
+        when certain instances are created, or if
+        peers_from_control_nodes changes.
+        In general, write_receptor_config should only
+        be called when necessary, as it will reload
+        receptor backend connections which is not trivial.
+        """
+        with mock.patch('awx.main.models.ha.schedule_write_receptor_config') as write_method:
+            # new control instance but nothing to peer to (no)
+            control = Instance.objects.create(hostname='control1', node_type='control')
+            write_method.assert_not_called()
+
+            # new hop node with peers_from_control_nodes False (no)
+            hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
+            hop1.delete()
+            write_method.assert_not_called()
+
+            # new hop node with peers_from_control_nodes True (yes)
+            hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=True)
+            write_method.assert_called()
+            write_method.reset_mock()
+
+            # new control instance but with something to peer to (yes)
+            Instance.objects.create(hostname='control2', node_type='control')
+            write_method.assert_called()
+            write_method.reset_mock()
+
+            # new hop node with peers_from_control_nodes False and peered to another hop node (no)
+            hop2 = Instance.objects.create(hostname='hop2', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
+            hop2.peers.add(hop1)
+            hop2.delete()
+            write_method.assert_not_called()
+
+            # changing peers_from_control_nodes to False (yes)
+            hop1.peers_from_control_nodes = False
+            hop1.save()
+            write_method.assert_called()
+            write_method.reset_mock()
+
+            # deleting hop node that has peers_from_control_nodes to False (no)
+            hop1.delete()
+            write_method.assert_not_called()
+
+            # deleting control nodes (no)
+            control.delete()
+            write_method.assert_not_called()
+
+    def test_write_receptor_config_data(self):
+        """
+        Assert the correct peers are included in data that will
+        be written to receptor.conf
+        """
+        from awx.main.tasks.receptor import RECEPTOR_CONFIG_STARTER
+
+        with mock.patch('awx.main.tasks.receptor.read_receptor_config', return_value=list(RECEPTOR_CONFIG_STARTER)):
+            from awx.main.tasks.receptor import generate_config_data
+
+            _, should_update = generate_config_data()
+            assert not should_update
+
+            # not peered, so config file should not be updated
+            for i in range(3):
+                Instance.objects.create(hostname=f"exNo-{i}", node_type='execution', listener_port=6789, peers_from_control_nodes=False)
+
+            _, should_update = generate_config_data()
+            assert not should_update
+
+            # peered, so config file should be updated
+            expected_peers = []
+            for i in range(3):
+                expected_peers.append(f"hop-{i}:6789")
+                Instance.objects.create(hostname=f"hop-{i}", node_type='hop', listener_port=6789, peers_from_control_nodes=True)
+
+            for i in range(3):
+                expected_peers.append(f"exYes-{i}:6789")
+                Instance.objects.create(hostname=f"exYes-{i}", node_type='execution', listener_port=6789, peers_from_control_nodes=True)
+
+            new_config, should_update = generate_config_data()
+            assert should_update
+
+            peers = []
+            for entry in new_config:
+                for key, value in entry.items():
+                    if key == "tcp-peer":
+                        peers.append(value['address'])
+
+            assert set(expected_peers) == set(peers)

awx/main/tests/functional/test_jobs.py

@@ -37,7 +37,7 @@ def test_orphan_unified_job_creation(instance, inventory):
 
 
 @pytest.mark.django_db
-@mock.patch('awx.main.tasks.system.inspect_execution_nodes', lambda *args, **kwargs: None)
+@mock.patch('awx.main.tasks.system.inspect_execution_and_hop_nodes', lambda *args, **kwargs: None)
 @mock.patch('awx.main.models.ha.get_cpu_effective_capacity', lambda cpu: 8)
 @mock.patch('awx.main.models.ha.get_mem_effective_capacity', lambda mem: 62)
 def test_job_capacity_and_with_inactive_node():

awx/settings/defaults.py

@@ -476,7 +476,7 @@ CELERYBEAT_SCHEDULE = {
 
 # Django Caching Configuration
 DJANGO_REDIS_IGNORE_EXCEPTIONS = True
-CACHES = {'default': {'BACKEND': 'awx.main.cache.AWXRedisCache', 'LOCATION': 'unix:/var/run/redis/redis.sock?db=1'}}
+CACHES = {'default': {'BACKEND': 'awx.main.cache.AWXRedisCache', 'LOCATION': 'unix:///var/run/redis/redis.sock?db=1'}}
 
 # Social Auth configuration.
 SOCIAL_AUTH_STRATEGY = 'social_django.strategy.DjangoStrategy'

awx/ui/public/installing.html

@@ -5,7 +5,11 @@
       <title data-cy="migration-title">{{ title }}</title>
       <meta
          http-equiv="Content-Security-Policy"
-         content="default-src 'self'; connect-src 'self' ws: wss:; style-src 'self' 'unsafe-inline'; script-src 'self' 'nonce-{{ csp_nonce }}' *.pendo.io; img-src 'self' *.pendo.io data:;"
+         content="default-src 'self';
+                  connect-src 'self' ws: wss:;
+                  style-src 'self' 'unsafe-inline';
+                  script-src 'self' 'nonce-{{ csp_nonce }}' *.pendo.io;
+                  img-src 'self' *.pendo.io data:;"
          />
       <meta charset="utf-8">
       <meta http-equiv="X-UA-Compatible" content="IE=edge" />

awx/ui/src/components/Lookup/Lookup.js

@@ -223,6 +223,10 @@ function Lookup(props) {
 const Item = shape({
   id: number.isRequired,
 });
+const InstanceItem = shape({
+  id: number.isRequired,
+  hostname: string.isRequired,
+});
 
 Lookup.propTypes = {
   id: string,
@@ -230,7 +234,13 @@ Lookup.propTypes = {
   modalDescription: oneOfType([string, node]),
   onChange: func.isRequired,
   onUpdate: func,
-  value: oneOfType([Item, arrayOf(Item), object]),
+  value: oneOfType([
+    Item,
+    arrayOf(Item),
+    object,
+    InstanceItem,
+    arrayOf(InstanceItem),
+  ]),
   multiple: bool,
   required: bool,
   onBlur: func,

awx/ui/src/components/Lookup/PeersLookup.js

@@ -0,0 +1,212 @@
+import React, { useCallback, useEffect } from 'react';
+import { arrayOf, string, func, bool, shape } from 'prop-types';
+import { withRouter } from 'react-router-dom';
+import { t } from '@lingui/macro';
+import { FormGroup, Chip } from '@patternfly/react-core';
+import { InstancesAPI } from 'api';
+import { Instance } from 'types';
+import { getSearchableKeys } from 'components/PaginatedTable';
+import { getQSConfig, parseQueryString, mergeParams } from 'util/qs';
+import useRequest from 'hooks/useRequest';
+import Popover from '../Popover';
+import OptionsList from '../OptionsList';
+import Lookup from './Lookup';
+import LookupErrorMessage from './shared/LookupErrorMessage';
+import FieldWithPrompt from '../FieldWithPrompt';
+
+const QS_CONFIG = getQSConfig('instances', {
+  page: 1,
+  page_size: 5,
+  order_by: 'hostname',
+});
+
+function PeersLookup({
+  id,
+  value,
+  onChange,
+  tooltip,
+  className,
+  required,
+  history,
+  fieldName,
+  multiple,
+  validate,
+  columns,
+  isPromptableField,
+  promptId,
+  promptName,
+  formLabel,
+  typePeers,
+  instance_details,
+}) {
+  const {
+    result: { instances, count, relatedSearchableKeys, searchableKeys },
+    request: fetchInstances,
+    error,
+    isLoading,
+  } = useRequest(
+    useCallback(async () => {
+      const params = parseQueryString(QS_CONFIG, history.location.search);
+      const peersFilter = {};
+      if (typePeers) {
+        peersFilter.not__node_type = ['control', 'hybrid'];
+        if (instance_details) {
+          if (instance_details.id) {
+            peersFilter.not__id = instance_details.id;
+            peersFilter.not__hostname = instance_details.peers;
+          }
+        }
+      }
+
+      const [{ data }, actionsResponse] = await Promise.all([
+        InstancesAPI.read(
+          mergeParams(params, {
+            ...peersFilter,
+          })
+        ),
+        InstancesAPI.readOptions(),
+      ]);
+      return {
+        instances: data.results,
+        count: data.count,
+        relatedSearchableKeys: (
+          actionsResponse?.data?.related_search_fields || []
+        ).map((val) => val.slice(0, -8)),
+        searchableKeys: getSearchableKeys(actionsResponse.data.actions?.GET),
+      };
+    }, [history.location, typePeers, instance_details]),
+    {
+      instances: [],
+      count: 0,
+      relatedSearchableKeys: [],
+      searchableKeys: [],
+    }
+  );
+
+  useEffect(() => {
+    fetchInstances();
+  }, [fetchInstances]);
+
+  const renderLookup = () => (
+    <>
+      <Lookup
+        id={fieldName}
+        header={formLabel}
+        value={value}
+        onChange={onChange}
+        onUpdate={fetchInstances}
+        fieldName={fieldName}
+        validate={validate}
+        qsConfig={QS_CONFIG}
+        multiple={multiple}
+        required={required}
+        isLoading={isLoading}
+        label={formLabel}
+        renderItemChip={({ item, removeItem, canDelete }) => (
+          <Chip
+            key={item.id}
+            onClick={() => removeItem(item)}
+            isReadOnly={!canDelete}
+          >
+            {item.hostname}
+          </Chip>
+        )}
+        renderOptionsList={({ state, dispatch, canDelete }) => (
+          <OptionsList
+            value={state.selectedItems}
+            options={instances}
+            optionCount={count}
+            columns={columns}
+            header={formLabel}
+            displayKey="hostname"
+            searchColumns={[
+              {
+                name: t`Hostname`,
+                key: 'hostname__icontains',
+                isDefault: true,
+              },
+            ]}
+            sortColumns={[
+              {
+                name: t`Hostname`,
+                key: 'hostname',
+              },
+            ]}
+            searchableKeys={searchableKeys}
+            relatedSearchableKeys={relatedSearchableKeys}
+            multiple={multiple}
+            label={formLabel}
+            name={fieldName}
+            qsConfig={QS_CONFIG}
+            readOnly={!canDelete}
+            selectItem={(item) => dispatch({ type: 'SELECT_ITEM', item })}
+            deselectItem={(item) => dispatch({ type: 'DESELECT_ITEM', item })}
+          />
+        )}
+      />
+      <LookupErrorMessage error={error} />
+    </>
+  );
+
+  return isPromptableField ? (
+    <FieldWithPrompt
+      fieldId={id}
+      label={formLabel}
+      promptId={promptId}
+      promptName={promptName}
+      tooltip={tooltip}
+    >
+      {renderLookup()}
+    </FieldWithPrompt>
+  ) : (
+    <FormGroup
+      className={className}
+      label={formLabel}
+      labelIcon={tooltip && <Popover content={tooltip} />}
+      fieldId={id}
+    >
+      {renderLookup()}
+    </FormGroup>
+  );
+}
+
+PeersLookup.propTypes = {
+  id: string,
+  value: arrayOf(Instance).isRequired,
+  tooltip: string,
+  onChange: func.isRequired,
+  className: string,
+  required: bool,
+  validate: func,
+  multiple: bool,
+  fieldName: string,
+  columns: arrayOf(Object),
+  formLabel: string,
+  instance_details: (Instance, shape({})),
+  typePeers: bool,
+};
+
+PeersLookup.defaultProps = {
+  id: 'instances',
+  tooltip: '',
+  className: '',
+  required: false,
+  validate: () => undefined,
+  fieldName: 'instances',
+  columns: [
+    {
+      key: 'hostname',
+      name: t`Hostname`,
+    },
+    {
+      key: 'node_type',
+      name: t`Node Type`,
+    },
+  ],
+  formLabel: t`Instances`,
+  instance_details: {},
+  multiple: true,
+  typePeers: false,
+};
+
+export default withRouter(PeersLookup);

awx/ui/src/components/Lookup/PeersLookup.test.js

@@ -0,0 +1,137 @@
+import React from 'react';
+import { act } from 'react-dom/test-utils';
+import { Formik } from 'formik';
+import { InstancesAPI } from 'api';
+import { mountWithContexts } from '../../../testUtils/enzymeHelpers';
+import PeersLookup from './PeersLookup';
+
+jest.mock('../../api');
+
+const mockedInstances = {
+  count: 1,
+  results: [
+    {
+      id: 2,
+      name: 'Foo',
+      image: 'quay.io/ansible/awx-ee',
+      pull: 'missing',
+    },
+  ],
+};
+
+const instances = [
+  {
+    id: 1,
+    hostname: 'awx_1',
+    type: 'instance',
+    url: '/api/v2/instances/1/',
+    related: {
+      named_url: '/api/v2/instances/awx_1/',
+      jobs: '/api/v2/instances/1/jobs/',
+      instance_groups: '/api/v2/instances/1/instance_groups/',
+      peers: '/api/v2/instances/1/peers/',
+    },
+    summary_fields: {
+      user_capabilities: {
+        edit: false,
+      },
+      links: [],
+    },
+    uuid: '00000000-0000-0000-0000-000000000000',
+    created: '2023-04-26T22:06:46.766198Z',
+    modified: '2023-04-26T22:06:46.766217Z',
+    last_seen: '2023-04-26T23:12:02.857732Z',
+    health_check_started: null,
+    health_check_pending: false,
+    last_health_check: '2023-04-26T23:01:13.941693Z',
+    errors: 'Instance received normal shutdown signal',
+    capacity_adjustment: '1.00',
+    version: '0.1.dev33237+g1fdef52',
+    capacity: 0,
+    consumed_capacity: 0,
+    percent_capacity_remaining: 0,
+    jobs_running: 0,
+    jobs_total: 0,
+    cpu: '8.0',
+    memory: 8011055104,
+    cpu_capacity: 0,
+    mem_capacity: 0,
+    enabled: true,
+    managed_by_policy: true,
+    node_type: 'hybrid',
+    node_state: 'installed',
+    ip_address: null,
+    listener_port: 27199,
+    peers: [],
+    peers_from_control_nodes: false,
+  },
+];
+
+describe('PeersLookup', () => {
+  let wrapper;
+
+  beforeEach(() => {
+    InstancesAPI.read.mockResolvedValue({
+      data: mockedInstances,
+    });
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('should render successfully without instance_details (for new added instance)', async () => {
+    InstancesAPI.readOptions.mockReturnValue({
+      data: {
+        actions: {
+          GET: {},
+          POST: {},
+        },
+        related_search_fields: [],
+      },
+    });
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <Formik>
+          <PeersLookup value={instances} onChange={() => {}} />
+        </Formik>
+      );
+    });
+    wrapper.update();
+    expect(InstancesAPI.read).toHaveBeenCalledTimes(1);
+    expect(wrapper.find('PeersLookup')).toHaveLength(1);
+    expect(wrapper.find('FormGroup[label="Instances"]').length).toBe(1);
+    expect(wrapper.find('Checkbox[aria-label="Prompt on launch"]').length).toBe(
+      0
+    );
+  });
+  test('should render successfully with instance_details for edit instance', async () => {
+    InstancesAPI.readOptions.mockReturnValue({
+      data: {
+        actions: {
+          GET: {},
+          POST: {},
+        },
+        related_search_fields: [],
+      },
+    });
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <Formik>
+          <PeersLookup
+            value={instances}
+            instance_details={instances[0]}
+            onChange={() => {}}
+          />
+        </Formik>
+      );
+    });
+    wrapper.update();
+    expect(InstancesAPI.read).toHaveBeenCalledTimes(1);
+    expect(wrapper.find('PeersLookup')).toHaveLength(1);
+    expect(wrapper.find('FormGroup[label="Instances"]').length).toBe(1);
+    expect(wrapper.find('Checkbox[aria-label="Prompt on launch"]').length).toBe(
+      0
+    );
+  });
+});

awx/ui/src/components/Lookup/index.js

@@ -8,3 +8,4 @@ export { default as ApplicationLookup } from './ApplicationLookup';
 export { default as HostFilterLookup } from './HostFilterLookup';
 export { default as OrganizationLookup } from './OrganizationLookup';
 export { default as ExecutionEnvironmentLookup } from './ExecutionEnvironmentLookup';
+export { default as PeersLookup } from './PeersLookup';

awx/ui/src/components/OptionsList/OptionsList.js

@@ -125,19 +125,24 @@ const Item = shape({
   name: string.isRequired,
   url: string,
 });
+const InstanceItem = shape({
+  id: oneOfType([number, string]).isRequired,
+  hostname: string.isRequired,
+  url: string,
+});
 OptionsList.propTypes = {
   deselectItem: func.isRequired,
   displayKey: string,
   isSelectedDraggable: bool,
   multiple: bool,
   optionCount: number.isRequired,
-  options: arrayOf(Item).isRequired,
+  options: oneOfType([arrayOf(Item), arrayOf(InstanceItem)]).isRequired,
   qsConfig: QSConfig.isRequired,
   renderItemChip: func,
   searchColumns: SearchColumns,
   selectItem: func.isRequired,
   sortColumns: SortColumns,
-  value: arrayOf(Item).isRequired,
+  value: oneOfType([arrayOf(Item), arrayOf(InstanceItem)]).isRequired,
 };
 OptionsList.defaultProps = {
   isSelectedDraggable: false,

awx/ui/src/screens/Instances/InstanceAdd/InstanceAdd.test.js

@@ -32,12 +32,10 @@ describe('<InstanceAdd />', () => {
     await waitForElement(wrapper, 'isLoading', (el) => el.length === 0);
     await act(async () => {
       wrapper.find('InstanceForm').prop('handleSubmit')({
-        name: 'new Foo',
         node_type: 'hop',
       });
     });
     expect(InstancesAPI.create).toHaveBeenCalledWith({
-      name: 'new Foo',
       node_type: 'hop',
     });
     expect(history.location.pathname).toBe('/instances/13/details');

awx/ui/src/screens/Instances/InstanceDetail/InstanceDetail.js

@@ -1,6 +1,6 @@
 import React, { useCallback, useEffect, useState } from 'react';
 
-import { useHistory, useParams } from 'react-router-dom';
+import { useHistory, useParams, Link } from 'react-router-dom';
 import { t, Plural } from '@lingui/macro';
 import {
   Button,
@@ -116,6 +116,7 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
       setBreadcrumb(instance);
     }
   }, [instance, setBreadcrumb]);
+
   const { error: healthCheckError, request: fetchHealthCheck } = useRequest(
     useCallback(async () => {
       const { status } = await InstancesAPI.healthCheck(id);
@@ -205,13 +206,42 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
             }
           />
           <Detail label={t`Node Type`} value={instance.node_type} />
+          <Detail label={t`Host`} value={instance.ip_address} />
+          <Detail label={t`Listener Port`} value={instance.listener_port} />
+          {(isExecutionNode || isHopNode) && (
+            <>
+              {instance.related?.install_bundle && (
+                <Detail
+                  label={t`Install Bundle`}
+                  value={
+                    <Tooltip content={t`Click to download bundle`}>
+                      <Button
+                        component="a"
+                        isSmall
+                        href={`${instance.related?.install_bundle}`}
+                        target="_blank"
+                        variant="secondary"
+                        dataCy="install-bundle-download-button"
+                        rel="noopener noreferrer"
+                      >
+                        <DownloadIcon />
+                      </Button>
+                    </Tooltip>
+                  }
+                />
+              )}
+              <Detail
+                label={t`Peers from control nodes`}
+                value={instance.peers_from_control_nodes ? t`On` : t`Off`}
+              />
+            </>
+          )}
           {!isHopNode && (
             <>
               <Detail
                 label={t`Policy Type`}
                 value={instance.managed_by_policy ? t`Auto` : t`Manual`}
               />
-              <Detail label={t`Host`} value={instance.ip_address} />
               <Detail label={t`Running Jobs`} value={instance.jobs_running} />
               <Detail label={t`Total Jobs`} value={instance.jobs_total} />
               {instanceGroups && (
@@ -246,26 +276,6 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
                 }
                 value={formatHealthCheckTimeStamp(instance.last_health_check)}
               />
-              {instance.related?.install_bundle && (
-                <Detail
-                  label={t`Install Bundle`}
-                  value={
-                    <Tooltip content={t`Click to download bundle`}>
-                      <Button
-                        component="a"
-                        isSmall
-                        href={`${instance.related?.install_bundle}`}
-                        target="_blank"
-                        variant="secondary"
-                        dataCy="install-bundle-download-button"
-                        rel="noopener noreferrer"
-                      >
-                        <DownloadIcon />
-                      </Button>
-                    </Tooltip>
-                  }
-                />
-              )}
               <Detail
                 label={t`Capacity Adjustment`}
                 dataCy="capacity-adjustment"
@@ -327,9 +337,20 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
             />
           )}
         </DetailList>
-        {!isHopNode && (
+        <CardActionsRow>
-          <CardActionsRow>
+          {config?.me?.is_superuser && isK8s && (isExecutionNode || isHopNode) && (
-            {config?.me?.is_superuser && isK8s && isExecutionNode && (
+            <Button
+              ouiaId="instance-detail-edit-button"
+              aria-label={t`edit`}
+              component={Link}
+              to={`/instances/${id}/edit`}
+            >
+              {t`Edit`}
+            </Button>
+          )}
+          {config?.me?.is_superuser &&
+            isK8s &&
+            (isExecutionNode || isHopNode) && (
               <RemoveInstanceButton
                 dataCy="remove-instance-button"
                 itemsToRemove={[instance]}
@@ -337,32 +358,31 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
                 onRemove={removeInstances}
               />
             )}
-            {isExecutionNode && (
+          {isExecutionNode && (
-              <Tooltip content={t`Run a health check on the instance`}>
+            <Tooltip content={t`Run a health check on the instance`}>
-                <Button
+              <Button
-                  isDisabled={
+                isDisabled={
-                    !config?.me?.is_superuser || instance.health_check_pending
+                  !config?.me?.is_superuser || instance.health_check_pending
-                  }
+                }
-                  variant="primary"
+                variant="primary"
-                  ouiaId="health-check-button"
+                ouiaId="health-check-button"
-                  onClick={fetchHealthCheck}
+                onClick={fetchHealthCheck}
-                  isLoading={instance.health_check_pending}
+                isLoading={instance.health_check_pending}
-                  spinnerAriaLabel={t`Running health check`}
+                spinnerAriaLabel={t`Running health check`}
-                >
+              >
-                  {instance.health_check_pending
+                {instance.health_check_pending
-                    ? t`Running health check`
+                  ? t`Running health check`
-                    : t`Run health check`}
+                  : t`Run health check`}
-                </Button>
+              </Button>
-              </Tooltip>
+            </Tooltip>
-            )}
+          )}
-            <InstanceToggle
+          <InstanceToggle
-              css="display: inline-flex;"
+            css="display: inline-flex;"
-              fetchInstances={fetchDetails}
+            fetchInstances={fetchDetails}
-              instance={instance}
+            instance={instance}
-              dataCy="enable-instance"
+            dataCy="enable-instance"
-            />
+          />
-          </CardActionsRow>
+        </CardActionsRow>
-        )}
 
         {error && (
           <AlertModal

awx/ui/src/screens/Instances/InstanceEdit/InstanceEdit.js

@@ -0,0 +1,105 @@
+import React, { useState, useCallback, useEffect } from 'react';
+
+import { t } from '@lingui/macro';
+import { useHistory, useParams, Link } from 'react-router-dom';
+import { Card, PageSection } from '@patternfly/react-core';
+import useRequest from 'hooks/useRequest';
+import ContentError from 'components/ContentError';
+import ContentLoading from 'components/ContentLoading';
+import { CardBody } from 'components/Card';
+import { InstancesAPI } from 'api';
+import InstanceForm from '../Shared/InstanceForm';
+
+function InstanceEdit({ setBreadcrumb }) {
+  const history = useHistory();
+  const { id } = useParams();
+  const [formError, setFormError] = useState();
+
+  const detailsUrl = `/instances/${id}/details`;
+
+  const handleSubmit = async (values) => {
+    try {
+      await InstancesAPI.update(id, values);
+      history.push(detailsUrl);
+    } catch (err) {
+      setFormError(err);
+    }
+  };
+
+  const handleCancel = () => {
+    history.push(detailsUrl);
+  };
+
+  const {
+    isLoading,
+    error,
+    request: fetchDetail,
+    result: { instance, peers },
+  } = useRequest(
+    useCallback(async () => {
+      const [{ data: instance_detail }, { data: peers_detail }] =
+        await Promise.all([
+          InstancesAPI.readDetail(id),
+          InstancesAPI.readPeers(id),
+        ]);
+      return {
+        instance: instance_detail,
+        peers: peers_detail.results,
+      };
+    }, [id]),
+    {
+      instance: {},
+      peers: [],
+    }
+  );
+
+  useEffect(() => {
+    fetchDetail();
+  }, [fetchDetail]);
+
+  useEffect(() => {
+    if (instance) {
+      setBreadcrumb(instance);
+    }
+  }, [instance, setBreadcrumb]);
+
+  if (isLoading) {
+    return (
+      <CardBody>
+        <ContentLoading />
+      </CardBody>
+    );
+  }
+
+  if (error) {
+    return (
+      <CardBody>
+        <ContentError error={error}>
+          {error?.response?.status === 404 && (
+            <span>
+              {t`Instance not found.`}{' '}
+              <Link to="/instances">{t`View all Instances.`}</Link>
+            </span>
+          )}
+        </ContentError>
+      </CardBody>
+    );
+  }
+
+  return (
+    <PageSection>
+      <Card>
+        <InstanceForm
+          instance={instance}
+          instance_peers={peers}
+          isEdit
+          submitError={formError}
+          handleSubmit={handleSubmit}
+          handleCancel={handleCancel}
+        />
+      </Card>
+    </PageSection>
+  );
+}
+
+export default InstanceEdit;

awx/ui/src/screens/Instances/InstanceEdit/InstanceEdit.test.js

@@ -0,0 +1,149 @@
+import React from 'react';
+import { act } from 'react-dom/test-utils';
+import { createMemoryHistory } from 'history';
+import useDebounce from 'hooks/useDebounce';
+import { InstancesAPI } from 'api';
+import {
+  mountWithContexts,
+  waitForElement,
+} from '../../../../testUtils/enzymeHelpers';
+
+import InstanceEdit from './InstanceEdit';
+
+jest.mock('../../../api');
+jest.mock('../../../hooks/useDebounce');
+jest.mock('react-router-dom', () => ({
+  ...jest.requireActual('react-router-dom'),
+  useParams: () => ({
+    id: 42,
+  }),
+}));
+
+const instanceData = {
+  id: 42,
+  hostname: 'awx_1',
+  type: 'instance',
+  url: '/api/v2/instances/1/',
+  related: {
+    named_url: '/api/v2/instances/awx_1/',
+    jobs: '/api/v2/instances/1/jobs/',
+    instance_groups: '/api/v2/instances/1/instance_groups/',
+    peers: '/api/v2/instances/1/peers/',
+  },
+  summary_fields: {
+    user_capabilities: {
+      edit: false,
+    },
+    links: [],
+  },
+  uuid: '00000000-0000-0000-0000-000000000000',
+  created: '2023-04-26T22:06:46.766198Z',
+  modified: '2023-04-26T22:06:46.766217Z',
+  last_seen: '2023-04-26T23:12:02.857732Z',
+  health_check_started: null,
+  health_check_pending: false,
+  last_health_check: '2023-04-26T23:01:13.941693Z',
+  errors: 'Instance received normal shutdown signal',
+  capacity_adjustment: '1.00',
+  version: '0.1.dev33237+g1fdef52',
+  capacity: 0,
+  consumed_capacity: 0,
+  percent_capacity_remaining: 0,
+  jobs_running: 0,
+  jobs_total: 0,
+  cpu: '8.0',
+  memory: 8011055104,
+  cpu_capacity: 0,
+  mem_capacity: 0,
+  enabled: true,
+  managed_by_policy: true,
+  node_type: 'hybrid',
+  node_state: 'installed',
+  ip_address: null,
+  listener_port: 27199,
+  peers: [],
+  peers_from_control_nodes: false,
+};
+
+const instanceDataWithPeers = {
+  results: [instanceData],
+};
+
+const updatedInstance = {
+  node_type: 'hop',
+  peers: ['test-peer'],
+};
+
+describe('<InstanceEdit/>', () => {
+  let wrapper;
+  let history;
+
+  beforeAll(async () => {
+    useDebounce.mockImplementation((fn) => fn);
+    history = createMemoryHistory();
+    InstancesAPI.readDetail.mockResolvedValue({ data: instanceData });
+    InstancesAPI.readPeers.mockResolvedValue({ data: instanceDataWithPeers });
+
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <InstanceEdit
+          instance={instanceData}
+          peers={instanceDataWithPeers}
+          isEdit
+          setBreadcrumb={() => {}}
+        />,
+        {
+          context: { router: { history } },
+        }
+      );
+    });
+    expect(InstancesAPI.readDetail).toBeCalledWith(42);
+  });
+
+  afterAll(() => {
+    jest.clearAllMocks();
+  });
+
+  test('initially renders successfully', async () => {
+    await waitForElement(wrapper, 'ContentLoading', (el) => el.length === 0);
+    expect(wrapper.find('InstanceEdit')).toHaveLength(1);
+  });
+
+  test('handleSubmit should call the api and redirect to details page', async () => {
+    await act(async () => {
+      wrapper.find('InstanceForm').invoke('handleSubmit')(updatedInstance);
+    });
+    expect(InstancesAPI.update).toHaveBeenCalledWith(42, updatedInstance);
+    expect(history.location.pathname).toEqual('/instances/42/details');
+  });
+
+  test('should navigate to instance details when cancel is clicked', async () => {
+    await act(async () => {
+      wrapper.find('button[aria-label="Cancel"]').simulate('click');
+    });
+    expect(history.location.pathname).toEqual('/instances/42/details');
+  });
+
+  test('should navigate to instance details after successful submission', async () => {
+    await act(async () => {
+      wrapper.find('InstanceForm').invoke('handleSubmit')(updatedInstance);
+    });
+    wrapper.update();
+    expect(wrapper.find('submitError').length).toBe(0);
+    expect(history.location.pathname).toEqual('/instances/42/details');
+  });
+
+  test('failed form submission should show an error message', async () => {
+    const error = {
+      response: {
+        data: { detail: 'An error occurred' },
+      },
+    };
+    InstancesAPI.update.mockImplementationOnce(() => Promise.reject(error));
+    await act(async () => {
+      wrapper.find('InstanceForm').invoke('handleSubmit')(updatedInstance);
+    });
+    wrapper.update();
+    expect(wrapper.find('FormSubmitError').length).toBe(1);
+  });
+});

awx/ui/src/screens/Instances/InstanceEdit/index.js

@@ -0,0 +1 @@
+export { default } from './InstanceEdit';

awx/ui/src/screens/Instances/InstanceList/InstanceListItem.js

@@ -138,7 +138,7 @@ function InstanceListItem({
             rowIndex,
             isSelected,
             onSelect,
-            disable: !isExecutionNode,
+            disable: !(isExecutionNode || isHopNode),
           }}
           dataLabel={t`Selected`}
         />

awx/ui/src/screens/Instances/InstancePeers/InstancePeerList.js

@@ -1,17 +1,24 @@
-import React, { useCallback, useEffect } from 'react';
+import React, { useCallback, useEffect, useState } from 'react';
 import { t } from '@lingui/macro';
 import { CardBody } from 'components/Card';
 import PaginatedTable, {
   getSearchableKeys,
   HeaderCell,
   HeaderRow,
+  ToolbarAddButton,
 } from 'components/PaginatedTable';
-import { getQSConfig, parseQueryString } from 'util/qs';
+import DisassociateButton from 'components/DisassociateButton';
+import AssociateModal from 'components/AssociateModal';
+import ErrorDetail from 'components/ErrorDetail';
+import AlertModal from 'components/AlertModal';
+import useToast, { AlertVariant } from 'hooks/useToast';
+import { getQSConfig, parseQueryString, mergeParams } from 'util/qs';
 import { useLocation, useParams } from 'react-router-dom';
-import useRequest from 'hooks/useRequest';
+import useRequest, { useDismissableError } from 'hooks/useRequest';
 import DataListToolbar from 'components/DataListToolbar';
 import { InstancesAPI } from 'api';
 import useExpanded from 'hooks/useExpanded';
+import useSelected from 'hooks/useSelected';
 import InstancePeerListItem from './InstancePeerListItem';
 
 const QS_CONFIG = getQSConfig('peer', {
@@ -20,27 +27,36 @@ const QS_CONFIG = getQSConfig('peer', {
   order_by: 'hostname',
 });
 
-function InstancePeerList() {
+function InstancePeerList({ setBreadcrumb }) {
   const location = useLocation();
   const { id } = useParams();
+  const [isModalOpen, setIsModalOpen] = useState(false);
+  const { addToast, Toast, toastProps } = useToast();
+  const readInstancesOptions = useCallback(
+    () => InstancesAPI.readOptions(id),
+    [id]
+  );
   const {
     isLoading,
     error: contentError,
     request: fetchPeers,
-    result: { peers, count, relatedSearchableKeys, searchableKeys },
+    result: { instance, peers, count, relatedSearchableKeys, searchableKeys },
   } = useRequest(
     useCallback(async () => {
       const params = parseQueryString(QS_CONFIG, location.search);
       const [
+        { data: detail },
         {
           data: { results, count: itemNumber },
         },
         actions,
       ] = await Promise.all([
+        InstancesAPI.readDetail(id),
         InstancesAPI.readPeers(id, params),
         InstancesAPI.readOptions(),
       ]);
       return {
+        instance: detail,
         peers: results,
         count: itemNumber,
         relatedSearchableKeys: (actions?.data?.related_search_fields || []).map(
@@ -50,6 +66,7 @@ function InstancePeerList() {
       };
     }, [id, location]),
     {
+      instance: {},
       peers: [],
       count: 0,
       relatedSearchableKeys: [],
@@ -61,18 +78,98 @@ function InstancePeerList() {
     fetchPeers();
   }, [fetchPeers]);
 
+  useEffect(() => {
+    if (instance) {
+      setBreadcrumb(instance);
+    }
+  }, [instance, setBreadcrumb]);
+
   const { expanded, isAllExpanded, handleExpand, expandAll } =
     useExpanded(peers);
+  const { selected, isAllSelected, handleSelect, clearSelected, selectAll } =
+    useSelected(peers);
+
+  const fetchInstancesToAssociate = useCallback(
+    (params) =>
+      InstancesAPI.read(
+        mergeParams(params, {
+          ...{ not__id: id },
+          ...{ not__node_type: ['control', 'hybrid'] },
+          ...{ not__hostname: instance.peers },
+        })
+      ),
+    [id, instance]
+  );
+
+  const {
+    isLoading: isAssociateLoading,
+    request: handlePeerAssociate,
+    error: associateError,
+  } = useRequest(
+    useCallback(
+      async (instancesPeerToAssociate) => {
+        const selected_hostname = instancesPeerToAssociate.map(
+          (obj) => obj.hostname
+        );
+        const new_peers = [
+          ...new Set([...instance.peers, ...selected_hostname]),
+        ];
+        await InstancesAPI.update(instance.id, { peers: new_peers });
+        fetchPeers();
+        addToast({
+          id: instancesPeerToAssociate,
+          title: t`${selected_hostname} added as a peer. Please be sure to run the install bundle for ${instance.hostname} again in order to see changes take effect.`,
+          variant: AlertVariant.success,
+          hasTimeout: true,
+        });
+      },
+      [instance, fetchPeers, addToast]
+    )
+  );
+
+  const {
+    isLoading: isDisassociateLoading,
+    request: handlePeersDiassociate,
+    error: disassociateError,
+  } = useRequest(
+    useCallback(async () => {
+      const new_peers = [];
+      const selected_hostname = selected.map((obj) => obj.hostname);
+      for (let i = 0; i < instance.peers.length; i++) {
+        if (!selected_hostname.includes(instance.peers[i])) {
+          new_peers.push(instance.peers[i]);
+        }
+      }
+      await InstancesAPI.update(instance.id, { peers: new_peers });
+      fetchPeers();
+      addToast({
+        title: t`${selected_hostname} removed. Please be sure to run the install bundle for ${instance.hostname} again in order to see changes take effect.`,
+        variant: AlertVariant.success,
+        hasTimeout: true,
+      });
+    }, [instance, selected, fetchPeers, addToast])
+  );
+
+  const { error, dismissError } = useDismissableError(
+    associateError || disassociateError
+  );
+
+  const isHopNode = instance.node_type === 'hop';
+  const isExecutionNode = instance.node_type === 'execution';
 
   return (
     <CardBody>
       <PaginatedTable
         contentError={contentError}
-        hasContentLoading={isLoading}
+        hasContentLoading={
+          isLoading || isDisassociateLoading || isAssociateLoading
+        }
         items={peers}
         itemCount={count}
         pluralizedItemName={t`Peers`}
         qsConfig={QS_CONFIG}
+        onRowClick={handleSelect}
+        clearSelected={clearSelected}
         toolbarSearchableKeys={searchableKeys}
         toolbarRelatedSearchableKeys={relatedSearchableKeys}
         toolbarSearchColumns={[
@@ -101,13 +198,36 @@ function InstancePeerList() {
         renderToolbar={(props) => (
           <DataListToolbar
             {...props}
+            isAllSelected={isAllSelected}
+            onSelectAll={selectAll}
             isAllExpanded={isAllExpanded}
             onExpandAll={expandAll}
             qsConfig={QS_CONFIG}
+            additionalControls={[
+              (isExecutionNode || isHopNode) && (
+                <ToolbarAddButton
+                  ouiaId="add-instance-peers-button"
+                  key="associate"
+                  defaultLabel={t`Associate`}
+                  onClick={() => setIsModalOpen(true)}
+                />
+              ),
+              (isExecutionNode || isHopNode) && (
+                <DisassociateButton
+                  verifyCannotDisassociate={false}
+                  key="disassociate"
+                  onDisassociate={handlePeersDiassociate}
+                  itemsToDisassociate={selected}
+                  modalTitle={t`Remove instance from peers?`}
+                />
+              ),
+            ]}
           />
         )}
         renderRow={(peer, index) => (
           <InstancePeerListItem
+            isSelected={selected.some((row) => row.id === peer.id)}
+            onSelect={() => handleSelect(peer)}
             isExpanded={expanded.some((row) => row.id === peer.id)}
             onExpand={() => handleExpand(peer)}
             key={peer.id}
@@ -116,6 +236,35 @@ function InstancePeerList() {
           />
         )}
       />
+      {isModalOpen && (
+        <AssociateModal
+          header={t`Instances`}
+          fetchRequest={fetchInstancesToAssociate}
+          isModalOpen={isModalOpen}
+          onAssociate={handlePeerAssociate}
+          onClose={() => setIsModalOpen(false)}
+          title={t`Select Instances`}
+          optionsRequest={readInstancesOptions}
+          displayKey="hostname"
+          columns={[
+            { key: 'hostname', name: t`Name` },
+            { key: 'node_type', name: t`Node Type` },
+          ]}
+        />
+      )}
+      <Toast {...toastProps} />
+      {error && (
+        <AlertModal
+          isOpen={error}
+          onClose={dismissError}
+          title={t`Error!`}
+          variant="error"
+        >
+          {associateError && t`Failed to associate peer.`}
+          {disassociateError && t`Failed to remove peers.`}
+          <ErrorDetail error={error} />
+        </AlertModal>
+      )}
     </CardBody>
   );
 }

awx/ui/src/screens/Instances/InstancePeers/InstancePeerListItem.js

@@ -10,6 +10,8 @@ import { Detail, DetailList } from 'components/DetailList';
 
 function InstancePeerListItem({
   peerInstance,
+  isSelected,
+  onSelect,
   isExpanded,
   onExpand,
   rowIndex,
@@ -33,7 +35,14 @@ function InstancePeerListItem({
             }}
           />
         )}
-        <Td />
+        <Td
+          select={{
+            rowIndex,
+            isSelected,
+            onSelect,
+          }}
+          dataLabel={t`Selected`}
+        />
         <Td id={labelId} dataLabel={t`Name`}>
           <Link to={`/instances/${peerInstance.id}/details`}>
             <b>{peerInstance.hostname}</b>

awx/ui/src/screens/Instances/Instances.js

@@ -7,6 +7,7 @@ import PersistentFilters from 'components/PersistentFilters';
 import { InstanceList } from './InstanceList';
 import Instance from './Instance';
 import InstanceAdd from './InstanceAdd';
+import InstanceEdit from './InstanceEdit';
 
 function Instances() {
   const [breadcrumbConfig, setBreadcrumbConfig] = useState({
@@ -20,8 +21,11 @@ function Instances() {
     }
     setBreadcrumbConfig({
       '/instances': t`Instances`,
+      '/instances/add': t`Create new Instance`,
       [`/instances/${instance.id}`]: `${instance.hostname}`,
       [`/instances/${instance.id}/details`]: t`Details`,
+      [`/instances/${instance.id}/peers`]: t`Peers`,
+      [`/instances/${instance.id}/edit`]: t`Edit Instance`,
     });
   }, []);
 
@@ -30,7 +34,10 @@ function Instances() {
       <ScreenHeader streamType="instance" breadcrumbConfig={breadcrumbConfig} />
       <Switch>
         <Route path="/instances/add">
-          <InstanceAdd />
+          <InstanceAdd setBreadcrumb={buildBreadcrumbConfig} />
+        </Route>
+        <Route path="/instances/:id/edit" key="edit">
+          <InstanceEdit setBreadcrumb={buildBreadcrumbConfig} />
         </Route>
         <Route path="/instances/:id">
           <Instance setBreadcrumb={buildBreadcrumbConfig} />

awx/ui/src/screens/Instances/Shared/InstanceForm.js

@@ -1,6 +1,6 @@
-import React from 'react';
+import React, { useCallback } from 'react';
 import { t } from '@lingui/macro';
-import { Formik } from 'formik';
+import { Formik, useField, useFormikContext } from 'formik';
 import { Form, FormGroup, CardBody } from '@patternfly/react-core';
 import { FormColumnLayout } from 'components/FormLayout';
 import FormField, {
@@ -8,9 +8,31 @@ import FormField, {
   CheckboxField,
 } from 'components/FormField';
 import FormActionGroup from 'components/FormActionGroup';
+import AnsibleSelect from 'components/AnsibleSelect';
+import { PeersLookup } from 'components/Lookup';
 import { required } from 'util/validators';
 
-function InstanceFormFields() {
+const INSTANCE_TYPES = [
+  { id: 'execution', name: t`Execution` },
+  { id: 'hop', name: t`Hop` },
+];
+
+function InstanceFormFields({ isEdit }) {
+  const [instanceTypeField, instanceTypeMeta, instanceTypeHelpers] = useField({
+    name: 'node_type',
+    validate: required(t`Set a value for this field`),
+  });
+
+  const { setFieldValue } = useFormikContext();
+
+  const [peersField, peersMeta, peersHelpers] = useField('peers');
+
+  const handlePeersUpdate = useCallback(
+    (value) => {
+      setFieldValue('peers', value);
+    },
+    [setFieldValue]
+  );
   return (
     <>
       <FormField
@@ -20,6 +42,7 @@ function InstanceFormFields() {
         type="text"
         validate={required(null)}
         isRequired
+        isDisabled={isEdit}
       />
       <FormField
         id="instance-description"
@@ -40,16 +63,47 @@ function InstanceFormFields() {
         label={t`Listener Port`}
         name="listener_port"
         type="number"
-        tooltip={t`Select the port that Receptor will listen on for incoming connections. Default is 27199.`}
+        tooltip={t`Select the port that Receptor will listen on for incoming connections, e.g. 27199.`}
-        isRequired
       />
-      <FormField
+      <FormGroup
-        id="instance-type"
+        fieldId="instance-type"
         label={t`Instance Type`}
-        name="node_type"
-        type="text"
         tooltip={t`Sets the role that this instance will play within mesh topology. Default is "execution."`}
-        isDisabled
+        validated={
+          !instanceTypeMeta.touched || !instanceTypeMeta.error
+            ? 'default'
+            : 'error'
+        }
+        helperTextInvalid={instanceTypeMeta.error}
+        isRequired
+      >
+        <AnsibleSelect
+          {...instanceTypeField}
+          id="node_type"
+          data={INSTANCE_TYPES.map((type) => ({
+            key: type.id,
+            value: type.id,
+            label: type.name,
+          }))}
+          onChange={(event, value) => {
+            instanceTypeHelpers.setValue(value);
+          }}
+          isDisabled={isEdit}
+        />
+      </FormGroup>
+      <PeersLookup
+        helperTextInvalid={peersMeta.error}
+        isValid={!peersMeta.touched || !peersMeta.error}
+        onBlur={() => peersHelpers.setTouched()}
+        onChange={handlePeersUpdate}
+        value={peersField.value}
+        tooltip={t`Select the Peers Instances.`}
+        fieldName="peers"
+        formLabel={t`Peers`}
+        multiple
+        typePeers
+        id="peers"
+        isRequired
       />
       <FormGroup fieldId="instance-option-checkboxes" label={t`Options`}>
         <CheckboxField
@@ -64,6 +118,12 @@ function InstanceFormFields() {
           label={t`Managed by Policy`}
           tooltip={t`Controls whether or not this instance is managed by policy. If enabled, the instance will be available for automatic assignment to and unassignment from instance groups based on policy rules.`}
         />
+        <CheckboxField
+          id="peers_from_control_nodes"
+          name="peers_from_control_nodes"
+          label={t`Peers from control nodes`}
+          tooltip={t`If enabled, control nodes will peer to this instance automatically. If disabled, instance will be connected only to associated peers.`}
+        />
       </FormGroup>
     </>
   );
@@ -71,6 +131,8 @@ function InstanceFormFields() {
 
 function InstanceForm({
   instance = {},
+  instance_peers = [],
+  isEdit = false,
   submitError,
   handleCancel,
   handleSubmit,
@@ -79,22 +141,29 @@ function InstanceForm({
     <CardBody>
       <Formik
         initialValues={{
-          hostname: '',
+          hostname: instance.hostname || '',
-          description: '',
+          description: instance.description || '',
-          node_type: 'execution',
+          node_type: instance.node_type || 'execution',
-          node_state: 'installed',
+          node_state: instance.node_state || 'installed',
-          listener_port: 27199,
+          listener_port: instance.listener_port,
-          enabled: true,
+          enabled: instance.enabled || true,
-          managed_by_policy: true,
+          managed_by_policy: instance.managed_by_policy || true,
+          peers_from_control_nodes: instance.peers_from_control_nodes || false,
+          peers: instance_peers,
         }}
         onSubmit={(values) => {
-          handleSubmit(values);
+          handleSubmit({
+            ...values,
+            listener_port:
+              values.listener_port === '' ? null : values.listener_port,
+            peers: values.peers.map((peer) => peer.hostname || peer),
+          });
         }}
       >
         {(formik) => (
           <Form autoComplete="off" onSubmit={formik.handleSubmit}>
             <FormColumnLayout>
-              <InstanceFormFields instance={instance} />
+              <InstanceFormFields isEdit={isEdit} />
               <FormSubmitError error={submitError} />
               <FormActionGroup
                 onCancel={handleCancel}

awx/ui/src/screens/Instances/Shared/InstanceForm.test.js

@@ -65,7 +65,7 @@ describe('<InstanceForm />', () => {
     expect(handleCancel).toBeCalled();
   });
 
-  test('should call handleSubmit when Cancel button is clicked', async () => {
+  test('should call handleSubmit when Save button is clicked', async () => {
     expect(handleSubmit).not.toHaveBeenCalled();
     await act(async () => {
       wrapper.find('input#hostname').simulate('change', {
@@ -74,9 +74,6 @@ describe('<InstanceForm />', () => {
       wrapper.find('input#instance-description').simulate('change', {
         target: { value: 'This is a repeat song', name: 'description' },
       });
-      wrapper.find('input#instance-port').simulate('change', {
-        target: { value: 'This is a repeat song', name: 'listener_port' },
-      });
     });
     wrapper.update();
     expect(
@@ -91,9 +88,10 @@ describe('<InstanceForm />', () => {
       enabled: true,
       managed_by_policy: true,
       hostname: 'new Foo',
-      listener_port: 'This is a repeat song',
       node_state: 'installed',
       node_type: 'execution',
+      peers_from_control_nodes: false,
+      peers: [],
     });
   });
 });

awx/ui/src/screens/Instances/Shared/RemoveInstanceButton.js

@@ -33,7 +33,8 @@ function RemoveInstanceButton({ itemsToRemove, onRemove, isK8s }) {
   const [removeDetails, setRemoveDetails] = useState(null);
   const [isLoading, setIsLoading] = useState(false);
 
-  const cannotRemove = (item) => item.node_type !== 'execution';
+  const cannotRemove = (item) =>
+    !(item.node_type === 'execution' || item.node_type === 'hop');
 
   const toggleModal = async (isOpen) => {
     setRemoveDetails(null);
@@ -175,7 +176,7 @@ function RemoveInstanceButton({ itemsToRemove, onRemove, isK8s }) {
             </Button>,
           ]}
         >
-          <div>{t`This action will remove the following instances:`}</div>
+          <div>{t`This action will remove the following instance and you may need to rerun the install bundle for any instance that was previously connected to:`}</div>
           {itemsToRemove.map((item) => (
             <span key={item.id} id={`item-to-be-removed-${item.id}`}>
               <strong>{item.hostname}</strong>

awx/ui/src/screens/TopologyView/Header.js

@@ -16,6 +16,7 @@ import {
   SearchPlusIcon,
   ExpandArrowsAltIcon,
   ExpandIcon,
+  RedoAltIcon,
 } from '@patternfly/react-icons';
 
 const Header = ({
@@ -26,6 +27,7 @@ const Header = ({
   zoomOut,
   resetZoom,
   zoomFit,
+  refresh,
   showZoomControls,
 }) => {
   const { light } = PageSectionVariants;
@@ -48,6 +50,18 @@ const Header = ({
           </Title>
         </div>
         <div>
+          <Tooltip content={t`Refresh`} position="top">
+            <Button
+              ouiaId="refresh-button"
+              aria-label={t`Refresh`}
+              variant="plain"
+              icon={<RedoAltIcon />}
+              onClick={refresh}
+              isDisabled={!showZoomControls}
+            >
+              <RedoAltIcon />
+            </Button>
+          </Tooltip>
           <Tooltip content={t`Zoom in`} position="top">
             <Button
               ouiaId="zoom-in-button"

awx/ui/src/screens/TopologyView/Legend.js

@@ -245,7 +245,7 @@ function Legend() {
                 y1="9"
                 x2="20"
                 y2="9"
-                stroke="#666"
+                stroke="#6A6E73"
                 strokeWidth="4"
               />
             </svg>
@@ -260,7 +260,7 @@ function Legend() {
                 y1="9"
                 x2="20"
                 y2="9"
-                stroke="#666"
+                stroke="#3E8635"
                 strokeWidth="4"
                 strokeDasharray="6"
               />

awx/ui/src/screens/TopologyView/MeshGraph.js

@@ -13,6 +13,7 @@ import Tooltip from './Tooltip';
 import ContentLoading from './ContentLoading';
 import {
   renderStateColor,
+  renderLinkStatusColor,
   renderLabelText,
   renderNodeType,
   renderNodeIcon,
@@ -177,7 +178,7 @@ function MeshGraph({
       mesh
         .append('defs')
         .selectAll('marker')
-        .data(['end', 'end-active'])
+        .data(['end', 'end-active', 'end-adding', 'end-removing'])
         .join('marker')
         .attr('id', String)
         .attr('viewBox', '0 -5 10 10')
@@ -187,8 +188,9 @@ function MeshGraph({
         .attr('orient', 'auto')
         .append('path')
         .attr('d', 'M0,-5L10,0L0,5');
-
+      mesh.select('#end').attr('refX', 23).attr('fill', '#6A6E73');
-      mesh.select('#end').attr('refX', 23).attr('fill', '#ccc');
+      mesh.select('#end-removing').attr('refX', 23).attr('fill', '#C9190B');
+      mesh.select('#end-adding').attr('refX', 23).attr('fill', '#3E8635');
       mesh.select('#end-active').attr('refX', 18).attr('fill', '#0066CC');
 
       // Add links
@@ -204,18 +206,24 @@ function MeshGraph({
         .attr('y1', (d) => d.source.y)
         .attr('x2', (d) => d.target.x)
         .attr('y2', (d) => d.target.y)
-        .attr('marker-end', 'url(#end)')
+        .attr('marker-end', (d) => {
+          if (d.link_state === 'adding') {
+            return 'url(#end-adding)';
+          }
+          if (d.link_state === 'removing') {
+            return 'url(#end-removing)';
+          }
+          return 'url(#end)';
+        })
         .attr('class', (_, i) => `link-${i}`)
         .attr('data-cy', (d) => `${d.source.hostname}-${d.target.hostname}`)
         .style('fill', 'none')
-        .style('stroke', (d) =>
+        .style('stroke', (d) => renderLinkStatusColor(d.link_state))
-          d.link_state === 'removing' ? '#C9190B' : '#CCC'
-        )
         .style('stroke-width', '2px')
         .style('stroke-dasharray', (d) => renderLinkState(d.link_state))
         .attr('pointer-events', 'none')
         .on('mouseover', function showPointer() {
-          d3.select(this).transition().style('cursor', 'pointer');
+          d3.select(this).style('cursor', 'pointer');
         });
       // add nodes
       const node = mesh
@@ -228,7 +236,7 @@ function MeshGraph({
         .append('g')
         .attr('data-cy', (d) => `node-${d.id}`)
         .on('mouseenter', function handleNodeHover(_, d) {
-          d3.select(this).transition().style('cursor', 'pointer');
+          d3.select(this).style('cursor', 'pointer');
           highlightSiblings(d);
         })
         .on('mouseleave', (_, d) => {
@@ -239,7 +247,8 @@ function MeshGraph({
         });
 
       // node circles
-      node
+      const nodeCircles = node.append('g');
+      nodeCircles
         .append('circle')
         .attr('r', DEFAULT_RADIUS)
         .attr('cx', (d) => d.x)
@@ -248,7 +257,8 @@ function MeshGraph({
         .attr('class', (d) => `id-${d.id}`)
         .attr('fill', DEFAULT_NODE_COLOR)
         .attr('stroke-dasharray', (d) => (d.enabled ? `1 0` : `5`))
-        .attr('stroke', DEFAULT_NODE_STROKE_COLOR);
+        .attr('stroke', (d) => renderStateColor(d.node_state));
+
       // node type labels
       node
         .append('text')
@@ -259,64 +269,62 @@ function MeshGraph({
         .attr('dominant-baseline', 'central')
         .attr('fill', DEFAULT_NODE_SYMBOL_TEXT_COLOR);
 
-      const placeholder = node.append('g').attr('class', 'placeholder');
+      // node hostname labels
-
+      const hostNames = node.append('g').attr('class', 'node-state-label');
-      placeholder
-        .append('text')
-        .text((d) => renderLabelText(d.node_state, d.hostname))
-        .attr('x', (d) => d.x)
-        .attr('y', (d) => d.y + 40)
-        .attr('fill', 'black')
-        .attr('font-size', '18px')
-        .attr('text-anchor', 'middle')
-        .each(function calculateLabelWidth() {
-          // eslint-disable-next-line react/no-this-in-sfc
-          const bbox = this.getBBox();
-          // eslint-disable-next-line react/no-this-in-sfc
-          d3.select(this.parentNode)
-            .append('path')
-            .attr('d', (d) => renderLabelIcons(d.node_state))
-            .attr('transform', (d) => renderIconPosition(d.node_state, bbox))
-            .style('fill', 'black');
-        });
-
-      placeholder.each(function calculateLabelWidth() {
-        // eslint-disable-next-line react/no-this-in-sfc
-        const bbox = this.getBBox();
-        // eslint-disable-next-line react/no-this-in-sfc
-        d3.select(this.parentNode)
-          .append('rect')
-          .attr('x', (d) => d.x - bbox.width / 2)
-          .attr('y', bbox.y + 5)
-          .attr('width', bbox.width)
-          .attr('height', bbox.height)
-          .attr('rx', 8)
-          .attr('ry', 8)
-          .style('fill', (d) => renderStateColor(d.node_state));
-      });
-
-      const hostNames = node.append('g');
       hostNames
         .append('text')
+        .attr('x', (d) => d.x)
+        .attr('y', (d) => d.y + 40)
         .text((d) => renderLabelText(d.node_state, d.hostname))
-        .attr('x', (d) => d.x + 6)
+        .attr('class', 'placeholder')
-        .attr('y', (d) => d.y + 42)
         .attr('fill', 'white')
-        .attr('font-size', DEFAULT_FONT_SIZE)
         .attr('text-anchor', 'middle')
         .each(function calculateLabelWidth() {
           // eslint-disable-next-line react/no-this-in-sfc
           const bbox = this.getBBox();
+          const padding = 10;
           // eslint-disable-next-line react/no-this-in-sfc
           d3.select(this.parentNode)
-            .append('path')
+            .append('rect')
-            .attr('class', (d) => `icon-${d.node_state}`)
+            .attr('x', bbox.x - padding / 2)
-            .attr('d', (d) => renderLabelIcons(d.node_state))
+            .attr('y', bbox.y)
-            .attr('transform', (d) => renderIconPosition(d.node_state, bbox))
+            .attr('width', bbox.width + padding)
-            .attr('fill', 'white');
+            .attr('height', bbox.height)
+            .style('stroke-width', 1)
+            .attr('rx', 4)
+            .attr('ry', 4)
+            .attr('fill', 'white')
+            .style('stroke', DEFAULT_NODE_STROKE_COLOR);
         });
-      svg.selectAll('g.placeholder').remove();
+      svg.selectAll('text.placeholder').remove();
+      hostNames
+        .append('text')
+        .attr('x', (d) => d.x)
+        .attr('y', (d) => d.y + 38)
+        .text((d) => renderLabelText(d.node_state, d.hostname))
+        .attr('font-size', DEFAULT_FONT_SIZE)
+        .attr('fill', 'black')
+        .attr('text-anchor', 'middle');
 
+      // add badge icons
+      const badges = nodeCircles.append('g').attr('class', 'node-state-badge');
+      badges.each(function drawStateBadge() {
+        // eslint-disable-next-line react/no-this-in-sfc
+        const bbox = this.parentNode.getBBox();
+        // eslint-disable-next-line react/no-this-in-sfc
+        d3.select(this)
+          .append('circle')
+          .attr('r', 9)
+          .attr('cx', bbox.x)
+          .attr('cy', bbox.y)
+          .attr('fill', (d) => renderStateColor(d.node_state));
+        d3.select(this)
+          .append('path')
+          .attr('class', (d) => `icon-${d.node_state}`)
+          .attr('d', (d) => renderLabelIcons(d.node_state))
+          .attr('transform', (d) => renderIconPosition(d.node_state, bbox))
+          .attr('fill', 'white');
+      });
       svg.call(zoom);
 
       function highlightSiblings(n) {
@@ -330,7 +338,6 @@ function MeshGraph({
         immediate.forEach((s) => {
           svg
             .selectAll(`.link-${s.index}`)
-            .transition()
             .style('stroke', '#0066CC')
             .style('stroke-width', '3px')
             .attr('marker-end', 'url(#end-active)');
@@ -346,13 +353,17 @@ function MeshGraph({
         immediate.forEach((s) => {
           svg
             .selectAll(`.link-${s.index}`)
-            .transition()
+            .style('stroke', (d) => renderLinkStatusColor(d.link_state))
-            .duration(50)
-            .style('stroke', (d) =>
-              d.link_state === 'removing' ? '#C9190B' : '#CCC'
-            )
             .style('stroke-width', '2px')
-            .attr('marker-end', 'url(#end)');
+            .attr('marker-end', (d) => {
+              if (d.link_state === 'adding') {
+                return 'url(#end-adding)';
+              }
+              if (d.link_state === 'removing') {
+                return 'url(#end-removing)';
+              }
+              return 'url(#end)';
+            });
         });
       }
 
@@ -361,7 +372,7 @@ function MeshGraph({
           // toggle rings
           svg
             .select(`circle.id-${n.id}`)
-            .attr('stroke', '#ccc')
+            .attr('stroke', (d) => renderStateColor(d.node_state))
             .attr('stroke-width', null);
           // show default empty state of tooltip
           setIsNodeSelected(false);
@@ -370,7 +381,7 @@ function MeshGraph({
         }
         svg
           .selectAll('circle')
-          .attr('stroke', '#ccc')
+          .attr('stroke', (d) => renderStateColor(d.node_state))
           .attr('stroke-width', null);
         svg
           .select(`circle.id-${n.id}`)

awx/ui/src/screens/TopologyView/TopologyView.js

@@ -45,6 +45,7 @@ function TopologyView() {
         zoomIn={zoomIn}
         zoomOut={zoomOut}
         zoomFit={zoomFit}
+        refresh={fetchMeshVisualizer}
         resetZoom={resetZoom}
         showZoomControls={showZoomControls}
       />

awx/ui/src/screens/TopologyView/constants.js

@@ -13,7 +13,7 @@ export const DEFAULT_NODE_COLOR = 'white';
 export const DEFAULT_NODE_HIGHLIGHT_COLOR = '#eee';
 export const DEFAULT_NODE_LABEL_TEXT_COLOR = 'white';
 export const DEFAULT_NODE_SYMBOL_TEXT_COLOR = 'black';
-export const DEFAULT_NODE_STROKE_COLOR = '#ccc';
+export const DEFAULT_NODE_STROKE_COLOR = '#6A6E73';
 export const DEFAULT_FONT_SIZE = '12px';
 export const LABEL_TEXT_MAX_LENGTH = 15;
 export const MARGIN = 15;
@@ -27,6 +27,12 @@ export const NODE_STATE_COLOR_KEY = {
   deprovisioning: '#666',
 };
 
+export const LINK_STATE_COLOR_KEY = {
+  established: '#6A6E73',
+  adding: '#3E8635',
+  removing: '#C9190B',
+};
+
 export const NODE_TYPE_SYMBOL_KEY = {
   hop: 'h',
   execution: 'Ex',

awx/ui/src/screens/TopologyView/utils/helpers.js

@@ -4,6 +4,7 @@ import { truncateString } from '../../../util/strings';
 import {
   NODE_STATE_COLOR_KEY,
   NODE_TYPE_SYMBOL_KEY,
+  LINK_STATE_COLOR_KEY,
   LABEL_TEXT_MAX_LENGTH,
   ICONS,
 } from '../constants';
@@ -20,6 +21,12 @@ export function renderStateColor(nodeState) {
   return NODE_STATE_COLOR_KEY[nodeState] ? NODE_STATE_COLOR_KEY[nodeState] : '';
 }
 
+export function renderLinkStatusColor(linkState) {
+  return LINK_STATE_COLOR_KEY[linkState]
+    ? LINK_STATE_COLOR_KEY[linkState]
+    : '#ccc';
+}
+
 export function renderLabelText(nodeState, name) {
   if (typeof nodeState === 'string' && typeof name === 'string') {
     return `${truncateString(name, LABEL_TEXT_MAX_LENGTH)}`;
@@ -45,8 +52,8 @@ export function renderLabelIcons(nodeState) {
       ready: 'checkmark',
       installed: 'clock',
       unavailable: 'exclaimation',
-      'provision-fail': 'exclaimation',
       'deprovision-fail': 'exclaimation',
+      'provision-fail': 'exclaimation',
       provisioning: 'plus',
       deprovisioning: 'minus',
     };
@@ -59,15 +66,17 @@ export function renderLabelIcons(nodeState) {
 export function renderIconPosition(nodeState, bbox) {
   if (nodeState) {
     const iconPositionMapper = {
-      ready: `translate(${bbox.x - 15}, ${bbox.y + 3}), scale(0.02)`,
+      ready: `translate(${bbox.x - 4.5}, ${bbox.y - 4.5}), scale(0.02)`,
-      installed: `translate(${bbox.x - 18}, ${bbox.y + 1}), scale(0.03)`,
+      installed: `translate(${bbox.x - 6.5}, ${bbox.y - 6.5}), scale(0.025)`,
-      unavailable: `translate(${bbox.x - 9}, ${bbox.y + 3}), scale(0.02)`,
+      unavailable: `translate(${bbox.x - 2}, ${bbox.y - 4.4}), scale(0.02)`,
-      'provision-fail': `translate(${bbox.x - 9}, ${bbox.y + 3}), scale(0.02)`,
+      'provision-fail': `translate(${bbox.x - 2}, ${bbox.y - 4}), scale(0.02)`,
-      'deprovision-fail': `translate(${bbox.x - 9}, ${
+      'deprovision-fail': `translate(${bbox.x - 2}, ${
-        bbox.y + 3
+        bbox.y - 4
+      }), scale(0.02)`,
+      provisioning: `translate(${bbox.x - 4.5}, ${bbox.y - 4.5}), scale(0.02)`,
+      deprovisioning: `translate(${bbox.x - 4.5}, ${
+        bbox.y - 4.5
       }), scale(0.02)`,
-      provisioning: `translate(${bbox.x - 12}, ${bbox.y + 3}), scale(0.02)`,
-      deprovisioning: `translate(${bbox.x - 12}, ${bbox.y + 3}), scale(0.02)`,
     };
     return iconPositionMapper[nodeState] ? iconPositionMapper[nodeState] : ``;
   }
@@ -101,7 +110,7 @@ export function getRandomInt(min, max) {
 const generateRandomLinks = (n, r) => {
   const links = [];
   function getRandomLinkState() {
-    return ['established', 'adding', 'removing'][getRandomInt(0, 2)];
+    return ['established', 'adding', 'removing'][getRandomInt(0, 3)];
   }
   for (let i = 0; i < r; i++) {
     const link = {
@@ -142,7 +151,7 @@ export const generateRandomNodes = (n) => {
       hostname: `node-${id}`,
       node_type: randomType,
       node_state: randomState,
-      enabled: Math.random() < 0.5,
+      enabled: Math.random() < 0.9,
     };
     nodes.push(node);
   }

awx/ui/src/screens/TopologyView/utils/helpers__RTL.test.js

@@ -73,7 +73,7 @@ describe('renderIconPosition', () => {
   const bbox = { x: 400, y: 400, width: 10, height: 20 };
   test('returns correct label icon', () => {
     expect(renderIconPosition('ready', bbox)).toBe(
-      `translate(${bbox.x - 15}, ${bbox.y + 3}), scale(0.02)`
+      `translate(${bbox.x - 4.5}, ${bbox.y - 4.5}), scale(0.02)`
     );
   });
   test('returns empty string if state is not found', () => {

awx/ui/src/types.js

@@ -122,7 +122,7 @@ export const InstanceGroup = shape({
 
 export const Instance = shape({
   id: number.isRequired,
-  name: string.isRequired,
+  hostname: string.isRequired,
 });
 
 export const Label = shape({

awx_collection/plugins/modules/credential.py

@@ -46,6 +46,7 @@ options:
     organization:
       description:
         - Organization name, ID, or named URL that should own the credential.
+        - This parameter is mutually exclusive with C(team) and C(user).
       type: str
     credential_type:
       description:
@@ -93,10 +94,12 @@ options:
     user:
       description:
         - User name, ID, or named URL that should own this credential.
+        - This parameter is mutually exclusive with C(organization) and C(team).
       type: str
     team:
       description:
         - Team name, ID, or named URL that should own this credential.
+        - This parameter is mutually exclusive with C(organization) and C(user).
       type: str
     state:
       description:
@@ -219,8 +222,13 @@ def main():
         state=dict(choices=['present', 'absent', 'exists'], default='present'),
     )
 
+    mutually_exclusive = [("organization", "user", "team")]
+
     # Create a module for ourselves
-    module = ControllerAPIModule(argument_spec=argument_spec)
+    module = ControllerAPIModule(
+        argument_spec=argument_spec,
+        mutually_exclusive=mutually_exclusive
+    )
 
     # Extract our parameters
     name = module.params.get('name')

awx_collection/plugins/modules/instance.py

@@ -47,6 +47,7 @@ options:
         - Role that this node plays in the mesh.
       choices:
         - execution
+        - hop
       required: False
       type: str
     node_state:
@@ -62,6 +63,18 @@ options:
         - Port that Receptor will listen for incoming connections on.
       required: False
       type: int
+    peers:
+      description:
+        - List of peers to connect outbound to. Only configurable for hop and execution nodes.
+        - To remove all current peers, set value to an empty list, [].
+      required: False
+      type: list
+      elements: str
+    peers_from_control_nodes:
+      description:
+        - If enabled, control plane nodes will automatically peer to this node.
+      required: False
+      type: bool
 extends_documentation_fragment: awx.awx.auth
 '''
 
@@ -88,9 +101,11 @@ def main():
         capacity_adjustment=dict(type='float'),
         enabled=dict(type='bool'),
         managed_by_policy=dict(type='bool'),
-        node_type=dict(type='str', choices=['execution']),
+        node_type=dict(type='str', choices=['execution', 'hop']),
         node_state=dict(type='str', choices=['deprovisioning', 'installed']),
         listener_port=dict(type='int'),
+        peers=dict(required=False, type='list', elements='str'),
+        peers_from_control_nodes=dict(required=False, type='bool'),
     )
 
     # Create a module for ourselves
@@ -104,7 +119,8 @@ def main():
     node_type = module.params.get('node_type')
     node_state = module.params.get('node_state')
     listener_port = module.params.get('listener_port')
-
+    peers = module.params.get('peers')
+    peers_from_control_nodes = module.params.get('peers_from_control_nodes')
     # Attempt to look up an existing item based on the provided data
     existing_item = module.get_one('instances', name_or_id=hostname)
 
@@ -122,6 +138,10 @@ def main():
         new_fields['node_state'] = node_state
     if listener_port is not None:
         new_fields['listener_port'] = listener_port
+    if peers is not None:
+        new_fields['peers'] = peers
+    if peers_from_control_nodes is not None:
+        new_fields['peers_from_control_nodes'] = peers_from_control_nodes
 
     module.create_or_update_if_needed(
         existing_item,

awx_collection/plugins/modules/license.py

@@ -52,7 +52,12 @@ EXAMPLES = '''
   license:
     manifest: "/tmp/my_manifest.zip"
 
-- name: Attach to a pool
+- name: Use the subscriptions module to fetch subscriptions from Red Hat or Red Hat Satellite
+  subscriptions:
+    username: "my_satellite_username"
+    password: "my_satellite_password"
+
+- name: Attach to a pool (requires fetching subscriptions at least once before)
   license:
     pool_id: 123456
 

awx_collection/plugins/modules/project.py

@@ -83,7 +83,7 @@ options:
       type: bool
     scm_update_on_launch:
       description:
-        - Before an update to the local repository before launching a job with this project.
+        - Perform an update to the local repository before launching a job with this project.
       type: bool
     scm_update_cache_timeout:
       description:

awx_collection/test/awx/test_instance.py

@@ -0,0 +1,53 @@
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+import pytest
+
+from awx.main.models import Instance
+from django.test.utils import override_settings
+
+
+@pytest.mark.django_db
+def test_peers_adding_and_removing(run_module, admin_user):
+    with override_settings(IS_K8S=True):
+        result = run_module(
+            'instance',
+            {'hostname': 'hopnode1', 'node_type': 'hop', 'peers_from_control_nodes': True, 'node_state': 'installed', 'listener_port': 27199},
+            admin_user,
+        )
+        assert result['changed']
+
+        hop_node_1 = Instance.objects.get(pk=result.get('id'))
+
+        assert hop_node_1.peers_from_control_nodes is True
+        assert hop_node_1.node_type == 'hop'
+
+        result = run_module(
+            'instance',
+            {'hostname': 'hopnode2', 'node_type': 'hop', 'peers_from_control_nodes': True, 'node_state': 'installed', 'listener_port': 27199},
+            admin_user,
+        )
+        assert result['changed']
+
+        hop_node_2 = Instance.objects.get(pk=result.get('id'))
+
+        result = run_module(
+            'instance',
+            {'hostname': 'executionnode', 'node_type': 'execution', 'node_state': 'installed', 'listener_port': 27199, 'peers': ['hopnode1', 'hopnode2']},
+            admin_user,
+        )
+        assert result['changed']
+
+        execution_node = Instance.objects.get(pk=result.get('id'))
+
+        assert set(execution_node.peers.all()) == {hop_node_1, hop_node_2}
+
+        result = run_module(
+            'instance',
+            {'hostname': 'executionnode', 'node_type': 'execution', 'node_state': 'installed', 'listener_port': 27199, 'peers': []},
+            admin_user,
+        )
+
+        assert result['changed']
+        assert set(execution_node.peers.all()) == set()

awx_collection/tests/integration/targets/instance/tasks/main.yml

@@ -70,3 +70,60 @@
         - "{{ hostname3 }}"
 
   when: IS_K8S
+
+- block:
+    - name: Create hop node 1
+      awx.awx.instance:
+        hostname: hopnode1
+        node_type: hop
+        node_state: installed
+        listener_port: 27199
+        peers_from_control_nodes: True
+      register: result
+
+    - assert:
+        that:
+          - result is changed
+
+    - name: Create hop node 2
+      awx.awx.instance:
+        hostname: hopnode2
+        node_type: hop
+        node_state: installed
+        listener_port: 27199
+        peers_from_control_nodes: True
+      register: result
+
+    - assert:
+        that:
+          - result is changed
+
+    - name: Create execution node
+      awx.awx.instance:
+        hostname: executionnode
+        node_type: execution
+        node_state: installed
+        listener_port: 27199
+        peers:
+          - "hopnode1"
+          - "hopnode2"
+      register: result
+
+    - assert:
+        that:
+          - result is changed
+
+    - name: Remove execution node peers
+      awx.awx.instance:
+        hostname: executionnode
+        node_type: execution
+        node_state: installed
+        listener_port: 27199
+        peers: []
+      register: result
+
+    - assert:
+        that:
+          - result is changed
+
+  when: IS_K8S

docs/execution_nodes.md

@@ -2,26 +2,25 @@
 
 Stand-alone execution nodes can be added to run alongside the Kubernetes deployment of AWX. These machines will not be a part of the AWX Kubernetes cluster. The control nodes running in the cluster will connect and submit work to these machines via Receptor. The machines be registered in AWX as type "execution" instances, meaning they will only be used to run AWX Jobs (i.e. they will not dispatch work or handle web requests as control nodes do).
 
-Below is an example of a single AWX pod connecting to two different execution nodes. For each execution node, the awx-ee container makes an outbound TCP connection to the machine via Receptor.
+Hop nodes can be added to sit between the control plane of AWX and stand alone execution nodes. These machines will not be a part of the AWX Kubernetes cluster. The machines will be registered in AWX as node type "hop", meaning they will only handle inbound / outbound traffic for otherwise unreachable nodes in a different or more strict network. 
+
+Below is an example of an AWX Task pod with two excution nodes. Traffic to execution node 2 flows through a hop node that is setup between it and the control plane.
 
 ```
-                                 AWX POD
+                                                     AWX TASK POD
-                             ┌──────────────┐
+                                                   ┌──────────────┐
-                             │              │
+                                                   │              │
-                             │ ┌──────────┐ │
+                                                   │ ┌──────────┐ │
-┌─────────────────┐          │ │ awx-task │ │
+┌─────────────────┐   ┌─────────────────┐          │ │ awx-task │ │
-│ execution node 1│◄────┐    │ ├──────────┤ │
+│execution node 2 ├──►│     hop node    │◄────┐    │ ├──────────┤ │
-├─────────────────┤     ├────┼─┤ awx-ee   │ │
+└─────────────────┘   ├─────────────────┤     ├────┼─┤ awx-ee   │ │
-│ execution node 2│◄────┘    │ ├──────────┤ │
+                      │ execution node 1│◄────┘    │ └──────────┘ │
-└─────────────────┘ Receptor │ │ awx-web  │ │
+                      └─────────────────┘ Receptor │              | 
-                      TCP    │ └──────────┘ │
+                                            TCP    └──────────────┘
-                     Peers   │              │
+                                           Peers                 
-                             └──────────────┘
+                                                   
 ```
 
-Note, if the AWX deployment is scaled up, the new AWX pod will also make TCP connections to each execution node.
-
-
 ## Overview
 Adding an execution instance involves a handful of steps:
 
@@ -34,7 +33,7 @@ Adding an execution instance involves a handful of steps:
 
 ### Start machine
 
-Bring a machine online with a compatible Red Hat family OS (e.g. RHEL 8 and 9). This machines needs a static IP, or a resolvable DNS hostname that the AWX cluster can access. The machine will also need an available open port to establish inbound TCP connections on (default is 27199).
+Bring a machine online with a compatible Red Hat family OS (e.g. RHEL 8 and 9). This machines needs a static IP, or a resolvable DNS hostname that the AWX cluster can access. If the listerner_port is defined, the machine will also need an available open port to establish inbound TCP connections on (e.g. 27199).
 
 In general the more CPU cores and memory the machine has, the more jobs that can be scheduled to run on that machine at once. See https://docs.ansible.com/automation-controller/4.2.1/html/userguide/jobs.html#at-capacity-determination-and-job-impact for more information on capacity.
 
@@ -43,9 +42,21 @@ In general the more CPU cores and memory the machine has, the more jobs that can
 
 Use the Instance page or `api/v2/instances` endpoint to add a new instance.
 - `hostname` ("Name" in UI) is the IP address or DNS name of your machine.
-- `node_type` is "execution"
+- `node_type` is "execution" or "hop"
 - `node_state` is "installed"
-- `listener_port` is an open port on the remote machine used to establish inbound TCP connections. Defaults to 27199.
+- `listener_port` is an open port on the remote machine used to establish inbound TCP connections. Defaults to null.
+- `peers` is a list of instance hostnames to connect outbound to.
+- `peers_from_control_nodes` boolean, if True, control plane nodes will automatically peer to this instance.
+
+Below is a table of configuartions for the [diagram](#adding-execution-nodes-to-awx) above.
+
+| instance name    | listener_port | peers_from_control_nodes | peers       |
+|------------------|---------------|-------------------------|--------------|
+| execution node 1 | 27199         | true                    | []           |
+| hop node         | 27199         | true                    | []           |
+| execution node 2 | null          | false                   | ["hop node"] |
+
+Listener port needs to be set if peers_from_control_nodes is enabled or the instance is a peer.
 
 
 ### Download the install bundle
@@ -65,8 +76,6 @@ Modify `inventory.yml`. Set the `ansible_user` and any other ansible variables t
 
 `ansible-playbook -i inventory.yml install_receptor.yml` to start installing Receptor on the remote machine.
 
-Note, the playbook will enable the [Copr ansible-awx/receptor repository](https://copr.fedorainfracloud.org/coprs/ansible-awx/receptor/) so that Receptor can be installed.
-
 
 ### Wait for instance to be Ready
 

requirements/requirements.in

@@ -8,7 +8,7 @@ boto3
 botocore
 channels
 channels-redis==3.4.1  # see UPGRADE BLOCKERs
-cryptography>=39.0.1 ## https://github.com/ansible/awx/security/dependabot/90
+cryptography>=41.0.2  # CVE-2023-38325
 Cython<3 # Since the bump to PyYAML 5.4.1 this is now a mandatory dep
 daphne
 distro
@@ -26,7 +26,7 @@ django-split-settings==1.0.0    # We hit a strange issue where the release proce
 djangorestframework
 djangorestframework-yaml
 filelock
-GitPython>=3.1.30  # CVE-2022-24439
+GitPython>=3.1.32  # CVE-2023-40267
 hiredis==2.0.0  # see UPGRADE BLOCKERs
 irc
 jinja2
@@ -39,6 +39,7 @@ prometheus_client
 psycopg
 psutil
 pygerduty
+pyopenssl>=23.2.0  # resolve dep conflict from cryptography pin above
 pyparsing==2.4.6  # Upgrading to v3 of pyparsing introduce errors on smart host filtering: Expected 'or' term, found 'or'  (at char 15), (line:1, col:16)
 python-daemon>3.0.0
 python-dsv-sdk

requirements/requirements.txt

@@ -77,7 +77,7 @@ click==8.1.3
     # via receptorctl
 constantly==15.1.0
     # via twisted
-cryptography==40.0.2
+cryptography==41.0.3
     # via
     #   -r /awx_devel/requirements/requirements.in
     #   adal
@@ -94,9 +94,7 @@ daphne==3.0.2
     #   -r /awx_devel/requirements/requirements.in
     #   channels
 dataclasses==0.6
-    # via
+    # via python-dsv-sdk
-    #   python-dsv-sdk
-    #   python-tss-sdk
 defusedxml==0.7.1
     # via
     #   python3-openid
@@ -157,7 +155,7 @@ frozenlist==1.3.3
     #   aiosignal
 gitdb==4.0.10
     # via gitpython
-gitpython==3.1.30
+gitpython==3.1.32
     # via -r /awx_devel/requirements/requirements.in
 google-auth==2.14.1
     # via kubernetes
@@ -303,8 +301,10 @@ pyjwt==2.6.0
     #   adal
     #   social-auth-core
     #   twilio
-pyopenssl==23.1.1
+pyopenssl==23.2.0
-    # via twisted
+    # via
+    #   -r /awx_devel/requirements/requirements.in
+    #   twisted
 pyparsing==2.4.6
     # via
     #   -r /awx_devel/requirements/requirements.in

tools/docker-compose/ansible/roles/sources/templates/nginx.locations.conf.j2

@@ -2,6 +2,10 @@ location {{ (ingress_path + '/static').replace('//', '/') }} {
     alias /var/lib/awx/public/static/;
 }
 
+location {{ (ingress_path + '/locales').replace('//', '/') }} {
+    alias /var/lib/awx/public/static/awx/locales;
+}
+
 location {{ (ingress_path + '/favicon.ico').replace('//', '/') }} {
     alias /awx_devel/awx/public/static/favicon.ico;
 }

tools/docker-compose/bootstrap_development.sh

@@ -40,7 +40,7 @@ echo "Admin password: ${DJANGO_SUPERUSER_PASSWORD}"
 awx-manage create_preload_data
 awx-manage register_default_execution_environments
 
-awx-manage provision_instance --hostname="$(hostname)" --node_type="$MAIN_NODE_TYPE"
+awx-manage provision_instance --hostname="$(hostname)" --node_type="$MAIN_NODE_TYPE" --listener_port=2222
 awx-manage register_queue --queuename=controlplane --instance_percent=100
 awx-manage register_queue --queuename=default --instance_percent=100
 
@@ -52,7 +52,7 @@ if [[ -n "$RUN_MIGRATIONS" ]]; then
     done
 
     if [[ $EXECUTION_NODE_COUNT > 0 ]]; then
-        awx-manage provision_instance --hostname="receptor-hop" --node_type="hop"
+        awx-manage provision_instance --hostname="receptor-hop" --node_type="hop" --listener_port=5555
         awx-manage register_peers "receptor-hop" --peers "awx_1"
         for (( e=1; e<=$EXECUTION_NODE_COUNT; e++ )); do
             awx-manage provision_instance --hostname="receptor-$e" --node_type="execution"