docs: update execution_nodes.md to follow changes for receptor_collection (#14247)

receptor_host_identifier can be left out
of group_vars and will default to the
'ansible_host' variable

.github/workflows/pr_body_check.yml

@@ -7,6 +7,7 @@ on:
     types: [opened, edited, reopened, synchronize]
 jobs:
   pr-check:
+    if: github.repository_owner == 'ansible' && endsWith(github.repository, 'awx')
     name: Scan PR description for semantic versioning keywords
     runs-on: ubuntu-latest
     permissions:

DATA_MIGRATION.md

@@ -4,6 +4,6 @@
 
 Early versions of AWX did not support seamless upgrades between major versions and required the use of a backup and restore tool to perform upgrades.
 
-Users who wish to upgrade modern AWX installations should follow the instructions at:
+As of version 18.0, `awx-operator` is the preferred install/upgrade method. Users who wish to upgrade modern AWX installations should follow the instructions at:
 
-https://github.com/ansible/awx/blob/devel/INSTALL.md#upgrading-from-previous-versions
+https://github.com/ansible/awx-operator/blob/devel/docs/upgrade/upgrading.md

awx/api/serializers.py

@@ -5356,10 +5356,16 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSeria
 class InstanceLinkSerializer(BaseSerializer):
     class Meta:
         model = InstanceLink
-        fields = ('source', 'target', 'link_state')
+        fields = ('id', 'url', 'related', 'source', 'target', 'link_state')
 
-    source = serializers.SlugRelatedField(slug_field="hostname", read_only=True)
-    target = serializers.SlugRelatedField(slug_field="hostname", read_only=True)
+    source = serializers.SlugRelatedField(slug_field="hostname", queryset=Instance.objects.all())
+    target = serializers.SlugRelatedField(slug_field="hostname", queryset=Instance.objects.all())
+
+    def get_related(self, obj):
+        res = super(InstanceLinkSerializer, self).get_related(obj)
+        res['source_instance'] = self.reverse('api:instance_detail', kwargs={'pk': obj.source.id})
+        res['target_instance'] = self.reverse('api:instance_detail', kwargs={'pk': obj.target.id})
+        return res
 
 
 class InstanceNodeSerializer(BaseSerializer):
@@ -5376,6 +5382,7 @@ class InstanceSerializer(BaseSerializer):
     jobs_running = serializers.IntegerField(help_text=_('Count of jobs in the running or waiting state that are targeted for this instance'), read_only=True)
     jobs_total = serializers.IntegerField(help_text=_('Count of all jobs that target this instance'), read_only=True)
     health_check_pending = serializers.SerializerMethodField()
+    peers = serializers.SlugRelatedField(many=True, required=False, slug_field="hostname", queryset=Instance.objects.all())
 
     class Meta:
         model = Instance
@@ -5412,6 +5419,8 @@ class InstanceSerializer(BaseSerializer):
             'node_state',
             'ip_address',
             'listener_port',
+            'peers',
+            'peers_from_control_nodes',
         )
         extra_kwargs = {
             'node_type': {'initial': Instance.Types.EXECUTION, 'default': Instance.Types.EXECUTION},
@@ -5464,22 +5473,57 @@ class InstanceSerializer(BaseSerializer):
     def get_health_check_pending(self, obj):
         return obj.health_check_pending
 
-    def validate(self, data):
-        if self.instance:
-            if self.instance.node_type == Instance.Types.HOP:
-                raise serializers.ValidationError("Hop node instances may not be changed.")
-        else:
-            if not settings.IS_K8S:
-                raise serializers.ValidationError("Can only create instances on Kubernetes or OpenShift.")
-        return data
+    def validate(self, attrs):
+        def get_field_from_model_or_attrs(fd):
+            return attrs.get(fd, self.instance and getattr(self.instance, fd) or None)
+
+        def check_peers_changed():
+            '''
+            return True if
+            - 'peers' in attrs
+            - instance peers matches peers in attrs
+            '''
+            return self.instance and 'peers' in attrs and set(self.instance.peers.all()) != set(attrs['peers'])
+
+        if not self.instance and not settings.IS_K8S:
+            raise serializers.ValidationError(_("Can only create instances on Kubernetes or OpenShift."))
+
+        node_type = get_field_from_model_or_attrs("node_type")
+        peers_from_control_nodes = get_field_from_model_or_attrs("peers_from_control_nodes")
+        listener_port = get_field_from_model_or_attrs("listener_port")
+        peers = attrs.get('peers', [])
+
+        if peers_from_control_nodes and node_type not in (Instance.Types.EXECUTION, Instance.Types.HOP):
+            raise serializers.ValidationError(_("peers_from_control_nodes can only be enabled for execution or hop nodes."))
+
+        if node_type in [Instance.Types.CONTROL, Instance.Types.HYBRID]:
+            if check_peers_changed():
+                raise serializers.ValidationError(
+                    _("Setting peers manually for control nodes is not allowed. Enable peers_from_control_nodes on the hop and execution nodes instead.")
+                )
+
+        if not listener_port and peers_from_control_nodes:
+            raise serializers.ValidationError(_("Field listener_port must be a valid integer when peers_from_control_nodes is enabled."))
+
+        if not listener_port and self.instance and self.instance.peers_from.exists():
+            raise serializers.ValidationError(_("Field listener_port must be a valid integer when other nodes peer to it."))
+
+        for peer in peers:
+            if peer.listener_port is None:
+                raise serializers.ValidationError(_("Field listener_port must be set on peer ") + peer.hostname + ".")
+
+        if not settings.IS_K8S:
+            if check_peers_changed():
+                raise serializers.ValidationError(_("Cannot change peers."))
+
+        return super().validate(attrs)
 
     def validate_node_type(self, value):
-        if not self.instance:
-            if value not in (Instance.Types.EXECUTION,):
-                raise serializers.ValidationError("Can only create execution nodes.")
-        else:
-            if self.instance.node_type != value:
-                raise serializers.ValidationError("Cannot change node type.")
+        if not self.instance and value not in [Instance.Types.HOP, Instance.Types.EXECUTION]:
+            raise serializers.ValidationError(_("Can only create execution or hop nodes."))
+
+        if self.instance and self.instance.node_type != value:
+            raise serializers.ValidationError(_("Cannot change node type."))
 
         return value
 
@@ -5487,30 +5531,41 @@ class InstanceSerializer(BaseSerializer):
         if self.instance:
             if value != self.instance.node_state:
                 if not settings.IS_K8S:
-                    raise serializers.ValidationError("Can only change the state on Kubernetes or OpenShift.")
+                    raise serializers.ValidationError(_("Can only change the state on Kubernetes or OpenShift."))
                 if value != Instance.States.DEPROVISIONING:
-                    raise serializers.ValidationError("Can only change instances to the 'deprovisioning' state.")
-                if self.instance.node_type not in (Instance.Types.EXECUTION,):
-                    raise serializers.ValidationError("Can only deprovision execution nodes.")
+                    raise serializers.ValidationError(_("Can only change instances to the 'deprovisioning' state."))
+                if self.instance.node_type not in (Instance.Types.EXECUTION, Instance.Types.HOP):
+                    raise serializers.ValidationError(_("Can only deprovision execution or hop nodes."))
         else:
             if value and value != Instance.States.INSTALLED:
-                raise serializers.ValidationError("Can only create instances in the 'installed' state.")
+                raise serializers.ValidationError(_("Can only create instances in the 'installed' state."))
 
         return value
 
     def validate_hostname(self, value):
         """
-        - Hostname cannot be "localhost" - but can be something like localhost.domain
-        - Cannot change the hostname of an-already instantiated & initialized Instance object
+        Cannot change the hostname
         """
         if self.instance and self.instance.hostname != value:
-            raise serializers.ValidationError("Cannot change hostname.")
+            raise serializers.ValidationError(_("Cannot change hostname."))
 
         return value
 
     def validate_listener_port(self, value):
-        if self.instance and self.instance.listener_port != value:
-            raise serializers.ValidationError("Cannot change listener port.")
+        """
+        Cannot change listener port, unless going from none to integer, and vice versa
+        """
+        if value and self.instance and self.instance.listener_port and self.instance.listener_port != value:
+            raise serializers.ValidationError(_("Cannot change listener port."))
+
+        return value
+
+    def validate_peers_from_control_nodes(self, value):
+        """
+        Can only enable for K8S based deployments
+        """
+        if value and not settings.IS_K8S:
+            raise serializers.ValidationError(_("Can only be enabled on Kubernetes or Openshift."))
 
         return value
 
@@ -5518,7 +5573,19 @@ class InstanceSerializer(BaseSerializer):
 class InstanceHealthCheckSerializer(BaseSerializer):
     class Meta:
         model = Instance
-        read_only_fields = ('uuid', 'hostname', 'version', 'last_health_check', 'errors', 'cpu', 'memory', 'cpu_capacity', 'mem_capacity', 'capacity')
+        read_only_fields = (
+            'uuid',
+            'hostname',
+            'ip_address',
+            'version',
+            'last_health_check',
+            'errors',
+            'cpu',
+            'memory',
+            'cpu_capacity',
+            'mem_capacity',
+            'capacity',
+        )
         fields = read_only_fields
 
 

awx/api/templates/instance_install_bundle/group_vars/all.yml

@@ -3,21 +3,35 @@ receptor_group: awx
 receptor_verify: true
 receptor_tls: true
 receptor_mintls13: false
+{% if instance.node_type == "execution" %}
 receptor_work_commands:
   ansible-runner:
     command: ansible-runner
     params: worker
     allowruntimeparams: true
     verifysignature: true
+additional_python_packages:
+  - ansible-runner
+{% endif %}
 custom_worksign_public_keyfile: receptor/work_public_key.pem
 custom_tls_certfile: receptor/tls/receptor.crt
 custom_tls_keyfile: receptor/tls/receptor.key
 custom_ca_certfile: receptor/tls/ca/mesh-CA.crt
 receptor_protocol: 'tcp'
+{% if instance.listener_port %}
 receptor_listener: true
 receptor_port: {{ instance.listener_port }}
-receptor_dependencies:
-  - python39-pip
+{% else %}
+receptor_listener: false
+{% endif %}
+{% if peers %}
+receptor_peers:
+{% for peer in peers %}
+  - host: {{ peer.host }}
+    port: {{ peer.port }}
+    protocol: tcp
+{% endfor %}
+{% endif %}
 {% verbatim %}
 podman_user: "{{ receptor_user }}"
 podman_group: "{{ receptor_group }}"

awx/api/templates/instance_install_bundle/install_receptor.yml

@@ -1,20 +1,16 @@
-{% verbatim %}
 ---
 - hosts: all
   become: yes
   tasks:
     - name: Create the receptor user
       user:
+{% verbatim %}
         name: "{{ receptor_user }}"
+{% endverbatim %}
         shell: /bin/bash
-    - name: Enable Copr repo for Receptor
-      command: dnf copr enable ansible-awx/receptor -y
+{% if instance.node_type == "execution" %}
     - import_role:
         name: ansible.receptor.podman
+{% endif %}
     - import_role:
         name: ansible.receptor.setup
-    - name: Install ansible-runner
-      pip:
-        name: ansible-runner
-        executable: pip3.9
-{% endverbatim %}

awx/api/templates/instance_install_bundle/requirements.yml

@@ -1,4 +1,4 @@
 ---
 collections:
   - name: ansible.receptor
-    version: 1.1.0
+    version: 2.0.0

awx/api/views/__init__.py

@@ -341,17 +341,18 @@ class InstanceDetail(RetrieveUpdateAPIView):
 
     def update_raw_data(self, data):
         # these fields are only valid on creation of an instance, so they unwanted on detail view
-        data.pop('listener_port', None)
         data.pop('node_type', None)
         data.pop('hostname', None)
+        data.pop('ip_address', None)
         return super(InstanceDetail, self).update_raw_data(data)
 
     def update(self, request, *args, **kwargs):
         r = super(InstanceDetail, self).update(request, *args, **kwargs)
         if status.is_success(r.status_code):
             obj = self.get_object()
-            obj.set_capacity_value()
-            obj.save(update_fields=['capacity'])
+            capacity_changed = obj.set_capacity_value()
+            if capacity_changed:
+                obj.save(update_fields=['capacity'])
             r.data = serializers.InstanceSerializer(obj, context=self.get_serializer_context()).to_representation(obj)
         return r
 

awx/api/views/instance_install_bundle.py

@@ -6,6 +6,8 @@ import io
 import ipaddress
 import os
 import tarfile
+import time
+import re
 
 import asn1
 from awx.api import serializers
@@ -40,6 +42,8 @@ RECEPTOR_OID = "1.3.6.1.4.1.2312.19.1"
 # │   │   └── receptor.key
 # │   └── work-public-key.pem
 # └── requirements.yml
+
+
 class InstanceInstallBundle(GenericAPIView):
     name = _('Install Bundle')
     model = models.Instance
@@ -49,9 +53,9 @@ class InstanceInstallBundle(GenericAPIView):
     def get(self, request, *args, **kwargs):
         instance_obj = self.get_object()
 
-        if instance_obj.node_type not in ('execution',):
+        if instance_obj.node_type not in ('execution', 'hop'):
             return Response(
-                data=dict(msg=_('Install bundle can only be generated for execution nodes.')),
+                data=dict(msg=_('Install bundle can only be generated for execution or hop nodes.')),
                 status=status.HTTP_400_BAD_REQUEST,
             )
 
@@ -66,37 +70,37 @@ class InstanceInstallBundle(GenericAPIView):
                 # generate and write the receptor key to receptor/tls/receptor.key in the tar file
                 key, cert = generate_receptor_tls(instance_obj)
 
+                def tar_addfile(tarinfo, filecontent):
+                    tarinfo.mtime = time.time()
+                    tarinfo.size = len(filecontent)
+                    tar.addfile(tarinfo, io.BytesIO(filecontent))
+
                 key_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/receptor/tls/receptor.key")
-                key_tarinfo.size = len(key)
-                tar.addfile(key_tarinfo, io.BytesIO(key))
+                tar_addfile(key_tarinfo, key)
 
                 cert_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/receptor/tls/receptor.crt")
                 cert_tarinfo.size = len(cert)
-                tar.addfile(cert_tarinfo, io.BytesIO(cert))
+                tar_addfile(cert_tarinfo, cert)
 
                 # generate and write install_receptor.yml to the tar file
-                playbook = generate_playbook().encode('utf-8')
+                playbook = generate_playbook(instance_obj).encode('utf-8')
                 playbook_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/install_receptor.yml")
-                playbook_tarinfo.size = len(playbook)
-                tar.addfile(playbook_tarinfo, io.BytesIO(playbook))
+                tar_addfile(playbook_tarinfo, playbook)
 
                 # generate and write inventory.yml to the tar file
                 inventory_yml = generate_inventory_yml(instance_obj).encode('utf-8')
                 inventory_yml_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/inventory.yml")
-                inventory_yml_tarinfo.size = len(inventory_yml)
-                tar.addfile(inventory_yml_tarinfo, io.BytesIO(inventory_yml))
+                tar_addfile(inventory_yml_tarinfo, inventory_yml)
 
                 # generate and write group_vars/all.yml to the tar file
                 group_vars = generate_group_vars_all_yml(instance_obj).encode('utf-8')
                 group_vars_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/group_vars/all.yml")
-                group_vars_tarinfo.size = len(group_vars)
-                tar.addfile(group_vars_tarinfo, io.BytesIO(group_vars))
+                tar_addfile(group_vars_tarinfo, group_vars)
 
                 # generate and write requirements.yml to the tar file
                 requirements_yml = generate_requirements_yml().encode('utf-8')
                 requirements_yml_tarinfo = tarfile.TarInfo(f"{instance_obj.hostname}_install_bundle/requirements.yml")
-                requirements_yml_tarinfo.size = len(requirements_yml)
-                tar.addfile(requirements_yml_tarinfo, io.BytesIO(requirements_yml))
+                tar_addfile(requirements_yml_tarinfo, requirements_yml)
 
             # respond with the tarfile
             f.seek(0)
@@ -105,8 +109,10 @@ class InstanceInstallBundle(GenericAPIView):
             return response
 
 
-def generate_playbook():
-    return render_to_string("instance_install_bundle/install_receptor.yml")
+def generate_playbook(instance_obj):
+    playbook_yaml = render_to_string("instance_install_bundle/install_receptor.yml", context=dict(instance=instance_obj))
+    # convert consecutive newlines with a single newline
+    return re.sub(r'\n+', '\n', playbook_yaml)
 
 
 def generate_requirements_yml():
@@ -118,7 +124,12 @@ def generate_inventory_yml(instance_obj):
 
 
 def generate_group_vars_all_yml(instance_obj):
-    return render_to_string("instance_install_bundle/group_vars/all.yml", context=dict(instance=instance_obj))
+    peers = []
+    for instance in instance_obj.peers.all():
+        peers.append(dict(host=instance.hostname, port=instance.listener_port))
+    all_yaml = render_to_string("instance_install_bundle/group_vars/all.yml", context=dict(instance=instance_obj, peers=peers))
+    # convert consecutive newlines with a single newline
+    return re.sub(r'\n+', '\n', all_yaml)
 
 
 def generate_receptor_tls(instance_obj):

awx/main/access.py

@@ -366,9 +366,9 @@ class BaseAccess(object):
             report_violation = lambda message: None
         else:
             report_violation = lambda message: logger.warning(message)
-        if validation_info.get('trial', False) is True or validation_info['instance_count'] == 10:  # basic 10 license
+        if validation_info.get('trial', False) is True:
 
-            def report_violation(message):
+            def report_violation(message):  # noqa
                 raise PermissionDenied(message)
 
         if check_expiration and validation_info.get('time_remaining', None) is None:

awx/main/analytics/collectors.py

@@ -613,3 +613,20 @@ def host_metric_table(since, full_path, until, **kwargs):
         since.isoformat(), until.isoformat(), since.isoformat(), until.isoformat()
     )
     return _copy_table(table='host_metric', query=host_metric_query, path=full_path)
+
+
+@register('host_metric_summary_monthly_table', '1.0', format='csv', description=_('HostMetricSummaryMonthly export, full sync'), expensive=trivial_slicing)
+def host_metric_summary_monthly_table(since, full_path, **kwargs):
+    query = '''
+    COPY (SELECT main_hostmetricsummarymonthly.id,
+                 main_hostmetricsummarymonthly.date,
+                 main_hostmetricsummarymonthly.license_capacity,
+                 main_hostmetricsummarymonthly.license_consumed,
+                 main_hostmetricsummarymonthly.hosts_added,
+                 main_hostmetricsummarymonthly.hosts_deleted,
+                 main_hostmetricsummarymonthly.indirectly_managed_hosts
+          FROM main_hostmetricsummarymonthly
+          ORDER BY main_hostmetricsummarymonthly.id ASC) TO STDOUT WITH CSV HEADER
+    '''
+
+    return _copy_table(table='host_metric_summary_monthly', query=query, path=full_path)

awx/main/credential_plugins/tss.py

@@ -1,7 +1,10 @@
 from .plugin import CredentialPlugin
 from django.utils.translation import gettext_lazy as _
 
-from thycotic.secrets.server import DomainPasswordGrantAuthorizer, PasswordGrantAuthorizer, SecretServer, ServerSecret
+try:
+    from delinea.secrets.server import DomainPasswordGrantAuthorizer, PasswordGrantAuthorizer, SecretServer, ServerSecret
+except ImportError:
+    from thycotic.secrets.server import DomainPasswordGrantAuthorizer, PasswordGrantAuthorizer, SecretServer, ServerSecret
 
 tss_inputs = {
     'fields': [

awx/main/dispatch/__init__.py

@@ -40,8 +40,12 @@ def get_task_queuename():
 
 
 class PubSub(object):
-    def __init__(self, conn):
+    def __init__(self, conn, select_timeout=None):
         self.conn = conn
+        if select_timeout is None:
+            self.select_timeout = 5
+        else:
+            self.select_timeout = select_timeout
 
     def listen(self, channel):
         with self.conn.cursor() as cur:
@@ -72,12 +76,12 @@ class PubSub(object):
             n = psycopg.connection.Notify(pgn.relname.decode(enc), pgn.extra.decode(enc), pgn.be_pid)
             yield n
 
-    def events(self, select_timeout=5, yield_timeouts=False):
+    def events(self, yield_timeouts=False):
         if not self.conn.autocommit:
             raise RuntimeError('Listening for events can only be done in autocommit mode')
 
         while True:
-            if select.select([self.conn], [], [], select_timeout) == NOT_READY:
+            if select.select([self.conn], [], [], self.select_timeout) == NOT_READY:
                 if yield_timeouts:
                     yield None
             else:
@@ -90,7 +94,7 @@ class PubSub(object):
 
 
 @contextmanager
-def pg_bus_conn(new_connection=False):
+def pg_bus_conn(new_connection=False, select_timeout=None):
     '''
     Any listeners probably want to establish a new database connection,
     separate from the Django connection used for queries, because that will prevent
@@ -115,7 +119,7 @@ def pg_bus_conn(new_connection=False):
             raise RuntimeError('Unexpectedly could not connect to postgres for pg_notify actions')
         conn = pg_connection.connection
 
-    pubsub = PubSub(conn)
+    pubsub = PubSub(conn, select_timeout=select_timeout)
     yield pubsub
     if new_connection:
         conn.close()

awx/main/dispatch/control.py

@@ -40,6 +40,9 @@ class Control(object):
     def cancel(self, task_ids, *args, **kwargs):
         return self.control_with_reply('cancel', *args, extra_data={'task_ids': task_ids}, **kwargs)
 
+    def schedule(self, *args, **kwargs):
+        return self.control_with_reply('schedule', *args, **kwargs)
+
     @classmethod
     def generate_reply_queue_name(cls):
         return f"reply_to_{str(uuid.uuid4()).replace('-','_')}"
@@ -52,14 +55,14 @@ class Control(object):
         if not connection.get_autocommit():
             raise RuntimeError('Control-with-reply messages can only be done in autocommit mode')
 
-        with pg_bus_conn() as conn:
+        with pg_bus_conn(select_timeout=timeout) as conn:
             conn.listen(reply_queue)
             send_data = {'control': command, 'reply_to': reply_queue}
             if extra_data:
                 send_data.update(extra_data)
             conn.notify(self.queuename, json.dumps(send_data))
 
-            for reply in conn.events(select_timeout=timeout, yield_timeouts=True):
+            for reply in conn.events(yield_timeouts=True):
                 if reply is None:
                     logger.error(f'{self.service} did not reply within {timeout}s')
                     raise RuntimeError(f"{self.service} did not reply within {timeout}s")

awx/main/dispatch/periodic.py

@@ -1,57 +1,142 @@
 import logging
-import os
 import time
-from multiprocessing import Process
+import yaml
+from datetime import datetime
 
-from django.conf import settings
-from django.db import connections
-from schedule import Scheduler
-from django_guid import set_guid
-from django_guid.utils import generate_guid
-
-from awx.main.dispatch.worker import TaskWorker
-from awx.main.utils.db import set_connection_name
 
 logger = logging.getLogger('awx.main.dispatch.periodic')
 
 
-class Scheduler(Scheduler):
-    def run_continuously(self):
-        idle_seconds = max(1, min(self.jobs).period.total_seconds() / 2)
+class ScheduledTask:
+    """
+    Class representing schedules, very loosely modeled after python schedule library Job
+    the idea of this class is to:
+     - only deal in relative times (time since the scheduler global start)
+     - only deal in integer math for target runtimes, but float for current relative time
 
-        def run():
-            ppid = os.getppid()
-            logger.warning('periodic beat started')
+    Missed schedule policy:
+    Invariant target times are maintained, meaning that if interval=10s offset=0
+    and it runs at t=7s, then it calls for next run in 3s.
+    However, if a complete interval has passed, that is counted as a missed run,
+    and missed runs are abandoned (no catch-up runs).
+    """
 
-            set_connection_name('periodic')  # set application_name to distinguish from other dispatcher processes
+    def __init__(self, name: str, data: dict):
+        # parameters need for schedule computation
+        self.interval = int(data['schedule'].total_seconds())
+        self.offset = 0  # offset relative to start time this schedule begins
+        self.index = 0  # number of periods of the schedule that has passed
 
-            while True:
-                if os.getppid() != ppid:
-                    # if the parent PID changes, this process has been orphaned
-                    # via e.g., segfault or sigkill, we should exit too
-                    pid = os.getpid()
-                    logger.warning(f'periodic beat exiting gracefully pid:{pid}')
-                    raise SystemExit()
-                try:
-                    for conn in connections.all():
-                        # If the database connection has a hiccup, re-establish a new
-                        # connection
-                        conn.close_if_unusable_or_obsolete()
-                    set_guid(generate_guid())
-                    self.run_pending()
-                except Exception:
-                    logger.exception('encountered an error while scheduling periodic tasks')
-                time.sleep(idle_seconds)
+        # parameters that do not affect scheduling logic
+        self.last_run = None  # time of last run, only used for debug
+        self.completed_runs = 0  # number of times schedule is known to run
+        self.name = name
+        self.data = data  # used by caller to know what to run
 
-        process = Process(target=run)
-        process.daemon = True
-        process.start()
+    @property
+    def next_run(self):
+        "Time until the next run with t=0 being the global_start of the scheduler class"
+        return (self.index + 1) * self.interval + self.offset
+
+    def due_to_run(self, relative_time):
+        return bool(self.next_run <= relative_time)
+
+    def expected_runs(self, relative_time):
+        return int((relative_time - self.offset) / self.interval)
+
+    def mark_run(self, relative_time):
+        self.last_run = relative_time
+        self.completed_runs += 1
+        new_index = self.expected_runs(relative_time)
+        if new_index > self.index + 1:
+            logger.warning(f'Missed {new_index - self.index - 1} schedules of {self.name}')
+        self.index = new_index
+
+    def missed_runs(self, relative_time):
+        "Number of times job was supposed to ran but failed to, only used for debug"
+        missed_ct = self.expected_runs(relative_time) - self.completed_runs
+        # if this is currently due to run do not count that as a missed run
+        if missed_ct and self.due_to_run(relative_time):
+            missed_ct -= 1
+        return missed_ct
 
 
-def run_continuously():
-    scheduler = Scheduler()
-    for task in settings.CELERYBEAT_SCHEDULE.values():
-        apply_async = TaskWorker.resolve_callable(task['task']).apply_async
-        total_seconds = task['schedule'].total_seconds()
-        scheduler.every(total_seconds).seconds.do(apply_async)
-    scheduler.run_continuously()
+class Scheduler:
+    def __init__(self, schedule):
+        """
+        Expects schedule in the form of a dictionary like
+        {
+            'job1': {'schedule': timedelta(seconds=50), 'other': 'stuff'}
+        }
+        Only the schedule nearest-second value is used for scheduling,
+        the rest of the data is for use by the caller to know what to run.
+        """
+        self.jobs = [ScheduledTask(name, data) for name, data in schedule.items()]
+        min_interval = min(job.interval for job in self.jobs)
+        num_jobs = len(self.jobs)
+
+        # this is intentionally oppioniated against spammy schedules
+        # a core goal is to spread out the scheduled tasks (for worker management)
+        # and high-frequency schedules just do not work with that
+        if num_jobs > min_interval:
+            raise RuntimeError(f'Number of schedules ({num_jobs}) is more than the shortest schedule interval ({min_interval} seconds).')
+
+        # even space out jobs over the base interval
+        for i, job in enumerate(self.jobs):
+            job.offset = (i * min_interval) // num_jobs
+
+        # internally times are all referenced relative to startup time, add grace period
+        self.global_start = time.time() + 2.0
+
+    def get_and_mark_pending(self):
+        relative_time = time.time() - self.global_start
+        to_run = []
+        for job in self.jobs:
+            if job.due_to_run(relative_time):
+                to_run.append(job)
+                logger.debug(f'scheduler found {job.name} to run, {relative_time - job.next_run} seconds after target')
+                job.mark_run(relative_time)
+        return to_run
+
+    def time_until_next_run(self):
+        relative_time = time.time() - self.global_start
+        next_job = min(self.jobs, key=lambda j: j.next_run)
+        delta = next_job.next_run - relative_time
+        if delta <= 0.1:
+            # careful not to give 0 or negative values to the select timeout, which has unclear interpretation
+            logger.warning(f'Scheduler next run of {next_job.name} is {-delta} seconds in the past')
+            return 0.1
+        elif delta > 20.0:
+            logger.warning(f'Scheduler next run unexpectedly over 20 seconds in future: {delta}')
+            return 20.0
+        logger.debug(f'Scheduler next run is {next_job.name} in {delta} seconds')
+        return delta
+
+    def debug(self, *args, **kwargs):
+        data = dict()
+        data['title'] = 'Scheduler status'
+
+        now = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S UTC')
+        start_time = datetime.fromtimestamp(self.global_start).strftime('%Y-%m-%d %H:%M:%S UTC')
+        relative_time = time.time() - self.global_start
+        data['started_time'] = start_time
+        data['current_time'] = now
+        data['current_time_relative'] = round(relative_time, 3)
+        data['total_schedules'] = len(self.jobs)
+
+        data['schedule_list'] = dict(
+            [
+                (
+                    job.name,
+                    dict(
+                        last_run_seconds_ago=round(relative_time - job.last_run, 3) if job.last_run else None,
+                        next_run_in_seconds=round(job.next_run - relative_time, 3),
+                        offset_in_seconds=job.offset,
+                        completed_runs=job.completed_runs,
+                        missed_runs=job.missed_runs(relative_time),
+                    ),
+                )
+                for job in sorted(self.jobs, key=lambda job: job.interval)
+            ]
+        )
+        return yaml.safe_dump(data, default_flow_style=False, sort_keys=False)

awx/main/dispatch/publish.py

@@ -73,15 +73,15 @@ class task:
                 return cls.apply_async(args, kwargs)
 
             @classmethod
-            def apply_async(cls, args=None, kwargs=None, queue=None, uuid=None, **kw):
+            def get_async_body(cls, args=None, kwargs=None, uuid=None, **kw):
+                """
+                Get the python dict to become JSON data in the pg_notify message
+                This same message gets passed over the dispatcher IPC queue to workers
+                If a task is submitted to a multiprocessing pool, skipping pg_notify, this might be used directly
+                """
                 task_id = uuid or str(uuid4())
                 args = args or []
                 kwargs = kwargs or {}
-                queue = queue or getattr(cls.queue, 'im_func', cls.queue)
-                if not queue:
-                    msg = f'{cls.name}: Queue value required and may not be None'
-                    logger.error(msg)
-                    raise ValueError(msg)
                 obj = {'uuid': task_id, 'args': args, 'kwargs': kwargs, 'task': cls.name, 'time_pub': time.time()}
                 guid = get_guid()
                 if guid:
@@ -89,6 +89,16 @@ class task:
                 if bind_kwargs:
                     obj['bind_kwargs'] = bind_kwargs
                 obj.update(**kw)
+                return obj
+
+            @classmethod
+            def apply_async(cls, args=None, kwargs=None, queue=None, uuid=None, **kw):
+                queue = queue or getattr(cls.queue, 'im_func', cls.queue)
+                if not queue:
+                    msg = f'{cls.name}: Queue value required and may not be None'
+                    logger.error(msg)
+                    raise ValueError(msg)
+                obj = cls.get_async_body(args=args, kwargs=kwargs, uuid=uuid, **kw)
                 if callable(queue):
                     queue = queue()
                 if not is_testing():
@@ -116,4 +126,5 @@ class task:
         setattr(fn, 'name', cls.name)
         setattr(fn, 'apply_async', cls.apply_async)
         setattr(fn, 'delay', cls.delay)
+        setattr(fn, 'get_async_body', cls.get_async_body)
         return fn

awx/main/dispatch/worker/base.py

@@ -11,11 +11,13 @@ import psycopg
 import time
 from uuid import UUID
 from queue import Empty as QueueEmpty
+from datetime import timedelta
 
 from django import db
 from django.conf import settings
 
 from awx.main.dispatch.pool import WorkerPool
+from awx.main.dispatch.periodic import Scheduler
 from awx.main.dispatch import pg_bus_conn
 from awx.main.utils.common import log_excess_runtime
 from awx.main.utils.db import set_connection_name
@@ -64,10 +66,12 @@ class AWXConsumerBase(object):
     def control(self, body):
         logger.warning(f'Received control signal:\n{body}')
         control = body.get('control')
-        if control in ('status', 'running', 'cancel'):
+        if control in ('status', 'schedule', 'running', 'cancel'):
             reply_queue = body['reply_to']
             if control == 'status':
                 msg = '\n'.join([self.listening_on, self.pool.debug()])
+            if control == 'schedule':
+                msg = self.scheduler.debug()
             elif control == 'running':
                 msg = []
                 for worker in self.pool.workers:
@@ -93,16 +97,11 @@ class AWXConsumerBase(object):
         else:
             logger.error('unrecognized control message: {}'.format(control))
 
-    def process_task(self, body):
+    def dispatch_task(self, body):
+        """This will place the given body into a worker queue to run method decorated as a task"""
         if isinstance(body, dict):
             body['time_ack'] = time.time()
 
-        if 'control' in body:
-            try:
-                return self.control(body)
-            except Exception:
-                logger.exception(f"Exception handling control message: {body}")
-                return
         if len(self.pool):
             if "uuid" in body and body['uuid']:
                 try:
@@ -116,6 +115,16 @@ class AWXConsumerBase(object):
         self.pool.write(queue, body)
         self.total_messages += 1
 
+    def process_task(self, body):
+        """Routes the task details in body as either a control task or a task-task"""
+        if 'control' in body:
+            try:
+                return self.control(body)
+            except Exception:
+                logger.exception(f"Exception handling control message: {body}")
+                return
+        self.dispatch_task(body)
+
     @log_excess_runtime(logger)
     def record_statistics(self):
         if time.time() - self.last_stats > 1:  # buffer stat recording to once per second
@@ -150,9 +159,9 @@ class AWXConsumerRedis(AWXConsumerBase):
 
 
 class AWXConsumerPG(AWXConsumerBase):
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, schedule=None, **kwargs):
         super().__init__(*args, **kwargs)
-        self.pg_max_wait = settings.DISPATCHER_DB_DOWNTOWN_TOLLERANCE
+        self.pg_max_wait = settings.DISPATCHER_DB_DOWNTIME_TOLERANCE
         # if no successful loops have ran since startup, then we should fail right away
         self.pg_is_down = True  # set so that we fail if we get database errors on startup
         init_time = time.time()
@@ -161,27 +170,53 @@ class AWXConsumerPG(AWXConsumerBase):
         self.subsystem_metrics = s_metrics.Metrics(auto_pipe_execute=False)
         self.last_metrics_gather = init_time
         self.listen_cumulative_time = 0.0
+        if schedule:
+            schedule = schedule.copy()
+        else:
+            schedule = {}
+        # add control tasks to be ran at regular schedules
+        # NOTE: if we run out of database connections, it is important to still run cleanup
+        # so that we scale down workers and free up connections
+        schedule['pool_cleanup'] = {'control': self.pool.cleanup, 'schedule': timedelta(seconds=60)}
+        # record subsystem metrics for the dispatcher
+        schedule['metrics_gather'] = {'control': self.record_metrics, 'schedule': timedelta(seconds=20)}
+        self.scheduler = Scheduler(schedule)
+
+    def record_metrics(self):
+        current_time = time.time()
+        self.pool.produce_subsystem_metrics(self.subsystem_metrics)
+        self.subsystem_metrics.set('dispatcher_availability', self.listen_cumulative_time / (current_time - self.last_metrics_gather))
+        self.subsystem_metrics.pipe_execute()
+        self.listen_cumulative_time = 0.0
+        self.last_metrics_gather = current_time
 
     def run_periodic_tasks(self):
-        self.record_statistics()  # maintains time buffer in method
+        """
+        Run general periodic logic, and return maximum time in seconds before
+        the next requested run
+        This may be called more often than that when events are consumed
+        so this should be very efficient in that
+        """
+        try:
+            self.record_statistics()  # maintains time buffer in method
+        except Exception as exc:
+            logger.warning(f'Failed to save dispatcher statistics {exc}')
 
-        current_time = time.time()
-        if current_time - self.last_cleanup > 60:  # same as cluster_node_heartbeat
-            # NOTE: if we run out of database connections, it is important to still run cleanup
-            # so that we scale down workers and free up connections
-            self.pool.cleanup()
-            self.last_cleanup = current_time
+        for job in self.scheduler.get_and_mark_pending():
+            if 'control' in job.data:
+                try:
+                    job.data['control']()
+                except Exception:
+                    logger.exception(f'Error running control task {job.data}')
+            elif 'task' in job.data:
+                body = self.worker.resolve_callable(job.data['task']).get_async_body()
+                # bypasses pg_notify for scheduled tasks
+                self.dispatch_task(body)
 
-        # record subsystem metrics for the dispatcher
-        if current_time - self.last_metrics_gather > 20:
-            try:
-                self.pool.produce_subsystem_metrics(self.subsystem_metrics)
-                self.subsystem_metrics.set('dispatcher_availability', self.listen_cumulative_time / (current_time - self.last_metrics_gather))
-                self.subsystem_metrics.pipe_execute()
-            except Exception:
-                logger.exception(f"encountered an error trying to store {self.name} metrics")
-            self.listen_cumulative_time = 0.0
-            self.last_metrics_gather = current_time
+        self.pg_is_down = False
+        self.listen_start = time.time()
+
+        return self.scheduler.time_until_next_run()
 
     def run(self, *args, **kwargs):
         super(AWXConsumerPG, self).run(*args, **kwargs)
@@ -197,14 +232,15 @@ class AWXConsumerPG(AWXConsumerBase):
                     if init is False:
                         self.worker.on_start()
                         init = True
-                    self.listen_start = time.time()
+                    # run_periodic_tasks run scheduled actions and gives time until next scheduled action
+                    # this is saved to the conn (PubSub) object in order to modify read timeout in-loop
+                    conn.select_timeout = self.run_periodic_tasks()
+                    # this is the main operational loop for awx-manage run_dispatcher
                     for e in conn.events(yield_timeouts=True):
-                        self.listen_cumulative_time += time.time() - self.listen_start
+                        self.listen_cumulative_time += time.time() - self.listen_start  # for metrics
                         if e is not None:
                             self.process_task(json.loads(e.payload))
-                        self.run_periodic_tasks()
-                        self.pg_is_down = False
-                        self.listen_start = time.time()
+                        conn.select_timeout = self.run_periodic_tasks()
                     if self.should_stop:
                         return
             except psycopg.InterfaceError:

awx/main/management/commands/provision_instance.py

@@ -25,17 +25,20 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('--hostname', dest='hostname', type=str, help="Hostname used during provisioning")
+        parser.add_argument('--listener_port', dest='listener_port', type=int, help="Receptor listener port")
         parser.add_argument('--node_type', type=str, default='hybrid', choices=['control', 'execution', 'hop', 'hybrid'], help="Instance Node type")
         parser.add_argument('--uuid', type=str, help="Instance UUID")
 
-    def _register_hostname(self, hostname, node_type, uuid):
+    def _register_hostname(self, hostname, node_type, uuid, listener_port):
         if not hostname:
             if not settings.AWX_AUTO_DEPROVISION_INSTANCES:
                 raise CommandError('Registering with values from settings only intended for use in K8s installs')
 
             from awx.main.management.commands.register_queue import RegisterQueue
 
-            (changed, instance) = Instance.objects.register(ip_address=os.environ.get('MY_POD_IP'), node_type='control', node_uuid=settings.SYSTEM_UUID)
+            (changed, instance) = Instance.objects.register(
+                ip_address=os.environ.get('MY_POD_IP'), listener_port=listener_port, node_type='control', node_uuid=settings.SYSTEM_UUID
+            )
             RegisterQueue(settings.DEFAULT_CONTROL_PLANE_QUEUE_NAME, 100, 0, [], is_container_group=False).register()
             RegisterQueue(
                 settings.DEFAULT_EXECUTION_QUEUE_NAME,
@@ -48,7 +51,7 @@ class Command(BaseCommand):
                 max_concurrent_jobs=settings.DEFAULT_EXECUTION_QUEUE_MAX_CONCURRENT_JOBS,
             ).register()
         else:
-            (changed, instance) = Instance.objects.register(hostname=hostname, node_type=node_type, node_uuid=uuid)
+            (changed, instance) = Instance.objects.register(hostname=hostname, node_type=node_type, node_uuid=uuid, listener_port=listener_port)
         if changed:
             print("Successfully registered instance {}".format(hostname))
         else:
@@ -58,6 +61,6 @@ class Command(BaseCommand):
     @transaction.atomic
     def handle(self, **options):
         self.changed = False
-        self._register_hostname(options.get('hostname'), options.get('node_type'), options.get('uuid'))
+        self._register_hostname(options.get('hostname'), options.get('node_type'), options.get('uuid'), options.get('listener_port'))
         if self.changed:
             print("(changed: True)")

awx/main/management/commands/run_dispatcher.py

@@ -3,15 +3,13 @@
 import logging
 import yaml
 
-from django.core.cache import cache as django_cache
+from django.conf import settings
 from django.core.management.base import BaseCommand
-from django.db import connection as django_connection
 
 from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.control import Control
 from awx.main.dispatch.pool import AutoscalePool
 from awx.main.dispatch.worker import AWXConsumerPG, TaskWorker
-from awx.main.dispatch import periodic
 
 logger = logging.getLogger('awx.main.dispatch')
 
@@ -21,6 +19,7 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('--status', dest='status', action='store_true', help='print the internal state of any running dispatchers')
+        parser.add_argument('--schedule', dest='schedule', action='store_true', help='print the current status of schedules being ran by dispatcher')
         parser.add_argument('--running', dest='running', action='store_true', help='print the UUIDs of any tasked managed by this dispatcher')
         parser.add_argument(
             '--reload',
@@ -42,6 +41,9 @@ class Command(BaseCommand):
         if options.get('status'):
             print(Control('dispatcher').status())
             return
+        if options.get('schedule'):
+            print(Control('dispatcher').schedule())
+            return
         if options.get('running'):
             print(Control('dispatcher').running())
             return
@@ -58,21 +60,11 @@ class Command(BaseCommand):
             print(Control('dispatcher').cancel(cancel_data))
             return
 
-        # It's important to close these because we're _about_ to fork, and we
-        # don't want the forked processes to inherit the open sockets
-        # for the DB and cache connections (that way lies race conditions)
-        django_connection.close()
-        django_cache.close()
-
-        # spawn a daemon thread to periodically enqueues scheduled tasks
-        # (like the node heartbeat)
-        periodic.run_continuously()
-
         consumer = None
 
         try:
             queues = ['tower_broadcast_all', 'tower_settings_change', get_task_queuename()]
-            consumer = AWXConsumerPG('dispatcher', TaskWorker(), queues, AutoscalePool(min_workers=4))
+            consumer = AWXConsumerPG('dispatcher', TaskWorker(), queues, AutoscalePool(min_workers=4), schedule=settings.CELERYBEAT_SCHEDULE)
             consumer.run()
         except KeyboardInterrupt:
             logger.debug('Terminating Task Dispatcher')

awx/main/managers.py

@@ -115,10 +115,13 @@ class InstanceManager(models.Manager):
             return node[0]
         raise RuntimeError("No instance found with the current cluster host id")
 
-    def register(self, node_uuid=None, hostname=None, ip_address=None, node_type='hybrid', defaults=None):
+    def register(self, node_uuid=None, hostname=None, ip_address="", listener_port=None, node_type='hybrid', defaults=None):
         if not hostname:
             hostname = settings.CLUSTER_HOST_ID
 
+        if not ip_address:
+            ip_address = ""
+
         with advisory_lock('instance_registration_%s' % hostname):
             if settings.AWX_AUTO_DEPROVISION_INSTANCES:
                 # detect any instances with the same IP address.
@@ -157,6 +160,9 @@ class InstanceManager(models.Manager):
                 if instance.node_type != node_type:
                     instance.node_type = node_type
                     update_fields.append('node_type')
+                if instance.listener_port != listener_port:
+                    instance.listener_port = listener_port
+                    update_fields.append('listener_port')
                 if update_fields:
                     instance.save(update_fields=update_fields)
                     return (True, instance)
@@ -167,12 +173,11 @@ class InstanceManager(models.Manager):
             create_defaults = {
                 'node_state': Instance.States.INSTALLED,
                 'capacity': 0,
-                'listener_port': 27199,
             }
             if defaults is not None:
                 create_defaults.update(defaults)
             uuid_option = {'uuid': node_uuid if node_uuid is not None else uuid.uuid4()}
             if node_type == 'execution' and 'version' not in create_defaults:
                 create_defaults['version'] = RECEPTOR_PENDING
-            instance = self.create(hostname=hostname, ip_address=ip_address, node_type=node_type, **create_defaults, **uuid_option)
+            instance = self.create(hostname=hostname, ip_address=ip_address, listener_port=listener_port, node_type=node_type, **create_defaults, **uuid_option)
         return (True, instance)

awx/main/migrations/0185_move_JSONBlob_to_JSONField.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2 on 2023-06-09 19:51
+# Generated by Django 4.2.3 on 2023-08-02 13:18
 
 import awx.main.models.notifications
 from django.db import migrations, models
@@ -11,16 +11,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AlterField(
-            model_name='activitystream',
-            name='deleted_actor',
-            field=models.JSONField(null=True),
-        ),
-        migrations.AlterField(
-            model_name='activitystream',
-            name='setting',
-            field=models.JSONField(blank=True, default=dict),
-        ),
         migrations.AlterField(
             model_name='instancegroup',
             name='policy_instance_list',
@@ -28,31 +18,11 @@ class Migration(migrations.Migration):
                 blank=True, default=list, help_text='List of exact-match Instances that will always be automatically assigned to this group'
             ),
         ),
-        migrations.AlterField(
-            model_name='job',
-            name='survey_passwords',
-            field=models.JSONField(blank=True, default=dict, editable=False),
-        ),
-        migrations.AlterField(
-            model_name='joblaunchconfig',
-            name='char_prompts',
-            field=models.JSONField(blank=True, default=dict),
-        ),
-        migrations.AlterField(
-            model_name='joblaunchconfig',
-            name='survey_passwords',
-            field=models.JSONField(blank=True, default=dict, editable=False),
-        ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='survey_spec',
             field=models.JSONField(blank=True, default=dict),
         ),
-        migrations.AlterField(
-            model_name='notification',
-            name='body',
-            field=models.JSONField(blank=True, default=dict),
-        ),
         migrations.AlterField(
             model_name='notificationtemplate',
             name='messages',
@@ -94,31 +64,6 @@ class Migration(migrations.Migration):
             name='survey_passwords',
             field=models.JSONField(blank=True, default=dict, editable=False),
         ),
-        migrations.AlterField(
-            model_name='unifiedjob',
-            name='job_env',
-            field=models.JSONField(blank=True, default=dict, editable=False),
-        ),
-        migrations.AlterField(
-            model_name='workflowjob',
-            name='char_prompts',
-            field=models.JSONField(blank=True, default=dict),
-        ),
-        migrations.AlterField(
-            model_name='workflowjob',
-            name='survey_passwords',
-            field=models.JSONField(blank=True, default=dict, editable=False),
-        ),
-        migrations.AlterField(
-            model_name='workflowjobnode',
-            name='char_prompts',
-            field=models.JSONField(blank=True, default=dict),
-        ),
-        migrations.AlterField(
-            model_name='workflowjobnode',
-            name='survey_passwords',
-            field=models.JSONField(blank=True, default=dict, editable=False),
-        ),
         migrations.AlterField(
             model_name='workflowjobtemplate',
             name='char_prompts',
@@ -139,4 +84,194 @@ class Migration(migrations.Migration):
             name='survey_passwords',
             field=models.JSONField(blank=True, default=dict, editable=False),
         ),
+        # These are potentially a problem.  Move the existing fields
+        # aside while pretending like they've been deleted, then add
+        # in fresh empty fields.  Make the old fields nullable where
+        # needed while we are at it, so that new rows don't hit
+        # IntegrityError.  We'll do the data migration out-of-band
+        # using a task.
+        migrations.RunSQL(  # Already nullable
+            "ALTER TABLE main_activitystream RENAME deleted_actor TO deleted_actor_old;",
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='activitystream',
+                    name='deleted_actor',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='activitystream',
+            name='deleted_actor',
+            field=models.JSONField(null=True),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_activitystream RENAME setting TO setting_old;
+            ALTER TABLE main_activitystream ALTER COLUMN setting_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='activitystream',
+                    name='setting',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='activitystream',
+            name='setting',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_job RENAME survey_passwords TO survey_passwords_old;
+            ALTER TABLE main_job ALTER COLUMN survey_passwords_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='job',
+                    name='survey_passwords',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='job',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_joblaunchconfig RENAME char_prompts TO char_prompts_old;
+            ALTER TABLE main_joblaunchconfig ALTER COLUMN char_prompts_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='joblaunchconfig',
+                    name='char_prompts',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='joblaunchconfig',
+            name='char_prompts',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_joblaunchconfig RENAME survey_passwords TO survey_passwords_old;
+            ALTER TABLE main_joblaunchconfig ALTER COLUMN survey_passwords_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='joblaunchconfig',
+                    name='survey_passwords',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='joblaunchconfig',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_notification RENAME body TO body_old;
+            ALTER TABLE main_notification ALTER COLUMN body_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='notification',
+                    name='body',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='notification',
+            name='body',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_unifiedjob RENAME job_env TO job_env_old;
+            ALTER TABLE main_unifiedjob ALTER COLUMN job_env_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='unifiedjob',
+                    name='job_env',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='unifiedjob',
+            name='job_env',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_workflowjob RENAME char_prompts TO char_prompts_old;
+            ALTER TABLE main_workflowjob ALTER COLUMN char_prompts_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='workflowjob',
+                    name='char_prompts',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='workflowjob',
+            name='char_prompts',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_workflowjob RENAME survey_passwords TO survey_passwords_old;
+            ALTER TABLE main_workflowjob ALTER COLUMN survey_passwords_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='workflowjob',
+                    name='survey_passwords',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='workflowjob',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_workflowjobnode RENAME char_prompts TO char_prompts_old;
+            ALTER TABLE main_workflowjobnode ALTER COLUMN char_prompts_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='workflowjobnode',
+                    name='char_prompts',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='workflowjobnode',
+            name='char_prompts',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_workflowjobnode RENAME survey_passwords TO survey_passwords_old;
+            ALTER TABLE main_workflowjobnode ALTER COLUMN survey_passwords_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='workflowjobnode',
+                    name='survey_passwords',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='workflowjobnode',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
     ]

awx/main/migrations/0187_hop_nodes.py

@@ -0,0 +1,75 @@
+# Generated by Django 4.2.3 on 2023-08-04 20:50
+
+import django.core.validators
+from django.db import migrations, models
+from django.conf import settings
+
+
+def automatically_peer_from_control_plane(apps, schema_editor):
+    if settings.IS_K8S:
+        Instance = apps.get_model('main', 'Instance')
+        Instance.objects.filter(node_type='execution').update(peers_from_control_nodes=True)
+        Instance.objects.filter(node_type='control').update(listener_port=None)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('main', '0186_drop_django_taggit'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='instancelink',
+            options={'ordering': ('id',)},
+        ),
+        migrations.AddField(
+            model_name='instance',
+            name='peers_from_control_nodes',
+            field=models.BooleanField(default=False, help_text='If True, control plane cluster nodes should automatically peer to it.'),
+        ),
+        migrations.AlterField(
+            model_name='instance',
+            name='ip_address',
+            field=models.CharField(blank=True, default='', max_length=50),
+        ),
+        migrations.AlterField(
+            model_name='instance',
+            name='listener_port',
+            field=models.PositiveIntegerField(
+                blank=True,
+                default=None,
+                help_text='Port that Receptor will listen for incoming connections on.',
+                null=True,
+                validators=[django.core.validators.MinValueValidator(1024), django.core.validators.MaxValueValidator(65535)],
+            ),
+        ),
+        migrations.AlterField(
+            model_name='instance',
+            name='peers',
+            field=models.ManyToManyField(related_name='peers_from', through='main.InstanceLink', to='main.instance'),
+        ),
+        migrations.AlterField(
+            model_name='instancelink',
+            name='link_state',
+            field=models.CharField(
+                choices=[('adding', 'Adding'), ('established', 'Established'), ('removing', 'Removing')],
+                default='adding',
+                help_text='Indicates the current life cycle stage of this peer link.',
+                max_length=16,
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name='instance',
+            constraint=models.UniqueConstraint(
+                condition=models.Q(('ip_address', ''), _negated=True),
+                fields=('ip_address',),
+                name='unique_ip_address_not_empty',
+                violation_error_message='Field ip_address must be unique.',
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name='instancelink',
+            constraint=models.CheckConstraint(check=models.Q(('source', models.F('target')), _negated=True), name='source_and_target_can_not_be_equal'),
+        ),
+        migrations.RunPython(automatically_peer_from_control_plane),
+    ]

awx/main/models/__init__.py

@@ -3,6 +3,7 @@
 
 # Django
 from django.conf import settings  # noqa
+from django.db import connection
 from django.db.models.signals import pre_delete  # noqa
 
 # AWX
@@ -99,6 +100,58 @@ User.add_to_class('can_access_with_errors', check_user_access_with_errors)
 User.add_to_class('accessible_objects', user_accessible_objects)
 
 
+def convert_jsonfields():
+    if connection.vendor != 'postgresql':
+        return
+
+    # fmt: off
+    fields = [
+        ('main_activitystream', 'id', (
+            'deleted_actor',
+            'setting',
+        )),
+        ('main_job', 'unifiedjob_ptr_id', (
+            'survey_passwords',
+        )),
+        ('main_joblaunchconfig', 'id', (
+            'char_prompts',
+            'survey_passwords',
+        )),
+        ('main_notification', 'id', (
+            'body',
+        )),
+        ('main_unifiedjob', 'id', (
+            'job_env',
+        )),
+        ('main_workflowjob', 'unifiedjob_ptr_id', (
+            'char_prompts',
+            'survey_passwords',
+        )),
+        ('main_workflowjobnode', 'id', (
+            'char_prompts',
+            'survey_passwords',
+        )),
+    ]
+    # fmt: on
+
+    with connection.cursor() as cursor:
+        for table, pkfield, columns in fields:
+            # Do the renamed old columns still exist?  If so, run the task.
+            old_columns = ','.join(f"'{column}_old'" for column in columns)
+            cursor.execute(
+                f"""
+                select count(1) from information_schema.columns
+                where
+                  table_name = %s and column_name in ({old_columns});
+                """,
+                (table,),
+            )
+            if cursor.fetchone()[0]:
+                from awx.main.tasks.system import migrate_jsonfield
+
+                migrate_jsonfield.apply_async([table, pkfield, columns])
+
+
 def cleanup_created_modified_by(sender, **kwargs):
     # work around a bug in django-polymorphic that doesn't properly
     # handle cascades for reverse foreign keys on the polymorphic base model

awx/main/models/credential/__init__.py

@@ -17,6 +17,7 @@ from jinja2 import sandbox
 from django.db import models
 from django.utils.translation import gettext_lazy as _, gettext_noop
 from django.core.exceptions import ValidationError
+from django.conf import settings
 from django.utils.encoding import force_str
 from django.utils.functional import cached_property
 from django.utils.timezone import now
@@ -30,7 +31,7 @@ from awx.main.fields import (
     CredentialTypeInjectorField,
     DynamicCredentialInputField,
 )
-from awx.main.utils import decrypt_field, classproperty
+from awx.main.utils import decrypt_field, classproperty, set_environ
 from awx.main.utils.safe_yaml import safe_dump
 from awx.main.utils.execution_environments import to_container_path
 from awx.main.validators import validate_ssh_private_key
@@ -1252,7 +1253,9 @@ class CredentialInputSource(PrimordialModel):
                 backend_kwargs[field_name] = value
 
         backend_kwargs.update(self.metadata)
-        return backend(**backend_kwargs)
+
+        with set_environ(**settings.AWX_TASK_ENV):
+            return backend(**backend_kwargs)
 
     def get_absolute_url(self, request=None):
         view_name = 'api:credential_input_source_detail'

awx/main/models/ha.py

@@ -12,13 +12,14 @@ from django.dispatch import receiver
 from django.utils.translation import gettext_lazy as _
 from django.conf import settings
 from django.utils.timezone import now, timedelta
-from django.db.models import Sum
+from django.db.models import Sum, Q
 
 import redis
 from solo.models import SingletonModel
 
 # AWX
 from awx import __version__ as awx_application_version
+from awx.main.utils import is_testing
 from awx.api.versioning import reverse
 from awx.main.fields import ImplicitRoleField
 from awx.main.managers import InstanceManager, UUID_DEFAULT
@@ -70,16 +71,33 @@ class InstanceLink(BaseModel):
         REMOVING = 'removing', _('Removing')
 
     link_state = models.CharField(
-        choices=States.choices, default=States.ESTABLISHED, max_length=16, help_text=_("Indicates the current life cycle stage of this peer link.")
+        choices=States.choices, default=States.ADDING, max_length=16, help_text=_("Indicates the current life cycle stage of this peer link.")
     )
 
     class Meta:
         unique_together = ('source', 'target')
+        ordering = ("id",)
+        constraints = [models.CheckConstraint(check=~models.Q(source=models.F('target')), name='source_and_target_can_not_be_equal')]
 
 
 class Instance(HasPolicyEditsMixin, BaseModel):
     """A model representing an AWX instance running against this database."""
 
+    class Meta:
+        app_label = 'main'
+        ordering = ("hostname",)
+        constraints = [
+            models.UniqueConstraint(
+                fields=["ip_address"],
+                condition=~Q(ip_address=""),  # don't apply to constraint to empty entries
+                name="unique_ip_address_not_empty",
+                violation_error_message=_("Field ip_address must be unique."),
+            )
+        ]
+
+    def __str__(self):
+        return self.hostname
+
     objects = InstanceManager()
 
     # Fields set in instance registration
@@ -87,10 +105,8 @@ class Instance(HasPolicyEditsMixin, BaseModel):
     hostname = models.CharField(max_length=250, unique=True)
     ip_address = models.CharField(
         blank=True,
-        null=True,
-        default=None,
+        default="",
         max_length=50,
-        unique=True,
     )
     # Auto-fields, implementation is different from BaseModel
     created = models.DateTimeField(auto_now_add=True)
@@ -169,16 +185,14 @@ class Instance(HasPolicyEditsMixin, BaseModel):
     )
     listener_port = models.PositiveIntegerField(
         blank=True,
-        default=27199,
-        validators=[MinValueValidator(1), MaxValueValidator(65535)],
+        null=True,
+        default=None,
+        validators=[MinValueValidator(1024), MaxValueValidator(65535)],
         help_text=_("Port that Receptor will listen for incoming connections on."),
     )
 
-    peers = models.ManyToManyField('self', symmetrical=False, through=InstanceLink, through_fields=('source', 'target'))
-
-    class Meta:
-        app_label = 'main'
-        ordering = ("hostname",)
+    peers = models.ManyToManyField('self', symmetrical=False, through=InstanceLink, through_fields=('source', 'target'), related_name='peers_from')
+    peers_from_control_nodes = models.BooleanField(default=False, help_text=_("If True, control plane cluster nodes should automatically peer to it."))
 
     POLICY_FIELDS = frozenset(('managed_by_policy', 'hostname', 'capacity_adjustment'))
 
@@ -279,6 +293,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
         return update_fields
 
     def set_capacity_value(self):
+        old_val = self.capacity
         """Sets capacity according to capacity adjustment rule (no save)"""
         if self.enabled and self.node_type != 'hop':
             lower_cap = min(self.mem_capacity, self.cpu_capacity)
@@ -286,6 +301,7 @@ class Instance(HasPolicyEditsMixin, BaseModel):
             self.capacity = lower_cap + (higher_cap - lower_cap) * self.capacity_adjustment
         else:
             self.capacity = 0
+        return int(self.capacity) != int(old_val)  # return True if value changed
 
     def refresh_capacity_fields(self):
         """Update derived capacity fields from cpu and memory (no save)"""
@@ -464,21 +480,50 @@ def on_instance_group_saved(sender, instance, created=False, raw=False, **kwargs
         instance.set_default_policy_fields()
 
 
+def schedule_write_receptor_config(broadcast=True):
+    from awx.main.tasks.receptor import write_receptor_config  # prevents circular import
+
+    # broadcast to all control instances to update their receptor configs
+    if broadcast:
+        connection.on_commit(lambda: write_receptor_config.apply_async(queue='tower_broadcast_all'))
+    else:
+        if not is_testing():
+            write_receptor_config()  # just run locally
+
+
 @receiver(post_save, sender=Instance)
 def on_instance_saved(sender, instance, created=False, raw=False, **kwargs):
-    if settings.IS_K8S and instance.node_type in (Instance.Types.EXECUTION,):
+    '''
+    Here we link control nodes to hop or execution nodes based on the
+    peers_from_control_nodes field.
+    write_receptor_config should be called on each control node when:
+    1. new node is created with peers_from_control_nodes enabled
+    2. a node changes its value of peers_from_control_nodes
+    3. a new control node comes online and has instances to peer to
+    '''
+    if created and settings.IS_K8S and instance.node_type in [Instance.Types.CONTROL, Instance.Types.HYBRID]:
+        inst = Instance.objects.filter(peers_from_control_nodes=True)
+        if set(instance.peers.all()) != set(inst):
+            instance.peers.set(inst)
+            schedule_write_receptor_config(broadcast=False)
+
+    if settings.IS_K8S and instance.node_type in [Instance.Types.HOP, Instance.Types.EXECUTION]:
         if instance.node_state == Instance.States.DEPROVISIONING:
             from awx.main.tasks.receptor import remove_deprovisioned_node  # prevents circular import
 
             # wait for jobs on the node to complete, then delete the
             # node and kick off write_receptor_config
             connection.on_commit(lambda: remove_deprovisioned_node.apply_async([instance.hostname]))
-
-        if instance.node_state == Instance.States.INSTALLED:
-            from awx.main.tasks.receptor import write_receptor_config  # prevents circular import
-
-            # broadcast to all control instances to update their receptor configs
-            connection.on_commit(lambda: write_receptor_config.apply_async(queue='tower_broadcast_all'))
+        else:
+            control_instances = set(Instance.objects.filter(node_type__in=[Instance.Types.CONTROL, Instance.Types.HYBRID]))
+            if instance.peers_from_control_nodes:
+                if (control_instances & set(instance.peers_from.all())) != set(control_instances):
+                    instance.peers_from.add(*control_instances)
+                    schedule_write_receptor_config()  # keep method separate to make pytest mocking easier
+            else:
+                if set(control_instances) & set(instance.peers_from.all()):
+                    instance.peers_from.remove(*control_instances)
+                    schedule_write_receptor_config()
 
     if created or instance.has_policy_changes():
         schedule_policy_task()
@@ -493,6 +538,8 @@ def on_instance_group_deleted(sender, instance, using, **kwargs):
 @receiver(post_delete, sender=Instance)
 def on_instance_deleted(sender, instance, using, **kwargs):
     schedule_policy_task()
+    if settings.IS_K8S and instance.node_type in (Instance.Types.EXECUTION, Instance.Types.HOP) and instance.peers_from_control_nodes:
+        schedule_write_receptor_config()
 
 
 class UnifiedJobTemplateInstanceGroupMembership(models.Model):

awx/main/tasks/callback.py

@@ -29,8 +29,9 @@ class RunnerCallback:
         self.safe_env = {}
         self.event_ct = 0
         self.model = model
-        self.update_attempts = int(settings.DISPATCHER_DB_DOWNTOWN_TOLLERANCE / 5)
+        self.update_attempts = int(settings.DISPATCHER_DB_DOWNTIME_TOLERANCE / 5)
         self.wrapup_event_dispatched = False
+        self.artifacts_processed = False
         self.extra_update_fields = {}
 
     def update_model(self, pk, _attempt=0, **updates):
@@ -211,6 +212,9 @@ class RunnerCallback:
             if result_traceback:
                 self.delay_update(result_traceback=result_traceback)
 
+    def artifacts_handler(self, artifact_dir):
+        self.artifacts_processed = True
+
 
 class RunnerCallbackForProjectUpdate(RunnerCallback):
     def __init__(self, *args, **kwargs):

awx/main/tasks/facts.py

@@ -9,6 +9,7 @@ from django.conf import settings
 from django.db.models.query import QuerySet
 from django.utils.encoding import smart_str
 from django.utils.timezone import now
+from django.db import OperationalError
 
 # AWX
 from awx.main.utils.common import log_excess_runtime
@@ -57,6 +58,28 @@ def start_fact_cache(hosts, destination, log_data, timeout=None, inventory_id=No
     return None
 
 
+def raw_update_hosts(host_list):
+    Host.objects.bulk_update(host_list, ['ansible_facts', 'ansible_facts_modified'])
+
+
+def update_hosts(host_list, max_tries=5):
+    if not host_list:
+        return
+    for i in range(max_tries):
+        try:
+            raw_update_hosts(host_list)
+        except OperationalError as exc:
+            # Deadlocks can happen if this runs at the same time as another large query
+            # inventory updates and updating last_job_host_summary are candidates for conflict
+            # but these would resolve easily on a retry
+            if i + 1 < max_tries:
+                logger.info(f'OperationalError (suspected deadlock) saving host facts retry {i}, message: {exc}')
+                continue
+            else:
+                raise
+        break
+
+
 @log_excess_runtime(
     logger,
     debug_cutoff=0.01,
@@ -111,7 +134,6 @@ def finish_fact_cache(hosts, destination, facts_write_time, log_data, job_id=Non
             system_tracking_logger.info('Facts cleared for inventory {} host {}'.format(smart_str(host.inventory.name), smart_str(host.name)))
             log_data['cleared_ct'] += 1
         if len(hosts_to_update) > 100:
-            Host.objects.bulk_update(hosts_to_update, ['ansible_facts', 'ansible_facts_modified'])
+            update_hosts(hosts_to_update)
             hosts_to_update = []
-    if hosts_to_update:
-        Host.objects.bulk_update(hosts_to_update, ['ansible_facts', 'ansible_facts_modified'])
+    update_hosts(hosts_to_update)

awx/main/tasks/jobs.py

@@ -112,7 +112,7 @@ class BaseTask(object):
 
     def __init__(self):
         self.cleanup_paths = []
-        self.update_attempts = int(settings.DISPATCHER_DB_DOWNTOWN_TOLLERANCE / 5)
+        self.update_attempts = int(settings.DISPATCHER_DB_DOWNTIME_TOLERANCE / 5)
         self.runner_callback = self.callback_class(model=self.model)
 
     def update_model(self, pk, _attempt=0, **updates):
@@ -1094,7 +1094,7 @@ class RunJob(SourceControlMixin, BaseTask):
             # actual `run()` call; this _usually_ means something failed in
             # the pre_run_hook method
             return
-        if self.should_use_fact_cache():
+        if self.should_use_fact_cache() and self.runner_callback.artifacts_processed:
             job.log_lifecycle("finish_job_fact_cache")
             finish_fact_cache(
                 job.get_hosts_for_fact_cache(),

awx/main/tasks/receptor.py

@@ -30,6 +30,7 @@ from awx.main.tasks.signals import signal_state, signal_callback, SignalExit
 from awx.main.models import Instance, InstanceLink, UnifiedJob
 from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.publish import task
+from awx.main.utils.pglock import advisory_lock
 
 # Receptorctl
 from receptorctl.socket_interface import ReceptorControl
@@ -464,6 +465,7 @@ class AWXReceptorJob:
             event_handler=self.task.runner_callback.event_handler,
             finished_callback=self.task.runner_callback.finished_callback,
             status_handler=self.task.runner_callback.status_handler,
+            artifacts_handler=self.task.runner_callback.artifacts_handler,
             **self.runner_params,
         )
 
@@ -674,26 +676,41 @@ RECEPTOR_CONFIG_STARTER = (
 )
 
 
-@task()
-def write_receptor_config():
-    lock = FileLock(__RECEPTOR_CONF_LOCKFILE)
-    with lock:
-        receptor_config = list(RECEPTOR_CONFIG_STARTER)
+def should_update_config(instances):
+    '''
+    checks that the list of instances matches the list of
+    tcp-peers in the config
+    '''
+    current_config = read_receptor_config()  # this gets receptor conf lock
+    current_peers = []
+    for config_entry in current_config:
+        for key, value in config_entry.items():
+            if key.endswith('-peer'):
+                current_peers.append(value['address'])
+    intended_peers = [f"{i.hostname}:{i.listener_port}" for i in instances]
+    logger.debug(f"Peers current {current_peers} intended {intended_peers}")
+    if set(current_peers) == set(intended_peers):
+        return False  # config file is already update to date
 
-        this_inst = Instance.objects.me()
-        instances = Instance.objects.filter(node_type=Instance.Types.EXECUTION)
-        existing_peers = {link.target_id for link in InstanceLink.objects.filter(source=this_inst)}
-        new_links = []
-        for instance in instances:
-            peer = {'tcp-peer': {'address': f'{instance.hostname}:{instance.listener_port}', 'tls': 'tlsclient'}}
-            receptor_config.append(peer)
-            if instance.id not in existing_peers:
-                new_links.append(InstanceLink(source=this_inst, target=instance, link_state=InstanceLink.States.ADDING))
+    return True
 
-        InstanceLink.objects.bulk_create(new_links)
 
-        with open(__RECEPTOR_CONF, 'w') as file:
-            yaml.dump(receptor_config, file, default_flow_style=False)
+def generate_config_data():
+    # returns two values
+    #   receptor config - based on current database peers
+    #   should_update   - If True, receptor_config differs from the receptor conf file on disk
+    instances = Instance.objects.filter(node_type__in=(Instance.Types.EXECUTION, Instance.Types.HOP), peers_from_control_nodes=True)
+
+    receptor_config = list(RECEPTOR_CONFIG_STARTER)
+    for instance in instances:
+        peer = {'tcp-peer': {'address': f'{instance.hostname}:{instance.listener_port}', 'tls': 'tlsclient'}}
+        receptor_config.append(peer)
+    should_update = should_update_config(instances)
+    return receptor_config, should_update
+
+
+def reload_receptor():
+    logger.warning("Receptor config changed, reloading receptor")
 
     # This needs to be outside of the lock because this function itself will acquire the lock.
     receptor_ctl = get_receptor_ctl()
@@ -709,8 +726,29 @@ def write_receptor_config():
     else:
         raise RuntimeError("Receptor reload failed")
 
-    links = InstanceLink.objects.filter(source=this_inst, target__in=instances, link_state=InstanceLink.States.ADDING)
-    links.update(link_state=InstanceLink.States.ESTABLISHED)
+
+@task()
+def write_receptor_config():
+    """
+    This task runs async on each control node, K8S only.
+    It is triggered whenever remote is added or removed, or if peers_from_control_nodes
+    is flipped.
+    It is possible for write_receptor_config to be called multiple times.
+    For example, if new instances are added in quick succession.
+    To prevent that case, each control node first grabs a DB advisory lock, specific
+    to just that control node (i.e. multiple control nodes can run this function
+    at the same time, since it only writes the local receptor config file)
+    """
+    with advisory_lock(f"{settings.CLUSTER_HOST_ID}_write_receptor_config", wait=True):
+        # Config file needs to be updated
+        receptor_config, should_update = generate_config_data()
+        if should_update:
+            lock = FileLock(__RECEPTOR_CONF_LOCKFILE)
+            with lock:
+                with open(__RECEPTOR_CONF, 'w') as file:
+                    yaml.dump(receptor_config, file, default_flow_style=False)
+
+            reload_receptor()
 
 
 @task(queue=get_task_queuename)
@@ -730,6 +768,3 @@ def remove_deprovisioned_node(hostname):
 
     # This will as a side effect also delete the InstanceLinks that are tied to it.
     Instance.objects.filter(hostname=hostname).delete()
-
-    # Update the receptor configs for all of the control-plane.
-    write_receptor_config.apply_async(queue='tower_broadcast_all')

awx/main/tasks/system.py

@@ -2,6 +2,7 @@
 from collections import namedtuple
 import functools
 import importlib
+import itertools
 import json
 import logging
 import os
@@ -14,7 +15,7 @@ from datetime import datetime
 
 # Django
 from django.conf import settings
-from django.db import transaction, DatabaseError, IntegrityError
+from django.db import connection, transaction, DatabaseError, IntegrityError
 from django.db.models.fields.related import ForeignKey
 from django.utils.timezone import now, timedelta
 from django.utils.encoding import smart_str
@@ -48,6 +49,7 @@ from awx.main.models import (
     SmartInventoryMembership,
     Job,
     HostMetric,
+    convert_jsonfields,
 )
 from awx.main.constants import ACTIVE_STATES
 from awx.main.dispatch.publish import task
@@ -86,6 +88,11 @@ def dispatch_startup():
     if settings.IS_K8S:
         write_receptor_config()
 
+    try:
+        convert_jsonfields()
+    except Exception:
+        logger.exception("Failed json field conversion, skipping.")
+
     startup_logger.debug("Syncing Schedules")
     for sch in Schedule.objects.all():
         try:
@@ -129,6 +136,52 @@ def inform_cluster_of_shutdown():
         logger.exception('Encountered problem with normal shutdown signal.')
 
 
+@task(queue=get_task_queuename)
+def migrate_jsonfield(table, pkfield, columns):
+    batchsize = 10000
+    with advisory_lock(f'json_migration_{table}', wait=False) as acquired:
+        if not acquired:
+            return
+
+        from django.db.migrations.executor import MigrationExecutor
+
+        # If Django is currently running migrations, wait until it is done.
+        while True:
+            executor = MigrationExecutor(connection)
+            if not executor.migration_plan(executor.loader.graph.leaf_nodes()):
+                break
+            time.sleep(120)
+
+        logger.warning(f"Migrating json fields for {table}: {', '.join(columns)}")
+
+        with connection.cursor() as cursor:
+            for i in itertools.count(0, batchsize):
+                # Are there even any rows in the table beyond this point?
+                cursor.execute(f"select count(1) from {table} where {pkfield} >= %s limit 1;", (i,))
+                if not cursor.fetchone()[0]:
+                    break
+
+                column_expr = ', '.join(f"{colname} = {colname}_old::jsonb" for colname in columns)
+                # If any of the old columns have non-null values, the data needs to be cast and copied over.
+                empty_expr = ' or '.join(f"{colname}_old is not null" for colname in columns)
+                cursor.execute(  # Only clobber the new fields if there is non-null data in the old ones.
+                    f"""
+                    update {table}
+                      set {column_expr}
+                      where {pkfield} >= %s and {pkfield} < %s
+                        and {empty_expr};
+                    """,
+                    (i, i + batchsize),
+                )
+                rows = cursor.rowcount
+                logger.debug(f"Batch {i} to {i + batchsize} copied on {table}, {rows} rows affected.")
+
+            column_expr = ', '.join(f"DROP COLUMN {column}_old" for column in columns)
+            cursor.execute(f"ALTER TABLE {table} {column_expr};")
+
+        logger.warning(f"Migration of {table} to jsonb is finished.")
+
+
 @task(queue=get_task_queuename)
 def apply_cluster_membership_policies():
     from awx.main.signals import disable_activity_stream
@@ -459,13 +512,37 @@ def execution_node_health_check(node):
     return data
 
 
-def inspect_execution_nodes(instance_list):
-    with advisory_lock('inspect_execution_nodes_lock', wait=False):
-        node_lookup = {inst.hostname: inst for inst in instance_list}
+def inspect_established_receptor_connections(mesh_status):
+    '''
+    Flips link state from ADDING to ESTABLISHED
+    If the InstanceLink source and target match the entries
+    in Known Connection Costs, flip to Established.
+    '''
+    from awx.main.models import InstanceLink
 
+    all_links = InstanceLink.objects.filter(link_state=InstanceLink.States.ADDING)
+    if not all_links.exists():
+        return
+    active_receptor_conns = mesh_status['KnownConnectionCosts']
+    update_links = []
+    for link in all_links:
+        if link.link_state != InstanceLink.States.REMOVING:
+            if link.target.hostname in active_receptor_conns.get(link.source.hostname, {}):
+                if link.link_state is not InstanceLink.States.ESTABLISHED:
+                    link.link_state = InstanceLink.States.ESTABLISHED
+                    update_links.append(link)
+
+    InstanceLink.objects.bulk_update(update_links, ['link_state'])
+
+
+def inspect_execution_and_hop_nodes(instance_list):
+    with advisory_lock('inspect_execution_and_hop_nodes_lock', wait=False):
+        node_lookup = {inst.hostname: inst for inst in instance_list}
         ctl = get_receptor_ctl()
         mesh_status = ctl.simple_command('status')
 
+        inspect_established_receptor_connections(mesh_status)
+
         nowtime = now()
         workers = mesh_status['Advertisements']
 
@@ -523,7 +600,7 @@ def cluster_node_heartbeat(dispatch_time=None, worker_tasks=None):
             this_inst = inst
             break
 
-    inspect_execution_nodes(instance_list)
+    inspect_execution_and_hop_nodes(instance_list)
 
     for inst in list(instance_list):
         if inst == this_inst:

awx/main/tests/functional/api/test_instance.py

@@ -84,5 +84,6 @@ def test_custom_hostname_regex(post, admin_user):
                 "hostname": value[0],
                 "node_type": "execution",
                 "node_state": "installed",
+                "peers": [],
             }
             post(url=url, user=admin_user, data=data, expect=value[1])

awx/main/tests/functional/api/test_instance_peers.py

@@ -0,0 +1,342 @@
+import pytest
+import yaml
+import itertools
+from unittest import mock
+
+from django.db.utils import IntegrityError
+
+from awx.api.versioning import reverse
+from awx.main.models import Instance
+from awx.api.views.instance_install_bundle import generate_group_vars_all_yml
+
+
+def has_peer(group_vars, peer):
+    peers = group_vars.get('receptor_peers', [])
+    for p in peers:
+        if f"{p['host']}:{p['port']}" == peer:
+            return True
+    return False
+
+
+@pytest.mark.django_db
+class TestPeers:
+    @pytest.fixture(autouse=True)
+    def configure_settings(self, settings):
+        settings.IS_K8S = True
+
+    @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
+    def test_prevent_peering_to_self(self, node_type):
+        """
+        cannot peer to self
+        """
+        control_instance = Instance.objects.create(hostname='abc', node_type=node_type)
+        with pytest.raises(IntegrityError):
+            control_instance.peers.add(control_instance)
+
+    @pytest.mark.parametrize('node_type', ['control', 'hybrid', 'hop', 'execution'])
+    def test_creating_node(self, node_type, admin_user, post):
+        """
+        can only add hop and execution nodes via API
+        """
+        post(
+            url=reverse('api:instance_list'),
+            data={"hostname": "abc", "node_type": node_type},
+            user=admin_user,
+            expect=400 if node_type in ['control', 'hybrid'] else 201,
+        )
+
+    def test_changing_node_type(self, admin_user, patch):
+        """
+        cannot change node type
+        """
+        hop = Instance.objects.create(hostname='abc', node_type="hop")
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
+            data={"node_type": "execution"},
+            user=admin_user,
+            expect=400,
+        )
+
+    @pytest.mark.parametrize('node_type', ['hop', 'execution'])
+    def test_listener_port_null(self, node_type, admin_user, post):
+        """
+        listener_port can be None
+        """
+        post(
+            url=reverse('api:instance_list'),
+            data={"hostname": "abc", "node_type": node_type, "listener_port": None},
+            user=admin_user,
+            expect=201,
+        )
+
+    @pytest.mark.parametrize('node_type, allowed', [('control', False), ('hybrid', False), ('hop', True), ('execution', True)])
+    def test_peers_from_control_nodes_allowed(self, node_type, allowed, post, admin_user):
+        """
+        only hop and execution nodes can have peers_from_control_nodes set to True
+        """
+        post(
+            url=reverse('api:instance_list'),
+            data={"hostname": "abc", "peers_from_control_nodes": True, "node_type": node_type, "listener_port": 6789},
+            user=admin_user,
+            expect=201 if allowed else 400,
+        )
+
+    def test_listener_port_is_required(self, admin_user, post):
+        """
+        if adding instance to peers list, that instance must have listener_port set
+        """
+        Instance.objects.create(hostname='abc', node_type="hop", listener_port=None)
+        post(
+            url=reverse('api:instance_list'),
+            data={"hostname": "ex", "peers_from_control_nodes": False, "node_type": "execution", "listener_port": None, "peers": ["abc"]},
+            user=admin_user,
+            expect=400,
+        )
+
+    def test_peers_from_control_nodes_listener_port_enabled(self, admin_user, post):
+        """
+        if peers_from_control_nodes is True, listener_port must an integer
+        Assert that all other combinations are allowed
+        """
+        for index, item in enumerate(itertools.product(['hop', 'execution'], [True, False], [None, 6789])):
+            node_type, peers_from, listener_port = item
+            # only disallowed case is when peers_from is True and listener port is None
+            disallowed = peers_from and not listener_port
+            post(
+                url=reverse('api:instance_list'),
+                data={"hostname": f"abc{index}", "peers_from_control_nodes": peers_from, "node_type": node_type, "listener_port": listener_port},
+                user=admin_user,
+                expect=400 if disallowed else 201,
+            )
+
+    @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
+    def test_disallow_modifying_peers_control_nodes(self, node_type, admin_user, patch):
+        """
+        for control nodes, peers field should not be
+        modified directly via patch.
+        """
+        control = Instance.objects.create(hostname='abc', node_type=node_type)
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop', peers_from_control_nodes=True, listener_port=6789)
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop', peers_from_control_nodes=False, listener_port=6789)
+        assert [hop1] == list(control.peers.all())  # only hop1 should be peered
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
+            data={"peers": ["hop2"]},
+            user=admin_user,
+            expect=400,  # cannot add peers directly
+        )
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
+            data={"peers": ["hop1"]},
+            user=admin_user,
+            expect=200,  # patching with current peers list should be okay
+        )
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
+            data={"peers": []},
+            user=admin_user,
+            expect=400,  # cannot remove peers directly
+        )
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': control.pk}),
+            data={},
+            user=admin_user,
+            expect=200,  # patching without data should be fine too
+        )
+        # patch hop2
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop2.pk}),
+            data={"peers_from_control_nodes": True},
+            user=admin_user,
+            expect=200,  # patching without data should be fine too
+        )
+        assert {hop1, hop2} == set(control.peers.all())  # hop1 and hop2 should now be peered from control node
+
+    def test_disallow_changing_hostname(self, admin_user, patch):
+        """
+        cannot change hostname
+        """
+        hop = Instance.objects.create(hostname='hop', node_type='hop')
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
+            data={"hostname": "hop2"},
+            user=admin_user,
+            expect=400,
+        )
+
+    def test_disallow_changing_node_state(self, admin_user, patch):
+        """
+        only allow setting to deprovisioning
+        """
+        hop = Instance.objects.create(hostname='hop', node_type='hop', node_state='installed')
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
+            data={"node_state": "deprovisioning"},
+            user=admin_user,
+            expect=200,
+        )
+        patch(
+            url=reverse('api:instance_detail', kwargs={'pk': hop.pk}),
+            data={"node_state": "ready"},
+            user=admin_user,
+            expect=400,
+        )
+
+    @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
+    def test_control_node_automatically_peers(self, node_type):
+        """
+        a new control node should automatically
+        peer to hop
+
+        peer to hop should be removed if hop is deleted
+        """
+
+        hop = Instance.objects.create(hostname='hop', node_type='hop', peers_from_control_nodes=True, listener_port=6789)
+        control = Instance.objects.create(hostname='abc', node_type=node_type)
+        assert hop in control.peers.all()
+        hop.delete()
+        assert not control.peers.exists()
+
+    @pytest.mark.parametrize('node_type', ['control', 'hybrid'])
+    def test_control_node_retains_other_peers(self, node_type):
+        """
+        if a new node comes online, other peer relationships should
+        remain intact
+        """
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=True)
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
+        hop1.peers.add(hop2)
+
+        # a control node is added
+        Instance.objects.create(hostname='control', node_type=node_type, listener_port=None)
+
+        assert hop1.peers.exists()
+
+    def test_group_vars(self, get, admin_user):
+        """
+        control > hop1 > hop2 < execution
+        """
+        control = Instance.objects.create(hostname='control', node_type='control', listener_port=None)
+        hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=True)
+        hop2 = Instance.objects.create(hostname='hop2', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
+        execution = Instance.objects.create(hostname='execution', node_type='execution', listener_port=6789)
+
+        execution.peers.add(hop2)
+        hop1.peers.add(hop2)
+
+        control_vars = yaml.safe_load(generate_group_vars_all_yml(control))
+        hop1_vars = yaml.safe_load(generate_group_vars_all_yml(hop1))
+        hop2_vars = yaml.safe_load(generate_group_vars_all_yml(hop2))
+        execution_vars = yaml.safe_load(generate_group_vars_all_yml(execution))
+
+        # control group vars assertions
+        assert has_peer(control_vars, 'hop1:6789')
+        assert not has_peer(control_vars, 'hop2:6789')
+        assert not has_peer(control_vars, 'execution:6789')
+        assert not control_vars.get('receptor_listener', False)
+
+        # hop1 group vars assertions
+        assert has_peer(hop1_vars, 'hop2:6789')
+        assert not has_peer(hop1_vars, 'execution:6789')
+        assert hop1_vars.get('receptor_listener', False)
+
+        # hop2 group vars assertions
+        assert not has_peer(hop2_vars, 'hop1:6789')
+        assert not has_peer(hop2_vars, 'execution:6789')
+        assert hop2_vars.get('receptor_listener', False)
+        assert hop2_vars.get('receptor_peers', []) == []
+
+        # execution group vars assertions
+        assert has_peer(execution_vars, 'hop2:6789')
+        assert not has_peer(execution_vars, 'hop1:6789')
+        assert execution_vars.get('receptor_listener', False)
+
+    def test_write_receptor_config_called(self):
+        """
+        Assert that write_receptor_config is called
+        when certain instances are created, or if
+        peers_from_control_nodes changes.
+        In general, write_receptor_config should only
+        be called when necessary, as it will reload
+        receptor backend connections which is not trivial.
+        """
+        with mock.patch('awx.main.models.ha.schedule_write_receptor_config') as write_method:
+            # new control instance but nothing to peer to (no)
+            control = Instance.objects.create(hostname='control1', node_type='control')
+            write_method.assert_not_called()
+
+            # new hop node with peers_from_control_nodes False (no)
+            hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
+            hop1.delete()
+            write_method.assert_not_called()
+
+            # new hop node with peers_from_control_nodes True (yes)
+            hop1 = Instance.objects.create(hostname='hop1', node_type='hop', listener_port=6789, peers_from_control_nodes=True)
+            write_method.assert_called()
+            write_method.reset_mock()
+
+            # new control instance but with something to peer to (yes)
+            Instance.objects.create(hostname='control2', node_type='control')
+            write_method.assert_called()
+            write_method.reset_mock()
+
+            # new hop node with peers_from_control_nodes False and peered to another hop node (no)
+            hop2 = Instance.objects.create(hostname='hop2', node_type='hop', listener_port=6789, peers_from_control_nodes=False)
+            hop2.peers.add(hop1)
+            hop2.delete()
+            write_method.assert_not_called()
+
+            # changing peers_from_control_nodes to False (yes)
+            hop1.peers_from_control_nodes = False
+            hop1.save()
+            write_method.assert_called()
+            write_method.reset_mock()
+
+            # deleting hop node that has peers_from_control_nodes to False (no)
+            hop1.delete()
+            write_method.assert_not_called()
+
+            # deleting control nodes (no)
+            control.delete()
+            write_method.assert_not_called()
+
+    def test_write_receptor_config_data(self):
+        """
+        Assert the correct peers are included in data that will
+        be written to receptor.conf
+        """
+        from awx.main.tasks.receptor import RECEPTOR_CONFIG_STARTER
+
+        with mock.patch('awx.main.tasks.receptor.read_receptor_config', return_value=list(RECEPTOR_CONFIG_STARTER)):
+            from awx.main.tasks.receptor import generate_config_data
+
+            _, should_update = generate_config_data()
+            assert not should_update
+
+            # not peered, so config file should not be updated
+            for i in range(3):
+                Instance.objects.create(hostname=f"exNo-{i}", node_type='execution', listener_port=6789, peers_from_control_nodes=False)
+
+            _, should_update = generate_config_data()
+            assert not should_update
+
+            # peered, so config file should be updated
+            expected_peers = []
+            for i in range(3):
+                expected_peers.append(f"hop-{i}:6789")
+                Instance.objects.create(hostname=f"hop-{i}", node_type='hop', listener_port=6789, peers_from_control_nodes=True)
+
+            for i in range(3):
+                expected_peers.append(f"exYes-{i}:6789")
+                Instance.objects.create(hostname=f"exYes-{i}", node_type='execution', listener_port=6789, peers_from_control_nodes=True)
+
+            new_config, should_update = generate_config_data()
+            assert should_update
+
+            peers = []
+            for entry in new_config:
+                for key, value in entry.items():
+                    if key == "tcp-peer":
+                        peers.append(value['address'])
+
+            assert set(expected_peers) == set(peers)

awx/main/tests/functional/test_dispatch.py

@@ -3,6 +3,7 @@ import multiprocessing
 import random
 import signal
 import time
+import yaml
 from unittest import mock
 
 from django.utils.timezone import now as tz_now
@@ -13,6 +14,7 @@ from awx.main.dispatch import reaper
 from awx.main.dispatch.pool import StatefulPoolWorker, WorkerPool, AutoscalePool
 from awx.main.dispatch.publish import task
 from awx.main.dispatch.worker import BaseWorker, TaskWorker
+from awx.main.dispatch.periodic import Scheduler
 
 
 '''
@@ -439,3 +441,76 @@ class TestJobReaper(object):
         assert job.started > ref_time
         assert job.status == 'running'
         assert job.job_explanation == ''
+
+
+@pytest.mark.django_db
+class TestScheduler:
+    def test_too_many_schedules_freak_out(self):
+        with pytest.raises(RuntimeError):
+            Scheduler({'job1': {'schedule': datetime.timedelta(seconds=1)}, 'job2': {'schedule': datetime.timedelta(seconds=1)}})
+
+    def test_spread_out(self):
+        scheduler = Scheduler(
+            {
+                'job1': {'schedule': datetime.timedelta(seconds=16)},
+                'job2': {'schedule': datetime.timedelta(seconds=16)},
+                'job3': {'schedule': datetime.timedelta(seconds=16)},
+                'job4': {'schedule': datetime.timedelta(seconds=16)},
+            }
+        )
+        assert [job.offset for job in scheduler.jobs] == [0, 4, 8, 12]
+
+    def test_missed_schedule(self, mocker):
+        scheduler = Scheduler({'job1': {'schedule': datetime.timedelta(seconds=10)}})
+        assert scheduler.jobs[0].missed_runs(time.time() - scheduler.global_start) == 0
+        mocker.patch('awx.main.dispatch.periodic.time.time', return_value=scheduler.global_start + 50)
+        scheduler.get_and_mark_pending()
+        assert scheduler.jobs[0].missed_runs(50) > 1
+
+    def test_advance_schedule(self, mocker):
+        scheduler = Scheduler(
+            {
+                'job1': {'schedule': datetime.timedelta(seconds=30)},
+                'joba': {'schedule': datetime.timedelta(seconds=20)},
+                'jobb': {'schedule': datetime.timedelta(seconds=20)},
+            }
+        )
+        for job in scheduler.jobs:
+            # HACK: the offsets automatically added make this a hard test to write... so remove offsets
+            job.offset = 0.0
+        mocker.patch('awx.main.dispatch.periodic.time.time', return_value=scheduler.global_start + 29)
+        to_run = scheduler.get_and_mark_pending()
+        assert set(job.name for job in to_run) == set(['joba', 'jobb'])
+        mocker.patch('awx.main.dispatch.periodic.time.time', return_value=scheduler.global_start + 39)
+        to_run = scheduler.get_and_mark_pending()
+        assert len(to_run) == 1
+        assert to_run[0].name == 'job1'
+
+    @staticmethod
+    def get_job(scheduler, name):
+        for job in scheduler.jobs:
+            if job.name == name:
+                return job
+
+    def test_scheduler_debug(self, mocker):
+        scheduler = Scheduler(
+            {
+                'joba': {'schedule': datetime.timedelta(seconds=20)},
+                'jobb': {'schedule': datetime.timedelta(seconds=50)},
+                'jobc': {'schedule': datetime.timedelta(seconds=500)},
+                'jobd': {'schedule': datetime.timedelta(seconds=20)},
+            }
+        )
+        rel_time = 119.9  # slightly under the 6th 20-second bin, to avoid offset problems
+        current_time = scheduler.global_start + rel_time
+        mocker.patch('awx.main.dispatch.periodic.time.time', return_value=current_time - 1.0e-8)
+        self.get_job(scheduler, 'jobb').mark_run(rel_time)
+        self.get_job(scheduler, 'jobd').mark_run(rel_time - 20.0)
+
+        output = scheduler.debug()
+        data = yaml.safe_load(output)
+        assert data['schedule_list']['jobc']['last_run_seconds_ago'] is None
+        assert data['schedule_list']['joba']['missed_runs'] == 4
+        assert data['schedule_list']['jobd']['missed_runs'] == 3
+        assert data['schedule_list']['jobd']['completed_runs'] == 1
+        assert data['schedule_list']['jobb']['next_run_in_seconds'] > 25.0

awx/main/tests/functional/test_jobs.py

@@ -6,6 +6,7 @@ import json
 from awx.main.models import (
     Job,
     Instance,
+    Host,
     JobHostSummary,
     InventoryUpdate,
     InventorySource,
@@ -18,6 +19,9 @@ from awx.main.models import (
     ExecutionEnvironment,
 )
 from awx.main.tasks.system import cluster_node_heartbeat
+from awx.main.tasks.facts import update_hosts
+
+from django.db import OperationalError
 from django.test.utils import override_settings
 
 
@@ -33,7 +37,7 @@ def test_orphan_unified_job_creation(instance, inventory):
 
 
 @pytest.mark.django_db
-@mock.patch('awx.main.tasks.system.inspect_execution_nodes', lambda *args, **kwargs: None)
+@mock.patch('awx.main.tasks.system.inspect_execution_and_hop_nodes', lambda *args, **kwargs: None)
 @mock.patch('awx.main.models.ha.get_cpu_effective_capacity', lambda cpu: 8)
 @mock.patch('awx.main.models.ha.get_mem_effective_capacity', lambda mem: 62)
 def test_job_capacity_and_with_inactive_node():
@@ -112,6 +116,51 @@ def test_job_notification_host_data(inventory, machine_credential, project, job_
     }
 
 
+@pytest.mark.django_db
+class TestAnsibleFactsSave:
+    current_call = 0
+
+    def test_update_hosts_deleted_host(self, inventory):
+        hosts = [Host.objects.create(inventory=inventory, name=f'foo{i}') for i in range(3)]
+        for host in hosts:
+            host.ansible_facts = {'foo': 'bar'}
+        last_pk = hosts[-1].pk
+        assert inventory.hosts.count() == 3
+        Host.objects.get(pk=last_pk).delete()
+        assert inventory.hosts.count() == 2
+        update_hosts(hosts)
+        assert inventory.hosts.count() == 2
+        for host in inventory.hosts.all():
+            host.refresh_from_db()
+            assert host.ansible_facts == {'foo': 'bar'}
+
+    def test_update_hosts_forever_deadlock(self, inventory, mocker):
+        hosts = [Host.objects.create(inventory=inventory, name=f'foo{i}') for i in range(3)]
+        for host in hosts:
+            host.ansible_facts = {'foo': 'bar'}
+        db_mock = mocker.patch('awx.main.tasks.facts.Host.objects.bulk_update')
+        db_mock.side_effect = OperationalError('deadlock detected')
+        with pytest.raises(OperationalError):
+            update_hosts(hosts)
+
+    def fake_bulk_update(self, host_list):
+        if self.current_call > 2:
+            return Host.objects.bulk_update(host_list, ['ansible_facts', 'ansible_facts_modified'])
+        self.current_call += 1
+        raise OperationalError('deadlock detected')
+
+    def test_update_hosts_resolved_deadlock(self, inventory, mocker):
+        hosts = [Host.objects.create(inventory=inventory, name=f'foo{i}') for i in range(3)]
+        for host in hosts:
+            host.ansible_facts = {'foo': 'bar'}
+        self.current_call = 0
+        mocker.patch('awx.main.tasks.facts.raw_update_hosts', new=self.fake_bulk_update)
+        update_hosts(hosts)
+        for host in inventory.hosts.all():
+            host.refresh_from_db()
+            assert host.ansible_facts == {'foo': 'bar'}
+
+
 @pytest.mark.django_db
 class TestLaunchConfig:
     def test_null_creation_from_prompts(self):

awx/main/utils/formatters.py

@@ -283,6 +283,7 @@ class LogstashFormatter(LogstashFormatterBase):
             message.update(self.get_debug_fields(record))
 
         if settings.LOG_AGGREGATOR_TYPE == 'splunk':
-            # splunk messages must have a top level "event" key
-            message = {'event': message}
+            # splunk messages must have a top level "event" key when using the /services/collector/event receiver.
+            # The event receiver wont scan an event for a timestamp field therefore a time field must also be supplied containing epoch timestamp
+            message = {'time': record.created, 'event': message}
         return self.serialize(message)

awx/main/utils/handlers.py

@@ -97,8 +97,6 @@ class SpecialInventoryHandler(logging.Handler):
         self.event_handler(dispatch_data)
 
 
-ColorHandler = logging.StreamHandler
-
 if settings.COLOR_LOGS is True:
     try:
         from logutils.colorize import ColorizingStreamHandler
@@ -133,3 +131,5 @@ if settings.COLOR_LOGS is True:
     except ImportError:
         # logutils is only used for colored logs in the dev environment
         pass
+else:
+    ColorHandler = logging.StreamHandler

awx/main/utils/licensing.py

@@ -175,7 +175,12 @@ class Licenser(object):
             license.setdefault('pool_id', sub['pool']['id'])
             license.setdefault('product_name', sub['pool']['productName'])
             license.setdefault('valid_key', True)
-            license.setdefault('license_type', 'enterprise')
+            if sub['pool']['productId'].startswith('S'):
+                license.setdefault('trial', True)
+                license.setdefault('license_type', 'trial')
+            else:
+                license.setdefault('trial', False)
+                license.setdefault('license_type', 'enterprise')
             license.setdefault('satellite', False)
             # Use the nearest end date
             endDate = parse_date(sub['endDate'])
@@ -287,7 +292,7 @@ class Licenser(object):
                     license['productId'] = sub['product_id']
                     license['quantity'] = int(sub['quantity'])
                     license['support_level'] = sub['support_level']
-                    license['usage'] = sub['usage']
+                    license['usage'] = sub.get('usage')
                     license['subscription_name'] = sub['name']
                     license['subscriptionId'] = sub['subscription_id']
                     license['accountNumber'] = sub['account_number']

awx/settings/defaults.py

@@ -210,7 +210,7 @@ JOB_EVENT_WORKERS = 4
 
 # The number of seconds to buffer callback receiver bulk
 # writes in memory before flushing via JobEvent.objects.bulk_create()
-JOB_EVENT_BUFFER_SECONDS = 0.1
+JOB_EVENT_BUFFER_SECONDS = 1
 
 # The interval at which callback receiver statistics should be
 # recorded
@@ -453,7 +453,7 @@ RECEPTOR_SERVICE_ADVERTISEMENT_PERIOD = 60  # https://github.com/ansible/recepto
 EXECUTION_NODE_REMEDIATION_CHECKS = 60 * 30  # once every 30 minutes check if an execution node errors have been resolved
 
 # Amount of time dispatcher will try to reconnect to database for jobs and consuming new work
-DISPATCHER_DB_DOWNTOWN_TOLLERANCE = 40
+DISPATCHER_DB_DOWNTIME_TOLERANCE = 40
 
 BROKER_URL = 'unix:///var/run/redis/redis.sock'
 CELERYBEAT_SCHEDULE = {
@@ -476,7 +476,7 @@ CELERYBEAT_SCHEDULE = {
 
 # Django Caching Configuration
 DJANGO_REDIS_IGNORE_EXCEPTIONS = True
-CACHES = {'default': {'BACKEND': 'awx.main.cache.AWXRedisCache', 'LOCATION': 'unix:/var/run/redis/redis.sock?db=1'}}
+CACHES = {'default': {'BACKEND': 'awx.main.cache.AWXRedisCache', 'LOCATION': 'unix:///var/run/redis/redis.sock?db=1'}}
 
 # Social Auth configuration.
 SOCIAL_AUTH_STRATEGY = 'social_django.strategy.DjangoStrategy'

awx/settings/development.py

@@ -28,8 +28,8 @@ SHELL_PLUS_PRINT_SQL = False
 
 # show colored logs in the dev environment
 # to disable this, set `COLOR_LOGS = False` in awx/settings/local_settings.py
-LOGGING['handlers']['console']['()'] = 'awx.main.utils.handlers.ColorHandler'  # noqa
 COLOR_LOGS = True
+LOGGING['handlers']['console']['()'] = 'awx.main.utils.handlers.ColorHandler'  # noqa
 
 ALLOWED_HOSTS = ['*']
 

awx/ui/public/installing.html

@@ -5,7 +5,11 @@
       <title data-cy="migration-title">{{ title }}</title>
       <meta
          http-equiv="Content-Security-Policy"
-         content="default-src 'self'; connect-src 'self' ws: wss:; style-src 'self' 'unsafe-inline'; script-src 'self' 'nonce-{{ csp_nonce }}' *.pendo.io; img-src 'self' *.pendo.io data:;"
+         content="default-src 'self';
+                  connect-src 'self' ws: wss:;
+                  style-src 'self' 'unsafe-inline';
+                  script-src 'self' 'nonce-{{ csp_nonce }}' *.pendo.io;
+                  img-src 'self' *.pendo.io data:;"
          />
       <meta charset="utf-8">
       <meta http-equiv="X-UA-Compatible" content="IE=edge" />

awx/ui/src/api/models/ConstructedInventories.js

@@ -6,5 +6,20 @@ class ConstructedInventories extends InstanceGroupsMixin(Base) {
     super(http);
     this.baseUrl = 'api/v2/constructed_inventories/';
   }
+
+  async readConstructedInventoryOptions(id, method) {
+    const {
+      data: { actions },
+    } = await this.http.options(`${this.baseUrl}${id}/`);
+
+    if (actions[method]) {
+      return actions[method];
+    }
+
+    throw new Error(
+      `You have insufficient access to this Constructed Inventory. 
+      Please contact your system administrator if there is an issue with your access.`
+    );
+  }
 }
 export default ConstructedInventories;

awx/ui/src/api/models/ConstructedInventories.test.js

@@ -0,0 +1,51 @@
+import ConstructedInventories from './ConstructedInventories';
+
+describe('ConstructedInventoriesAPI', () => {
+  const constructedInventoryId = 1;
+  const constructedInventoryMethod = 'PUT';
+  let ConstructedInventoriesAPI;
+  let mockHttp;
+
+  beforeEach(() => {
+    const optionsPromise = () =>
+      Promise.resolve({
+        data: {
+          actions: {
+            PUT: {},
+          },
+        },
+      });
+    mockHttp = {
+      options: jest.fn(optionsPromise),
+    };
+    ConstructedInventoriesAPI = new ConstructedInventories(mockHttp);
+  });
+
+  afterEach(() => {
+    jest.resetAllMocks();
+  });
+
+  test('readConstructedInventoryOptions calls options with the expected params', async () => {
+    await ConstructedInventoriesAPI.readConstructedInventoryOptions(
+      constructedInventoryId,
+      constructedInventoryMethod
+    );
+    expect(mockHttp.options).toHaveBeenCalledTimes(1);
+    expect(mockHttp.options).toHaveBeenCalledWith(
+      `api/v2/constructed_inventories/${constructedInventoryId}/`
+    );
+  });
+
+  test('readConstructedInventory should throw an error if action method is missing', async () => {
+    try {
+      await ConstructedInventoriesAPI.readConstructedInventoryOptions(
+        constructedInventoryId,
+        'POST'
+      );
+    } catch (error) {
+      expect(error.message).toContain(
+        'You have insufficient access to this Constructed Inventory.'
+      );
+    }
+  });
+});

awx/ui/src/components/LaunchPrompt/LaunchPrompt.js

@@ -77,7 +77,7 @@ function PromptModalForm({
     }
 
     if (launchConfig.ask_labels_on_launch) {
-      const { labelIds } = createNewLabels(
+      const { labelIds } = await createNewLabels(
         values.labels,
         resource.organization
       );

awx/ui/src/components/Lookup/Lookup.js

@@ -223,6 +223,10 @@ function Lookup(props) {
 const Item = shape({
   id: number.isRequired,
 });
+const InstanceItem = shape({
+  id: number.isRequired,
+  hostname: string.isRequired,
+});
 
 Lookup.propTypes = {
   id: string,
@@ -230,7 +234,13 @@ Lookup.propTypes = {
   modalDescription: oneOfType([string, node]),
   onChange: func.isRequired,
   onUpdate: func,
-  value: oneOfType([Item, arrayOf(Item), object]),
+  value: oneOfType([
+    Item,
+    arrayOf(Item),
+    object,
+    InstanceItem,
+    arrayOf(InstanceItem),
+  ]),
   multiple: bool,
   required: bool,
   onBlur: func,

awx/ui/src/components/Lookup/PeersLookup.js

@@ -0,0 +1,212 @@
+import React, { useCallback, useEffect } from 'react';
+import { arrayOf, string, func, bool, shape } from 'prop-types';
+import { withRouter } from 'react-router-dom';
+import { t } from '@lingui/macro';
+import { FormGroup, Chip } from '@patternfly/react-core';
+import { InstancesAPI } from 'api';
+import { Instance } from 'types';
+import { getSearchableKeys } from 'components/PaginatedTable';
+import { getQSConfig, parseQueryString, mergeParams } from 'util/qs';
+import useRequest from 'hooks/useRequest';
+import Popover from '../Popover';
+import OptionsList from '../OptionsList';
+import Lookup from './Lookup';
+import LookupErrorMessage from './shared/LookupErrorMessage';
+import FieldWithPrompt from '../FieldWithPrompt';
+
+const QS_CONFIG = getQSConfig('instances', {
+  page: 1,
+  page_size: 5,
+  order_by: 'hostname',
+});
+
+function PeersLookup({
+  id,
+  value,
+  onChange,
+  tooltip,
+  className,
+  required,
+  history,
+  fieldName,
+  multiple,
+  validate,
+  columns,
+  isPromptableField,
+  promptId,
+  promptName,
+  formLabel,
+  typePeers,
+  instance_details,
+}) {
+  const {
+    result: { instances, count, relatedSearchableKeys, searchableKeys },
+    request: fetchInstances,
+    error,
+    isLoading,
+  } = useRequest(
+    useCallback(async () => {
+      const params = parseQueryString(QS_CONFIG, history.location.search);
+      const peersFilter = {};
+      if (typePeers) {
+        peersFilter.not__node_type = ['control', 'hybrid'];
+        if (instance_details) {
+          if (instance_details.id) {
+            peersFilter.not__id = instance_details.id;
+            peersFilter.not__hostname = instance_details.peers;
+          }
+        }
+      }
+
+      const [{ data }, actionsResponse] = await Promise.all([
+        InstancesAPI.read(
+          mergeParams(params, {
+            ...peersFilter,
+          })
+        ),
+        InstancesAPI.readOptions(),
+      ]);
+      return {
+        instances: data.results,
+        count: data.count,
+        relatedSearchableKeys: (
+          actionsResponse?.data?.related_search_fields || []
+        ).map((val) => val.slice(0, -8)),
+        searchableKeys: getSearchableKeys(actionsResponse.data.actions?.GET),
+      };
+    }, [history.location, typePeers, instance_details]),
+    {
+      instances: [],
+      count: 0,
+      relatedSearchableKeys: [],
+      searchableKeys: [],
+    }
+  );
+
+  useEffect(() => {
+    fetchInstances();
+  }, [fetchInstances]);
+
+  const renderLookup = () => (
+    <>
+      <Lookup
+        id={fieldName}
+        header={formLabel}
+        value={value}
+        onChange={onChange}
+        onUpdate={fetchInstances}
+        fieldName={fieldName}
+        validate={validate}
+        qsConfig={QS_CONFIG}
+        multiple={multiple}
+        required={required}
+        isLoading={isLoading}
+        label={formLabel}
+        renderItemChip={({ item, removeItem, canDelete }) => (
+          <Chip
+            key={item.id}
+            onClick={() => removeItem(item)}
+            isReadOnly={!canDelete}
+          >
+            {item.hostname}
+          </Chip>
+        )}
+        renderOptionsList={({ state, dispatch, canDelete }) => (
+          <OptionsList
+            value={state.selectedItems}
+            options={instances}
+            optionCount={count}
+            columns={columns}
+            header={formLabel}
+            displayKey="hostname"
+            searchColumns={[
+              {
+                name: t`Hostname`,
+                key: 'hostname__icontains',
+                isDefault: true,
+              },
+            ]}
+            sortColumns={[
+              {
+                name: t`Hostname`,
+                key: 'hostname',
+              },
+            ]}
+            searchableKeys={searchableKeys}
+            relatedSearchableKeys={relatedSearchableKeys}
+            multiple={multiple}
+            label={formLabel}
+            name={fieldName}
+            qsConfig={QS_CONFIG}
+            readOnly={!canDelete}
+            selectItem={(item) => dispatch({ type: 'SELECT_ITEM', item })}
+            deselectItem={(item) => dispatch({ type: 'DESELECT_ITEM', item })}
+          />
+        )}
+      />
+      <LookupErrorMessage error={error} />
+    </>
+  );
+
+  return isPromptableField ? (
+    <FieldWithPrompt
+      fieldId={id}
+      label={formLabel}
+      promptId={promptId}
+      promptName={promptName}
+      tooltip={tooltip}
+    >
+      {renderLookup()}
+    </FieldWithPrompt>
+  ) : (
+    <FormGroup
+      className={className}
+      label={formLabel}
+      labelIcon={tooltip && <Popover content={tooltip} />}
+      fieldId={id}
+    >
+      {renderLookup()}
+    </FormGroup>
+  );
+}
+
+PeersLookup.propTypes = {
+  id: string,
+  value: arrayOf(Instance).isRequired,
+  tooltip: string,
+  onChange: func.isRequired,
+  className: string,
+  required: bool,
+  validate: func,
+  multiple: bool,
+  fieldName: string,
+  columns: arrayOf(Object),
+  formLabel: string,
+  instance_details: (Instance, shape({})),
+  typePeers: bool,
+};
+
+PeersLookup.defaultProps = {
+  id: 'instances',
+  tooltip: '',
+  className: '',
+  required: false,
+  validate: () => undefined,
+  fieldName: 'instances',
+  columns: [
+    {
+      key: 'hostname',
+      name: t`Hostname`,
+    },
+    {
+      key: 'node_type',
+      name: t`Node Type`,
+    },
+  ],
+  formLabel: t`Instances`,
+  instance_details: {},
+  multiple: true,
+  typePeers: false,
+};
+
+export default withRouter(PeersLookup);

awx/ui/src/components/Lookup/PeersLookup.test.js

@@ -0,0 +1,137 @@
+import React from 'react';
+import { act } from 'react-dom/test-utils';
+import { Formik } from 'formik';
+import { InstancesAPI } from 'api';
+import { mountWithContexts } from '../../../testUtils/enzymeHelpers';
+import PeersLookup from './PeersLookup';
+
+jest.mock('../../api');
+
+const mockedInstances = {
+  count: 1,
+  results: [
+    {
+      id: 2,
+      name: 'Foo',
+      image: 'quay.io/ansible/awx-ee',
+      pull: 'missing',
+    },
+  ],
+};
+
+const instances = [
+  {
+    id: 1,
+    hostname: 'awx_1',
+    type: 'instance',
+    url: '/api/v2/instances/1/',
+    related: {
+      named_url: '/api/v2/instances/awx_1/',
+      jobs: '/api/v2/instances/1/jobs/',
+      instance_groups: '/api/v2/instances/1/instance_groups/',
+      peers: '/api/v2/instances/1/peers/',
+    },
+    summary_fields: {
+      user_capabilities: {
+        edit: false,
+      },
+      links: [],
+    },
+    uuid: '00000000-0000-0000-0000-000000000000',
+    created: '2023-04-26T22:06:46.766198Z',
+    modified: '2023-04-26T22:06:46.766217Z',
+    last_seen: '2023-04-26T23:12:02.857732Z',
+    health_check_started: null,
+    health_check_pending: false,
+    last_health_check: '2023-04-26T23:01:13.941693Z',
+    errors: 'Instance received normal shutdown signal',
+    capacity_adjustment: '1.00',
+    version: '0.1.dev33237+g1fdef52',
+    capacity: 0,
+    consumed_capacity: 0,
+    percent_capacity_remaining: 0,
+    jobs_running: 0,
+    jobs_total: 0,
+    cpu: '8.0',
+    memory: 8011055104,
+    cpu_capacity: 0,
+    mem_capacity: 0,
+    enabled: true,
+    managed_by_policy: true,
+    node_type: 'hybrid',
+    node_state: 'installed',
+    ip_address: null,
+    listener_port: 27199,
+    peers: [],
+    peers_from_control_nodes: false,
+  },
+];
+
+describe('PeersLookup', () => {
+  let wrapper;
+
+  beforeEach(() => {
+    InstancesAPI.read.mockResolvedValue({
+      data: mockedInstances,
+    });
+  });
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  test('should render successfully without instance_details (for new added instance)', async () => {
+    InstancesAPI.readOptions.mockReturnValue({
+      data: {
+        actions: {
+          GET: {},
+          POST: {},
+        },
+        related_search_fields: [],
+      },
+    });
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <Formik>
+          <PeersLookup value={instances} onChange={() => {}} />
+        </Formik>
+      );
+    });
+    wrapper.update();
+    expect(InstancesAPI.read).toHaveBeenCalledTimes(1);
+    expect(wrapper.find('PeersLookup')).toHaveLength(1);
+    expect(wrapper.find('FormGroup[label="Instances"]').length).toBe(1);
+    expect(wrapper.find('Checkbox[aria-label="Prompt on launch"]').length).toBe(
+      0
+    );
+  });
+  test('should render successfully with instance_details for edit instance', async () => {
+    InstancesAPI.readOptions.mockReturnValue({
+      data: {
+        actions: {
+          GET: {},
+          POST: {},
+        },
+        related_search_fields: [],
+      },
+    });
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <Formik>
+          <PeersLookup
+            value={instances}
+            instance_details={instances[0]}
+            onChange={() => {}}
+          />
+        </Formik>
+      );
+    });
+    wrapper.update();
+    expect(InstancesAPI.read).toHaveBeenCalledTimes(1);
+    expect(wrapper.find('PeersLookup')).toHaveLength(1);
+    expect(wrapper.find('FormGroup[label="Instances"]').length).toBe(1);
+    expect(wrapper.find('Checkbox[aria-label="Prompt on launch"]').length).toBe(
+      0
+    );
+  });
+});

awx/ui/src/components/Lookup/index.js

@@ -8,3 +8,4 @@ export { default as ApplicationLookup } from './ApplicationLookup';
 export { default as HostFilterLookup } from './HostFilterLookup';
 export { default as OrganizationLookup } from './OrganizationLookup';
 export { default as ExecutionEnvironmentLookup } from './ExecutionEnvironmentLookup';
+export { default as PeersLookup } from './PeersLookup';

awx/ui/src/components/OptionsList/OptionsList.js

@@ -125,19 +125,24 @@ const Item = shape({
   name: string.isRequired,
   url: string,
 });
+const InstanceItem = shape({
+  id: oneOfType([number, string]).isRequired,
+  hostname: string.isRequired,
+  url: string,
+});
 OptionsList.propTypes = {
   deselectItem: func.isRequired,
   displayKey: string,
   isSelectedDraggable: bool,
   multiple: bool,
   optionCount: number.isRequired,
-  options: arrayOf(Item).isRequired,
+  options: oneOfType([arrayOf(Item), arrayOf(InstanceItem)]).isRequired,
   qsConfig: QSConfig.isRequired,
   renderItemChip: func,
   searchColumns: SearchColumns,
   selectItem: func.isRequired,
   sortColumns: SortColumns,
-  value: arrayOf(Item).isRequired,
+  value: oneOfType([arrayOf(Item), arrayOf(InstanceItem)]).isRequired,
 };
 OptionsList.defaultProps = {
   isSelectedDraggable: false,

awx/ui/src/components/ResourceAccessList/ResourceAccessList.js

@@ -1,10 +1,10 @@
 import React, { useCallback, useEffect, useState } from 'react';
 import { useLocation } from 'react-router-dom';
 import { t } from '@lingui/macro';
-import { RolesAPI, TeamsAPI, UsersAPI, OrganizationsAPI } from 'api';
+import { RolesAPI, TeamsAPI, UsersAPI } from 'api';
 import { getQSConfig, parseQueryString } from 'util/qs';
 import useRequest, { useDeleteItems } from 'hooks/useRequest';
-import { useUserProfile, useConfig } from 'contexts/Config';
+import { useUserProfile } from 'contexts/Config';
 import AddResourceRole from '../AddRole/AddResourceRole';
 import AlertModal from '../AlertModal';
 import DataListToolbar from '../DataListToolbar';
@@ -25,8 +25,7 @@ const QS_CONFIG = getQSConfig('access', {
 });
 
 function ResourceAccessList({ apiModel, resource }) {
-  const { isSuperUser, isOrgAdmin } = useUserProfile();
-  const { me } = useConfig();
+  const { isSuperUser } = useUserProfile();
   const [submitError, setSubmitError] = useState(null);
   const [deletionRecord, setDeletionRecord] = useState(null);
   const [deletionRole, setDeletionRole] = useState(null);
@@ -34,42 +33,15 @@ function ResourceAccessList({ apiModel, resource }) {
   const [showDeleteModal, setShowDeleteModal] = useState(false);
   const location = useLocation();
 
-  const {
-    isLoading: isFetchingOrgAdmins,
-    error: errorFetchingOrgAdmins,
-    request: fetchOrgAdmins,
-    result: { isCredentialOrgAdmin },
-  } = useRequest(
-    useCallback(async () => {
-      if (
-        isSuperUser ||
-        resource.type !== 'credential' ||
-        !isOrgAdmin ||
-        !resource?.organization
-      ) {
-        return false;
-      }
-      const {
-        data: { count },
-      } = await OrganizationsAPI.readAdmins(resource.organization, {
-        id: me.id,
-      });
-      return { isCredentialOrgAdmin: !!count };
-    }, [me.id, isOrgAdmin, isSuperUser, resource.type, resource.organization]),
-    {
-      isCredentialOrgAdmin: false,
-    }
-  );
-
-  useEffect(() => {
-    fetchOrgAdmins();
-  }, [fetchOrgAdmins]);
-
   let canAddAdditionalControls = false;
   if (isSuperUser) {
     canAddAdditionalControls = true;
   }
-  if (resource.type === 'credential' && isOrgAdmin && isCredentialOrgAdmin) {
+  if (
+    resource.type === 'credential' &&
+    resource?.summary_fields?.user_capabilities?.edit &&
+    resource?.organization
+  ) {
     canAddAdditionalControls = true;
   }
   if (resource.type !== 'credential') {
@@ -195,8 +167,8 @@ function ResourceAccessList({ apiModel, resource }) {
   return (
     <>
       <PaginatedTable
-        error={contentError || errorFetchingOrgAdmins}
-        hasContentLoading={isLoading || isDeleteLoading || isFetchingOrgAdmins}
+        error={contentError}
+        hasContentLoading={isLoading || isDeleteLoading}
         items={accessRecords}
         itemCount={itemCount}
         pluralizedItemName={t`Roles`}

awx/ui/src/components/ResourceAccessList/ResourceAccessList.test.js

@@ -463,7 +463,7 @@ describe('<ResourceAccessList />', () => {
     expect(wrapper.find('ToolbarAddButton').length).toEqual(1);
   });
 
-  test('should not show add button for non system admin & non org admin', async () => {
+  test('should not show add button for a user without edit permissions on the credential', async () => {
     useUserProfile.mockImplementation(() => {
       return {
         isSuperUser: false,
@@ -476,7 +476,21 @@ describe('<ResourceAccessList />', () => {
     let wrapper;
     await act(async () => {
       wrapper = mountWithContexts(
-        <ResourceAccessList resource={credential} apiModel={CredentialsAPI} />,
+        <ResourceAccessList
+          resource={{
+            ...credential,
+            summary_fields: {
+              ...credential.summary_fields,
+              user_capabilities: {
+                edit: false,
+                delete: false,
+                copy: false,
+                use: false,
+              },
+            },
+          }}
+          apiModel={CredentialsAPI}
+        />,
         { context: { router: { credentialHistory } } }
       );
     });

awx/ui/src/components/StatusLabel/StatusLabel.js

@@ -47,7 +47,7 @@ export default function StatusLabel({ status, tooltipContent = '', children }) {
     unreachable: t`Unreachable`,
     running: t`Running`,
     pending: t`Pending`,
-    skipped: t`Skipped'`,
+    skipped: t`Skipped`,
     timedOut: t`Timed out`,
     waiting: t`Waiting`,
     disabled: t`Disabled`,

awx/ui/src/locales/en/messages.po

@@ -8722,8 +8722,8 @@ msgid "Skipped"
 msgstr "Skipped"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
-msgstr "Skipped'"
+msgid "Skipped"
+msgstr "Skipped"
 
 #: components/NotificationList/NotificationList.js:200
 #: screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js:141

awx/ui/src/locales/es/messages.po

@@ -8190,8 +8190,8 @@ msgid "Skipped"
 msgstr "Omitido"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
-msgstr "Omitido'"
+msgid "Skipped"
+msgstr "Omitido"
 
 #: components/NotificationList/NotificationList.js:200
 #: screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js:141

awx/ui/src/locales/fr/messages.po

@@ -8078,7 +8078,7 @@ msgid "Skipped"
 msgstr "Ignoré"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
+msgid "Skipped"
 msgstr "Ignoré"
 
 #: components/NotificationList/NotificationList.js:200

awx/ui/src/locales/ja/messages.po

@@ -8118,8 +8118,8 @@ msgid "Skipped"
 msgstr "スキップ済"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
-msgstr "スキップ済'"
+msgid "Skipped"
+msgstr "スキップ済"
 
 #: components/NotificationList/NotificationList.js:200
 #: screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js:141

awx/ui/src/locales/ko/messages.po

@@ -8072,8 +8072,8 @@ msgid "Skipped"
 msgstr "건너뜀"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
-msgstr "건너뜀'"
+msgid "Skipped"
+msgstr "건너뜀"
 
 #: components/NotificationList/NotificationList.js:200
 #: screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js:141

awx/ui/src/locales/nl/messages.po

@@ -8096,8 +8096,8 @@ msgid "Skipped"
 msgstr "Overgeslagen"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
-msgstr "Overgeslagen'"
+msgid "Skipped"
+msgstr "Overgeslagen"
 
 #: components/NotificationList/NotificationList.js:200
 #: screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js:141

awx/ui/src/locales/zh/messages.po

@@ -8072,8 +8072,8 @@ msgid "Skipped"
 msgstr "跳过"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
-msgstr "跳过'"
+msgid "Skipped"
+msgstr "跳过"
 
 #: components/NotificationList/NotificationList.js:200
 #: screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js:141

awx/ui/src/locales/zu/messages.po

@@ -8503,7 +8503,7 @@ msgid "Skipped"
 msgstr ""
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
+msgid "Skipped"
 msgstr ""
 
 #: components/NotificationList/NotificationList.js:200

awx/ui/src/screens/Instances/InstanceAdd/InstanceAdd.test.js

@@ -32,12 +32,10 @@ describe('<InstanceAdd />', () => {
     await waitForElement(wrapper, 'isLoading', (el) => el.length === 0);
     await act(async () => {
       wrapper.find('InstanceForm').prop('handleSubmit')({
-        name: 'new Foo',
         node_type: 'hop',
       });
     });
     expect(InstancesAPI.create).toHaveBeenCalledWith({
-      name: 'new Foo',
       node_type: 'hop',
     });
     expect(history.location.pathname).toBe('/instances/13/details');

awx/ui/src/screens/Instances/InstanceDetail/InstanceDetail.js

@@ -1,6 +1,6 @@
 import React, { useCallback, useEffect, useState } from 'react';
 
-import { useHistory, useParams } from 'react-router-dom';
+import { useHistory, useParams, Link } from 'react-router-dom';
 import { t, Plural } from '@lingui/macro';
 import {
   Button,
@@ -116,6 +116,7 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
       setBreadcrumb(instance);
     }
   }, [instance, setBreadcrumb]);
+
   const { error: healthCheckError, request: fetchHealthCheck } = useRequest(
     useCallback(async () => {
       const { status } = await InstancesAPI.healthCheck(id);
@@ -205,13 +206,42 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
             }
           />
           <Detail label={t`Node Type`} value={instance.node_type} />
+          <Detail label={t`Host`} value={instance.ip_address} />
+          <Detail label={t`Listener Port`} value={instance.listener_port} />
+          {(isExecutionNode || isHopNode) && (
+            <>
+              {instance.related?.install_bundle && (
+                <Detail
+                  label={t`Install Bundle`}
+                  value={
+                    <Tooltip content={t`Click to download bundle`}>
+                      <Button
+                        component="a"
+                        isSmall
+                        href={`${instance.related?.install_bundle}`}
+                        target="_blank"
+                        variant="secondary"
+                        dataCy="install-bundle-download-button"
+                        rel="noopener noreferrer"
+                      >
+                        <DownloadIcon />
+                      </Button>
+                    </Tooltip>
+                  }
+                />
+              )}
+              <Detail
+                label={t`Peers from control nodes`}
+                value={instance.peers_from_control_nodes ? t`On` : t`Off`}
+              />
+            </>
+          )}
           {!isHopNode && (
             <>
               <Detail
                 label={t`Policy Type`}
                 value={instance.managed_by_policy ? t`Auto` : t`Manual`}
               />
-              <Detail label={t`Host`} value={instance.ip_address} />
               <Detail label={t`Running Jobs`} value={instance.jobs_running} />
               <Detail label={t`Total Jobs`} value={instance.jobs_total} />
               {instanceGroups && (
@@ -246,26 +276,6 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
                 }
                 value={formatHealthCheckTimeStamp(instance.last_health_check)}
               />
-              {instance.related?.install_bundle && (
-                <Detail
-                  label={t`Install Bundle`}
-                  value={
-                    <Tooltip content={t`Click to download bundle`}>
-                      <Button
-                        component="a"
-                        isSmall
-                        href={`${instance.related?.install_bundle}`}
-                        target="_blank"
-                        variant="secondary"
-                        dataCy="install-bundle-download-button"
-                        rel="noopener noreferrer"
-                      >
-                        <DownloadIcon />
-                      </Button>
-                    </Tooltip>
-                  }
-                />
-              )}
               <Detail
                 label={t`Capacity Adjustment`}
                 dataCy="capacity-adjustment"
@@ -327,9 +337,20 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
             />
           )}
         </DetailList>
-        {!isHopNode && (
-          <CardActionsRow>
-            {config?.me?.is_superuser && isK8s && isExecutionNode && (
+        <CardActionsRow>
+          {config?.me?.is_superuser && isK8s && (isExecutionNode || isHopNode) && (
+            <Button
+              ouiaId="instance-detail-edit-button"
+              aria-label={t`edit`}
+              component={Link}
+              to={`/instances/${id}/edit`}
+            >
+              {t`Edit`}
+            </Button>
+          )}
+          {config?.me?.is_superuser &&
+            isK8s &&
+            (isExecutionNode || isHopNode) && (
               <RemoveInstanceButton
                 dataCy="remove-instance-button"
                 itemsToRemove={[instance]}
@@ -337,32 +358,31 @@ function InstanceDetail({ setBreadcrumb, isK8s }) {
                 onRemove={removeInstances}
               />
             )}
-            {isExecutionNode && (
-              <Tooltip content={t`Run a health check on the instance`}>
-                <Button
-                  isDisabled={
-                    !config?.me?.is_superuser || instance.health_check_pending
-                  }
-                  variant="primary"
-                  ouiaId="health-check-button"
-                  onClick={fetchHealthCheck}
-                  isLoading={instance.health_check_pending}
-                  spinnerAriaLabel={t`Running health check`}
-                >
-                  {instance.health_check_pending
-                    ? t`Running health check`
-                    : t`Run health check`}
-                </Button>
-              </Tooltip>
-            )}
-            <InstanceToggle
-              css="display: inline-flex;"
-              fetchInstances={fetchDetails}
-              instance={instance}
-              dataCy="enable-instance"
-            />
-          </CardActionsRow>
-        )}
+          {isExecutionNode && (
+            <Tooltip content={t`Run a health check on the instance`}>
+              <Button
+                isDisabled={
+                  !config?.me?.is_superuser || instance.health_check_pending
+                }
+                variant="primary"
+                ouiaId="health-check-button"
+                onClick={fetchHealthCheck}
+                isLoading={instance.health_check_pending}
+                spinnerAriaLabel={t`Running health check`}
+              >
+                {instance.health_check_pending
+                  ? t`Running health check`
+                  : t`Run health check`}
+              </Button>
+            </Tooltip>
+          )}
+          <InstanceToggle
+            css="display: inline-flex;"
+            fetchInstances={fetchDetails}
+            instance={instance}
+            dataCy="enable-instance"
+          />
+        </CardActionsRow>
 
         {error && (
           <AlertModal

awx/ui/src/screens/Instances/InstanceEdit/InstanceEdit.js

@@ -0,0 +1,105 @@
+import React, { useState, useCallback, useEffect } from 'react';
+
+import { t } from '@lingui/macro';
+import { useHistory, useParams, Link } from 'react-router-dom';
+import { Card, PageSection } from '@patternfly/react-core';
+import useRequest from 'hooks/useRequest';
+import ContentError from 'components/ContentError';
+import ContentLoading from 'components/ContentLoading';
+import { CardBody } from 'components/Card';
+import { InstancesAPI } from 'api';
+import InstanceForm from '../Shared/InstanceForm';
+
+function InstanceEdit({ setBreadcrumb }) {
+  const history = useHistory();
+  const { id } = useParams();
+  const [formError, setFormError] = useState();
+
+  const detailsUrl = `/instances/${id}/details`;
+
+  const handleSubmit = async (values) => {
+    try {
+      await InstancesAPI.update(id, values);
+      history.push(detailsUrl);
+    } catch (err) {
+      setFormError(err);
+    }
+  };
+
+  const handleCancel = () => {
+    history.push(detailsUrl);
+  };
+
+  const {
+    isLoading,
+    error,
+    request: fetchDetail,
+    result: { instance, peers },
+  } = useRequest(
+    useCallback(async () => {
+      const [{ data: instance_detail }, { data: peers_detail }] =
+        await Promise.all([
+          InstancesAPI.readDetail(id),
+          InstancesAPI.readPeers(id),
+        ]);
+      return {
+        instance: instance_detail,
+        peers: peers_detail.results,
+      };
+    }, [id]),
+    {
+      instance: {},
+      peers: [],
+    }
+  );
+
+  useEffect(() => {
+    fetchDetail();
+  }, [fetchDetail]);
+
+  useEffect(() => {
+    if (instance) {
+      setBreadcrumb(instance);
+    }
+  }, [instance, setBreadcrumb]);
+
+  if (isLoading) {
+    return (
+      <CardBody>
+        <ContentLoading />
+      </CardBody>
+    );
+  }
+
+  if (error) {
+    return (
+      <CardBody>
+        <ContentError error={error}>
+          {error?.response?.status === 404 && (
+            <span>
+              {t`Instance not found.`}{' '}
+              <Link to="/instances">{t`View all Instances.`}</Link>
+            </span>
+          )}
+        </ContentError>
+      </CardBody>
+    );
+  }
+
+  return (
+    <PageSection>
+      <Card>
+        <InstanceForm
+          instance={instance}
+          instance_peers={peers}
+          isEdit
+          submitError={formError}
+          handleSubmit={handleSubmit}
+          handleCancel={handleCancel}
+        />
+      </Card>
+    </PageSection>
+  );
+}
+
+export default InstanceEdit;

awx/ui/src/screens/Instances/InstanceEdit/InstanceEdit.test.js

@@ -0,0 +1,149 @@
+import React from 'react';
+import { act } from 'react-dom/test-utils';
+import { createMemoryHistory } from 'history';
+import useDebounce from 'hooks/useDebounce';
+import { InstancesAPI } from 'api';
+import {
+  mountWithContexts,
+  waitForElement,
+} from '../../../../testUtils/enzymeHelpers';
+
+import InstanceEdit from './InstanceEdit';
+
+jest.mock('../../../api');
+jest.mock('../../../hooks/useDebounce');
+jest.mock('react-router-dom', () => ({
+  ...jest.requireActual('react-router-dom'),
+  useParams: () => ({
+    id: 42,
+  }),
+}));
+
+const instanceData = {
+  id: 42,
+  hostname: 'awx_1',
+  type: 'instance',
+  url: '/api/v2/instances/1/',
+  related: {
+    named_url: '/api/v2/instances/awx_1/',
+    jobs: '/api/v2/instances/1/jobs/',
+    instance_groups: '/api/v2/instances/1/instance_groups/',
+    peers: '/api/v2/instances/1/peers/',
+  },
+  summary_fields: {
+    user_capabilities: {
+      edit: false,
+    },
+    links: [],
+  },
+  uuid: '00000000-0000-0000-0000-000000000000',
+  created: '2023-04-26T22:06:46.766198Z',
+  modified: '2023-04-26T22:06:46.766217Z',
+  last_seen: '2023-04-26T23:12:02.857732Z',
+  health_check_started: null,
+  health_check_pending: false,
+  last_health_check: '2023-04-26T23:01:13.941693Z',
+  errors: 'Instance received normal shutdown signal',
+  capacity_adjustment: '1.00',
+  version: '0.1.dev33237+g1fdef52',
+  capacity: 0,
+  consumed_capacity: 0,
+  percent_capacity_remaining: 0,
+  jobs_running: 0,
+  jobs_total: 0,
+  cpu: '8.0',
+  memory: 8011055104,
+  cpu_capacity: 0,
+  mem_capacity: 0,
+  enabled: true,
+  managed_by_policy: true,
+  node_type: 'hybrid',
+  node_state: 'installed',
+  ip_address: null,
+  listener_port: 27199,
+  peers: [],
+  peers_from_control_nodes: false,
+};
+
+const instanceDataWithPeers = {
+  results: [instanceData],
+};
+
+const updatedInstance = {
+  node_type: 'hop',
+  peers: ['test-peer'],
+};
+
+describe('<InstanceEdit/>', () => {
+  let wrapper;
+  let history;
+
+  beforeAll(async () => {
+    useDebounce.mockImplementation((fn) => fn);
+    history = createMemoryHistory();
+    InstancesAPI.readDetail.mockResolvedValue({ data: instanceData });
+    InstancesAPI.readPeers.mockResolvedValue({ data: instanceDataWithPeers });
+
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <InstanceEdit
+          instance={instanceData}
+          peers={instanceDataWithPeers}
+          isEdit
+          setBreadcrumb={() => {}}
+        />,
+        {
+          context: { router: { history } },
+        }
+      );
+    });
+    expect(InstancesAPI.readDetail).toBeCalledWith(42);
+  });
+
+  afterAll(() => {
+    jest.clearAllMocks();
+  });
+
+  test('initially renders successfully', async () => {
+    await waitForElement(wrapper, 'ContentLoading', (el) => el.length === 0);
+    expect(wrapper.find('InstanceEdit')).toHaveLength(1);
+  });
+
+  test('handleSubmit should call the api and redirect to details page', async () => {
+    await act(async () => {
+      wrapper.find('InstanceForm').invoke('handleSubmit')(updatedInstance);
+    });
+    expect(InstancesAPI.update).toHaveBeenCalledWith(42, updatedInstance);
+    expect(history.location.pathname).toEqual('/instances/42/details');
+  });
+
+  test('should navigate to instance details when cancel is clicked', async () => {
+    await act(async () => {
+      wrapper.find('button[aria-label="Cancel"]').simulate('click');
+    });
+    expect(history.location.pathname).toEqual('/instances/42/details');
+  });
+
+  test('should navigate to instance details after successful submission', async () => {
+    await act(async () => {
+      wrapper.find('InstanceForm').invoke('handleSubmit')(updatedInstance);
+    });
+    wrapper.update();
+    expect(wrapper.find('submitError').length).toBe(0);
+    expect(history.location.pathname).toEqual('/instances/42/details');
+  });
+
+  test('failed form submission should show an error message', async () => {
+    const error = {
+      response: {
+        data: { detail: 'An error occurred' },
+      },
+    };
+    InstancesAPI.update.mockImplementationOnce(() => Promise.reject(error));
+    await act(async () => {
+      wrapper.find('InstanceForm').invoke('handleSubmit')(updatedInstance);
+    });
+    wrapper.update();
+    expect(wrapper.find('FormSubmitError').length).toBe(1);
+  });
+});

awx/ui/src/screens/Instances/InstanceEdit/index.js

@@ -0,0 +1 @@
+export { default } from './InstanceEdit';

awx/ui/src/screens/Instances/InstanceList/InstanceListItem.js

@@ -138,7 +138,7 @@ function InstanceListItem({
             rowIndex,
             isSelected,
             onSelect,
-            disable: !isExecutionNode,
+            disable: !(isExecutionNode || isHopNode),
           }}
           dataLabel={t`Selected`}
         />

awx/ui/src/screens/Instances/InstancePeers/InstancePeerList.js

@@ -1,17 +1,24 @@
-import React, { useCallback, useEffect } from 'react';
+import React, { useCallback, useEffect, useState } from 'react';
 import { t } from '@lingui/macro';
 import { CardBody } from 'components/Card';
 import PaginatedTable, {
   getSearchableKeys,
   HeaderCell,
   HeaderRow,
+  ToolbarAddButton,
 } from 'components/PaginatedTable';
-import { getQSConfig, parseQueryString } from 'util/qs';
+import DisassociateButton from 'components/DisassociateButton';
+import AssociateModal from 'components/AssociateModal';
+import ErrorDetail from 'components/ErrorDetail';
+import AlertModal from 'components/AlertModal';
+import useToast, { AlertVariant } from 'hooks/useToast';
+import { getQSConfig, parseQueryString, mergeParams } from 'util/qs';
 import { useLocation, useParams } from 'react-router-dom';
-import useRequest from 'hooks/useRequest';
+import useRequest, { useDismissableError } from 'hooks/useRequest';
 import DataListToolbar from 'components/DataListToolbar';
 import { InstancesAPI } from 'api';
 import useExpanded from 'hooks/useExpanded';
+import useSelected from 'hooks/useSelected';
 import InstancePeerListItem from './InstancePeerListItem';
 
 const QS_CONFIG = getQSConfig('peer', {
@@ -20,27 +27,36 @@ const QS_CONFIG = getQSConfig('peer', {
   order_by: 'hostname',
 });
 
-function InstancePeerList() {
+function InstancePeerList({ setBreadcrumb }) {
   const location = useLocation();
   const { id } = useParams();
+  const [isModalOpen, setIsModalOpen] = useState(false);
+  const { addToast, Toast, toastProps } = useToast();
+  const readInstancesOptions = useCallback(
+    () => InstancesAPI.readOptions(id),
+    [id]
+  );
   const {
     isLoading,
     error: contentError,
     request: fetchPeers,
-    result: { peers, count, relatedSearchableKeys, searchableKeys },
+    result: { instance, peers, count, relatedSearchableKeys, searchableKeys },
   } = useRequest(
     useCallback(async () => {
       const params = parseQueryString(QS_CONFIG, location.search);
       const [
+        { data: detail },
         {
           data: { results, count: itemNumber },
         },
         actions,
       ] = await Promise.all([
+        InstancesAPI.readDetail(id),
         InstancesAPI.readPeers(id, params),
         InstancesAPI.readOptions(),
       ]);
       return {
+        instance: detail,
         peers: results,
         count: itemNumber,
         relatedSearchableKeys: (actions?.data?.related_search_fields || []).map(
@@ -50,6 +66,7 @@ function InstancePeerList() {
       };
     }, [id, location]),
     {
+      instance: {},
       peers: [],
       count: 0,
       relatedSearchableKeys: [],
@@ -61,18 +78,98 @@ function InstancePeerList() {
     fetchPeers();
   }, [fetchPeers]);
 
+  useEffect(() => {
+    if (instance) {
+      setBreadcrumb(instance);
+    }
+  }, [instance, setBreadcrumb]);
+
   const { expanded, isAllExpanded, handleExpand, expandAll } =
     useExpanded(peers);
+  const { selected, isAllSelected, handleSelect, clearSelected, selectAll } =
+    useSelected(peers);
+
+  const fetchInstancesToAssociate = useCallback(
+    (params) =>
+      InstancesAPI.read(
+        mergeParams(params, {
+          ...{ not__id: id },
+          ...{ not__node_type: ['control', 'hybrid'] },
+          ...{ not__hostname: instance.peers },
+        })
+      ),
+    [id, instance]
+  );
+
+  const {
+    isLoading: isAssociateLoading,
+    request: handlePeerAssociate,
+    error: associateError,
+  } = useRequest(
+    useCallback(
+      async (instancesPeerToAssociate) => {
+        const selected_hostname = instancesPeerToAssociate.map(
+          (obj) => obj.hostname
+        );
+        const new_peers = [
+          ...new Set([...instance.peers, ...selected_hostname]),
+        ];
+        await InstancesAPI.update(instance.id, { peers: new_peers });
+        fetchPeers();
+        addToast({
+          id: instancesPeerToAssociate,
+          title: t`${selected_hostname} added as a peer. Please be sure to run the install bundle for ${instance.hostname} again in order to see changes take effect.`,
+          variant: AlertVariant.success,
+          hasTimeout: true,
+        });
+      },
+      [instance, fetchPeers, addToast]
+    )
+  );
+
+  const {
+    isLoading: isDisassociateLoading,
+    request: handlePeersDiassociate,
+    error: disassociateError,
+  } = useRequest(
+    useCallback(async () => {
+      const new_peers = [];
+      const selected_hostname = selected.map((obj) => obj.hostname);
+      for (let i = 0; i < instance.peers.length; i++) {
+        if (!selected_hostname.includes(instance.peers[i])) {
+          new_peers.push(instance.peers[i]);
+        }
+      }
+      await InstancesAPI.update(instance.id, { peers: new_peers });
+      fetchPeers();
+      addToast({
+        title: t`${selected_hostname} removed. Please be sure to run the install bundle for ${instance.hostname} again in order to see changes take effect.`,
+        variant: AlertVariant.success,
+        hasTimeout: true,
+      });
+    }, [instance, selected, fetchPeers, addToast])
+  );
+
+  const { error, dismissError } = useDismissableError(
+    associateError || disassociateError
+  );
+
+  const isHopNode = instance.node_type === 'hop';
+  const isExecutionNode = instance.node_type === 'execution';
 
   return (
     <CardBody>
       <PaginatedTable
         contentError={contentError}
-        hasContentLoading={isLoading}
+        hasContentLoading={
+          isLoading || isDisassociateLoading || isAssociateLoading
+        }
         items={peers}
         itemCount={count}
         pluralizedItemName={t`Peers`}
         qsConfig={QS_CONFIG}
+        onRowClick={handleSelect}
+        clearSelected={clearSelected}
         toolbarSearchableKeys={searchableKeys}
         toolbarRelatedSearchableKeys={relatedSearchableKeys}
         toolbarSearchColumns={[
@@ -101,13 +198,36 @@ function InstancePeerList() {
         renderToolbar={(props) => (
           <DataListToolbar
             {...props}
+            isAllSelected={isAllSelected}
+            onSelectAll={selectAll}
             isAllExpanded={isAllExpanded}
             onExpandAll={expandAll}
             qsConfig={QS_CONFIG}
+            additionalControls={[
+              (isExecutionNode || isHopNode) && (
+                <ToolbarAddButton
+                  ouiaId="add-instance-peers-button"
+                  key="associate"
+                  defaultLabel={t`Associate`}
+                  onClick={() => setIsModalOpen(true)}
+                />
+              ),
+              (isExecutionNode || isHopNode) && (
+                <DisassociateButton
+                  verifyCannotDisassociate={false}
+                  key="disassociate"
+                  onDisassociate={handlePeersDiassociate}
+                  itemsToDisassociate={selected}
+                  modalTitle={t`Remove instance from peers?`}
+                />
+              ),
+            ]}
           />
         )}
         renderRow={(peer, index) => (
           <InstancePeerListItem
+            isSelected={selected.some((row) => row.id === peer.id)}
+            onSelect={() => handleSelect(peer)}
             isExpanded={expanded.some((row) => row.id === peer.id)}
             onExpand={() => handleExpand(peer)}
             key={peer.id}
@@ -116,6 +236,35 @@ function InstancePeerList() {
           />
         )}
       />
+      {isModalOpen && (
+        <AssociateModal
+          header={t`Instances`}
+          fetchRequest={fetchInstancesToAssociate}
+          isModalOpen={isModalOpen}
+          onAssociate={handlePeerAssociate}
+          onClose={() => setIsModalOpen(false)}
+          title={t`Select Instances`}
+          optionsRequest={readInstancesOptions}
+          displayKey="hostname"
+          columns={[
+            { key: 'hostname', name: t`Name` },
+            { key: 'node_type', name: t`Node Type` },
+          ]}
+        />
+      )}
+      <Toast {...toastProps} />
+      {error && (
+        <AlertModal
+          isOpen={error}
+          onClose={dismissError}
+          title={t`Error!`}
+          variant="error"
+        >
+          {associateError && t`Failed to associate peer.`}
+          {disassociateError && t`Failed to remove peers.`}
+          <ErrorDetail error={error} />
+        </AlertModal>
+      )}
     </CardBody>
   );
 }

awx/ui/src/screens/Instances/InstancePeers/InstancePeerListItem.js

@@ -10,6 +10,8 @@ import { Detail, DetailList } from 'components/DetailList';
 
 function InstancePeerListItem({
   peerInstance,
+  isSelected,
+  onSelect,
   isExpanded,
   onExpand,
   rowIndex,
@@ -33,7 +35,14 @@ function InstancePeerListItem({
             }}
           />
         )}
-        <Td />
+        <Td
+          select={{
+            rowIndex,
+            isSelected,
+            onSelect,
+          }}
+          dataLabel={t`Selected`}
+        />
         <Td id={labelId} dataLabel={t`Name`}>
           <Link to={`/instances/${peerInstance.id}/details`}>
             <b>{peerInstance.hostname}</b>

awx/ui/src/screens/Instances/Instances.js

@@ -7,6 +7,7 @@ import PersistentFilters from 'components/PersistentFilters';
 import { InstanceList } from './InstanceList';
 import Instance from './Instance';
 import InstanceAdd from './InstanceAdd';
+import InstanceEdit from './InstanceEdit';
 
 function Instances() {
   const [breadcrumbConfig, setBreadcrumbConfig] = useState({
@@ -20,8 +21,11 @@ function Instances() {
     }
     setBreadcrumbConfig({
       '/instances': t`Instances`,
+      '/instances/add': t`Create new Instance`,
       [`/instances/${instance.id}`]: `${instance.hostname}`,
       [`/instances/${instance.id}/details`]: t`Details`,
+      [`/instances/${instance.id}/peers`]: t`Peers`,
+      [`/instances/${instance.id}/edit`]: t`Edit Instance`,
     });
   }, []);
 
@@ -30,7 +34,10 @@ function Instances() {
       <ScreenHeader streamType="instance" breadcrumbConfig={breadcrumbConfig} />
       <Switch>
         <Route path="/instances/add">
-          <InstanceAdd />
+          <InstanceAdd setBreadcrumb={buildBreadcrumbConfig} />
+        </Route>
+        <Route path="/instances/:id/edit" key="edit">
+          <InstanceEdit setBreadcrumb={buildBreadcrumbConfig} />
         </Route>
         <Route path="/instances/:id">
           <Instance setBreadcrumb={buildBreadcrumbConfig} />

awx/ui/src/screens/Instances/Shared/InstanceForm.js

@@ -1,6 +1,6 @@
-import React from 'react';
+import React, { useCallback } from 'react';
 import { t } from '@lingui/macro';
-import { Formik } from 'formik';
+import { Formik, useField, useFormikContext } from 'formik';
 import { Form, FormGroup, CardBody } from '@patternfly/react-core';
 import { FormColumnLayout } from 'components/FormLayout';
 import FormField, {
@@ -8,9 +8,31 @@ import FormField, {
   CheckboxField,
 } from 'components/FormField';
 import FormActionGroup from 'components/FormActionGroup';
+import AnsibleSelect from 'components/AnsibleSelect';
+import { PeersLookup } from 'components/Lookup';
 import { required } from 'util/validators';
 
-function InstanceFormFields() {
+const INSTANCE_TYPES = [
+  { id: 'execution', name: t`Execution` },
+  { id: 'hop', name: t`Hop` },
+];
+
+function InstanceFormFields({ isEdit }) {
+  const [instanceTypeField, instanceTypeMeta, instanceTypeHelpers] = useField({
+    name: 'node_type',
+    validate: required(t`Set a value for this field`),
+  });
+
+  const { setFieldValue } = useFormikContext();
+
+  const [peersField, peersMeta, peersHelpers] = useField('peers');
+
+  const handlePeersUpdate = useCallback(
+    (value) => {
+      setFieldValue('peers', value);
+    },
+    [setFieldValue]
+  );
   return (
     <>
       <FormField
@@ -20,6 +42,7 @@ function InstanceFormFields() {
         type="text"
         validate={required(null)}
         isRequired
+        isDisabled={isEdit}
       />
       <FormField
         id="instance-description"
@@ -40,16 +63,47 @@ function InstanceFormFields() {
         label={t`Listener Port`}
         name="listener_port"
         type="number"
-        tooltip={t`Select the port that Receptor will listen on for incoming connections. Default is 27199.`}
-        isRequired
+        tooltip={t`Select the port that Receptor will listen on for incoming connections, e.g. 27199.`}
       />
-      <FormField
-        id="instance-type"
+      <FormGroup
+        fieldId="instance-type"
         label={t`Instance Type`}
-        name="node_type"
-        type="text"
         tooltip={t`Sets the role that this instance will play within mesh topology. Default is "execution."`}
-        isDisabled
+        validated={
+          !instanceTypeMeta.touched || !instanceTypeMeta.error
+            ? 'default'
+            : 'error'
+        }
+        helperTextInvalid={instanceTypeMeta.error}
+        isRequired
+      >
+        <AnsibleSelect
+          {...instanceTypeField}
+          id="node_type"
+          data={INSTANCE_TYPES.map((type) => ({
+            key: type.id,
+            value: type.id,
+            label: type.name,
+          }))}
+          onChange={(event, value) => {
+            instanceTypeHelpers.setValue(value);
+          }}
+          isDisabled={isEdit}
+        />
+      </FormGroup>
+      <PeersLookup
+        helperTextInvalid={peersMeta.error}
+        isValid={!peersMeta.touched || !peersMeta.error}
+        onBlur={() => peersHelpers.setTouched()}
+        onChange={handlePeersUpdate}
+        value={peersField.value}
+        tooltip={t`Select the Peers Instances.`}
+        fieldName="peers"
+        formLabel={t`Peers`}
+        multiple
+        typePeers
+        id="peers"
+        isRequired
       />
       <FormGroup fieldId="instance-option-checkboxes" label={t`Options`}>
         <CheckboxField
@@ -64,6 +118,12 @@ function InstanceFormFields() {
           label={t`Managed by Policy`}
           tooltip={t`Controls whether or not this instance is managed by policy. If enabled, the instance will be available for automatic assignment to and unassignment from instance groups based on policy rules.`}
         />
+        <CheckboxField
+          id="peers_from_control_nodes"
+          name="peers_from_control_nodes"
+          label={t`Peers from control nodes`}
+          tooltip={t`If enabled, control nodes will peer to this instance automatically. If disabled, instance will be connected only to associated peers.`}
+        />
       </FormGroup>
     </>
   );
@@ -71,6 +131,8 @@ function InstanceFormFields() {
 
 function InstanceForm({
   instance = {},
+  instance_peers = [],
+  isEdit = false,
   submitError,
   handleCancel,
   handleSubmit,
@@ -79,22 +141,29 @@ function InstanceForm({
     <CardBody>
       <Formik
         initialValues={{
-          hostname: '',
-          description: '',
-          node_type: 'execution',
-          node_state: 'installed',
-          listener_port: 27199,
-          enabled: true,
-          managed_by_policy: true,
+          hostname: instance.hostname || '',
+          description: instance.description || '',
+          node_type: instance.node_type || 'execution',
+          node_state: instance.node_state || 'installed',
+          listener_port: instance.listener_port,
+          enabled: instance.enabled || true,
+          managed_by_policy: instance.managed_by_policy || true,
+          peers_from_control_nodes: instance.peers_from_control_nodes || false,
+          peers: instance_peers,
         }}
         onSubmit={(values) => {
-          handleSubmit(values);
+          handleSubmit({
+            ...values,
+            listener_port:
+              values.listener_port === '' ? null : values.listener_port,
+            peers: values.peers.map((peer) => peer.hostname || peer),
+          });
         }}
       >
         {(formik) => (
           <Form autoComplete="off" onSubmit={formik.handleSubmit}>
             <FormColumnLayout>
-              <InstanceFormFields instance={instance} />
+              <InstanceFormFields isEdit={isEdit} />
               <FormSubmitError error={submitError} />
               <FormActionGroup
                 onCancel={handleCancel}

awx/ui/src/screens/Instances/Shared/InstanceForm.test.js

@@ -65,7 +65,7 @@ describe('<InstanceForm />', () => {
     expect(handleCancel).toBeCalled();
   });
 
-  test('should call handleSubmit when Cancel button is clicked', async () => {
+  test('should call handleSubmit when Save button is clicked', async () => {
     expect(handleSubmit).not.toHaveBeenCalled();
     await act(async () => {
       wrapper.find('input#hostname').simulate('change', {
@@ -74,9 +74,6 @@ describe('<InstanceForm />', () => {
       wrapper.find('input#instance-description').simulate('change', {
         target: { value: 'This is a repeat song', name: 'description' },
       });
-      wrapper.find('input#instance-port').simulate('change', {
-        target: { value: 'This is a repeat song', name: 'listener_port' },
-      });
     });
     wrapper.update();
     expect(
@@ -91,9 +88,10 @@ describe('<InstanceForm />', () => {
       enabled: true,
       managed_by_policy: true,
       hostname: 'new Foo',
-      listener_port: 'This is a repeat song',
       node_state: 'installed',
       node_type: 'execution',
+      peers_from_control_nodes: false,
+      peers: [],
     });
   });
 });

awx/ui/src/screens/Instances/Shared/RemoveInstanceButton.js

@@ -33,7 +33,8 @@ function RemoveInstanceButton({ itemsToRemove, onRemove, isK8s }) {
   const [removeDetails, setRemoveDetails] = useState(null);
   const [isLoading, setIsLoading] = useState(false);
 
-  const cannotRemove = (item) => item.node_type !== 'execution';
+  const cannotRemove = (item) =>
+    !(item.node_type === 'execution' || item.node_type === 'hop');
 
   const toggleModal = async (isOpen) => {
     setRemoveDetails(null);
@@ -175,7 +176,7 @@ function RemoveInstanceButton({ itemsToRemove, onRemove, isK8s }) {
             </Button>,
           ]}
         >
-          <div>{t`This action will remove the following instances:`}</div>
+          <div>{t`This action will remove the following instance and you may need to rerun the install bundle for any instance that was previously connected to:`}</div>
           {itemsToRemove.map((item) => (
             <span key={item.id} id={`item-to-be-removed-${item.id}`}>
               <strong>{item.hostname}</strong>

awx/ui/src/screens/Inventory/ConstructedInventoryAdd/ConstructedInventoryAdd.js

@@ -1,14 +1,43 @@
-import React, { useState } from 'react';
+import React, { useCallback, useEffect, useState } from 'react';
 import { useHistory } from 'react-router-dom';
 import { Card, PageSection } from '@patternfly/react-core';
 import { ConstructedInventoriesAPI, InventoriesAPI } from 'api';
+import useRequest from 'hooks/useRequest';
 import { CardBody } from 'components/Card';
+import ContentError from 'components/ContentError';
+import ContentLoading from 'components/ContentLoading';
 import ConstructedInventoryForm from '../shared/ConstructedInventoryForm';
 
 function ConstructedInventoryAdd() {
   const history = useHistory();
   const [submitError, setSubmitError] = useState(null);
 
+  const {
+    isLoading: isLoadingOptions,
+    error: optionsError,
+    request: fetchOptions,
+    result: options,
+  } = useRequest(
+    useCallback(async () => {
+      const res = await ConstructedInventoriesAPI.readOptions();
+      const { data } = res;
+      return data.actions.POST;
+    }, []),
+    null
+  );
+
+  useEffect(() => {
+    fetchOptions();
+  }, [fetchOptions]);
+
+  if (isLoadingOptions || (!options && !optionsError)) {
+    return <ContentLoading />;
+  }
+
+  if (optionsError) {
+    return <ContentError error={optionsError} />;
+  }
+
   const handleCancel = () => {
     history.push('/inventories');
   };
@@ -48,6 +77,7 @@ function ConstructedInventoryAdd() {
             onCancel={handleCancel}
             onSubmit={handleSubmit}
             submitError={submitError}
+            options={options}
           />
         </CardBody>
       </Card>

awx/ui/src/screens/Inventory/ConstructedInventoryAdd/ConstructedInventoryAdd.test.js

@@ -55,6 +55,7 @@ describe('<ConstructedInventoryAdd />', () => {
         context: { router: { history } },
       });
     });
+    await waitForElement(wrapper, 'ContentLoading', (el) => el.length === 0);
   });
 
   afterEach(() => {

awx/ui/src/screens/Inventory/ConstructedInventoryEdit/ConstructedInventoryEdit.js

@@ -20,6 +20,27 @@ function ConstructedInventoryEdit({ inventory }) {
   const detailsUrl = `/inventories/constructed_inventory/${inventory.id}/details`;
   const constructedInventoryId = inventory.id;
 
+  const {
+    isLoading: isLoadingOptions,
+    error: optionsError,
+    request: fetchOptions,
+    result: options,
+  } = useRequest(
+    useCallback(
+      () =>
+        ConstructedInventoriesAPI.readConstructedInventoryOptions(
+          constructedInventoryId,
+          'PUT'
+        ),
+      [constructedInventoryId]
+    ),
+    null
+  );
+
+  useEffect(() => {
+    fetchOptions();
+  }, [fetchOptions]);
+
   const {
     result: { initialInstanceGroups, initialInputInventories },
     request: fetchedRelatedData,
@@ -44,6 +65,7 @@ function ConstructedInventoryEdit({ inventory }) {
       isLoading: true,
     }
   );
+
   useEffect(() => {
     fetchedRelatedData();
   }, [fetchedRelatedData]);
@@ -99,12 +121,12 @@ function ConstructedInventoryEdit({ inventory }) {
 
   const handleCancel = () => history.push(detailsUrl);
 
-  if (isLoading) {
-    return <ContentLoading />;
+  if (contentError || optionsError) {
+    return <ContentError error={contentError || optionsError} />;
   }
 
-  if (contentError) {
-    return <ContentError error={contentError} />;
+  if (isLoading || isLoadingOptions || (!options && !optionsError)) {
+    return <ContentLoading />;
   }
 
   return (
@@ -116,6 +138,7 @@ function ConstructedInventoryEdit({ inventory }) {
         constructedInventory={inventory}
         instanceGroups={initialInstanceGroups}
         inputInventories={initialInputInventories}
+        options={options}
       />
     </CardBody>
   );

awx/ui/src/screens/Inventory/ConstructedInventoryEdit/ConstructedInventoryEdit.test.js

@@ -51,27 +51,22 @@ describe('<ConstructedInventoryEdit />', () => {
   };
 
   beforeEach(async () => {
-    ConstructedInventoriesAPI.readOptions.mockResolvedValue({
-      data: {
-        related: {},
-        actions: {
-          POST: {
-            limit: {
-              label: 'Limit',
-              help_text: '',
-            },
-            update_cache_timeout: {
-              label: 'Update cache timeout',
-              help_text: 'help',
-            },
-            verbosity: {
-              label: 'Verbosity',
-              help_text: '',
-            },
-          },
+    ConstructedInventoriesAPI.readConstructedInventoryOptions.mockResolvedValue(
+      {
+        limit: {
+          label: 'Limit',
+          help_text: '',
         },
-      },
-    });
+        update_cache_timeout: {
+          label: 'Update cache timeout',
+          help_text: 'help',
+        },
+        verbosity: {
+          label: 'Verbosity',
+          help_text: '',
+        },
+      }
+    );
     InventoriesAPI.readInstanceGroups.mockResolvedValue({
       data: {
         results: associatedInstanceGroups,
@@ -169,6 +164,21 @@ describe('<ConstructedInventoryEdit />', () => {
     expect(wrapper.find('ContentError').length).toBe(1);
   });
 
+  test('should throw content error if user has insufficient options permissions', async () => {
+    expect(wrapper.find('ContentError').length).toBe(0);
+    ConstructedInventoriesAPI.readConstructedInventoryOptions.mockImplementationOnce(
+      () => Promise.reject(new Error())
+    );
+    await act(async () => {
+      wrapper = mountWithContexts(
+        <ConstructedInventoryEdit inventory={mockInv} />
+      );
+    });
+
+    await waitForElement(wrapper, 'ContentLoading', (el) => el.length === 0);
+    expect(wrapper.find('ContentError').length).toBe(1);
+  });
+
   test('unsuccessful form submission should show an error message', async () => {
     const error = {
       response: {

awx/ui/src/screens/Inventory/shared/ConstructedInventoryForm.js

@@ -1,14 +1,10 @@
-import React, { useCallback, useEffect } from 'react';
+import React, { useCallback } from 'react';
 import { Formik, useField, useFormikContext } from 'formik';
 import { func, shape } from 'prop-types';
 import { t } from '@lingui/macro';
-import { ConstructedInventoriesAPI } from 'api';
 import { minMaxValue, required } from 'util/validators';
-import useRequest from 'hooks/useRequest';
 import { Form, FormGroup } from '@patternfly/react-core';
 import { VariablesField } from 'components/CodeEditor';
-import ContentError from 'components/ContentError';
-import ContentLoading from 'components/ContentLoading';
 import FormActionGroup from 'components/FormActionGroup/FormActionGroup';
 import FormField, { FormSubmitError } from 'components/FormField';
 import { FormFullWidthLayout, FormColumnLayout } from 'components/FormLayout';
@@ -165,6 +161,7 @@ function ConstructedInventoryForm({
   onCancel,
   onSubmit,
   submitError,
+  options,
 }) {
   const initialValues = {
     kind: 'constructed',
@@ -179,32 +176,6 @@ function ConstructedInventoryForm({
     source_vars: constructedInventory?.source_vars || '---',
   };
 
-  const {
-    isLoading,
-    error,
-    request: fetchOptions,
-    result: options,
-  } = useRequest(
-    useCallback(async () => {
-      const res = await ConstructedInventoriesAPI.readOptions();
-      const { data } = res;
-      return data.actions.POST;
-    }, []),
-    null
-  );
-
-  useEffect(() => {
-    fetchOptions();
-  }, [fetchOptions]);
-
-  if (isLoading || (!options && !error)) {
-    return <ContentLoading />;
-  }
-
-  if (error) {
-    return <ContentError error={error} />;
-  }
-
   return (
     <Formik initialValues={initialValues} onSubmit={onSubmit}>
       {(formik) => (

awx/ui/src/screens/Inventory/shared/ConstructedInventoryForm.test.js

@@ -1,6 +1,5 @@
 import React from 'react';
 import { act } from 'react-dom/test-utils';
-import { ConstructedInventoriesAPI } from 'api';
 import {
   mountWithContexts,
   waitForElement,
@@ -19,38 +18,35 @@ const mockFormValues = {
   inputInventories: [{ id: 100, name: 'East' }],
 };
 
+const options = {
+  limit: {
+    label: 'Limit',
+    help_text: '',
+  },
+  update_cache_timeout: {
+    label: 'Update cache timeout',
+    help_text: 'help',
+  },
+  verbosity: {
+    label: 'Verbosity',
+    help_text: '',
+  },
+};
+
 describe('<ConstructedInventoryForm />', () => {
   let wrapper;
   const onSubmit = jest.fn();
 
   beforeEach(async () => {
-    ConstructedInventoriesAPI.readOptions.mockResolvedValue({
-      data: {
-        related: {},
-        actions: {
-          POST: {
-            limit: {
-              label: 'Limit',
-              help_text: '',
-            },
-            update_cache_timeout: {
-              label: 'Update cache timeout',
-              help_text: 'help',
-            },
-            verbosity: {
-              label: 'Verbosity',
-              help_text: '',
-            },
-          },
-        },
-      },
-    });
     await act(async () => {
       wrapper = mountWithContexts(
-        <ConstructedInventoryForm onCancel={() => {}} onSubmit={onSubmit} />
+        <ConstructedInventoryForm
+          onCancel={() => {}}
+          onSubmit={onSubmit}
+          options={options}
+        />
       );
     });
-    await waitForElement(wrapper, 'ContentLoading', (el) => el.length === 0);
   });
 
   afterEach(() => {
@@ -104,20 +100,4 @@ describe('<ConstructedInventoryForm />', () => {
       'The plugin parameter is required.'
     );
   });
-
-  test('should throw content error when option request fails', async () => {
-    let newWrapper;
-    ConstructedInventoriesAPI.readOptions.mockImplementationOnce(() =>
-      Promise.reject(new Error())
-    );
-    await act(async () => {
-      newWrapper = mountWithContexts(
-        <ConstructedInventoryForm onCancel={() => {}} onSubmit={() => {}} />
-      );
-    });
-    expect(newWrapper.find('ContentError').length).toBe(0);
-    newWrapper.update();
-    expect(newWrapper.find('ContentError').length).toBe(1);
-    jest.clearAllMocks();
-  });
 });

awx/ui/src/screens/TopologyView/Header.js

@@ -16,6 +16,7 @@ import {
   SearchPlusIcon,
   ExpandArrowsAltIcon,
   ExpandIcon,
+  RedoAltIcon,
 } from '@patternfly/react-icons';
 
 const Header = ({
@@ -26,6 +27,7 @@ const Header = ({
   zoomOut,
   resetZoom,
   zoomFit,
+  refresh,
   showZoomControls,
 }) => {
   const { light } = PageSectionVariants;
@@ -48,6 +50,18 @@ const Header = ({
           </Title>
         </div>
         <div>
+          <Tooltip content={t`Refresh`} position="top">
+            <Button
+              ouiaId="refresh-button"
+              aria-label={t`Refresh`}
+              variant="plain"
+              icon={<RedoAltIcon />}
+              onClick={refresh}
+              isDisabled={!showZoomControls}
+            >
+              <RedoAltIcon />
+            </Button>
+          </Tooltip>
           <Tooltip content={t`Zoom in`} position="top">
             <Button
               ouiaId="zoom-in-button"

awx/ui/src/screens/TopologyView/Legend.js

@@ -245,7 +245,7 @@ function Legend() {
                 y1="9"
                 x2="20"
                 y2="9"
-                stroke="#666"
+                stroke="#6A6E73"
                 strokeWidth="4"
               />
             </svg>
@@ -260,7 +260,7 @@ function Legend() {
                 y1="9"
                 x2="20"
                 y2="9"
-                stroke="#666"
+                stroke="#3E8635"
                 strokeWidth="4"
                 strokeDasharray="6"
               />

awx/ui/src/screens/TopologyView/MeshGraph.js

@@ -13,6 +13,7 @@ import Tooltip from './Tooltip';
 import ContentLoading from './ContentLoading';
 import {
   renderStateColor,
+  renderLinkStatusColor,
   renderLabelText,
   renderNodeType,
   renderNodeIcon,
@@ -177,7 +178,7 @@ function MeshGraph({
       mesh
         .append('defs')
         .selectAll('marker')
-        .data(['end', 'end-active'])
+        .data(['end', 'end-active', 'end-adding', 'end-removing'])
         .join('marker')
         .attr('id', String)
         .attr('viewBox', '0 -5 10 10')
@@ -187,8 +188,9 @@ function MeshGraph({
         .attr('orient', 'auto')
         .append('path')
         .attr('d', 'M0,-5L10,0L0,5');
-
-      mesh.select('#end').attr('refX', 23).attr('fill', '#ccc');
+      mesh.select('#end').attr('refX', 23).attr('fill', '#6A6E73');
+      mesh.select('#end-removing').attr('refX', 23).attr('fill', '#C9190B');
+      mesh.select('#end-adding').attr('refX', 23).attr('fill', '#3E8635');
       mesh.select('#end-active').attr('refX', 18).attr('fill', '#0066CC');
 
       // Add links
@@ -204,18 +206,24 @@ function MeshGraph({
         .attr('y1', (d) => d.source.y)
         .attr('x2', (d) => d.target.x)
         .attr('y2', (d) => d.target.y)
-        .attr('marker-end', 'url(#end)')
+        .attr('marker-end', (d) => {
+          if (d.link_state === 'adding') {
+            return 'url(#end-adding)';
+          }
+          if (d.link_state === 'removing') {
+            return 'url(#end-removing)';
+          }
+          return 'url(#end)';
+        })
         .attr('class', (_, i) => `link-${i}`)
         .attr('data-cy', (d) => `${d.source.hostname}-${d.target.hostname}`)
         .style('fill', 'none')
-        .style('stroke', (d) =>
-          d.link_state === 'removing' ? '#C9190B' : '#CCC'
-        )
+        .style('stroke', (d) => renderLinkStatusColor(d.link_state))
         .style('stroke-width', '2px')
         .style('stroke-dasharray', (d) => renderLinkState(d.link_state))
         .attr('pointer-events', 'none')
         .on('mouseover', function showPointer() {
-          d3.select(this).transition().style('cursor', 'pointer');
+          d3.select(this).style('cursor', 'pointer');
         });
       // add nodes
       const node = mesh
@@ -228,7 +236,7 @@ function MeshGraph({
         .append('g')
         .attr('data-cy', (d) => `node-${d.id}`)
         .on('mouseenter', function handleNodeHover(_, d) {
-          d3.select(this).transition().style('cursor', 'pointer');
+          d3.select(this).style('cursor', 'pointer');
           highlightSiblings(d);
         })
         .on('mouseleave', (_, d) => {
@@ -239,7 +247,8 @@ function MeshGraph({
         });
 
       // node circles
-      node
+      const nodeCircles = node.append('g');
+      nodeCircles
         .append('circle')
         .attr('r', DEFAULT_RADIUS)
         .attr('cx', (d) => d.x)
@@ -248,7 +257,8 @@ function MeshGraph({
         .attr('class', (d) => `id-${d.id}`)
         .attr('fill', DEFAULT_NODE_COLOR)
         .attr('stroke-dasharray', (d) => (d.enabled ? `1 0` : `5`))
-        .attr('stroke', DEFAULT_NODE_STROKE_COLOR);
+        .attr('stroke', (d) => renderStateColor(d.node_state));
+
       // node type labels
       node
         .append('text')
@@ -259,64 +269,62 @@ function MeshGraph({
         .attr('dominant-baseline', 'central')
         .attr('fill', DEFAULT_NODE_SYMBOL_TEXT_COLOR);
 
-      const placeholder = node.append('g').attr('class', 'placeholder');
-
-      placeholder
-        .append('text')
-        .text((d) => renderLabelText(d.node_state, d.hostname))
-        .attr('x', (d) => d.x)
-        .attr('y', (d) => d.y + 40)
-        .attr('fill', 'black')
-        .attr('font-size', '18px')
-        .attr('text-anchor', 'middle')
-        .each(function calculateLabelWidth() {
-          // eslint-disable-next-line react/no-this-in-sfc
-          const bbox = this.getBBox();
-          // eslint-disable-next-line react/no-this-in-sfc
-          d3.select(this.parentNode)
-            .append('path')
-            .attr('d', (d) => renderLabelIcons(d.node_state))
-            .attr('transform', (d) => renderIconPosition(d.node_state, bbox))
-            .style('fill', 'black');
-        });
-
-      placeholder.each(function calculateLabelWidth() {
-        // eslint-disable-next-line react/no-this-in-sfc
-        const bbox = this.getBBox();
-        // eslint-disable-next-line react/no-this-in-sfc
-        d3.select(this.parentNode)
-          .append('rect')
-          .attr('x', (d) => d.x - bbox.width / 2)
-          .attr('y', bbox.y + 5)
-          .attr('width', bbox.width)
-          .attr('height', bbox.height)
-          .attr('rx', 8)
-          .attr('ry', 8)
-          .style('fill', (d) => renderStateColor(d.node_state));
-      });
-
-      const hostNames = node.append('g');
+      // node hostname labels
+      const hostNames = node.append('g').attr('class', 'node-state-label');
       hostNames
         .append('text')
+        .attr('x', (d) => d.x)
+        .attr('y', (d) => d.y + 40)
         .text((d) => renderLabelText(d.node_state, d.hostname))
-        .attr('x', (d) => d.x + 6)
-        .attr('y', (d) => d.y + 42)
+        .attr('class', 'placeholder')
         .attr('fill', 'white')
-        .attr('font-size', DEFAULT_FONT_SIZE)
         .attr('text-anchor', 'middle')
         .each(function calculateLabelWidth() {
           // eslint-disable-next-line react/no-this-in-sfc
           const bbox = this.getBBox();
+          const padding = 10;
           // eslint-disable-next-line react/no-this-in-sfc
           d3.select(this.parentNode)
-            .append('path')
-            .attr('class', (d) => `icon-${d.node_state}`)
-            .attr('d', (d) => renderLabelIcons(d.node_state))
-            .attr('transform', (d) => renderIconPosition(d.node_state, bbox))
-            .attr('fill', 'white');
+            .append('rect')
+            .attr('x', bbox.x - padding / 2)
+            .attr('y', bbox.y)
+            .attr('width', bbox.width + padding)
+            .attr('height', bbox.height)
+            .style('stroke-width', 1)
+            .attr('rx', 4)
+            .attr('ry', 4)
+            .attr('fill', 'white')
+            .style('stroke', DEFAULT_NODE_STROKE_COLOR);
         });
-      svg.selectAll('g.placeholder').remove();
+      svg.selectAll('text.placeholder').remove();
+      hostNames
+        .append('text')
+        .attr('x', (d) => d.x)
+        .attr('y', (d) => d.y + 38)
+        .text((d) => renderLabelText(d.node_state, d.hostname))
+        .attr('font-size', DEFAULT_FONT_SIZE)
+        .attr('fill', 'black')
+        .attr('text-anchor', 'middle');
 
+      // add badge icons
+      const badges = nodeCircles.append('g').attr('class', 'node-state-badge');
+      badges.each(function drawStateBadge() {
+        // eslint-disable-next-line react/no-this-in-sfc
+        const bbox = this.parentNode.getBBox();
+        // eslint-disable-next-line react/no-this-in-sfc
+        d3.select(this)
+          .append('circle')
+          .attr('r', 9)
+          .attr('cx', bbox.x)
+          .attr('cy', bbox.y)
+          .attr('fill', (d) => renderStateColor(d.node_state));
+        d3.select(this)
+          .append('path')
+          .attr('class', (d) => `icon-${d.node_state}`)
+          .attr('d', (d) => renderLabelIcons(d.node_state))
+          .attr('transform', (d) => renderIconPosition(d.node_state, bbox))
+          .attr('fill', 'white');
+      });
       svg.call(zoom);
 
       function highlightSiblings(n) {
@@ -330,7 +338,6 @@ function MeshGraph({
         immediate.forEach((s) => {
           svg
             .selectAll(`.link-${s.index}`)
-            .transition()
             .style('stroke', '#0066CC')
             .style('stroke-width', '3px')
             .attr('marker-end', 'url(#end-active)');
@@ -346,13 +353,17 @@ function MeshGraph({
         immediate.forEach((s) => {
           svg
             .selectAll(`.link-${s.index}`)
-            .transition()
-            .duration(50)
-            .style('stroke', (d) =>
-              d.link_state === 'removing' ? '#C9190B' : '#CCC'
-            )
+            .style('stroke', (d) => renderLinkStatusColor(d.link_state))
             .style('stroke-width', '2px')
-            .attr('marker-end', 'url(#end)');
+            .attr('marker-end', (d) => {
+              if (d.link_state === 'adding') {
+                return 'url(#end-adding)';
+              }
+              if (d.link_state === 'removing') {
+                return 'url(#end-removing)';
+              }
+              return 'url(#end)';
+            });
         });
       }
 
@@ -361,7 +372,7 @@ function MeshGraph({
           // toggle rings
           svg
             .select(`circle.id-${n.id}`)
-            .attr('stroke', '#ccc')
+            .attr('stroke', (d) => renderStateColor(d.node_state))
             .attr('stroke-width', null);
           // show default empty state of tooltip
           setIsNodeSelected(false);
@@ -370,7 +381,7 @@ function MeshGraph({
         }
         svg
           .selectAll('circle')
-          .attr('stroke', '#ccc')
+          .attr('stroke', (d) => renderStateColor(d.node_state))
           .attr('stroke-width', null);
         svg
           .select(`circle.id-${n.id}`)

awx/ui/src/screens/TopologyView/TopologyView.js

@@ -45,6 +45,7 @@ function TopologyView() {
         zoomIn={zoomIn}
         zoomOut={zoomOut}
         zoomFit={zoomFit}
+        refresh={fetchMeshVisualizer}
         resetZoom={resetZoom}
         showZoomControls={showZoomControls}
       />

awx/ui/src/screens/TopologyView/constants.js

@@ -13,7 +13,7 @@ export const DEFAULT_NODE_COLOR = 'white';
 export const DEFAULT_NODE_HIGHLIGHT_COLOR = '#eee';
 export const DEFAULT_NODE_LABEL_TEXT_COLOR = 'white';
 export const DEFAULT_NODE_SYMBOL_TEXT_COLOR = 'black';
-export const DEFAULT_NODE_STROKE_COLOR = '#ccc';
+export const DEFAULT_NODE_STROKE_COLOR = '#6A6E73';
 export const DEFAULT_FONT_SIZE = '12px';
 export const LABEL_TEXT_MAX_LENGTH = 15;
 export const MARGIN = 15;
@@ -27,6 +27,12 @@ export const NODE_STATE_COLOR_KEY = {
   deprovisioning: '#666',
 };
 
+export const LINK_STATE_COLOR_KEY = {
+  established: '#6A6E73',
+  adding: '#3E8635',
+  removing: '#C9190B',
+};
+
 export const NODE_TYPE_SYMBOL_KEY = {
   hop: 'h',
   execution: 'Ex',

awx/ui/src/screens/TopologyView/utils/helpers.js

@@ -4,6 +4,7 @@ import { truncateString } from '../../../util/strings';
 import {
   NODE_STATE_COLOR_KEY,
   NODE_TYPE_SYMBOL_KEY,
+  LINK_STATE_COLOR_KEY,
   LABEL_TEXT_MAX_LENGTH,
   ICONS,
 } from '../constants';
@@ -20,6 +21,12 @@ export function renderStateColor(nodeState) {
   return NODE_STATE_COLOR_KEY[nodeState] ? NODE_STATE_COLOR_KEY[nodeState] : '';
 }
 
+export function renderLinkStatusColor(linkState) {
+  return LINK_STATE_COLOR_KEY[linkState]
+    ? LINK_STATE_COLOR_KEY[linkState]
+    : '#ccc';
+}
+
 export function renderLabelText(nodeState, name) {
   if (typeof nodeState === 'string' && typeof name === 'string') {
     return `${truncateString(name, LABEL_TEXT_MAX_LENGTH)}`;
@@ -45,8 +52,8 @@ export function renderLabelIcons(nodeState) {
       ready: 'checkmark',
       installed: 'clock',
       unavailable: 'exclaimation',
-      'provision-fail': 'exclaimation',
       'deprovision-fail': 'exclaimation',
+      'provision-fail': 'exclaimation',
       provisioning: 'plus',
       deprovisioning: 'minus',
     };
@@ -59,15 +66,17 @@ export function renderLabelIcons(nodeState) {
 export function renderIconPosition(nodeState, bbox) {
   if (nodeState) {
     const iconPositionMapper = {
-      ready: `translate(${bbox.x - 15}, ${bbox.y + 3}), scale(0.02)`,
-      installed: `translate(${bbox.x - 18}, ${bbox.y + 1}), scale(0.03)`,
-      unavailable: `translate(${bbox.x - 9}, ${bbox.y + 3}), scale(0.02)`,
-      'provision-fail': `translate(${bbox.x - 9}, ${bbox.y + 3}), scale(0.02)`,
-      'deprovision-fail': `translate(${bbox.x - 9}, ${
-        bbox.y + 3
+      ready: `translate(${bbox.x - 4.5}, ${bbox.y - 4.5}), scale(0.02)`,
+      installed: `translate(${bbox.x - 6.5}, ${bbox.y - 6.5}), scale(0.025)`,
+      unavailable: `translate(${bbox.x - 2}, ${bbox.y - 4.4}), scale(0.02)`,
+      'provision-fail': `translate(${bbox.x - 2}, ${bbox.y - 4}), scale(0.02)`,
+      'deprovision-fail': `translate(${bbox.x - 2}, ${
+        bbox.y - 4
+      }), scale(0.02)`,
+      provisioning: `translate(${bbox.x - 4.5}, ${bbox.y - 4.5}), scale(0.02)`,
+      deprovisioning: `translate(${bbox.x - 4.5}, ${
+        bbox.y - 4.5
       }), scale(0.02)`,
-      provisioning: `translate(${bbox.x - 12}, ${bbox.y + 3}), scale(0.02)`,
-      deprovisioning: `translate(${bbox.x - 12}, ${bbox.y + 3}), scale(0.02)`,
     };
     return iconPositionMapper[nodeState] ? iconPositionMapper[nodeState] : ``;
   }
@@ -101,7 +110,7 @@ export function getRandomInt(min, max) {
 const generateRandomLinks = (n, r) => {
   const links = [];
   function getRandomLinkState() {
-    return ['established', 'adding', 'removing'][getRandomInt(0, 2)];
+    return ['established', 'adding', 'removing'][getRandomInt(0, 3)];
   }
   for (let i = 0; i < r; i++) {
     const link = {
@@ -142,7 +151,7 @@ export const generateRandomNodes = (n) => {
       hostname: `node-${id}`,
       node_type: randomType,
       node_state: randomState,
-      enabled: Math.random() < 0.5,
+      enabled: Math.random() < 0.9,
     };
     nodes.push(node);
   }

awx/ui/src/screens/TopologyView/utils/helpers__RTL.test.js

@@ -73,7 +73,7 @@ describe('renderIconPosition', () => {
   const bbox = { x: 400, y: 400, width: 10, height: 20 };
   test('returns correct label icon', () => {
     expect(renderIconPosition('ready', bbox)).toBe(
-      `translate(${bbox.x - 15}, ${bbox.y + 3}), scale(0.02)`
+      `translate(${bbox.x - 4.5}, ${bbox.y - 4.5}), scale(0.02)`
     );
   });
   test('returns empty string if state is not found', () => {

awx/ui/src/types.js

@@ -122,7 +122,7 @@ export const InstanceGroup = shape({
 
 export const Instance = shape({
   id: number.isRequired,
-  name: string.isRequired,
+  hostname: string.isRequired,
 });
 
 export const Label = shape({

awx/ui_next/Makefile

@@ -5,10 +5,13 @@ UI_NEXT_DIR := $(patsubst %/,%,$(dir $(lastword $(MAKEFILE_LIST))))
 #  NOTE: you will not be able to build within the docker-compose development environment if you use this option
 UI_NEXT_LOCAL ?=
 
-# Git repo and branch to the UI_NEXT repo
+## Git repo and branch to the UI_NEXT repo
 UI_NEXT_GIT_REPO ?= https://github.com/ansible/ansible-ui.git
 UI_NEXT_GIT_BRANCH ?= main
 
+## Product name to display on the UI used in UI_NEXT build process
+PRODUCT ?= AWX
+
 .PHONY: ui-next
 ## Default build target of ui-next Makefile, builds ui-next/build
 ui-next: ui-next/build
@@ -32,7 +35,8 @@ ui-next/src/build: $(UI_NEXT_DIR)/src/build/awx
 ## True target for ui-next/src/build. Build ui_next from source.
 $(UI_NEXT_DIR)/src/build/awx: $(UI_NEXT_DIR)/src $(UI_NEXT_DIR)/src/node_modules/webpack
 	@echo "=== Building ui_next ==="
-	@cd $(UI_NEXT_DIR)/src && npm run build:awx
+	@cd $(UI_NEXT_DIR)/src && PRODUCT="$(PRODUCT)" PUBLIC_PATH=/static/awx/ npm run build:awx
+	@mv $(UI_NEXT_DIR)/src/build/awx/index.html $(UI_NEXT_DIR)/src/build/awx/index_awx.html
 
 .PHONY: ui-next/src
 ## Clone or link src of UI_NEXT to ui-next/src, will re-clone/link/update if necessary.

awx_collection/plugins/doc_fragments/auth.py

@@ -50,6 +50,11 @@ options:
     - If value not set, will try environment variable C(CONTROLLER_VERIFY_SSL) and then config files
     type: bool
     aliases: [ tower_verify_ssl ]
+  request_timeout:
+    description:
+    - Specify the timeout Ansible should use in requests to the controller host.
+    - Defaults to 10s, but this is handled by the shared module_utils code
+    type: float
   controller_config_file:
     description:
     - Path to the controller config file.

awx_collection/plugins/doc_fragments/auth_plugin.py

@@ -68,6 +68,14 @@ options:
         why: Collection name change
         alternatives: 'CONTROLLER_VERIFY_SSL'
     aliases: [ validate_certs ]
+  request_timeout:
+    description:
+    - Specify the timeout Ansible should use in requests to the controller host.
+    - Defaults to 10 seconds
+    - This will not work with the export or import modules.
+    type: float
+    env:
+    - name: CONTROLLER_REQUEST_TIMEOUT
 
 notes:
 - If no I(config_file) is provided we will attempt to use the tower-cli library

awx_collection/plugins/lookup/schedule_rruleset.py

@@ -214,7 +214,7 @@ class LookupModule(LookupBase):
         if not isinstance(rule[field_name], list):
             rule[field_name] = rule[field_name].split(',')
         for value in rule[field_name]:
-            value = value.strip()
+            value = value.strip().lower()
             if value not in valid_list:
                 raise AnsibleError('In rule {0} {1} must only contain values in {2}'.format(rule_number, field_name, ', '.join(valid_list.keys())))
             return_values.append(valid_list[value])

awx_collection/plugins/module_utils/controller_api.py

@@ -51,6 +51,7 @@ class ControllerModule(AnsibleModule):
         controller_username=dict(required=False, aliases=['tower_username'], fallback=(env_fallback, ['CONTROLLER_USERNAME', 'TOWER_USERNAME'])),
         controller_password=dict(no_log=True, aliases=['tower_password'], required=False, fallback=(env_fallback, ['CONTROLLER_PASSWORD', 'TOWER_PASSWORD'])),
         validate_certs=dict(type='bool', aliases=['tower_verify_ssl'], required=False, fallback=(env_fallback, ['CONTROLLER_VERIFY_SSL', 'TOWER_VERIFY_SSL'])),
+        request_timeout=dict(type='float', required=False, fallback=(env_fallback, ['CONTROLLER_REQUEST_TIMEOUT'])),
         controller_oauthtoken=dict(
             type='raw', no_log=True, aliases=['tower_oauthtoken'], required=False, fallback=(env_fallback, ['CONTROLLER_OAUTH_TOKEN', 'TOWER_OAUTH_TOKEN'])
         ),
@@ -63,12 +64,14 @@ class ControllerModule(AnsibleModule):
         'username': 'controller_username',
         'password': 'controller_password',
         'verify_ssl': 'validate_certs',
+        'request_timeout': 'request_timeout',
         'oauth_token': 'controller_oauthtoken',
     }
     host = '127.0.0.1'
     username = None
     password = None
     verify_ssl = True
+    request_timeout = 10
     oauth_token = None
     oauth_token_id = None
     authenticated = False
@@ -304,7 +307,7 @@ class ControllerAPIModule(ControllerModule):
         kwargs['supports_check_mode'] = True
 
         super().__init__(argument_spec=argument_spec, direct_params=direct_params, error_callback=error_callback, warn_callback=warn_callback, **kwargs)
-        self.session = Request(cookies=CookieJar(), validate_certs=self.verify_ssl)
+        self.session = Request(cookies=CookieJar(), timeout=self.request_timeout, validate_certs=self.verify_ssl)
 
         if 'update_secrets' in self.params:
             self.update_secrets = self.params.pop('update_secrets')
@@ -500,7 +503,14 @@ class ControllerAPIModule(ControllerModule):
             data = dumps(kwargs.get('data', {}))
 
         try:
-            response = self.session.open(method, url.geturl(), headers=headers, validate_certs=self.verify_ssl, follow_redirects=True, data=data)
+            response = self.session.open(
+                method, url.geturl(),
+                headers=headers,
+                timeout=self.request_timeout,
+                validate_certs=self.verify_ssl,
+                follow_redirects=True,
+                data=data
+            )
         except (SSLValidationError) as ssl_err:
             self.fail_json(msg="Could not establish a secure connection to your host ({1}): {0}.".format(url.netloc, ssl_err))
         except (ConnectionError) as con_err:
@@ -612,6 +622,7 @@ class ControllerAPIModule(ControllerModule):
                     'POST',
                     api_token_url,
                     validate_certs=self.verify_ssl,
+                    timeout=self.request_timeout,
                     follow_redirects=True,
                     force_basic_auth=True,
                     url_username=self.username,
@@ -988,6 +999,7 @@ class ControllerAPIModule(ControllerModule):
                     'DELETE',
                     api_token_url,
                     validate_certs=self.verify_ssl,
+                    timeout=self.request_timeout,
                     follow_redirects=True,
                     force_basic_auth=True,
                     url_username=self.username,

awx_collection/plugins/modules/credential.py

@@ -46,6 +46,7 @@ options:
     organization:
       description:
         - Organization name, ID, or named URL that should own the credential.
+        - This parameter is mutually exclusive with C(team) and C(user).
       type: str
     credential_type:
       description:
@@ -93,10 +94,12 @@ options:
     user:
       description:
         - User name, ID, or named URL that should own this credential.
+        - This parameter is mutually exclusive with C(organization) and C(team).
       type: str
     team:
       description:
         - Team name, ID, or named URL that should own this credential.
+        - This parameter is mutually exclusive with C(organization) and C(user).
       type: str
     state:
       description:
@@ -219,8 +222,13 @@ def main():
         state=dict(choices=['present', 'absent', 'exists'], default='present'),
     )
 
+    mutually_exclusive = [("organization", "user", "team")]
+
     # Create a module for ourselves
-    module = ControllerAPIModule(argument_spec=argument_spec)
+    module = ControllerAPIModule(
+        argument_spec=argument_spec,
+        mutually_exclusive=mutually_exclusive
+    )
 
     # Extract our parameters
     name = module.params.get('name')

awx_collection/plugins/modules/instance.py

@@ -47,6 +47,7 @@ options:
         - Role that this node plays in the mesh.
       choices:
         - execution
+        - hop
       required: False
       type: str
     node_state:
@@ -62,6 +63,18 @@ options:
         - Port that Receptor will listen for incoming connections on.
       required: False
       type: int
+    peers:
+      description:
+        - List of peers to connect outbound to. Only configurable for hop and execution nodes.
+        - To remove all current peers, set value to an empty list, [].
+      required: False
+      type: list
+      elements: str
+    peers_from_control_nodes:
+      description:
+        - If enabled, control plane nodes will automatically peer to this node.
+      required: False
+      type: bool
 extends_documentation_fragment: awx.awx.auth
 '''
 
@@ -88,9 +101,11 @@ def main():
         capacity_adjustment=dict(type='float'),
         enabled=dict(type='bool'),
         managed_by_policy=dict(type='bool'),
-        node_type=dict(type='str', choices=['execution']),
+        node_type=dict(type='str', choices=['execution', 'hop']),
         node_state=dict(type='str', choices=['deprovisioning', 'installed']),
         listener_port=dict(type='int'),
+        peers=dict(required=False, type='list', elements='str'),
+        peers_from_control_nodes=dict(required=False, type='bool'),
     )
 
     # Create a module for ourselves
@@ -104,7 +119,8 @@ def main():
     node_type = module.params.get('node_type')
     node_state = module.params.get('node_state')
     listener_port = module.params.get('listener_port')
-
+    peers = module.params.get('peers')
+    peers_from_control_nodes = module.params.get('peers_from_control_nodes')
     # Attempt to look up an existing item based on the provided data
     existing_item = module.get_one('instances', name_or_id=hostname)
 
@@ -122,6 +138,10 @@ def main():
         new_fields['node_state'] = node_state
     if listener_port is not None:
         new_fields['listener_port'] = listener_port
+    if peers is not None:
+        new_fields['peers'] = peers
+    if peers_from_control_nodes is not None:
+        new_fields['peers_from_control_nodes'] = peers_from_control_nodes
 
     module.create_or_update_if_needed(
         existing_item,

awx_collection/plugins/modules/license.py

@@ -52,7 +52,12 @@ EXAMPLES = '''
   license:
     manifest: "/tmp/my_manifest.zip"
 
-- name: Attach to a pool
+- name: Use the subscriptions module to fetch subscriptions from Red Hat or Red Hat Satellite
+  subscriptions:
+    username: "my_satellite_username"
+    password: "my_satellite_password"
+
+- name: Attach to a pool (requires fetching subscriptions at least once before)
   license:
     pool_id: 123456
 

awx_collection/plugins/modules/project.py

@@ -83,7 +83,7 @@ options:
       type: bool
     scm_update_on_launch:
       description:
-        - Before an update to the local repository before launching a job with this project.
+        - Perform an update to the local repository before launching a job with this project.
       type: bool
     scm_update_cache_timeout:
       description:

awx_collection/test/awx/test_instance.py

@@ -0,0 +1,53 @@
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+import pytest
+
+from awx.main.models import Instance
+from django.test.utils import override_settings
+
+
+@pytest.mark.django_db
+def test_peers_adding_and_removing(run_module, admin_user):
+    with override_settings(IS_K8S=True):
+        result = run_module(
+            'instance',
+            {'hostname': 'hopnode1', 'node_type': 'hop', 'peers_from_control_nodes': True, 'node_state': 'installed', 'listener_port': 27199},
+            admin_user,
+        )
+        assert result['changed']
+
+        hop_node_1 = Instance.objects.get(pk=result.get('id'))
+
+        assert hop_node_1.peers_from_control_nodes is True
+        assert hop_node_1.node_type == 'hop'
+
+        result = run_module(
+            'instance',
+            {'hostname': 'hopnode2', 'node_type': 'hop', 'peers_from_control_nodes': True, 'node_state': 'installed', 'listener_port': 27199},
+            admin_user,
+        )
+        assert result['changed']
+
+        hop_node_2 = Instance.objects.get(pk=result.get('id'))
+
+        result = run_module(
+            'instance',
+            {'hostname': 'executionnode', 'node_type': 'execution', 'node_state': 'installed', 'listener_port': 27199, 'peers': ['hopnode1', 'hopnode2']},
+            admin_user,
+        )
+        assert result['changed']
+
+        execution_node = Instance.objects.get(pk=result.get('id'))
+
+        assert set(execution_node.peers.all()) == {hop_node_1, hop_node_2}
+
+        result = run_module(
+            'instance',
+            {'hostname': 'executionnode', 'node_type': 'execution', 'node_state': 'installed', 'listener_port': 27199, 'peers': []},
+            admin_user,
+        )
+
+        assert result['changed']
+        assert set(execution_node.peers.all()) == set()

awx_collection/tests/integration/targets/instance/tasks/main.yml

@@ -70,3 +70,60 @@
         - "{{ hostname3 }}"
 
   when: IS_K8S
+
+- block:
+    - name: Create hop node 1
+      awx.awx.instance:
+        hostname: hopnode1
+        node_type: hop
+        node_state: installed
+        listener_port: 27199
+        peers_from_control_nodes: True
+      register: result
+
+    - assert:
+        that:
+          - result is changed
+
+    - name: Create hop node 2
+      awx.awx.instance:
+        hostname: hopnode2
+        node_type: hop
+        node_state: installed
+        listener_port: 27199
+        peers_from_control_nodes: True
+      register: result
+
+    - assert:
+        that:
+          - result is changed
+
+    - name: Create execution node
+      awx.awx.instance:
+        hostname: executionnode
+        node_type: execution
+        node_state: installed
+        listener_port: 27199
+        peers:
+          - "hopnode1"
+          - "hopnode2"
+      register: result
+
+    - assert:
+        that:
+          - result is changed
+
+    - name: Remove execution node peers
+      awx.awx.instance:
+        hostname: executionnode
+        node_type: execution
+        node_state: installed
+        listener_port: 27199
+        peers: []
+      register: result
+
+    - assert:
+        that:
+          - result is changed
+
+  when: IS_K8S

awx_collection/tests/integration/targets/lookup_rruleset/tasks/main.yml

@@ -356,3 +356,19 @@
     that:
       - results is success
       - "'DTSTART;TZID=UTC:20220430T103045 RRULE:FREQ=MONTHLY;BYMONTHDAY=12,13,14,15,16,17,18;BYDAY=SA;INTERVAL=1' == complex_rule"
+
+- name: mondays, Tuesdays, and WEDNESDAY with case-insensitivity
+  set_fact:
+    complex_rule: "{{ query(ruleset_plugin_name, '2022-04-30 10:30:45', rules=rrules, timezone='UTC' ) }}"
+  ignore_errors: True
+  register: results
+  vars:
+    rrules:
+      - frequency: 'day'
+        interval: 1
+        byweekday: monday, Tuesday, WEDNESDAY
+
+- assert:
+    that:
+      - results is success
+      - "'DTSTART;TZID=UTC:20220430T103045 RRULE:FREQ=DAILY;BYDAY=MO,TU,WE;INTERVAL=1' == complex_rule"

awx_collection/tests/integration/targets/project/tasks/main.yml

@@ -53,6 +53,23 @@
         that:
           - result is not changed
 
+    - name: Create a git project and wait with short request timeout.
+      project:
+        name: "{{ project_name1 }}"
+        organization: Default
+        scm_type: git
+        scm_url: https://github.com/ansible/test-playbooks
+        wait: true
+        state: exists
+        request_timeout: .001
+      register: result
+      ignore_errors: true
+
+    - assert:
+        that:
+          - result is failed
+          - "'timed out' in result.msg"
+
     - name: Delete a git project without credentials and wait
       project:
         name: "{{ project_name1 }}"

awxkit/awxkit/api/pages/api.py

@@ -162,7 +162,7 @@ class ApiV2(base.Base):
                 export_key = 'create_approval_template'
                 rel_option_endpoint = _page.related.get('create_approval_template')
 
-            rel_post_fields = self._cache.get_post_fields(rel_option_endpoint)
+            rel_post_fields = utils.get_post_fields(rel_option_endpoint, self._cache)
             if rel_post_fields is None:
                 log.debug("%s is a read-only endpoint.", rel_endpoint)
                 continue
@@ -202,7 +202,7 @@ class ApiV2(base.Base):
         return utils.remove_encrypted(fields)
 
     def _export_list(self, endpoint):
-        post_fields = self._cache.get_post_fields(endpoint)
+        post_fields = utils.get_post_fields(endpoint, self._cache)
         if post_fields is None:
             return None
 
@@ -267,7 +267,7 @@ class ApiV2(base.Base):
 
     def _import_list(self, endpoint, assets):
         log.debug("_import_list -- endpoint: %s, assets: %s", endpoint.endpoint, repr(assets))
-        post_fields = self._cache.get_post_fields(endpoint)
+        post_fields = utils.get_post_fields(endpoint, self._cache)
 
         changed = False
 

awxkit/awxkit/api/pages/page.py

@@ -495,7 +495,6 @@ class TentativePage(str):
 class PageCache(object):
     def __init__(self):
         self.options = {}
-        self.post_fields = {}
         self.pages_by_url = {}
         self.pages_by_natural_key = {}
 
@@ -517,29 +516,6 @@ class PageCache(object):
 
         return self.options.setdefault(url, options)
 
-    def get_post_fields(self, page):
-        url = page.endpoint if isinstance(page, Page) else str(page)
-        key = get_registered_page(url)
-        if key in self.post_fields:
-            return self.post_fields[key]
-
-        options_page = self.get_options(page)
-
-        if options_page is None:
-            return None
-
-        if 'POST' not in options_page.r.headers.get('Allow', ''):
-            return None
-
-        if 'POST' in options_page.json['actions']:
-            post_fields = options_page.json['actions']['POST']
-        else:
-            log.warning("Insufficient privileges on %s, inferring POST fields from description.", options_page.endpoint)
-            post_fields = utils.parse_description(options_page.json['description'])
-        self.post_fields[key] = post_fields
-
-        return post_fields
-
     def set_page(self, page):
         log.debug("set_page: %s %s", type(page), page.endpoint)
         self.pages_by_url[page.endpoint] = page

awxkit/awxkit/api/utils.py

@@ -31,3 +31,18 @@ def remove_encrypted(value):
     if isinstance(value, dict):
         return {k: remove_encrypted(v) for k, v in value.items()}
     return value
+
+
+def get_post_fields(page, cache):
+    options_page = cache.get_options(page)
+    if options_page is None:
+        return None
+
+    if 'POST' not in options_page.r.headers.get('Allow', ''):
+        return None
+
+    if 'POST' in options_page.json['actions']:
+        return options_page.json['actions']['POST']
+    else:
+        log.warning("Insufficient privileges on %s, inferring POST fields from description.", options_page.endpoint)
+        return parse_description(options_page.json['description'])