make the default JOB_EVENT_BUFFER_SECONDS 1 seconds (#14335)

Dispatcher refactoring to get pg_notify publish payload
  as separate method

Refactor periodic module under dispatcher entirely
  Use real numbers for schedule reference time
  Run based on due_to_run method

Review comments about naming and code comments

.github/pr_labeler.yml

@@ -15,5 +15,5 @@
 
 "dependencies":
   - any: ["awx/ui/package.json"]
-  - any: ["awx/requirements/*.txt"]
-  - any: ["awx/requirements/requirements.in"]
+  - any: ["requirements/*.txt"]
+  - any: ["requirements/requirements.in"]

.github/workflows/devel_images.yml

@@ -48,8 +48,11 @@ jobs:
           DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-dev-build
           DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} COMPOSE_TAG=${GITHUB_REF##*/} make awx-kube-build
 
-      - name: Push image
+      - name: Push development images
         run: |
           docker push ghcr.io/${OWNER_LC}/awx_devel:${GITHUB_REF##*/}
           docker push ghcr.io/${OWNER_LC}/awx_kube_devel:${GITHUB_REF##*/}
-          docker push ghcr.io/${OWNER_LC}/awx:${GITHUB_REF##*/}
+
+      - name: Push AWX k8s image, only for upstream and feature branches
+        run: docker push ghcr.io/${OWNER_LC}/awx:${GITHUB_REF##*/}
+        if: endsWith(github.repository, '/awx')

.github/workflows/pr_body_check_jira.yml

@@ -0,0 +1,35 @@
+---
+name: Check body for reference to jira
+on:
+  pull_request:
+    branches:
+      - release_**
+jobs:
+  pr-check:
+    if: github.repository_owner == 'ansible' && github.repository != 'awx'
+    name: Scan PR description for JIRA links
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Check for JIRA lines
+        env:
+          PR_BODY: ${{ github.event.pull_request.body }}
+        run: |
+          echo "$PR_BODY" | grep "JIRA: None" > no_jira
+          echo "$PR_BODY" | grep "JIRA: https://.*[0-9]+"> jira
+          exit 0
+        # We exit 0 and set the shell to prevent the returns from the greps from failing this step
+        # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
+        shell: bash {0}
+
+      - name: Check for exactly one item
+        run: |
+          if [ $(cat no_jira jira | wc -l) != 1 ] ; then
+            echo "The PR body must contain exactly one of [ 'JIRA: None' or 'JIRA: <one or more links>' ]"
+            echo "We counted $(cat no_jira jira | wc -l)"
+            exit 255;
+          else
+            exit 0;
+          fi

DATA_MIGRATION.md

@@ -4,6 +4,6 @@
 
 Early versions of AWX did not support seamless upgrades between major versions and required the use of a backup and restore tool to perform upgrades.
 
-Users who wish to upgrade modern AWX installations should follow the instructions at:
+As of version 18.0, `awx-operator` is the preferred install/upgrade method. Users who wish to upgrade modern AWX installations should follow the instructions at:
 
-https://github.com/ansible/awx/blob/devel/INSTALL.md#upgrading-from-previous-versions
+https://github.com/ansible/awx-operator/blob/devel/docs/upgrade/upgrading.md

Makefile

@@ -1,6 +1,7 @@
 -include awx/ui_next/Makefile
 
 PYTHON := $(notdir $(shell for i in python3.9 python3; do command -v $$i; done|sed 1q))
+SHELL := bash
 DOCKER_COMPOSE ?= docker-compose
 OFFICIAL ?= no
 NODE ?= node

awx/api/generics.py

@@ -232,7 +232,8 @@ class APIView(views.APIView):
 
         response = super(APIView, self).finalize_response(request, response, *args, **kwargs)
         time_started = getattr(self, 'time_started', None)
-        response['X-API-Product-Version'] = get_awx_version()
+        if request.user.is_authenticated:
+            response['X-API-Product-Version'] = get_awx_version()
         response['X-API-Product-Name'] = server_product_name()
 
         response['X-API-Node'] = settings.CLUSTER_HOST_ID

awx/api/urls/urls.py

@@ -30,7 +30,7 @@ from awx.api.views import (
     OAuth2TokenList,
     ApplicationOAuth2TokenList,
     OAuth2ApplicationDetail,
-    # HostMetricSummaryMonthlyList, # It will be enabled in future version of the AWX
+    HostMetricSummaryMonthlyList,
 )
 
 from awx.api.views.bulk import (
@@ -123,8 +123,7 @@ v2_urls = [
     re_path(r'^constructed_inventories/', include(constructed_inventory_urls)),
     re_path(r'^hosts/', include(host_urls)),
     re_path(r'^host_metrics/', include(host_metric_urls)),
-    # It will be enabled in future version of the AWX
-    # re_path(r'^host_metric_summary_monthly/$', HostMetricSummaryMonthlyList.as_view(), name='host_metric_summary_monthly_list'),
+    re_path(r'^host_metric_summary_monthly/$', HostMetricSummaryMonthlyList.as_view(), name='host_metric_summary_monthly_list'),
     re_path(r'^groups/', include(group_urls)),
     re_path(r'^inventory_sources/', include(inventory_source_urls)),
     re_path(r'^inventory_updates/', include(inventory_update_urls)),

awx/api/views/__init__.py

@@ -1564,16 +1564,15 @@ class HostMetricDetail(RetrieveDestroyAPIView):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-# It will be enabled in future version of the AWX
-# class HostMetricSummaryMonthlyList(ListAPIView):
-#     name = _("Host Metrics Summary Monthly")
-#     model = models.HostMetricSummaryMonthly
-#     serializer_class = serializers.HostMetricSummaryMonthlySerializer
-#     permission_classes = (IsSystemAdminOrAuditor,)
-#     search_fields = ('date',)
-#
-#     def get_queryset(self):
-#         return self.model.objects.all()
+class HostMetricSummaryMonthlyList(ListAPIView):
+    name = _("Host Metrics Summary Monthly")
+    model = models.HostMetricSummaryMonthly
+    serializer_class = serializers.HostMetricSummaryMonthlySerializer
+    permission_classes = (IsSystemAdminOrAuditor,)
+    search_fields = ('date',)
+
+    def get_queryset(self):
+        return self.model.objects.all()
 
 
 class HostList(HostRelatedSearchMixin, ListCreateAPIView):

awx/api/views/root.py

@@ -107,8 +107,7 @@ class ApiVersionRootView(APIView):
         data['groups'] = reverse('api:group_list', request=request)
         data['hosts'] = reverse('api:host_list', request=request)
         data['host_metrics'] = reverse('api:host_metric_list', request=request)
-        # It will be enabled in future version of the AWX
-        # data['host_metric_summary_monthly'] = reverse('api:host_metric_summary_monthly_list', request=request)
+        data['host_metric_summary_monthly'] = reverse('api:host_metric_summary_monthly_list', request=request)
         data['job_templates'] = reverse('api:job_template_list', request=request)
         data['jobs'] = reverse('api:job_list', request=request)
         data['ad_hoc_commands'] = reverse('api:ad_hoc_command_list', request=request)

awx/main/access.py

@@ -366,9 +366,9 @@ class BaseAccess(object):
             report_violation = lambda message: None
         else:
             report_violation = lambda message: logger.warning(message)
-        if validation_info.get('trial', False) is True or validation_info['instance_count'] == 10:  # basic 10 license
+        if validation_info.get('trial', False) is True:
 
-            def report_violation(message):
+            def report_violation(message):  # noqa
                 raise PermissionDenied(message)
 
         if check_expiration and validation_info.get('time_remaining', None) is None:

awx/main/analytics/collectors.py

@@ -613,3 +613,20 @@ def host_metric_table(since, full_path, until, **kwargs):
         since.isoformat(), until.isoformat(), since.isoformat(), until.isoformat()
     )
     return _copy_table(table='host_metric', query=host_metric_query, path=full_path)
+
+
+@register('host_metric_summary_monthly_table', '1.0', format='csv', description=_('HostMetricSummaryMonthly export, full sync'), expensive=trivial_slicing)
+def host_metric_summary_monthly_table(since, full_path, **kwargs):
+    query = '''
+    COPY (SELECT main_hostmetricsummarymonthly.id,
+                 main_hostmetricsummarymonthly.date,
+                 main_hostmetricsummarymonthly.license_capacity,
+                 main_hostmetricsummarymonthly.license_consumed,
+                 main_hostmetricsummarymonthly.hosts_added,
+                 main_hostmetricsummarymonthly.hosts_deleted,
+                 main_hostmetricsummarymonthly.indirectly_managed_hosts
+          FROM main_hostmetricsummarymonthly
+          ORDER BY main_hostmetricsummarymonthly.id ASC) TO STDOUT WITH CSV HEADER
+    '''
+
+    return _copy_table(table='host_metric_summary_monthly', query=query, path=full_path)

awx/main/cache.py

@@ -0,0 +1,87 @@
+import functools
+
+from django.conf import settings
+from django.core.cache.backends.base import DEFAULT_TIMEOUT
+from django.core.cache.backends.redis import RedisCache
+
+from redis.exceptions import ConnectionError, ResponseError, TimeoutError
+import socket
+
+# This list comes from what django-redis ignores and the behavior we are trying
+# to retain while dropping the dependency on django-redis.
+IGNORED_EXCEPTIONS = (TimeoutError, ResponseError, ConnectionError, socket.timeout)
+
+CONNECTION_INTERRUPTED_SENTINEL = object()
+
+
+def optionally_ignore_exceptions(func=None, return_value=None):
+    if func is None:
+        return functools.partial(optionally_ignore_exceptions, return_value=return_value)
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except IGNORED_EXCEPTIONS as e:
+            if settings.DJANGO_REDIS_IGNORE_EXCEPTIONS:
+                return return_value
+            raise e.__cause__ or e
+
+    return wrapper
+
+
+class AWXRedisCache(RedisCache):
+    """
+    We just want to wrap the upstream RedisCache class so that we can ignore
+    the exceptions that it raises when the cache is unavailable.
+    """
+
+    @optionally_ignore_exceptions
+    def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
+        return super().add(key, value, timeout, version)
+
+    @optionally_ignore_exceptions(return_value=CONNECTION_INTERRUPTED_SENTINEL)
+    def _get(self, key, default=None, version=None):
+        return super().get(key, default, version)
+
+    def get(self, key, default=None, version=None):
+        value = self._get(key, default, version)
+        if value is CONNECTION_INTERRUPTED_SENTINEL:
+            return default
+        return value
+
+    @optionally_ignore_exceptions
+    def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
+        return super().set(key, value, timeout, version)
+
+    @optionally_ignore_exceptions
+    def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None):
+        return super().touch(key, timeout, version)
+
+    @optionally_ignore_exceptions
+    def delete(self, key, version=None):
+        return super().delete(key, version)
+
+    @optionally_ignore_exceptions
+    def get_many(self, keys, version=None):
+        return super().get_many(keys, version)
+
+    @optionally_ignore_exceptions
+    def has_key(self, key, version=None):
+        return super().has_key(key, version)
+
+    @optionally_ignore_exceptions
+    def incr(self, key, delta=1, version=None):
+        return super().incr(key, delta, version)
+
+    @optionally_ignore_exceptions
+    def set_many(self, data, timeout=DEFAULT_TIMEOUT, version=None):
+        return super().set_many(data, timeout, version)
+
+    @optionally_ignore_exceptions
+    def delete_many(self, keys, version=None):
+        return super().delete_many(keys, version)
+
+    @optionally_ignore_exceptions
+    def clear(self):
+        return super().clear()

awx/main/conf.py

@@ -862,6 +862,15 @@ register(
     category_slug='system',
 )
 
+register(
+    'HOST_METRIC_SUMMARY_TASK_LAST_TS',
+    field_class=fields.DateTimeField,
+    label=_('Last computing date of HostMetricSummaryMonthly'),
+    allow_null=True,
+    category=_('System'),
+    category_slug='system',
+)
+
 register(
     'AWX_CLEANUP_PATHS',
     field_class=fields.BooleanField,

awx/main/dispatch/__init__.py

@@ -40,8 +40,12 @@ def get_task_queuename():
 
 
 class PubSub(object):
-    def __init__(self, conn):
+    def __init__(self, conn, select_timeout=None):
         self.conn = conn
+        if select_timeout is None:
+            self.select_timeout = 5
+        else:
+            self.select_timeout = select_timeout
 
     def listen(self, channel):
         with self.conn.cursor() as cur:
@@ -55,16 +59,33 @@ class PubSub(object):
         with self.conn.cursor() as cur:
             cur.execute('SELECT pg_notify(%s, %s);', (channel, payload))
 
-    def events(self, select_timeout=5, yield_timeouts=False):
+    @staticmethod
+    def current_notifies(conn):
+        """
+        Altered version of .notifies method from psycopg library
+        This removes the outer while True loop so that we only process
+        queued notifications
+        """
+        with conn.lock:
+            try:
+                ns = conn.wait(psycopg.generators.notifies(conn.pgconn))
+            except psycopg.errors._NO_TRACEBACK as ex:
+                raise ex.with_traceback(None)
+        enc = psycopg._encodings.pgconn_encoding(conn.pgconn)
+        for pgn in ns:
+            n = psycopg.connection.Notify(pgn.relname.decode(enc), pgn.extra.decode(enc), pgn.be_pid)
+            yield n
+
+    def events(self, yield_timeouts=False):
         if not self.conn.autocommit:
             raise RuntimeError('Listening for events can only be done in autocommit mode')
 
         while True:
-            if select.select([self.conn], [], [], select_timeout) == NOT_READY:
+            if select.select([self.conn], [], [], self.select_timeout) == NOT_READY:
                 if yield_timeouts:
                     yield None
             else:
-                notification_generator = self.conn.notifies()
+                notification_generator = self.current_notifies(self.conn)
                 for notification in notification_generator:
                     yield notification
 
@@ -73,7 +94,7 @@ class PubSub(object):
 
 
 @contextmanager
-def pg_bus_conn(new_connection=False):
+def pg_bus_conn(new_connection=False, select_timeout=None):
     '''
     Any listeners probably want to establish a new database connection,
     separate from the Django connection used for queries, because that will prevent
@@ -98,7 +119,7 @@ def pg_bus_conn(new_connection=False):
             raise RuntimeError('Unexpectedly could not connect to postgres for pg_notify actions')
         conn = pg_connection.connection
 
-    pubsub = PubSub(conn)
+    pubsub = PubSub(conn, select_timeout=select_timeout)
     yield pubsub
     if new_connection:
         conn.close()

awx/main/dispatch/control.py

@@ -40,6 +40,9 @@ class Control(object):
     def cancel(self, task_ids, *args, **kwargs):
         return self.control_with_reply('cancel', *args, extra_data={'task_ids': task_ids}, **kwargs)
 
+    def schedule(self, *args, **kwargs):
+        return self.control_with_reply('schedule', *args, **kwargs)
+
     @classmethod
     def generate_reply_queue_name(cls):
         return f"reply_to_{str(uuid.uuid4()).replace('-','_')}"
@@ -52,14 +55,14 @@ class Control(object):
         if not connection.get_autocommit():
             raise RuntimeError('Control-with-reply messages can only be done in autocommit mode')
 
-        with pg_bus_conn() as conn:
+        with pg_bus_conn(select_timeout=timeout) as conn:
             conn.listen(reply_queue)
             send_data = {'control': command, 'reply_to': reply_queue}
             if extra_data:
                 send_data.update(extra_data)
             conn.notify(self.queuename, json.dumps(send_data))
 
-            for reply in conn.events(select_timeout=timeout, yield_timeouts=True):
+            for reply in conn.events(yield_timeouts=True):
                 if reply is None:
                     logger.error(f'{self.service} did not reply within {timeout}s')
                     raise RuntimeError(f"{self.service} did not reply within {timeout}s")

awx/main/dispatch/periodic.py

@@ -1,57 +1,142 @@
 import logging
-import os
 import time
-from multiprocessing import Process
+import yaml
+from datetime import datetime
 
-from django.conf import settings
-from django.db import connections
-from schedule import Scheduler
-from django_guid import set_guid
-from django_guid.utils import generate_guid
-
-from awx.main.dispatch.worker import TaskWorker
-from awx.main.utils.db import set_connection_name
 
 logger = logging.getLogger('awx.main.dispatch.periodic')
 
 
-class Scheduler(Scheduler):
-    def run_continuously(self):
-        idle_seconds = max(1, min(self.jobs).period.total_seconds() / 2)
+class ScheduledTask:
+    """
+    Class representing schedules, very loosely modeled after python schedule library Job
+    the idea of this class is to:
+     - only deal in relative times (time since the scheduler global start)
+     - only deal in integer math for target runtimes, but float for current relative time
 
-        def run():
-            ppid = os.getppid()
-            logger.warning('periodic beat started')
+    Missed schedule policy:
+    Invariant target times are maintained, meaning that if interval=10s offset=0
+    and it runs at t=7s, then it calls for next run in 3s.
+    However, if a complete interval has passed, that is counted as a missed run,
+    and missed runs are abandoned (no catch-up runs).
+    """
 
-            set_connection_name('periodic')  # set application_name to distinguish from other dispatcher processes
+    def __init__(self, name: str, data: dict):
+        # parameters need for schedule computation
+        self.interval = int(data['schedule'].total_seconds())
+        self.offset = 0  # offset relative to start time this schedule begins
+        self.index = 0  # number of periods of the schedule that has passed
 
-            while True:
-                if os.getppid() != ppid:
-                    # if the parent PID changes, this process has been orphaned
-                    # via e.g., segfault or sigkill, we should exit too
-                    pid = os.getpid()
-                    logger.warning(f'periodic beat exiting gracefully pid:{pid}')
-                    raise SystemExit()
-                try:
-                    for conn in connections.all():
-                        # If the database connection has a hiccup, re-establish a new
-                        # connection
-                        conn.close_if_unusable_or_obsolete()
-                    set_guid(generate_guid())
-                    self.run_pending()
-                except Exception:
-                    logger.exception('encountered an error while scheduling periodic tasks')
-                time.sleep(idle_seconds)
+        # parameters that do not affect scheduling logic
+        self.last_run = None  # time of last run, only used for debug
+        self.completed_runs = 0  # number of times schedule is known to run
+        self.name = name
+        self.data = data  # used by caller to know what to run
 
-        process = Process(target=run)
-        process.daemon = True
-        process.start()
+    @property
+    def next_run(self):
+        "Time until the next run with t=0 being the global_start of the scheduler class"
+        return (self.index + 1) * self.interval + self.offset
+
+    def due_to_run(self, relative_time):
+        return bool(self.next_run <= relative_time)
+
+    def expected_runs(self, relative_time):
+        return int((relative_time - self.offset) / self.interval)
+
+    def mark_run(self, relative_time):
+        self.last_run = relative_time
+        self.completed_runs += 1
+        new_index = self.expected_runs(relative_time)
+        if new_index > self.index + 1:
+            logger.warning(f'Missed {new_index - self.index - 1} schedules of {self.name}')
+        self.index = new_index
+
+    def missed_runs(self, relative_time):
+        "Number of times job was supposed to ran but failed to, only used for debug"
+        missed_ct = self.expected_runs(relative_time) - self.completed_runs
+        # if this is currently due to run do not count that as a missed run
+        if missed_ct and self.due_to_run(relative_time):
+            missed_ct -= 1
+        return missed_ct
 
 
-def run_continuously():
-    scheduler = Scheduler()
-    for task in settings.CELERYBEAT_SCHEDULE.values():
-        apply_async = TaskWorker.resolve_callable(task['task']).apply_async
-        total_seconds = task['schedule'].total_seconds()
-        scheduler.every(total_seconds).seconds.do(apply_async)
-    scheduler.run_continuously()
+class Scheduler:
+    def __init__(self, schedule):
+        """
+        Expects schedule in the form of a dictionary like
+        {
+            'job1': {'schedule': timedelta(seconds=50), 'other': 'stuff'}
+        }
+        Only the schedule nearest-second value is used for scheduling,
+        the rest of the data is for use by the caller to know what to run.
+        """
+        self.jobs = [ScheduledTask(name, data) for name, data in schedule.items()]
+        min_interval = min(job.interval for job in self.jobs)
+        num_jobs = len(self.jobs)
+
+        # this is intentionally oppioniated against spammy schedules
+        # a core goal is to spread out the scheduled tasks (for worker management)
+        # and high-frequency schedules just do not work with that
+        if num_jobs > min_interval:
+            raise RuntimeError(f'Number of schedules ({num_jobs}) is more than the shortest schedule interval ({min_interval} seconds).')
+
+        # even space out jobs over the base interval
+        for i, job in enumerate(self.jobs):
+            job.offset = (i * min_interval) // num_jobs
+
+        # internally times are all referenced relative to startup time, add grace period
+        self.global_start = time.time() + 2.0
+
+    def get_and_mark_pending(self):
+        relative_time = time.time() - self.global_start
+        to_run = []
+        for job in self.jobs:
+            if job.due_to_run(relative_time):
+                to_run.append(job)
+                logger.debug(f'scheduler found {job.name} to run, {relative_time - job.next_run} seconds after target')
+                job.mark_run(relative_time)
+        return to_run
+
+    def time_until_next_run(self):
+        relative_time = time.time() - self.global_start
+        next_job = min(self.jobs, key=lambda j: j.next_run)
+        delta = next_job.next_run - relative_time
+        if delta <= 0.1:
+            # careful not to give 0 or negative values to the select timeout, which has unclear interpretation
+            logger.warning(f'Scheduler next run of {next_job.name} is {-delta} seconds in the past')
+            return 0.1
+        elif delta > 20.0:
+            logger.warning(f'Scheduler next run unexpectedly over 20 seconds in future: {delta}')
+            return 20.0
+        logger.debug(f'Scheduler next run is {next_job.name} in {delta} seconds')
+        return delta
+
+    def debug(self, *args, **kwargs):
+        data = dict()
+        data['title'] = 'Scheduler status'
+
+        now = datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S UTC')
+        start_time = datetime.fromtimestamp(self.global_start).strftime('%Y-%m-%d %H:%M:%S UTC')
+        relative_time = time.time() - self.global_start
+        data['started_time'] = start_time
+        data['current_time'] = now
+        data['current_time_relative'] = round(relative_time, 3)
+        data['total_schedules'] = len(self.jobs)
+
+        data['schedule_list'] = dict(
+            [
+                (
+                    job.name,
+                    dict(
+                        last_run_seconds_ago=round(relative_time - job.last_run, 3) if job.last_run else None,
+                        next_run_in_seconds=round(job.next_run - relative_time, 3),
+                        offset_in_seconds=job.offset,
+                        completed_runs=job.completed_runs,
+                        missed_runs=job.missed_runs(relative_time),
+                    ),
+                )
+                for job in sorted(self.jobs, key=lambda job: job.interval)
+            ]
+        )
+        return yaml.safe_dump(data, default_flow_style=False, sort_keys=False)

awx/main/dispatch/pool.py

@@ -417,16 +417,16 @@ class AutoscalePool(WorkerPool):
                 # the task manager to never do more work
                 current_task = w.current_task
                 if current_task and isinstance(current_task, dict):
-                    endings = ['tasks.task_manager', 'tasks.dependency_manager', 'tasks.workflow_manager']
+                    endings = ('tasks.task_manager', 'tasks.dependency_manager', 'tasks.workflow_manager')
                     current_task_name = current_task.get('task', '')
-                    if any(current_task_name.endswith(e) for e in endings):
+                    if current_task_name.endswith(endings):
                         if 'started' not in current_task:
                             w.managed_tasks[current_task['uuid']]['started'] = time.time()
                         age = time.time() - current_task['started']
                         w.managed_tasks[current_task['uuid']]['age'] = age
                         if age > self.task_manager_timeout:
-                            logger.error(f'{current_task_name} has held the advisory lock for {age}, sending SIGTERM to {w.pid}')
-                            os.kill(w.pid, signal.SIGTERM)
+                            logger.error(f'{current_task_name} has held the advisory lock for {age}, sending SIGUSR1 to {w.pid}')
+                            os.kill(w.pid, signal.SIGUSR1)
 
         for m in orphaned:
             # if all the workers are dead, spawn at least one

awx/main/dispatch/publish.py

@@ -73,15 +73,15 @@ class task:
                 return cls.apply_async(args, kwargs)
 
             @classmethod
-            def apply_async(cls, args=None, kwargs=None, queue=None, uuid=None, **kw):
+            def get_async_body(cls, args=None, kwargs=None, uuid=None, **kw):
+                """
+                Get the python dict to become JSON data in the pg_notify message
+                This same message gets passed over the dispatcher IPC queue to workers
+                If a task is submitted to a multiprocessing pool, skipping pg_notify, this might be used directly
+                """
                 task_id = uuid or str(uuid4())
                 args = args or []
                 kwargs = kwargs or {}
-                queue = queue or getattr(cls.queue, 'im_func', cls.queue)
-                if not queue:
-                    msg = f'{cls.name}: Queue value required and may not be None'
-                    logger.error(msg)
-                    raise ValueError(msg)
                 obj = {'uuid': task_id, 'args': args, 'kwargs': kwargs, 'task': cls.name, 'time_pub': time.time()}
                 guid = get_guid()
                 if guid:
@@ -89,6 +89,16 @@ class task:
                 if bind_kwargs:
                     obj['bind_kwargs'] = bind_kwargs
                 obj.update(**kw)
+                return obj
+
+            @classmethod
+            def apply_async(cls, args=None, kwargs=None, queue=None, uuid=None, **kw):
+                queue = queue or getattr(cls.queue, 'im_func', cls.queue)
+                if not queue:
+                    msg = f'{cls.name}: Queue value required and may not be None'
+                    logger.error(msg)
+                    raise ValueError(msg)
+                obj = cls.get_async_body(args=args, kwargs=kwargs, uuid=uuid, **kw)
                 if callable(queue):
                     queue = queue()
                 if not is_testing():
@@ -116,4 +126,5 @@ class task:
         setattr(fn, 'name', cls.name)
         setattr(fn, 'apply_async', cls.apply_async)
         setattr(fn, 'delay', cls.delay)
+        setattr(fn, 'get_async_body', cls.get_async_body)
         return fn

awx/main/dispatch/worker/base.py

@@ -11,11 +11,13 @@ import psycopg
 import time
 from uuid import UUID
 from queue import Empty as QueueEmpty
+from datetime import timedelta
 
 from django import db
 from django.conf import settings
 
 from awx.main.dispatch.pool import WorkerPool
+from awx.main.dispatch.periodic import Scheduler
 from awx.main.dispatch import pg_bus_conn
 from awx.main.utils.common import log_excess_runtime
 from awx.main.utils.db import set_connection_name
@@ -64,10 +66,12 @@ class AWXConsumerBase(object):
     def control(self, body):
         logger.warning(f'Received control signal:\n{body}')
         control = body.get('control')
-        if control in ('status', 'running', 'cancel'):
+        if control in ('status', 'schedule', 'running', 'cancel'):
             reply_queue = body['reply_to']
             if control == 'status':
                 msg = '\n'.join([self.listening_on, self.pool.debug()])
+            if control == 'schedule':
+                msg = self.scheduler.debug()
             elif control == 'running':
                 msg = []
                 for worker in self.pool.workers:
@@ -93,16 +97,11 @@ class AWXConsumerBase(object):
         else:
             logger.error('unrecognized control message: {}'.format(control))
 
-    def process_task(self, body):
+    def dispatch_task(self, body):
+        """This will place the given body into a worker queue to run method decorated as a task"""
         if isinstance(body, dict):
             body['time_ack'] = time.time()
 
-        if 'control' in body:
-            try:
-                return self.control(body)
-            except Exception:
-                logger.exception(f"Exception handling control message: {body}")
-                return
         if len(self.pool):
             if "uuid" in body and body['uuid']:
                 try:
@@ -116,15 +115,24 @@ class AWXConsumerBase(object):
         self.pool.write(queue, body)
         self.total_messages += 1
 
+    def process_task(self, body):
+        """Routes the task details in body as either a control task or a task-task"""
+        if 'control' in body:
+            try:
+                return self.control(body)
+            except Exception:
+                logger.exception(f"Exception handling control message: {body}")
+                return
+        self.dispatch_task(body)
+
     @log_excess_runtime(logger)
     def record_statistics(self):
         if time.time() - self.last_stats > 1:  # buffer stat recording to once per second
             try:
                 self.redis.set(f'awx_{self.name}_statistics', self.pool.debug())
-                self.last_stats = time.time()
             except Exception:
                 logger.exception(f"encountered an error communicating with redis to store {self.name} statistics")
-                self.last_stats = time.time()
+            self.last_stats = time.time()
 
     def run(self, *args, **kwargs):
         signal.signal(signal.SIGINT, self.stop)
@@ -151,7 +159,7 @@ class AWXConsumerRedis(AWXConsumerBase):
 
 
 class AWXConsumerPG(AWXConsumerBase):
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, schedule=None, **kwargs):
         super().__init__(*args, **kwargs)
         self.pg_max_wait = settings.DISPATCHER_DB_DOWNTOWN_TOLLERANCE
         # if no successful loops have ran since startup, then we should fail right away
@@ -162,24 +170,53 @@ class AWXConsumerPG(AWXConsumerBase):
         self.subsystem_metrics = s_metrics.Metrics(auto_pipe_execute=False)
         self.last_metrics_gather = init_time
         self.listen_cumulative_time = 0.0
+        if schedule:
+            schedule = schedule.copy()
+        else:
+            schedule = {}
+        # add control tasks to be ran at regular schedules
+        # NOTE: if we run out of database connections, it is important to still run cleanup
+        # so that we scale down workers and free up connections
+        schedule['pool_cleanup'] = {'control': self.pool.cleanup, 'schedule': timedelta(seconds=60)}
+        # record subsystem metrics for the dispatcher
+        schedule['metrics_gather'] = {'control': self.record_metrics, 'schedule': timedelta(seconds=20)}
+        self.scheduler = Scheduler(schedule)
+
+    def record_metrics(self):
+        current_time = time.time()
+        self.pool.produce_subsystem_metrics(self.subsystem_metrics)
+        self.subsystem_metrics.set('dispatcher_availability', self.listen_cumulative_time / (current_time - self.last_metrics_gather))
+        self.subsystem_metrics.pipe_execute()
+        self.listen_cumulative_time = 0.0
+        self.last_metrics_gather = current_time
 
     def run_periodic_tasks(self):
-        self.record_statistics()  # maintains time buffer in method
+        """
+        Run general periodic logic, and return maximum time in seconds before
+        the next requested run
+        This may be called more often than that when events are consumed
+        so this should be very efficient in that
+        """
+        try:
+            self.record_statistics()  # maintains time buffer in method
+        except Exception as exc:
+            logger.warning(f'Failed to save dispatcher statistics {exc}')
 
-        current_time = time.time()
-        if current_time - self.last_cleanup > 60:  # same as cluster_node_heartbeat
-            # NOTE: if we run out of database connections, it is important to still run cleanup
-            # so that we scale down workers and free up connections
-            self.pool.cleanup()
-            self.last_cleanup = current_time
+        for job in self.scheduler.get_and_mark_pending():
+            if 'control' in job.data:
+                try:
+                    job.data['control']()
+                except Exception:
+                    logger.exception(f'Error running control task {job.data}')
+            elif 'task' in job.data:
+                body = self.worker.resolve_callable(job.data['task']).get_async_body()
+                # bypasses pg_notify for scheduled tasks
+                self.dispatch_task(body)
 
-        # record subsystem metrics for the dispatcher
-        if current_time - self.last_metrics_gather > 20:
-            self.pool.produce_subsystem_metrics(self.subsystem_metrics)
-            self.subsystem_metrics.set('dispatcher_availability', self.listen_cumulative_time / (current_time - self.last_metrics_gather))
-            self.subsystem_metrics.pipe_execute()
-            self.listen_cumulative_time = 0.0
-            self.last_metrics_gather = current_time
+        self.pg_is_down = False
+        self.listen_start = time.time()
+
+        return self.scheduler.time_until_next_run()
 
     def run(self, *args, **kwargs):
         super(AWXConsumerPG, self).run(*args, **kwargs)
@@ -195,14 +232,15 @@ class AWXConsumerPG(AWXConsumerBase):
                     if init is False:
                         self.worker.on_start()
                         init = True
-                    self.listen_start = time.time()
+                    # run_periodic_tasks run scheduled actions and gives time until next scheduled action
+                    # this is saved to the conn (PubSub) object in order to modify read timeout in-loop
+                    conn.select_timeout = self.run_periodic_tasks()
+                    # this is the main operational loop for awx-manage run_dispatcher
                     for e in conn.events(yield_timeouts=True):
-                        self.listen_cumulative_time += time.time() - self.listen_start
+                        self.listen_cumulative_time += time.time() - self.listen_start  # for metrics
                         if e is not None:
                             self.process_task(json.loads(e.payload))
-                        self.run_periodic_tasks()
-                        self.pg_is_down = False
-                        self.listen_start = time.time()
+                        conn.select_timeout = self.run_periodic_tasks()
                     if self.should_stop:
                         return
             except psycopg.InterfaceError:
@@ -250,8 +288,8 @@ class BaseWorker(object):
                     break
             except QueueEmpty:
                 continue
-            except Exception as e:
-                logger.error("Exception on worker {}, restarting: ".format(idx) + str(e))
+            except Exception:
+                logger.exception("Exception on worker {}, reconnecting: ".format(idx))
                 continue
             try:
                 for conn in db.connections.all():

awx/main/management/commands/cleanup_host_metrics.py

@@ -17,6 +17,6 @@ class Command(BaseCommand):
         months_ago = options.get('months-ago') or None
 
         if not months_ago:
-            months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_THRESHOLD', 12)
+            months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_SOFT_THRESHOLD', 12)
 
         HostMetric.cleanup_task(months_ago)

awx/main/management/commands/host_metric_summary_monthly.py

@@ -0,0 +1,9 @@
+from django.core.management.base import BaseCommand
+from awx.main.tasks.host_metrics import HostMetricSummaryMonthlyTask
+
+
+class Command(BaseCommand):
+    help = 'Computing of HostMetricSummaryMonthly'
+
+    def handle(self, *args, **options):
+        HostMetricSummaryMonthlyTask().execute()

awx/main/management/commands/run_dispatcher.py

@@ -3,15 +3,13 @@
 import logging
 import yaml
 
-from django.core.cache import cache as django_cache
+from django.conf import settings
 from django.core.management.base import BaseCommand
-from django.db import connection as django_connection
 
 from awx.main.dispatch import get_task_queuename
 from awx.main.dispatch.control import Control
 from awx.main.dispatch.pool import AutoscalePool
 from awx.main.dispatch.worker import AWXConsumerPG, TaskWorker
-from awx.main.dispatch import periodic
 
 logger = logging.getLogger('awx.main.dispatch')
 
@@ -21,6 +19,7 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument('--status', dest='status', action='store_true', help='print the internal state of any running dispatchers')
+        parser.add_argument('--schedule', dest='schedule', action='store_true', help='print the current status of schedules being ran by dispatcher')
         parser.add_argument('--running', dest='running', action='store_true', help='print the UUIDs of any tasked managed by this dispatcher')
         parser.add_argument(
             '--reload',
@@ -42,6 +41,9 @@ class Command(BaseCommand):
         if options.get('status'):
             print(Control('dispatcher').status())
             return
+        if options.get('schedule'):
+            print(Control('dispatcher').schedule())
+            return
         if options.get('running'):
             print(Control('dispatcher').running())
             return
@@ -58,21 +60,11 @@ class Command(BaseCommand):
             print(Control('dispatcher').cancel(cancel_data))
             return
 
-        # It's important to close these because we're _about_ to fork, and we
-        # don't want the forked processes to inherit the open sockets
-        # for the DB and cache connections (that way lies race conditions)
-        django_connection.close()
-        django_cache.close()
-
-        # spawn a daemon thread to periodically enqueues scheduled tasks
-        # (like the node heartbeat)
-        periodic.run_continuously()
-
         consumer = None
 
         try:
             queues = ['tower_broadcast_all', 'tower_settings_change', get_task_queuename()]
-            consumer = AWXConsumerPG('dispatcher', TaskWorker(), queues, AutoscalePool(min_workers=4))
+            consumer = AWXConsumerPG('dispatcher', TaskWorker(), queues, AutoscalePool(min_workers=4), schedule=settings.CELERYBEAT_SCHEDULE)
             consumer.run()
         except KeyboardInterrupt:
             logger.debug('Terminating Task Dispatcher')

awx/main/migrations/0001_initial.py

@@ -9,13 +9,11 @@ from django.db import migrations, models
 import django.utils.timezone
 import django.db.models.deletion
 from django.conf import settings
-import taggit.managers
 import awx.main.fields
 
 
 class Migration(migrations.Migration):
     dependencies = [
-        ('taggit', '0002_auto_20150616_2121'),
         ('contenttypes', '0002_remove_content_type_name'),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
     ]
@@ -184,12 +182,6 @@ class Migration(migrations.Migration):
                         null=True,
                     ),
                 ),
-                (
-                    'tags',
-                    taggit.managers.TaggableManager(
-                        to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-                    ),
-                ),
             ],
             options={
                 'ordering': ('kind', 'name'),
@@ -529,12 +521,6 @@ class Migration(migrations.Migration):
                         null=True,
                     ),
                 ),
-                (
-                    'tags',
-                    taggit.managers.TaggableManager(
-                        to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-                    ),
-                ),
                 ('users', models.ManyToManyField(related_name='organizations', to=settings.AUTH_USER_MODEL, blank=True)),
             ],
             options={
@@ -589,12 +575,6 @@ class Migration(migrations.Migration):
                         null=True,
                     ),
                 ),
-                (
-                    'tags',
-                    taggit.managers.TaggableManager(
-                        to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-                    ),
-                ),
             ],
         ),
         migrations.CreateModel(
@@ -644,12 +624,6 @@ class Migration(migrations.Migration):
                         null=True,
                     ),
                 ),
-                (
-                    'tags',
-                    taggit.managers.TaggableManager(
-                        to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-                    ),
-                ),
             ],
             options={
                 'ordering': ['-next_run'],
@@ -687,12 +661,6 @@ class Migration(migrations.Migration):
                     ),
                 ),
                 ('organization', models.ForeignKey(related_name='teams', on_delete=django.db.models.deletion.SET_NULL, to='main.Organization', null=True)),
-                (
-                    'tags',
-                    taggit.managers.TaggableManager(
-                        to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-                    ),
-                ),
                 ('users', models.ManyToManyField(related_name='teams', to=settings.AUTH_USER_MODEL, blank=True)),
             ],
             options={
@@ -1267,13 +1235,6 @@ class Migration(migrations.Migration):
                 null=True,
             ),
         ),
-        migrations.AddField(
-            model_name='unifiedjobtemplate',
-            name='tags',
-            field=taggit.managers.TaggableManager(
-                to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-            ),
-        ),
         migrations.AddField(
             model_name='unifiedjob',
             name='created_by',
@@ -1319,13 +1280,6 @@ class Migration(migrations.Migration):
             name='schedule',
             field=models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to='main.Schedule', null=True),
         ),
-        migrations.AddField(
-            model_name='unifiedjob',
-            name='tags',
-            field=taggit.managers.TaggableManager(
-                to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-            ),
-        ),
         migrations.AddField(
             model_name='unifiedjob',
             name='unified_job_template',
@@ -1370,13 +1324,6 @@ class Migration(migrations.Migration):
                 help_text='Organization containing this inventory.',
             ),
         ),
-        migrations.AddField(
-            model_name='inventory',
-            name='tags',
-            field=taggit.managers.TaggableManager(
-                to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-            ),
-        ),
         migrations.AddField(
             model_name='host',
             name='inventory',
@@ -1407,13 +1354,6 @@ class Migration(migrations.Migration):
                 null=True,
             ),
         ),
-        migrations.AddField(
-            model_name='host',
-            name='tags',
-            field=taggit.managers.TaggableManager(
-                to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-            ),
-        ),
         migrations.AddField(
             model_name='group',
             name='hosts',
@@ -1441,13 +1381,6 @@ class Migration(migrations.Migration):
             name='parents',
             field=models.ManyToManyField(related_name='children', to='main.Group', blank=True),
         ),
-        migrations.AddField(
-            model_name='group',
-            name='tags',
-            field=taggit.managers.TaggableManager(
-                to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-            ),
-        ),
         migrations.AddField(
             model_name='custominventoryscript',
             name='organization',
@@ -1459,13 +1392,6 @@ class Migration(migrations.Migration):
                 null=True,
             ),
         ),
-        migrations.AddField(
-            model_name='custominventoryscript',
-            name='tags',
-            field=taggit.managers.TaggableManager(
-                to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-            ),
-        ),
         migrations.AddField(
             model_name='credential',
             name='team',

awx/main/migrations/0002_squashed_v300_release.py

@@ -12,8 +12,6 @@ import django.db.models.deletion
 from django.conf import settings
 from django.utils.timezone import now
 
-import taggit.managers
-
 
 def create_system_job_templates(apps, schema_editor):
     """
@@ -125,7 +123,6 @@ class Migration(migrations.Migration):
     ]
 
     dependencies = [
-        ('taggit', '0002_auto_20150616_2121'),
         ('contenttypes', '0002_remove_content_type_name'),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
         ('main', '0001_initial'),
@@ -256,12 +253,6 @@ class Migration(migrations.Migration):
                     'organization',
                     models.ForeignKey(related_name='notification_templates', on_delete=django.db.models.deletion.SET_NULL, to='main.Organization', null=True),
                 ),
-                (
-                    'tags',
-                    taggit.managers.TaggableManager(
-                        to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-                    ),
-                ),
             ],
         ),
         migrations.AddField(
@@ -721,12 +712,6 @@ class Migration(migrations.Migration):
                         help_text='Organization this label belongs to.',
                     ),
                 ),
-                (
-                    'tags',
-                    taggit.managers.TaggableManager(
-                        to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-                    ),
-                ),
             ],
             options={
                 'ordering': ('organization', 'name'),

awx/main/migrations/0006_v320_release.py

@@ -5,7 +5,6 @@ from __future__ import unicode_literals
 # Django
 from django.db import connection, migrations, models, OperationalError, ProgrammingError
 from django.conf import settings
-import taggit.managers
 
 # AWX
 import awx.main.fields
@@ -317,10 +316,6 @@ class Migration(migrations.Migration):
             model_name='permission',
             name='project',
         ),
-        migrations.RemoveField(
-            model_name='permission',
-            name='tags',
-        ),
         migrations.RemoveField(
             model_name='permission',
             name='team',
@@ -510,12 +505,6 @@ class Migration(migrations.Migration):
                         null=True,
                     ),
                 ),
-                (
-                    'tags',
-                    taggit.managers.TaggableManager(
-                        to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags'
-                    ),
-                ),
             ],
             options={
                 'ordering': ('kind', 'name'),

awx/main/migrations/0067_v350_credential_plugins.py

@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
-import taggit.managers
 
 # AWX
 import awx.main.fields
@@ -20,7 +19,6 @@ def setup_tower_managed_defaults(apps, schema_editor):
 class Migration(migrations.Migration):
     dependencies = [
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-        ('taggit', '0002_auto_20150616_2121'),
         ('main', '0066_v350_inventorysource_custom_virtualenv'),
     ]
 
@@ -60,12 +58,6 @@ class Migration(migrations.Migration):
                     'source_credential',
                     models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='target_input_sources', to='main.Credential'),
                 ),
-                (
-                    'tags',
-                    taggit.managers.TaggableManager(
-                        blank=True, help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags'
-                    ),
-                ),
                 (
                     'target_credential',
                     models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='input_sources', to='main.Credential'),

awx/main/migrations/0124_execution_environments.py

@@ -4,12 +4,10 @@ from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
 import django.db.models.expressions
-import taggit.managers
 
 
 class Migration(migrations.Migration):
     dependencies = [
-        ('taggit', '0003_taggeditem_add_unique_index'),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
         ('main', '0123_drop_hg_support'),
     ]
@@ -69,12 +67,6 @@ class Migration(migrations.Migration):
                         to='main.Organization',
                     ),
                 ),
-                (
-                    'tags',
-                    taggit.managers.TaggableManager(
-                        blank=True, help_text='A comma-separated list of tags.', through='taggit.TaggedItem', to='taggit.Tag', verbose_name='Tags'
-                    ),
-                ),
             ],
             options={
                 'ordering': (django.db.models.expressions.OrderBy(django.db.models.expressions.F('organization_id'), nulls_first=True), 'image'),

awx/main/migrations/0185_move_JSONBlob_to_JSONField.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2 on 2023-06-09 19:51
+# Generated by Django 4.2.3 on 2023-08-02 13:18
 
 import awx.main.models.notifications
 from django.db import migrations, models
@@ -11,16 +11,6 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AlterField(
-            model_name='activitystream',
-            name='deleted_actor',
-            field=models.JSONField(null=True),
-        ),
-        migrations.AlterField(
-            model_name='activitystream',
-            name='setting',
-            field=models.JSONField(blank=True, default=dict),
-        ),
         migrations.AlterField(
             model_name='instancegroup',
             name='policy_instance_list',
@@ -28,31 +18,11 @@ class Migration(migrations.Migration):
                 blank=True, default=list, help_text='List of exact-match Instances that will always be automatically assigned to this group'
             ),
         ),
-        migrations.AlterField(
-            model_name='job',
-            name='survey_passwords',
-            field=models.JSONField(blank=True, default=dict, editable=False),
-        ),
-        migrations.AlterField(
-            model_name='joblaunchconfig',
-            name='char_prompts',
-            field=models.JSONField(blank=True, default=dict),
-        ),
-        migrations.AlterField(
-            model_name='joblaunchconfig',
-            name='survey_passwords',
-            field=models.JSONField(blank=True, default=dict, editable=False),
-        ),
         migrations.AlterField(
             model_name='jobtemplate',
             name='survey_spec',
             field=models.JSONField(blank=True, default=dict),
         ),
-        migrations.AlterField(
-            model_name='notification',
-            name='body',
-            field=models.JSONField(blank=True, default=dict),
-        ),
         migrations.AlterField(
             model_name='notificationtemplate',
             name='messages',
@@ -94,31 +64,6 @@ class Migration(migrations.Migration):
             name='survey_passwords',
             field=models.JSONField(blank=True, default=dict, editable=False),
         ),
-        migrations.AlterField(
-            model_name='unifiedjob',
-            name='job_env',
-            field=models.JSONField(blank=True, default=dict, editable=False),
-        ),
-        migrations.AlterField(
-            model_name='workflowjob',
-            name='char_prompts',
-            field=models.JSONField(blank=True, default=dict),
-        ),
-        migrations.AlterField(
-            model_name='workflowjob',
-            name='survey_passwords',
-            field=models.JSONField(blank=True, default=dict, editable=False),
-        ),
-        migrations.AlterField(
-            model_name='workflowjobnode',
-            name='char_prompts',
-            field=models.JSONField(blank=True, default=dict),
-        ),
-        migrations.AlterField(
-            model_name='workflowjobnode',
-            name='survey_passwords',
-            field=models.JSONField(blank=True, default=dict, editable=False),
-        ),
         migrations.AlterField(
             model_name='workflowjobtemplate',
             name='char_prompts',
@@ -139,4 +84,194 @@ class Migration(migrations.Migration):
             name='survey_passwords',
             field=models.JSONField(blank=True, default=dict, editable=False),
         ),
+        # These are potentially a problem.  Move the existing fields
+        # aside while pretending like they've been deleted, then add
+        # in fresh empty fields.  Make the old fields nullable where
+        # needed while we are at it, so that new rows don't hit
+        # IntegrityError.  We'll do the data migration out-of-band
+        # using a task.
+        migrations.RunSQL(  # Already nullable
+            "ALTER TABLE main_activitystream RENAME deleted_actor TO deleted_actor_old;",
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='activitystream',
+                    name='deleted_actor',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='activitystream',
+            name='deleted_actor',
+            field=models.JSONField(null=True),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_activitystream RENAME setting TO setting_old;
+            ALTER TABLE main_activitystream ALTER COLUMN setting_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='activitystream',
+                    name='setting',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='activitystream',
+            name='setting',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_job RENAME survey_passwords TO survey_passwords_old;
+            ALTER TABLE main_job ALTER COLUMN survey_passwords_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='job',
+                    name='survey_passwords',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='job',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_joblaunchconfig RENAME char_prompts TO char_prompts_old;
+            ALTER TABLE main_joblaunchconfig ALTER COLUMN char_prompts_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='joblaunchconfig',
+                    name='char_prompts',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='joblaunchconfig',
+            name='char_prompts',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_joblaunchconfig RENAME survey_passwords TO survey_passwords_old;
+            ALTER TABLE main_joblaunchconfig ALTER COLUMN survey_passwords_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='joblaunchconfig',
+                    name='survey_passwords',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='joblaunchconfig',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_notification RENAME body TO body_old;
+            ALTER TABLE main_notification ALTER COLUMN body_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='notification',
+                    name='body',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='notification',
+            name='body',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_unifiedjob RENAME job_env TO job_env_old;
+            ALTER TABLE main_unifiedjob ALTER COLUMN job_env_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='unifiedjob',
+                    name='job_env',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='unifiedjob',
+            name='job_env',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_workflowjob RENAME char_prompts TO char_prompts_old;
+            ALTER TABLE main_workflowjob ALTER COLUMN char_prompts_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='workflowjob',
+                    name='char_prompts',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='workflowjob',
+            name='char_prompts',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_workflowjob RENAME survey_passwords TO survey_passwords_old;
+            ALTER TABLE main_workflowjob ALTER COLUMN survey_passwords_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='workflowjob',
+                    name='survey_passwords',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='workflowjob',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_workflowjobnode RENAME char_prompts TO char_prompts_old;
+            ALTER TABLE main_workflowjobnode ALTER COLUMN char_prompts_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='workflowjobnode',
+                    name='char_prompts',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='workflowjobnode',
+            name='char_prompts',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.RunSQL(
+            """
+            ALTER TABLE main_workflowjobnode RENAME survey_passwords TO survey_passwords_old;
+            ALTER TABLE main_workflowjobnode ALTER COLUMN survey_passwords_old DROP NOT NULL;
+            """,
+            state_operations=[
+                migrations.RemoveField(
+                    model_name='workflowjobnode',
+                    name='survey_passwords',
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='workflowjobnode',
+            name='survey_passwords',
+            field=models.JSONField(blank=True, default=dict, editable=False),
+        ),
     ]

awx/main/migrations/0186_drop_django_taggit.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+def delete_taggit_contenttypes(apps, schema_editor):
+    ContentType = apps.get_model('contenttypes', 'ContentType')
+    ContentType.objects.filter(app_label='taggit').delete()
+
+
+def delete_taggit_migration_records(apps, schema_editor):
+    recorder = migrations.recorder.MigrationRecorder(connection=schema_editor.connection)
+    recorder.migration_qs.filter(app='taggit').delete()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('main', '0185_move_JSONBlob_to_JSONField'),
+    ]
+
+    operations = [
+        migrations.RunSQL("DROP TABLE IF EXISTS taggit_tag CASCADE;"),
+        migrations.RunSQL("DROP TABLE IF EXISTS taggit_taggeditem CASCADE;"),
+        migrations.RunPython(delete_taggit_contenttypes),
+        migrations.RunPython(delete_taggit_migration_records),
+    ]

awx/main/models/__init__.py

@@ -3,6 +3,7 @@
 
 # Django
 from django.conf import settings  # noqa
+from django.db import connection
 from django.db.models.signals import pre_delete  # noqa
 
 # AWX
@@ -99,6 +100,58 @@ User.add_to_class('can_access_with_errors', check_user_access_with_errors)
 User.add_to_class('accessible_objects', user_accessible_objects)
 
 
+def convert_jsonfields():
+    if connection.vendor != 'postgresql':
+        return
+
+    # fmt: off
+    fields = [
+        ('main_activitystream', 'id', (
+            'deleted_actor',
+            'setting',
+        )),
+        ('main_job', 'unifiedjob_ptr_id', (
+            'survey_passwords',
+        )),
+        ('main_joblaunchconfig', 'id', (
+            'char_prompts',
+            'survey_passwords',
+        )),
+        ('main_notification', 'id', (
+            'body',
+        )),
+        ('main_unifiedjob', 'id', (
+            'job_env',
+        )),
+        ('main_workflowjob', 'unifiedjob_ptr_id', (
+            'char_prompts',
+            'survey_passwords',
+        )),
+        ('main_workflowjobnode', 'id', (
+            'char_prompts',
+            'survey_passwords',
+        )),
+    ]
+    # fmt: on
+
+    with connection.cursor() as cursor:
+        for table, pkfield, columns in fields:
+            # Do the renamed old columns still exist?  If so, run the task.
+            old_columns = ','.join(f"'{column}_old'" for column in columns)
+            cursor.execute(
+                f"""
+                select count(1) from information_schema.columns
+                where
+                  table_name = %s and column_name in ({old_columns});
+                """,
+                (table,),
+            )
+            if cursor.fetchone()[0]:
+                from awx.main.tasks.system import migrate_jsonfield
+
+                migrate_jsonfield.apply_async([table, pkfield, columns])
+
+
 def cleanup_created_modified_by(sender, **kwargs):
     # work around a bug in django-polymorphic that doesn't properly
     # handle cascades for reverse foreign keys on the polymorphic base model

awx/main/models/base.py

@@ -7,9 +7,6 @@ from django.core.exceptions import ValidationError, ObjectDoesNotExist
 from django.utils.translation import gettext_lazy as _
 from django.utils.timezone import now
 
-# Django-Taggit
-from taggit.managers import TaggableManager
-
 # Django-CRUM
 from crum import get_current_user
 
@@ -301,8 +298,6 @@ class PrimordialModel(HasEditsMixin, CreatedModifiedModel):
         on_delete=models.SET_NULL,
     )
 
-    tags = TaggableManager(blank=True)
-
     def __init__(self, *args, **kwargs):
         r = super(PrimordialModel, self).__init__(*args, **kwargs)
         if self.pk:

awx/main/models/inventory.py

@@ -899,18 +899,18 @@ class HostMetric(models.Model):
 
             last_automation_before = now() - dateutil.relativedelta.relativedelta(months=months_ago)
 
-            logger.info(f'Cleanup [HostMetric]: soft-deleting records last automated before {last_automation_before}')
+            logger.info(f'cleanup_host_metrics: soft-deleting records last automated before {last_automation_before}')
             HostMetric.active_objects.filter(last_automation__lt=last_automation_before).update(
                 deleted=True, deleted_counter=models.F('deleted_counter') + 1, last_deleted=now()
             )
             settings.CLEANUP_HOST_METRICS_LAST_TS = now()
         except (TypeError, ValueError):
-            logger.error(f"Cleanup [HostMetric]: months_ago({months_ago}) has to be a positive integer value")
+            logger.error(f"cleanup_host_metrics: months_ago({months_ago}) has to be a positive integer value")
 
 
 class HostMetricSummaryMonthly(models.Model):
     """
-    HostMetric summaries computed by scheduled task <TODO> monthly
+    HostMetric summaries computed by scheduled task 'awx.main.tasks.system.host_metric_summary_monthly' monthly
     """
 
     date = models.DateField(unique=True)

awx/main/registrar.py

@@ -3,8 +3,6 @@
 
 from django.db.models.signals import pre_save, post_save, pre_delete, m2m_changed
 
-from taggit.managers import TaggableManager
-
 
 class ActivityStreamRegistrar(object):
     def __init__(self):
@@ -21,8 +19,6 @@ class ActivityStreamRegistrar(object):
             pre_delete.connect(activity_stream_delete, sender=model, dispatch_uid=str(self.__class__) + str(model) + "_delete")
 
             for m2mfield in model._meta.many_to_many:
-                if isinstance(m2mfield, TaggableManager):
-                    continue  # Special case for taggit app
                 try:
                     m2m_attr = getattr(model, m2mfield.name)
                     m2m_changed.connect(

awx/main/scheduler/task_manager.py

@@ -25,7 +25,6 @@ from awx.main.models import (
     InventoryUpdate,
     Job,
     Project,
-    ProjectUpdate,
     UnifiedJob,
     WorkflowApproval,
     WorkflowJob,
@@ -102,27 +101,33 @@ class TaskBase:
 
     def record_aggregate_metrics(self, *args):
         if not is_testing():
-            # increment task_manager_schedule_calls regardless if the other
-            # metrics are recorded
-            s_metrics.Metrics(auto_pipe_execute=True).inc(f"{self.prefix}__schedule_calls", 1)
-            # Only record metrics if the last time recording was more
-            # than SUBSYSTEM_METRICS_TASK_MANAGER_RECORD_INTERVAL ago.
-            # Prevents a short-duration task manager that runs directly after a
-            # long task manager to override useful metrics.
-            current_time = time.time()
-            time_last_recorded = current_time - self.subsystem_metrics.decode(f"{self.prefix}_recorded_timestamp")
-            if time_last_recorded > settings.SUBSYSTEM_METRICS_TASK_MANAGER_RECORD_INTERVAL:
-                logger.debug(f"recording {self.prefix} metrics, last recorded {time_last_recorded} seconds ago")
-                self.subsystem_metrics.set(f"{self.prefix}_recorded_timestamp", current_time)
-                self.subsystem_metrics.pipe_execute()
-            else:
-                logger.debug(f"skipping recording {self.prefix} metrics, last recorded {time_last_recorded} seconds ago")
+            try:
+                # increment task_manager_schedule_calls regardless if the other
+                # metrics are recorded
+                s_metrics.Metrics(auto_pipe_execute=True).inc(f"{self.prefix}__schedule_calls", 1)
+                # Only record metrics if the last time recording was more
+                # than SUBSYSTEM_METRICS_TASK_MANAGER_RECORD_INTERVAL ago.
+                # Prevents a short-duration task manager that runs directly after a
+                # long task manager to override useful metrics.
+                current_time = time.time()
+                time_last_recorded = current_time - self.subsystem_metrics.decode(f"{self.prefix}_recorded_timestamp")
+                if time_last_recorded > settings.SUBSYSTEM_METRICS_TASK_MANAGER_RECORD_INTERVAL:
+                    logger.debug(f"recording {self.prefix} metrics, last recorded {time_last_recorded} seconds ago")
+                    self.subsystem_metrics.set(f"{self.prefix}_recorded_timestamp", current_time)
+                    self.subsystem_metrics.pipe_execute()
+                else:
+                    logger.debug(f"skipping recording {self.prefix} metrics, last recorded {time_last_recorded} seconds ago")
+            except Exception:
+                logger.exception(f"Error saving metrics for {self.prefix}")
 
     def record_aggregate_metrics_and_exit(self, *args):
         self.record_aggregate_metrics()
         sys.exit(1)
 
     def schedule(self):
+        # Always be able to restore the original signal handler if we finish
+        original_sigusr1 = signal.getsignal(signal.SIGUSR1)
+
         # Lock
         with task_manager_bulk_reschedule():
             with advisory_lock(f"{self.prefix}_lock", wait=False) as acquired:
@@ -131,9 +136,14 @@ class TaskBase:
                         logger.debug(f"Not running {self.prefix} scheduler, another task holds lock")
                         return
                     logger.debug(f"Starting {self.prefix} Scheduler")
-                    # if sigterm due to timeout, still record metrics
-                    signal.signal(signal.SIGTERM, self.record_aggregate_metrics_and_exit)
-                    self._schedule()
+                    # if sigusr1 due to timeout, still record metrics
+                    signal.signal(signal.SIGUSR1, self.record_aggregate_metrics_and_exit)
+                    try:
+                        self._schedule()
+                    finally:
+                        # Reset the signal handler back to the default just in case anything
+                        # else uses the same signal for other purposes
+                        signal.signal(signal.SIGUSR1, original_sigusr1)
                     commit_start = time.time()
 
                 if self.prefix == "task_manager":
@@ -154,7 +164,6 @@ class WorkflowManager(TaskBase):
                 logger.warning("Workflow manager has reached time out while processing running workflows, exiting loop early")
                 ScheduleWorkflowManager().schedule()
                 # Do not process any more workflow jobs. Stop here.
-                # Maybe we should schedule another WorkflowManager run
                 break
             dag = WorkflowDAG(workflow_job)
             status_changed = False
@@ -169,8 +178,8 @@ class WorkflowManager(TaskBase):
                     workflow_job.save(update_fields=['status', 'start_args'])
                     status_changed = True
             else:
-                workflow_nodes = dag.mark_dnr_nodes()
-                WorkflowJobNode.objects.bulk_update(workflow_nodes, ['do_not_run'])
+                dnr_nodes = dag.mark_dnr_nodes()
+                WorkflowJobNode.objects.bulk_update(dnr_nodes, ['do_not_run'])
                 # If workflow is now done, we do special things to mark it as done.
                 is_done = dag.is_workflow_done()
                 if is_done:
@@ -250,6 +259,7 @@ class WorkflowManager(TaskBase):
                         job.status = 'failed'
                         job.save(update_fields=['status', 'job_explanation'])
                         job.websocket_emit_status('failed')
+                        ScheduleWorkflowManager().schedule()
 
                     # TODO: should we emit a status on the socket here similar to tasks.py awx_periodic_scheduler() ?
                     # emit_websocket_notification('/socket.io/jobs', '', dict(id=))
@@ -270,184 +280,115 @@ class WorkflowManager(TaskBase):
 class DependencyManager(TaskBase):
     def __init__(self):
         super().__init__(prefix="dependency_manager")
+        self.all_projects = {}
+        self.all_inventory_sources = {}
 
-    def create_project_update(self, task, project_id=None):
-        if project_id is None:
-            project_id = task.project_id
-        project_task = Project.objects.get(id=project_id).create_project_update(_eager_fields=dict(launch_type='dependency'))
-
-        # Project created 1 seconds behind
-        project_task.created = task.created - timedelta(seconds=1)
-        project_task.status = 'pending'
-        project_task.save()
-        logger.debug('Spawned {} as dependency of {}'.format(project_task.log_format, task.log_format))
-        return project_task
-
-    def create_inventory_update(self, task, inventory_source_task):
-        inventory_task = InventorySource.objects.get(id=inventory_source_task.id).create_inventory_update(_eager_fields=dict(launch_type='dependency'))
-
-        inventory_task.created = task.created - timedelta(seconds=2)
-        inventory_task.status = 'pending'
-        inventory_task.save()
-        logger.debug('Spawned {} as dependency of {}'.format(inventory_task.log_format, task.log_format))
-
-        return inventory_task
-
-    def add_dependencies(self, task, dependencies):
-        with disable_activity_stream():
-            task.dependent_jobs.add(*dependencies)
-
-    def get_inventory_source_tasks(self):
+    def cache_projects_and_sources(self, task_list):
+        project_ids = set()
         inventory_ids = set()
-        for task in self.all_tasks:
+        for task in task_list:
             if isinstance(task, Job):
-                inventory_ids.add(task.inventory_id)
-        self.all_inventory_sources = [invsrc for invsrc in InventorySource.objects.filter(inventory_id__in=inventory_ids, update_on_launch=True)]
+                if task.project_id:
+                    project_ids.add(task.project_id)
+                if task.inventory_id:
+                    inventory_ids.add(task.inventory_id)
+            elif isinstance(task, InventoryUpdate):
+                if task.inventory_source and task.inventory_source.source_project_id:
+                    project_ids.add(task.inventory_source.source_project_id)
 
-    def get_latest_inventory_update(self, inventory_source):
-        latest_inventory_update = InventoryUpdate.objects.filter(inventory_source=inventory_source).order_by("-created")
-        if not latest_inventory_update.exists():
-            return None
-        return latest_inventory_update.first()
+        for proj in Project.objects.filter(id__in=project_ids, scm_update_on_launch=True):
+            self.all_projects[proj.id] = proj
 
-    def should_update_inventory_source(self, job, latest_inventory_update):
-        now = tz_now()
+        for invsrc in InventorySource.objects.filter(inventory_id__in=inventory_ids, update_on_launch=True):
+            self.all_inventory_sources.setdefault(invsrc.inventory_id, [])
+            self.all_inventory_sources[invsrc.inventory_id].append(invsrc)
 
-        if latest_inventory_update is None:
+    @staticmethod
+    def should_update_again(update, cache_timeout):
+        '''
+        If it has never updated, we need to update
+        If there is already an update in progress then we do not need to a new create one
+        If the last update failed, we always need to try and update again
+        If current time is more than cache_timeout after last update, then we need a new one
+        '''
+        if (update is None) or (update.status in ['failed', 'canceled', 'error']):
             return True
-        '''
-        If there's already a inventory update utilizing this job that's about to run
-        then we don't need to create one
-        '''
-        if latest_inventory_update.status in ['waiting', 'pending', 'running']:
+        if update.status in ['waiting', 'pending', 'running']:
             return False
 
-        timeout_seconds = timedelta(seconds=latest_inventory_update.inventory_source.update_cache_timeout)
-        if (latest_inventory_update.finished + timeout_seconds) < now:
-            return True
-        if latest_inventory_update.inventory_source.update_on_launch is True and latest_inventory_update.status in ['failed', 'canceled', 'error']:
-            return True
-        return False
+        return bool(((update.finished + timedelta(seconds=cache_timeout))) < tz_now())
 
-    def get_latest_project_update(self, project_id):
-        latest_project_update = ProjectUpdate.objects.filter(project=project_id, job_type='check').order_by("-created")
-        if not latest_project_update.exists():
-            return None
-        return latest_project_update.first()
-
-    def should_update_related_project(self, job, latest_project_update):
-        now = tz_now()
-
-        if latest_project_update is None:
-            return True
-
-        if latest_project_update.status in ['failed', 'canceled']:
-            return True
-
-        '''
-        If there's already a project update utilizing this job that's about to run
-        then we don't need to create one
-        '''
-        if latest_project_update.status in ['waiting', 'pending', 'running']:
-            return False
-
-        '''
-        If the latest project update has a created time == job_created_time-1
-        then consider the project update found. This is so we don't enter an infinite loop
-        of updating the project when cache timeout is 0.
-        '''
-        if (
-            latest_project_update.project.scm_update_cache_timeout == 0
-            and latest_project_update.launch_type == 'dependency'
-            and latest_project_update.created == job.created - timedelta(seconds=1)
-        ):
-            return False
-        '''
-        Normal Cache Timeout Logic
-        '''
-        timeout_seconds = timedelta(seconds=latest_project_update.project.scm_update_cache_timeout)
-        if (latest_project_update.finished + timeout_seconds) < now:
-            return True
-        return False
+    def get_or_create_project_update(self, project_id):
+        project = self.all_projects.get(project_id, None)
+        if project is not None:
+            latest_project_update = project.project_updates.filter(job_type='check').order_by("-created").first()
+            if self.should_update_again(latest_project_update, project.scm_update_cache_timeout):
+                project_task = project.create_project_update(_eager_fields=dict(launch_type='dependency'))
+                project_task.signal_start()
+                return [project_task]
+            else:
+                return [latest_project_update]
+        return []
 
     def gen_dep_for_job(self, task):
-        created_dependencies = []
-        dependencies = []
-        # TODO: Can remove task.project None check after scan-job-default-playbook is removed
-        if task.project is not None and task.project.scm_update_on_launch is True:
-            latest_project_update = self.get_latest_project_update(task.project_id)
-            if self.should_update_related_project(task, latest_project_update):
-                latest_project_update = self.create_project_update(task)
-                created_dependencies.append(latest_project_update)
-            dependencies.append(latest_project_update)
+        dependencies = self.get_or_create_project_update(task.project_id)
 
-        # Inventory created 2 seconds behind job
         try:
             start_args = json.loads(decrypt_field(task, field_name="start_args"))
         except ValueError:
             start_args = dict()
-        # generator for inventory sources related to this task
-        task_inv_sources = (invsrc for invsrc in self.all_inventory_sources if invsrc.inventory_id == task.inventory_id)
-        for inventory_source in task_inv_sources:
+        # generator for update-on-launch inventory sources related to this task
+        for inventory_source in self.all_inventory_sources.get(task.inventory_id, []):
             if "inventory_sources_already_updated" in start_args and inventory_source.id in start_args['inventory_sources_already_updated']:
                 continue
-            if not inventory_source.update_on_launch:
-                continue
-            latest_inventory_update = self.get_latest_inventory_update(inventory_source)
-            if self.should_update_inventory_source(task, latest_inventory_update):
-                inventory_task = self.create_inventory_update(task, inventory_source)
-                created_dependencies.append(inventory_task)
+            latest_inventory_update = inventory_source.inventory_updates.order_by("-created").first()
+            if self.should_update_again(latest_inventory_update, inventory_source.update_cache_timeout):
+                inventory_task = inventory_source.create_inventory_update(_eager_fields=dict(launch_type='dependency'))
+                inventory_task.signal_start()
                 dependencies.append(inventory_task)
             else:
                 dependencies.append(latest_inventory_update)
 
-        if dependencies:
-            self.add_dependencies(task, dependencies)
-
-        return created_dependencies
+        return dependencies
 
     def gen_dep_for_inventory_update(self, inventory_task):
-        created_dependencies = []
         if inventory_task.source == "scm":
             invsrc = inventory_task.inventory_source
-            if not invsrc.source_project.scm_update_on_launch:
-                return created_dependencies
-
-            latest_src_project_update = self.get_latest_project_update(invsrc.source_project_id)
-            if self.should_update_related_project(inventory_task, latest_src_project_update):
-                latest_src_project_update = self.create_project_update(inventory_task, project_id=invsrc.source_project_id)
-                created_dependencies.append(latest_src_project_update)
-            self.add_dependencies(inventory_task, [latest_src_project_update])
-            latest_src_project_update.scm_inventory_updates.add(inventory_task)
-        return created_dependencies
+            if invsrc:
+                return self.get_or_create_project_update(invsrc.source_project_id)
+        return []
 
     @timeit
     def generate_dependencies(self, undeped_tasks):
-        created_dependencies = []
+        dependencies = []
+        self.cache_projects_and_sources(undeped_tasks)
         for task in undeped_tasks:
             task.log_lifecycle("acknowledged")
             if type(task) is Job:
-                created_dependencies += self.gen_dep_for_job(task)
+                job_deps = self.gen_dep_for_job(task)
             elif type(task) is InventoryUpdate:
-                created_dependencies += self.gen_dep_for_inventory_update(task)
+                job_deps = self.gen_dep_for_inventory_update(task)
             else:
                 continue
+            if job_deps:
+                dependencies += job_deps
+                with disable_activity_stream():
+                    task.dependent_jobs.add(*dependencies)
+                logger.debug(f'Linked {[dep.log_format for dep in dependencies]} as dependencies of {task.log_format}')
+
         UnifiedJob.objects.filter(pk__in=[task.pk for task in undeped_tasks]).update(dependencies_processed=True)
 
-        return created_dependencies
-
-    def process_tasks(self):
-        deps = self.generate_dependencies(self.all_tasks)
-        self.generate_dependencies(deps)
-        self.subsystem_metrics.inc(f"{self.prefix}_pending_processed", len(self.all_tasks) + len(deps))
+        return dependencies
 
     @timeit
     def _schedule(self):
         self.get_tasks(dict(status__in=["pending"], dependencies_processed=False))
 
         if len(self.all_tasks) > 0:
-            self.get_inventory_source_tasks()
-            self.process_tasks()
+            deps = self.generate_dependencies(self.all_tasks)
+            undeped_deps = [dep for dep in deps if dep.dependencies_processed is False]
+            self.generate_dependencies(undeped_deps)
+            self.subsystem_metrics.inc(f"{self.prefix}_pending_processed", len(self.all_tasks) + len(undeped_deps))
             ScheduleTaskManager().schedule()
 
 

awx/main/tasks/__init__.py

@@ -1 +1 @@
-from . import jobs, receptor, system  # noqa
+from . import host_metrics, jobs, receptor, system  # noqa

awx/main/tasks/callback.py

@@ -31,6 +31,7 @@ class RunnerCallback:
         self.model = model
         self.update_attempts = int(settings.DISPATCHER_DB_DOWNTOWN_TOLLERANCE / 5)
         self.wrapup_event_dispatched = False
+        self.artifacts_processed = False
         self.extra_update_fields = {}
 
     def update_model(self, pk, _attempt=0, **updates):
@@ -211,6 +212,9 @@ class RunnerCallback:
             if result_traceback:
                 self.delay_update(result_traceback=result_traceback)
 
+    def artifacts_handler(self, artifact_dir):
+        self.artifacts_processed = True
+
 
 class RunnerCallbackForProjectUpdate(RunnerCallback):
     def __init__(self, *args, **kwargs):

awx/main/tasks/facts.py

@@ -9,6 +9,7 @@ from django.conf import settings
 from django.db.models.query import QuerySet
 from django.utils.encoding import smart_str
 from django.utils.timezone import now
+from django.db import OperationalError
 
 # AWX
 from awx.main.utils.common import log_excess_runtime
@@ -57,6 +58,28 @@ def start_fact_cache(hosts, destination, log_data, timeout=None, inventory_id=No
     return None
 
 
+def raw_update_hosts(host_list):
+    Host.objects.bulk_update(host_list, ['ansible_facts', 'ansible_facts_modified'])
+
+
+def update_hosts(host_list, max_tries=5):
+    if not host_list:
+        return
+    for i in range(max_tries):
+        try:
+            raw_update_hosts(host_list)
+        except OperationalError as exc:
+            # Deadlocks can happen if this runs at the same time as another large query
+            # inventory updates and updating last_job_host_summary are candidates for conflict
+            # but these would resolve easily on a retry
+            if i + 1 < max_tries:
+                logger.info(f'OperationalError (suspected deadlock) saving host facts retry {i}, message: {exc}')
+                continue
+            else:
+                raise
+        break
+
+
 @log_excess_runtime(
     logger,
     debug_cutoff=0.01,
@@ -111,7 +134,6 @@ def finish_fact_cache(hosts, destination, facts_write_time, log_data, job_id=Non
             system_tracking_logger.info('Facts cleared for inventory {} host {}'.format(smart_str(host.inventory.name), smart_str(host.name)))
             log_data['cleared_ct'] += 1
         if len(hosts_to_update) > 100:
-            Host.objects.bulk_update(hosts_to_update, ['ansible_facts', 'ansible_facts_modified'])
+            update_hosts(hosts_to_update)
             hosts_to_update = []
-    if hosts_to_update:
-        Host.objects.bulk_update(hosts_to_update, ['ansible_facts', 'ansible_facts_modified'])
+    update_hosts(hosts_to_update)

awx/main/tasks/host_metrics.py

@@ -0,0 +1,205 @@
+import datetime
+from dateutil.relativedelta import relativedelta
+import logging
+
+from django.conf import settings
+from django.db.models import Count
+from django.db.models.functions import TruncMonth
+from django.utils.timezone import now
+from rest_framework.fields import DateTimeField
+from awx.main.dispatch import get_task_queuename
+from awx.main.dispatch.publish import task
+from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly
+from awx.conf.license import get_license
+
+logger = logging.getLogger('awx.main.tasks.host_metric_summary_monthly')
+
+
+@task(queue=get_task_queuename)
+def host_metric_summary_monthly():
+    """Run cleanup host metrics summary monthly task each week"""
+    if _is_run_threshold_reached(
+        getattr(settings, 'HOST_METRIC_SUMMARY_TASK_LAST_TS', None), getattr(settings, 'HOST_METRIC_SUMMARY_TASK_INTERVAL', 7) * 86400
+    ):
+        logger.info(f"Executing host_metric_summary_monthly, last ran at {getattr(settings, 'HOST_METRIC_SUMMARY_TASK_LAST_TS', '---')}")
+        HostMetricSummaryMonthlyTask().execute()
+        logger.info("Finished host_metric_summary_monthly")
+
+
+def _is_run_threshold_reached(setting, threshold_seconds):
+    last_time = DateTimeField().to_internal_value(setting) if setting else DateTimeField().to_internal_value('1970-01-01')
+
+    return (now() - last_time).total_seconds() > threshold_seconds
+
+
+class HostMetricSummaryMonthlyTask:
+    """
+    This task computes last [threshold] months of HostMetricSummaryMonthly table
+    [threshold] is setting CLEANUP_HOST_METRICS_HARD_THRESHOLD
+    Each record in the table represents changes in HostMetric table in one month
+    It always overrides all the months newer than <threshold>, never updates older months
+    Algorithm:
+    - hosts_added are HostMetric records with first_automation in given month
+    - hosts_deleted are HostMetric records with deleted=True and last_deleted in given month
+    - - HostMetrics soft-deleted before <threshold> also increases hosts_deleted in their last_deleted month
+    - license_consumed is license_consumed(previous month) + hosts_added - hosts_deleted
+    - - license_consumed for HostMetricSummaryMonthly.date < [threshold] is computed also from
+        all HostMetrics.first_automation < [threshold]
+    - license_capacity is set only for current month, and it's never updated (value taken from current subscription)
+    """
+
+    def __init__(self):
+        self.host_metrics = {}
+        self.processed_month = self._get_first_month()
+        self.existing_summaries = None
+        self.existing_summaries_idx = 0
+        self.existing_summaries_cnt = 0
+        self.records_to_create = []
+        self.records_to_update = []
+
+    def execute(self):
+        self._load_existing_summaries()
+        self._load_hosts_added()
+        self._load_hosts_deleted()
+
+        # Get first month after last hard delete
+        month = self._get_first_month()
+        license_consumed = self._get_license_consumed_before(month)
+
+        # Fill record for each month
+        while month <= datetime.date.today().replace(day=1):
+            summary = self._find_or_create_summary(month)
+            # Update summary and update license_consumed by hosts added/removed this month
+            self._update_summary(summary, month, license_consumed)
+            license_consumed = summary.license_consumed
+
+            month = month + relativedelta(months=1)
+
+        # Create/Update stats
+        HostMetricSummaryMonthly.objects.bulk_create(self.records_to_create, batch_size=1000)
+        HostMetricSummaryMonthly.objects.bulk_update(self.records_to_update, ['license_consumed', 'hosts_added', 'hosts_deleted'], batch_size=1000)
+
+        # Set timestamp of last run
+        settings.HOST_METRIC_SUMMARY_TASK_LAST_TS = now()
+
+    def _get_license_consumed_before(self, month):
+        license_consumed = 0
+        for metric_month, metric in self.host_metrics.items():
+            if metric_month < month:
+                hosts_added = metric.get('hosts_added', 0)
+                hosts_deleted = metric.get('hosts_deleted', 0)
+                license_consumed = license_consumed + hosts_added - hosts_deleted
+            else:
+                break
+        return license_consumed
+
+    def _load_existing_summaries(self):
+        """Find all summaries newer than host metrics delete threshold"""
+        self.existing_summaries = HostMetricSummaryMonthly.objects.filter(date__gte=self._get_first_month()).order_by('date')
+        self.existing_summaries_idx = 0
+        self.existing_summaries_cnt = len(self.existing_summaries)
+
+    def _load_hosts_added(self):
+        """Aggregates hosts added each month, by the 'first_automation' timestamp"""
+        #
+        # -- SQL translation (for better code readability)
+        # SELECT date_trunc('month', first_automation) as month,
+        #        count(first_automation) AS hosts_added
+        # FROM main_hostmetric
+        # GROUP BY month
+        # ORDER by month;
+        result = (
+            HostMetric.objects.annotate(month=TruncMonth('first_automation'))
+            .values('month')
+            .annotate(hosts_added=Count('first_automation'))
+            .values('month', 'hosts_added')
+            .order_by('month')
+        )
+
+        for host_metric in list(result):
+            month = host_metric['month']
+            if month:
+                beginning_of_month = datetime.date(month.year, month.month, 1)
+                if self.host_metrics.get(beginning_of_month) is None:
+                    self.host_metrics[beginning_of_month] = {}
+                self.host_metrics[beginning_of_month]['hosts_added'] = host_metric['hosts_added']
+
+    def _load_hosts_deleted(self):
+        """
+        Aggregates hosts deleted each month, by the 'last_deleted' timestamp.
+        Host metrics have to be deleted NOW to be counted as deleted before
+        (by intention - statistics can change retrospectively by re-automation of previously deleted host)
+        """
+        #
+        # -- SQL translation (for better code readability)
+        # SELECT date_trunc('month', last_deleted) as month,
+        #       count(last_deleted) AS hosts_deleted
+        # FROM main_hostmetric
+        # WHERE deleted = True
+        # GROUP BY 1 # equal to "GROUP BY month"
+        # ORDER by month;
+        result = (
+            HostMetric.objects.annotate(month=TruncMonth('last_deleted'))
+            .values('month')
+            .annotate(hosts_deleted=Count('last_deleted'))
+            .values('month', 'hosts_deleted')
+            .filter(deleted=True)
+            .order_by('month')
+        )
+        for host_metric in list(result):
+            month = host_metric['month']
+            if month:
+                beginning_of_month = datetime.date(month.year, month.month, 1)
+                if self.host_metrics.get(beginning_of_month) is None:
+                    self.host_metrics[beginning_of_month] = {}
+                self.host_metrics[beginning_of_month]['hosts_deleted'] = host_metric['hosts_deleted']
+
+    def _find_or_create_summary(self, month):
+        summary = self._find_summary(month)
+
+        if not summary:
+            summary = HostMetricSummaryMonthly(date=month)
+            self.records_to_create.append(summary)
+        else:
+            self.records_to_update.append(summary)
+        return summary
+
+    def _find_summary(self, month):
+        """
+        Existing summaries are ordered by month ASC.
+        This method is called with month in ascending order too => only 1 traversing is enough
+        """
+        summary = None
+        while not summary and self.existing_summaries_idx < self.existing_summaries_cnt:
+            tmp = self.existing_summaries[self.existing_summaries_idx]
+            if tmp.date < month:
+                self.existing_summaries_idx += 1
+            elif tmp.date == month:
+                summary = tmp
+            elif tmp.date > month:
+                break
+        return summary
+
+    def _update_summary(self, summary, month, license_consumed):
+        """Updates the metric with hosts added and deleted and set license info for current month"""
+        # Get month counts from host metrics, zero if not found
+        hosts_added, hosts_deleted = 0, 0
+        if metric := self.host_metrics.get(month, None):
+            hosts_added = metric.get('hosts_added', 0)
+            hosts_deleted = metric.get('hosts_deleted', 0)
+
+        summary.license_consumed = license_consumed + hosts_added - hosts_deleted
+        summary.hosts_added = hosts_added
+        summary.hosts_deleted = hosts_deleted
+
+        # Set subscription count for current month
+        if month == datetime.date.today().replace(day=1):
+            license_info = get_license()
+            summary.license_capacity = license_info.get('instance_count', 0)
+        return summary
+
+    @staticmethod
+    def _get_first_month():
+        """Returns first month after host metrics hard delete threshold"""
+        threshold = getattr(settings, 'CLEANUP_HOST_METRICS_HARD_THRESHOLD', 36)
+        return datetime.date.today().replace(day=1) - relativedelta(months=int(threshold) - 1)

awx/main/tasks/jobs.py

@@ -1094,7 +1094,7 @@ class RunJob(SourceControlMixin, BaseTask):
             # actual `run()` call; this _usually_ means something failed in
             # the pre_run_hook method
             return
-        if self.should_use_fact_cache():
+        if self.should_use_fact_cache() and self.runner_callback.artifacts_processed:
             job.log_lifecycle("finish_job_fact_cache")
             finish_fact_cache(
                 job.get_hosts_for_fact_cache(),

awx/main/tasks/receptor.py

@@ -464,6 +464,7 @@ class AWXReceptorJob:
             event_handler=self.task.runner_callback.event_handler,
             finished_callback=self.task.runner_callback.finished_callback,
             status_handler=self.task.runner_callback.status_handler,
+            artifacts_handler=self.task.runner_callback.artifacts_handler,
             **self.runner_params,
         )
 

awx/main/tasks/system.py

@@ -2,6 +2,7 @@
 from collections import namedtuple
 import functools
 import importlib
+import itertools
 import json
 import logging
 import os
@@ -14,7 +15,7 @@ from datetime import datetime
 
 # Django
 from django.conf import settings
-from django.db import transaction, DatabaseError, IntegrityError
+from django.db import connection, transaction, DatabaseError, IntegrityError
 from django.db.models.fields.related import ForeignKey
 from django.utils.timezone import now, timedelta
 from django.utils.encoding import smart_str
@@ -48,6 +49,7 @@ from awx.main.models import (
     SmartInventoryMembership,
     Job,
     HostMetric,
+    convert_jsonfields,
 )
 from awx.main.constants import ACTIVE_STATES
 from awx.main.dispatch.publish import task
@@ -86,6 +88,11 @@ def dispatch_startup():
     if settings.IS_K8S:
         write_receptor_config()
 
+    try:
+        convert_jsonfields()
+    except Exception:
+        logger.exception("Failed json field conversion, skipping.")
+
     startup_logger.debug("Syncing Schedules")
     for sch in Schedule.objects.all():
         try:
@@ -129,6 +136,52 @@ def inform_cluster_of_shutdown():
         logger.exception('Encountered problem with normal shutdown signal.')
 
 
+@task(queue=get_task_queuename)
+def migrate_jsonfield(table, pkfield, columns):
+    batchsize = 10000
+    with advisory_lock(f'json_migration_{table}', wait=False) as acquired:
+        if not acquired:
+            return
+
+        from django.db.migrations.executor import MigrationExecutor
+
+        # If Django is currently running migrations, wait until it is done.
+        while True:
+            executor = MigrationExecutor(connection)
+            if not executor.migration_plan(executor.loader.graph.leaf_nodes()):
+                break
+            time.sleep(120)
+
+        logger.warning(f"Migrating json fields for {table}: {', '.join(columns)}")
+
+        with connection.cursor() as cursor:
+            for i in itertools.count(0, batchsize):
+                # Are there even any rows in the table beyond this point?
+                cursor.execute(f"select count(1) from {table} where {pkfield} >= %s limit 1;", (i,))
+                if not cursor.fetchone()[0]:
+                    break
+
+                column_expr = ', '.join(f"{colname} = {colname}_old::jsonb" for colname in columns)
+                # If any of the old columns have non-null values, the data needs to be cast and copied over.
+                empty_expr = ' or '.join(f"{colname}_old is not null" for colname in columns)
+                cursor.execute(  # Only clobber the new fields if there is non-null data in the old ones.
+                    f"""
+                    update {table}
+                      set {column_expr}
+                      where {pkfield} >= %s and {pkfield} < %s
+                        and {empty_expr};
+                    """,
+                    (i, i + batchsize),
+                )
+                rows = cursor.rowcount
+                logger.debug(f"Batch {i} to {i + batchsize} copied on {table}, {rows} rows affected.")
+
+            column_expr = ', '.join(f"DROP COLUMN {column}_old" for column in columns)
+            cursor.execute(f"ALTER TABLE {table} {column_expr};")
+
+        logger.warning(f"Migration of {table} to jsonb is finished.")
+
+
 @task(queue=get_task_queuename)
 def apply_cluster_membership_policies():
     from awx.main.signals import disable_activity_stream
@@ -316,13 +369,8 @@ def send_notifications(notification_list, job_id=None):
 @task(queue=get_task_queuename)
 def gather_analytics():
     from awx.conf.models import Setting
-    from rest_framework.fields import DateTimeField
 
-    last_gather = Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_GATHER').first()
-    last_time = DateTimeField().to_internal_value(last_gather.value) if last_gather and last_gather.value else None
-    gather_time = now()
-
-    if not last_time or ((gather_time - last_time).total_seconds() > settings.AUTOMATION_ANALYTICS_GATHER_INTERVAL):
+    if is_run_threshold_reached(Setting.objects.filter(key='AUTOMATION_ANALYTICS_LAST_GATHER').first(), settings.AUTOMATION_ANALYTICS_GATHER_INTERVAL):
         analytics.gather()
 
 
@@ -381,16 +429,25 @@ def cleanup_images_and_files():
 
 @task(queue=get_task_queuename)
 def cleanup_host_metrics():
+    """Run cleanup host metrics ~each month"""
+    # TODO: move whole method to host_metrics in follow-up PR
     from awx.conf.models import Setting
+
+    if is_run_threshold_reached(
+        Setting.objects.filter(key='CLEANUP_HOST_METRICS_LAST_TS').first(), getattr(settings, 'CLEANUP_HOST_METRICS_INTERVAL', 30) * 86400
+    ):
+        months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_SOFT_THRESHOLD', 12)
+        logger.info("Executing cleanup_host_metrics")
+        HostMetric.cleanup_task(months_ago)
+        logger.info("Finished cleanup_host_metrics")
+
+
+def is_run_threshold_reached(setting, threshold_seconds):
     from rest_framework.fields import DateTimeField
 
-    last_cleanup = Setting.objects.filter(key='CLEANUP_HOST_METRICS_LAST_TS').first()
-    last_time = DateTimeField().to_internal_value(last_cleanup.value) if last_cleanup and last_cleanup.value else None
+    last_time = DateTimeField().to_internal_value(setting.value) if setting and setting.value else DateTimeField().to_internal_value('1970-01-01')
 
-    cleanup_interval_secs = getattr(settings, 'CLEANUP_HOST_METRICS_INTERVAL', 30) * 86400
-    if not last_time or ((now() - last_time).total_seconds() > cleanup_interval_secs):
-        months_ago = getattr(settings, 'CLEANUP_HOST_METRICS_THRESHOLD', 12)
-        HostMetric.cleanup_task(months_ago)
+    return (now() - last_time).total_seconds() > threshold_seconds
 
 
 @task(queue=get_task_queuename)
@@ -839,10 +896,7 @@ def delete_inventory(inventory_id, user_id, retries=5):
             user = None
     with ignore_inventory_computed_fields(), ignore_inventory_group_removal(), impersonate(user):
         try:
-            i = Inventory.objects.get(id=inventory_id)
-            for host in i.hosts.iterator():
-                host.job_events_as_primary_host.update(host=None)
-            i.delete()
+            Inventory.objects.get(id=inventory_id).delete()
             emit_channel_notification('inventories-status_changed', {'group_name': 'inventories', 'inventory_id': inventory_id, 'status': 'deleted'})
             logger.debug('Deleted inventory {} as user {}.'.format(inventory_id, user_id))
         except Inventory.DoesNotExist:

awx/main/tests/factories/fixtures.py

@@ -1,6 +1,9 @@
 import json
 
 from django.contrib.auth.models import User
+from django.core.exceptions import ValidationError
+
+from unittest import mock
 
 from awx.main.models import (
     Organization,
@@ -20,6 +23,7 @@ from awx.main.models import (
     WorkflowJobNode,
     WorkflowJobTemplateNode,
 )
+from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly
 
 # mk methods should create only a single object of a single type.
 # they should also have the option of being persisted or not.
@@ -248,3 +252,42 @@ def mk_workflow_job_node(unified_job_template=None, success_nodes=None, failure_
     if persisted:
         workflow_node.save()
     return workflow_node
+
+
+def mk_host_metric(hostname, first_automation, last_automation=None, last_deleted=None, deleted=False, persisted=True):
+    ok, idx = False, 1
+    while not ok:
+        try:
+            with mock.patch("django.utils.timezone.now") as mock_now:
+                mock_now.return_value = first_automation
+                metric = HostMetric(
+                    hostname=hostname or f"host-{first_automation}-{idx}",
+                    first_automation=first_automation,
+                    last_automation=last_automation or first_automation,
+                    last_deleted=last_deleted,
+                    deleted=deleted,
+                )
+                metric.validate_unique()
+                if persisted:
+                    metric.save()
+                ok = True
+        except ValidationError as e:
+            # Repeat create for auto-generated hostname
+            if not hostname and e.message_dict.get('hostname', None):
+                idx += 1
+            else:
+                raise e
+
+
+def mk_host_metric_summary(date, license_consumed=0, license_capacity=0, hosts_added=0, hosts_deleted=0, indirectly_managed_hosts=0, persisted=True):
+    summary = HostMetricSummaryMonthly(
+        date=date,
+        license_consumed=license_consumed,
+        license_capacity=license_capacity,
+        hosts_added=hosts_added,
+        hosts_deleted=hosts_deleted,
+        indirectly_managed_hosts=indirectly_managed_hosts,
+    )
+    if persisted:
+        summary.save()
+    return summary

awx/main/tests/functional/commands/test_host_metric_summary_monthly.py

@@ -0,0 +1,382 @@
+import pytest
+import datetime
+from dateutil.relativedelta import relativedelta
+from django.conf import settings
+from django.utils import timezone
+
+
+from awx.main.management.commands.host_metric_summary_monthly import Command
+from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly
+from awx.main.tests.factories.fixtures import mk_host_metric, mk_host_metric_summary
+
+
+@pytest.fixture
+def threshold():
+    return int(getattr(settings, 'CLEANUP_HOST_METRICS_HARD_THRESHOLD', 36))
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize("metrics_cnt", [0, 1, 2, 3])
+@pytest.mark.parametrize("mode", ["old_data", "actual_data", "all_data"])
+def test_summaries_counts(threshold, metrics_cnt, mode):
+    assert HostMetricSummaryMonthly.objects.count() == 0
+
+    for idx in range(metrics_cnt):
+        if mode == "old_data" or mode == "all_data":
+            mk_host_metric(None, months_ago(threshold + idx, "dt"))
+        elif mode == "actual_data" or mode == "all_data":
+            mk_host_metric(None, (months_ago(threshold - idx, "dt")))
+
+    Command().handle()
+
+    # Number of records is equal to host metrics' hard cleanup months
+    assert HostMetricSummaryMonthly.objects.count() == threshold
+
+    # Records start with date in the month following to the threshold month
+    date = months_ago(threshold - 1)
+    for metric in list(HostMetricSummaryMonthly.objects.order_by('date').all()):
+        assert metric.date == date
+        date += relativedelta(months=1)
+
+    # Older record are untouched
+    mk_host_metric_summary(date=months_ago(threshold + 10))
+    Command().handle()
+
+    assert HostMetricSummaryMonthly.objects.count() == threshold + 1
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize("mode", ["old_data", "actual_data", "all_data"])
+def test_summary_values(threshold, mode):
+    tester = {"old_data": MetricsTesterOldData(threshold), "actual_data": MetricsTesterActualData(threshold), "all_data": MetricsTesterCombinedData(threshold)}[
+        mode
+    ]
+
+    for iteration in ["create_metrics", "add_old_summaries", "change_metrics", "delete_metrics", "add_metrics"]:
+        getattr(tester, iteration)()  # call method by string
+
+        # Operation is idempotent, repeat twice
+        for _ in range(2):
+            Command().handle()
+            # call assert method by string
+            getattr(tester, f"assert_{iteration}")()
+
+
+class MetricsTester:
+    def __init__(self, threshold, ignore_asserts=False):
+        self.threshold = threshold
+        self.expected_summaries = {}
+        self.ignore_asserts = ignore_asserts
+
+    def add_old_summaries(self):
+        """These records don't correspond with Host metrics"""
+        mk_host_metric_summary(self.below(4), license_consumed=100, hosts_added=10, hosts_deleted=5)
+        mk_host_metric_summary(self.below(3), license_consumed=105, hosts_added=20, hosts_deleted=10)
+        mk_host_metric_summary(self.below(2), license_consumed=115, hosts_added=60, hosts_deleted=75)
+
+    def assert_add_old_summaries(self):
+        """Old summary records should be untouched"""
+        self.expected_summaries[self.below(4)] = {"date": self.below(4), "license_consumed": 100, "hosts_added": 10, "hosts_deleted": 5}
+        self.expected_summaries[self.below(3)] = {"date": self.below(3), "license_consumed": 105, "hosts_added": 20, "hosts_deleted": 10}
+        self.expected_summaries[self.below(2)] = {"date": self.below(2), "license_consumed": 115, "hosts_added": 60, "hosts_deleted": 75}
+
+        self.assert_host_metric_summaries()
+
+    def assert_host_metric_summaries(self):
+        """Ignore asserts when old/actual test object is used only as a helper for Combined test"""
+        if self.ignore_asserts:
+            return True
+
+        for summary in list(HostMetricSummaryMonthly.objects.order_by('date').all()):
+            assert self.expected_summaries.get(summary.date, None) is not None
+
+            assert self.expected_summaries[summary.date] == {
+                "date": summary.date,
+                "license_consumed": summary.license_consumed,
+                "hosts_added": summary.hosts_added,
+                "hosts_deleted": summary.hosts_deleted,
+            }
+
+    def below(self, months, fmt="date"):
+        """months below threshold, returns first date of that month"""
+        date = months_ago(self.threshold + months)
+        if fmt == "dt":
+            return timezone.make_aware(datetime.datetime.combine(date, datetime.datetime.min.time()))
+        else:
+            return date
+
+    def above(self, months, fmt="date"):
+        """months above threshold, returns first date of that month"""
+        date = months_ago(self.threshold - months)
+        if fmt == "dt":
+            return timezone.make_aware(datetime.datetime.combine(date, datetime.datetime.min.time()))
+        else:
+            return date
+
+
+class MetricsTesterOldData(MetricsTester):
+    def create_metrics(self):
+        """Creates 7 host metrics older than delete threshold"""
+        mk_host_metric("host_1", first_automation=self.below(3, "dt"))
+        mk_host_metric("host_2", first_automation=self.below(2, "dt"))
+        mk_host_metric("host_3", first_automation=self.below(2, "dt"), last_deleted=self.above(2, "dt"), deleted=False)
+        mk_host_metric("host_4", first_automation=self.below(2, "dt"), last_deleted=self.above(2, "dt"), deleted=True)
+        mk_host_metric("host_5", first_automation=self.below(2, "dt"), last_deleted=self.below(2, "dt"), deleted=True)
+        mk_host_metric("host_6", first_automation=self.below(1, "dt"), last_deleted=self.below(1, "dt"), deleted=False)
+        mk_host_metric("host_7", first_automation=self.below(1, "dt"))
+
+    def assert_create_metrics(self):
+        """
+        Month 1 is computed from older host metrics,
+        Month 2 has deletion (host_4)
+        Other months are unchanged (same as month 2)
+        """
+        self.expected_summaries = {
+            self.above(1): {"date": self.above(1), "license_consumed": 6, "hosts_added": 0, "hosts_deleted": 0},
+            self.above(2): {"date": self.above(2), "license_consumed": 5, "hosts_added": 0, "hosts_deleted": 1},
+        }
+        # no change in months 3+
+        idx = 3
+        month = self.above(idx)
+        while month <= beginning_of_the_month():
+            self.expected_summaries[self.above(idx)] = {"date": self.above(idx), "license_consumed": 5, "hosts_added": 0, "hosts_deleted": 0}
+            month += relativedelta(months=1)
+            idx += 1
+
+        self.assert_host_metric_summaries()
+
+    def add_old_summaries(self):
+        super().add_old_summaries()
+
+    def assert_add_old_summaries(self):
+        super().assert_add_old_summaries()
+
+    @staticmethod
+    def change_metrics():
+        """Hosts 1,2 soft deleted, host_4 automated again (undeleted)"""
+        HostMetric.objects.filter(hostname='host_1').update(last_deleted=beginning_of_the_month("dt"), deleted=True)
+        HostMetric.objects.filter(hostname='host_2').update(last_deleted=timezone.now(), deleted=True)
+        HostMetric.objects.filter(hostname='host_4').update(deleted=False)
+
+    def assert_change_metrics(self):
+        """
+        Summaries since month 2 were changed (host_4 restored == automated again)
+        Current month has 2 deletions (host_1, host_2)
+        """
+        self.expected_summaries[self.above(2)] |= {'hosts_deleted': 0}
+        for idx in range(2, self.threshold):
+            self.expected_summaries[self.above(idx)] |= {'license_consumed': 6}
+        self.expected_summaries[beginning_of_the_month()] |= {'license_consumed': 4, 'hosts_deleted': 2}
+
+        self.assert_host_metric_summaries()
+
+    @staticmethod
+    def delete_metrics():
+        """Deletes metric deleted before the threshold"""
+        HostMetric.objects.filter(hostname='host_5').delete()
+
+    def assert_delete_metrics(self):
+        """No change"""
+        self.assert_host_metric_summaries()
+
+    @staticmethod
+    def add_metrics():
+        """Adds new metrics"""
+        mk_host_metric("host_24", first_automation=beginning_of_the_month("dt"))
+        mk_host_metric("host_25", first_automation=beginning_of_the_month("dt"))  # timezone.now())
+
+    def assert_add_metrics(self):
+        """Summary in current month is updated"""
+        self.expected_summaries[beginning_of_the_month()]['license_consumed'] = 6
+        self.expected_summaries[beginning_of_the_month()]['hosts_added'] = 2
+
+        self.assert_host_metric_summaries()
+
+
+class MetricsTesterActualData(MetricsTester):
+    def create_metrics(self):
+        """Creates 16 host metrics newer than delete threshold"""
+        mk_host_metric("host_8", first_automation=self.above(1, "dt"))
+        mk_host_metric("host_9", first_automation=self.above(1, "dt"), last_deleted=self.above(1, "dt"))
+        mk_host_metric("host_10", first_automation=self.above(1, "dt"), last_deleted=self.above(1, "dt"), deleted=True)
+        mk_host_metric("host_11", first_automation=self.above(1, "dt"), last_deleted=self.above(2, "dt"))
+        mk_host_metric("host_12", first_automation=self.above(1, "dt"), last_deleted=self.above(2, "dt"), deleted=True)
+        mk_host_metric("host_13", first_automation=self.above(2, "dt"))
+        mk_host_metric("host_14", first_automation=self.above(2, "dt"), last_deleted=self.above(2, "dt"))
+        mk_host_metric("host_15", first_automation=self.above(2, "dt"), last_deleted=self.above(2, "dt"), deleted=True)
+        mk_host_metric("host_16", first_automation=self.above(2, "dt"), last_deleted=self.above(3, "dt"))
+        mk_host_metric("host_17", first_automation=self.above(2, "dt"), last_deleted=self.above(3, "dt"), deleted=True)
+        mk_host_metric("host_18", first_automation=self.above(4, "dt"))
+        # next one shouldn't happen in real (deleted=True, last_deleted = NULL)
+        mk_host_metric("host_19", first_automation=self.above(4, "dt"), deleted=True)
+        mk_host_metric("host_20", first_automation=self.above(4, "dt"), last_deleted=self.above(4, "dt"))
+        mk_host_metric("host_21", first_automation=self.above(4, "dt"), last_deleted=self.above(4, "dt"), deleted=True)
+        mk_host_metric("host_22", first_automation=self.above(4, "dt"), last_deleted=self.above(5, "dt"))
+        mk_host_metric("host_23", first_automation=self.above(4, "dt"), last_deleted=self.above(5, "dt"), deleted=True)
+
+    def assert_create_metrics(self):
+        self.expected_summaries = {
+            self.above(1): {"date": self.above(1), "license_consumed": 4, "hosts_added": 5, "hosts_deleted": 1},
+            self.above(2): {"date": self.above(2), "license_consumed": 7, "hosts_added": 5, "hosts_deleted": 2},
+            self.above(3): {"date": self.above(3), "license_consumed": 6, "hosts_added": 0, "hosts_deleted": 1},
+            self.above(4): {"date": self.above(4), "license_consumed": 11, "hosts_added": 6, "hosts_deleted": 1},
+            self.above(5): {"date": self.above(5), "license_consumed": 10, "hosts_added": 0, "hosts_deleted": 1},
+        }
+        # no change in months 6+
+        idx = 6
+        month = self.above(idx)
+        while month <= beginning_of_the_month():
+            self.expected_summaries[self.above(idx)] = {"date": self.above(idx), "license_consumed": 10, "hosts_added": 0, "hosts_deleted": 0}
+            month += relativedelta(months=1)
+            idx += 1
+
+        self.assert_host_metric_summaries()
+
+    def add_old_summaries(self):
+        super().add_old_summaries()
+
+    def assert_add_old_summaries(self):
+        super().assert_add_old_summaries()
+
+    @staticmethod
+    def change_metrics():
+        """
+        - Hosts 12, 19, 21 were automated again (undeleted)
+        - Host 16 was soft deleted
+        - Host 17 was undeleted and soft deleted again
+        """
+        HostMetric.objects.filter(hostname='host_12').update(deleted=False)
+        HostMetric.objects.filter(hostname='host_16').update(last_deleted=timezone.now(), deleted=True)
+        HostMetric.objects.filter(hostname='host_17').update(last_deleted=beginning_of_the_month("dt"), deleted=True)
+        HostMetric.objects.filter(hostname='host_19').update(deleted=False)
+        HostMetric.objects.filter(hostname='host_21').update(deleted=False)
+
+    def assert_change_metrics(self):
+        """
+        Summaries since month 2 were changed
+        Current month has 2 deletions (host_16, host_17)
+        """
+        self.expected_summaries[self.above(2)] |= {'license_consumed': 8, 'hosts_deleted': 1}
+        self.expected_summaries[self.above(3)] |= {'license_consumed': 8, 'hosts_deleted': 0}
+        self.expected_summaries[self.above(4)] |= {'license_consumed': 14, 'hosts_deleted': 0}
+
+        # month 5 had hosts_deleted 1 => license_consumed == 14 - 1
+        for idx in range(5, self.threshold):
+            self.expected_summaries[self.above(idx)] |= {'license_consumed': 13}
+        self.expected_summaries[beginning_of_the_month()] |= {'license_consumed': 11, 'hosts_deleted': 2}
+
+        self.assert_host_metric_summaries()
+
+    def delete_metrics(self):
+        """Hard cleanup can't delete metrics newer than threshold. No change"""
+        pass
+
+    def assert_delete_metrics(self):
+        """No change"""
+        self.assert_host_metric_summaries()
+
+    @staticmethod
+    def add_metrics():
+        """Adds new metrics"""
+        mk_host_metric("host_26", first_automation=beginning_of_the_month("dt"))
+        mk_host_metric("host_27", first_automation=timezone.now())
+
+    def assert_add_metrics(self):
+        """
+        Two metrics were deleted in current month by change_metrics()
+        Two metrics are added now
+        => license_consumed is equal to the previous month (13 - 2 + 2)
+        """
+        self.expected_summaries[beginning_of_the_month()] |= {'license_consumed': 13, 'hosts_added': 2}
+
+        self.assert_host_metric_summaries()
+
+
+class MetricsTesterCombinedData(MetricsTester):
+    def __init__(self, threshold):
+        super().__init__(threshold)
+        self.old_data = MetricsTesterOldData(threshold, ignore_asserts=True)
+        self.actual_data = MetricsTesterActualData(threshold, ignore_asserts=True)
+
+    def assert_host_metric_summaries(self):
+        self._combine_expected_summaries()
+        super().assert_host_metric_summaries()
+
+    def create_metrics(self):
+        self.old_data.create_metrics()
+        self.actual_data.create_metrics()
+
+    def assert_create_metrics(self):
+        self.old_data.assert_create_metrics()
+        self.actual_data.assert_create_metrics()
+
+        self.assert_host_metric_summaries()
+
+    def add_old_summaries(self):
+        super().add_old_summaries()
+
+    def assert_add_old_summaries(self):
+        self.old_data.assert_add_old_summaries()
+        self.actual_data.assert_add_old_summaries()
+
+        self.assert_host_metric_summaries()
+
+    def change_metrics(self):
+        self.old_data.change_metrics()
+        self.actual_data.change_metrics()
+
+    def assert_change_metrics(self):
+        self.old_data.assert_change_metrics()
+        self.actual_data.assert_change_metrics()
+
+        self.assert_host_metric_summaries()
+
+    def delete_metrics(self):
+        self.old_data.delete_metrics()
+        self.actual_data.delete_metrics()
+
+    def assert_delete_metrics(self):
+        self.old_data.assert_delete_metrics()
+        self.actual_data.assert_delete_metrics()
+
+        self.assert_host_metric_summaries()
+
+    def add_metrics(self):
+        self.old_data.add_metrics()
+        self.actual_data.add_metrics()
+
+    def assert_add_metrics(self):
+        self.old_data.assert_add_metrics()
+        self.actual_data.assert_add_metrics()
+
+        self.assert_host_metric_summaries()
+
+    def _combine_expected_summaries(self):
+        """
+        Expected summaries are sum of expected values for tests with old and actual data
+        Except data older than hard delete threshold (these summaries are untouched by task => the same in all tests)
+        """
+        for date, summary in self.old_data.expected_summaries.items():
+            if date <= months_ago(self.threshold):
+                license_consumed = summary['license_consumed']
+                hosts_added = summary['hosts_added']
+                hosts_deleted = summary['hosts_deleted']
+            else:
+                license_consumed = summary['license_consumed'] + self.actual_data.expected_summaries[date]['license_consumed']
+                hosts_added = summary['hosts_added'] + self.actual_data.expected_summaries[date]['hosts_added']
+                hosts_deleted = summary['hosts_deleted'] + self.actual_data.expected_summaries[date]['hosts_deleted']
+            self.expected_summaries[date] = {'date': date, 'license_consumed': license_consumed, 'hosts_added': hosts_added, 'hosts_deleted': hosts_deleted}
+
+
+def months_ago(num, fmt="date"):
+    if num is None:
+        return None
+    return beginning_of_the_month(fmt) - relativedelta(months=num)
+
+
+def beginning_of_the_month(fmt="date"):
+    date = datetime.date.today().replace(day=1)
+    if fmt == "dt":
+        return timezone.make_aware(datetime.datetime.combine(date, datetime.datetime.min.time()))
+    else:
+        return date

awx/main/tests/functional/task_management/test_scheduler.py

@@ -331,15 +331,13 @@ def test_single_job_dependencies_project_launch(controlplane_instance_group, job
     p.save(skip_update=True)
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         dm = DependencyManager()
-        with mock.patch.object(DependencyManager, "create_project_update", wraps=dm.create_project_update) as mock_pu:
-            dm.schedule()
-            mock_pu.assert_called_once_with(j)
-            pu = [x for x in p.project_updates.all()]
-            assert len(pu) == 1
-            TaskManager().schedule()
-            TaskManager.start_task.assert_called_once_with(pu[0], controlplane_instance_group, instance)
-            pu[0].status = "successful"
-            pu[0].save()
+        dm.schedule()
+        pu = [x for x in p.project_updates.all()]
+        assert len(pu) == 1
+        TaskManager().schedule()
+        TaskManager.start_task.assert_called_once_with(pu[0], controlplane_instance_group, instance)
+        pu[0].status = "successful"
+        pu[0].save()
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
         TaskManager.start_task.assert_called_once_with(j, controlplane_instance_group, instance)
@@ -359,15 +357,14 @@ def test_single_job_dependencies_inventory_update_launch(controlplane_instance_g
     i.inventory_sources.add(ii)
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         dm = DependencyManager()
-        with mock.patch.object(DependencyManager, "create_inventory_update", wraps=dm.create_inventory_update) as mock_iu:
-            dm.schedule()
-            mock_iu.assert_called_once_with(j, ii)
-            iu = [x for x in ii.inventory_updates.all()]
-            assert len(iu) == 1
-            TaskManager().schedule()
-            TaskManager.start_task.assert_called_once_with(iu[0], controlplane_instance_group, instance)
-            iu[0].status = "successful"
-            iu[0].save()
+        dm.schedule()
+        assert ii.inventory_updates.count() == 1
+        iu = [x for x in ii.inventory_updates.all()]
+        assert len(iu) == 1
+        TaskManager().schedule()
+        TaskManager.start_task.assert_called_once_with(iu[0], controlplane_instance_group, instance)
+        iu[0].status = "successful"
+        iu[0].save()
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
         TaskManager.start_task.assert_called_once_with(j, controlplane_instance_group, instance)
@@ -382,11 +379,11 @@ def test_inventory_update_launches_project_update(controlplane_instance_group, s
     iu = ii.create_inventory_update()
     iu.status = "pending"
     iu.save()
+    assert project.project_updates.count() == 0
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         dm = DependencyManager()
-        with mock.patch.object(DependencyManager, "create_project_update", wraps=dm.create_project_update) as mock_pu:
-            dm.schedule()
-            mock_pu.assert_called_with(iu, project_id=project.id)
+        dm.schedule()
+    assert project.project_updates.count() == 1
 
 
 @pytest.mark.django_db
@@ -407,9 +404,8 @@ def test_job_dependency_with_already_updated(controlplane_instance_group, job_te
     j.save()
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         dm = DependencyManager()
-        with mock.patch.object(DependencyManager, "create_inventory_update", wraps=dm.create_inventory_update) as mock_iu:
-            dm.schedule()
-            mock_iu.assert_not_called()
+        dm.schedule()
+        assert ii.inventory_updates.count() == 0
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
         TaskManager.start_task.assert_called_once_with(j, controlplane_instance_group, instance)
@@ -442,7 +438,9 @@ def test_shared_dependencies_launch(controlplane_instance_group, job_template_fa
         TaskManager().schedule()
         pu = p.project_updates.first()
         iu = ii.inventory_updates.first()
-        TaskManager.start_task.assert_has_calls([mock.call(iu, controlplane_instance_group, instance), mock.call(pu, controlplane_instance_group, instance)])
+        TaskManager.start_task.assert_has_calls(
+            [mock.call(iu, controlplane_instance_group, instance), mock.call(pu, controlplane_instance_group, instance)], any_order=True
+        )
         pu.status = "successful"
         pu.finished = pu.created + timedelta(seconds=1)
         pu.save()
@@ -451,7 +449,9 @@ def test_shared_dependencies_launch(controlplane_instance_group, job_template_fa
         iu.save()
     with mock.patch("awx.main.scheduler.TaskManager.start_task"):
         TaskManager().schedule()
-        TaskManager.start_task.assert_has_calls([mock.call(j1, controlplane_instance_group, instance), mock.call(j2, controlplane_instance_group, instance)])
+        TaskManager.start_task.assert_has_calls(
+            [mock.call(j1, controlplane_instance_group, instance), mock.call(j2, controlplane_instance_group, instance)], any_order=True
+        )
     pu = [x for x in p.project_updates.all()]
     iu = [x for x in ii.inventory_updates.all()]
     assert len(pu) == 1

awx/main/tests/functional/test_dispatch.py

@@ -3,6 +3,7 @@ import multiprocessing
 import random
 import signal
 import time
+import yaml
 from unittest import mock
 
 from django.utils.timezone import now as tz_now
@@ -13,6 +14,7 @@ from awx.main.dispatch import reaper
 from awx.main.dispatch.pool import StatefulPoolWorker, WorkerPool, AutoscalePool
 from awx.main.dispatch.publish import task
 from awx.main.dispatch.worker import BaseWorker, TaskWorker
+from awx.main.dispatch.periodic import Scheduler
 
 
 '''
@@ -439,3 +441,76 @@ class TestJobReaper(object):
         assert job.started > ref_time
         assert job.status == 'running'
         assert job.job_explanation == ''
+
+
+@pytest.mark.django_db
+class TestScheduler:
+    def test_too_many_schedules_freak_out(self):
+        with pytest.raises(RuntimeError):
+            Scheduler({'job1': {'schedule': datetime.timedelta(seconds=1)}, 'job2': {'schedule': datetime.timedelta(seconds=1)}})
+
+    def test_spread_out(self):
+        scheduler = Scheduler(
+            {
+                'job1': {'schedule': datetime.timedelta(seconds=16)},
+                'job2': {'schedule': datetime.timedelta(seconds=16)},
+                'job3': {'schedule': datetime.timedelta(seconds=16)},
+                'job4': {'schedule': datetime.timedelta(seconds=16)},
+            }
+        )
+        assert [job.offset for job in scheduler.jobs] == [0, 4, 8, 12]
+
+    def test_missed_schedule(self, mocker):
+        scheduler = Scheduler({'job1': {'schedule': datetime.timedelta(seconds=10)}})
+        assert scheduler.jobs[0].missed_runs(time.time() - scheduler.global_start) == 0
+        mocker.patch('awx.main.dispatch.periodic.time.time', return_value=scheduler.global_start + 50)
+        scheduler.get_and_mark_pending()
+        assert scheduler.jobs[0].missed_runs(50) > 1
+
+    def test_advance_schedule(self, mocker):
+        scheduler = Scheduler(
+            {
+                'job1': {'schedule': datetime.timedelta(seconds=30)},
+                'joba': {'schedule': datetime.timedelta(seconds=20)},
+                'jobb': {'schedule': datetime.timedelta(seconds=20)},
+            }
+        )
+        for job in scheduler.jobs:
+            # HACK: the offsets automatically added make this a hard test to write... so remove offsets
+            job.offset = 0.0
+        mocker.patch('awx.main.dispatch.periodic.time.time', return_value=scheduler.global_start + 29)
+        to_run = scheduler.get_and_mark_pending()
+        assert set(job.name for job in to_run) == set(['joba', 'jobb'])
+        mocker.patch('awx.main.dispatch.periodic.time.time', return_value=scheduler.global_start + 39)
+        to_run = scheduler.get_and_mark_pending()
+        assert len(to_run) == 1
+        assert to_run[0].name == 'job1'
+
+    @staticmethod
+    def get_job(scheduler, name):
+        for job in scheduler.jobs:
+            if job.name == name:
+                return job
+
+    def test_scheduler_debug(self, mocker):
+        scheduler = Scheduler(
+            {
+                'joba': {'schedule': datetime.timedelta(seconds=20)},
+                'jobb': {'schedule': datetime.timedelta(seconds=50)},
+                'jobc': {'schedule': datetime.timedelta(seconds=500)},
+                'jobd': {'schedule': datetime.timedelta(seconds=20)},
+            }
+        )
+        rel_time = 119.9  # slightly under the 6th 20-second bin, to avoid offset problems
+        current_time = scheduler.global_start + rel_time
+        mocker.patch('awx.main.dispatch.periodic.time.time', return_value=current_time - 1.0e-8)
+        self.get_job(scheduler, 'jobb').mark_run(rel_time)
+        self.get_job(scheduler, 'jobd').mark_run(rel_time - 20.0)
+
+        output = scheduler.debug()
+        data = yaml.safe_load(output)
+        assert data['schedule_list']['jobc']['last_run_seconds_ago'] is None
+        assert data['schedule_list']['joba']['missed_runs'] == 4
+        assert data['schedule_list']['jobd']['missed_runs'] == 3
+        assert data['schedule_list']['jobd']['completed_runs'] == 1
+        assert data['schedule_list']['jobb']['next_run_in_seconds'] > 25.0

awx/main/tests/functional/test_jobs.py

@@ -6,6 +6,7 @@ import json
 from awx.main.models import (
     Job,
     Instance,
+    Host,
     JobHostSummary,
     InventoryUpdate,
     InventorySource,
@@ -18,6 +19,9 @@ from awx.main.models import (
     ExecutionEnvironment,
 )
 from awx.main.tasks.system import cluster_node_heartbeat
+from awx.main.tasks.facts import update_hosts
+
+from django.db import OperationalError
 from django.test.utils import override_settings
 
 
@@ -112,6 +116,51 @@ def test_job_notification_host_data(inventory, machine_credential, project, job_
     }
 
 
+@pytest.mark.django_db
+class TestAnsibleFactsSave:
+    current_call = 0
+
+    def test_update_hosts_deleted_host(self, inventory):
+        hosts = [Host.objects.create(inventory=inventory, name=f'foo{i}') for i in range(3)]
+        for host in hosts:
+            host.ansible_facts = {'foo': 'bar'}
+        last_pk = hosts[-1].pk
+        assert inventory.hosts.count() == 3
+        Host.objects.get(pk=last_pk).delete()
+        assert inventory.hosts.count() == 2
+        update_hosts(hosts)
+        assert inventory.hosts.count() == 2
+        for host in inventory.hosts.all():
+            host.refresh_from_db()
+            assert host.ansible_facts == {'foo': 'bar'}
+
+    def test_update_hosts_forever_deadlock(self, inventory, mocker):
+        hosts = [Host.objects.create(inventory=inventory, name=f'foo{i}') for i in range(3)]
+        for host in hosts:
+            host.ansible_facts = {'foo': 'bar'}
+        db_mock = mocker.patch('awx.main.tasks.facts.Host.objects.bulk_update')
+        db_mock.side_effect = OperationalError('deadlock detected')
+        with pytest.raises(OperationalError):
+            update_hosts(hosts)
+
+    def fake_bulk_update(self, host_list):
+        if self.current_call > 2:
+            return Host.objects.bulk_update(host_list, ['ansible_facts', 'ansible_facts_modified'])
+        self.current_call += 1
+        raise OperationalError('deadlock detected')
+
+    def test_update_hosts_resolved_deadlock(self, inventory, mocker):
+        hosts = [Host.objects.create(inventory=inventory, name=f'foo{i}') for i in range(3)]
+        for host in hosts:
+            host.ansible_facts = {'foo': 'bar'}
+        self.current_call = 0
+        mocker.patch('awx.main.tasks.facts.raw_update_hosts', new=self.fake_bulk_update)
+        update_hosts(hosts)
+        for host in inventory.hosts.all():
+            host.refresh_from_db()
+            assert host.ansible_facts == {'foo': 'bar'}
+
+
 @pytest.mark.django_db
 class TestLaunchConfig:
     def test_null_creation_from_prompts(self):

awx/main/utils/handlers.py

@@ -97,8 +97,6 @@ class SpecialInventoryHandler(logging.Handler):
         self.event_handler(dispatch_data)
 
 
-ColorHandler = logging.StreamHandler
-
 if settings.COLOR_LOGS is True:
     try:
         from logutils.colorize import ColorizingStreamHandler
@@ -133,3 +131,5 @@ if settings.COLOR_LOGS is True:
     except ImportError:
         # logutils is only used for colored logs in the dev environment
         pass
+else:
+    ColorHandler = logging.StreamHandler

awx/main/utils/licensing.py

@@ -175,7 +175,12 @@ class Licenser(object):
             license.setdefault('pool_id', sub['pool']['id'])
             license.setdefault('product_name', sub['pool']['productName'])
             license.setdefault('valid_key', True)
-            license.setdefault('license_type', 'enterprise')
+            if sub['pool']['productId'].startswith('S'):
+                license.setdefault('trial', True)
+                license.setdefault('license_type', 'trial')
+            else:
+                license.setdefault('trial', False)
+                license.setdefault('license_type', 'enterprise')
             license.setdefault('satellite', False)
             # Use the nearest end date
             endDate = parse_date(sub['endDate'])
@@ -287,7 +292,7 @@ class Licenser(object):
                     license['productId'] = sub['product_id']
                     license['quantity'] = int(sub['quantity'])
                     license['support_level'] = sub['support_level']
-                    license['usage'] = sub['usage']
+                    license['usage'] = sub.get('usage')
                     license['subscription_name'] = sub['name']
                     license['subscriptionId'] = sub['subscription_id']
                     license['accountNumber'] = sub['account_number']

awx/playbooks/project_update.yml

@@ -255,6 +255,9 @@
           tags:
             - install_collections
             - install_roles
+      module_defaults:
+        ansible.builtin.command:
+          chdir: "{{ project_path | quote }}"
 
       # We combine our additional_galaxy_env into galaxy_task_env so that our values are preferred over anything a user would set
       environment: "{{ galaxy_task_env | combine(additional_galaxy_env) }}"

awx/settings/defaults.py

@@ -210,7 +210,7 @@ JOB_EVENT_WORKERS = 4
 
 # The number of seconds to buffer callback receiver bulk
 # writes in memory before flushing via JobEvent.objects.bulk_create()
-JOB_EVENT_BUFFER_SECONDS = 0.1
+JOB_EVENT_BUFFER_SECONDS = 1
 
 # The interval at which callback receiver statistics should be
 # recorded
@@ -327,7 +327,6 @@ INSTALLED_APPS = [
     'rest_framework',
     'django_extensions',
     'polymorphic',
-    'taggit',
     'social_django',
     'django_guid',
     'corsheaders',
@@ -471,12 +470,13 @@ CELERYBEAT_SCHEDULE = {
     'receptor_reaper': {'task': 'awx.main.tasks.system.awx_receptor_workunit_reaper', 'schedule': timedelta(seconds=60)},
     'send_subsystem_metrics': {'task': 'awx.main.analytics.analytics_tasks.send_subsystem_metrics', 'schedule': timedelta(seconds=20)},
     'cleanup_images': {'task': 'awx.main.tasks.system.cleanup_images_and_files', 'schedule': timedelta(hours=3)},
-    'cleanup_host_metrics': {'task': 'awx.main.tasks.system.cleanup_host_metrics', 'schedule': timedelta(days=1)},
+    'cleanup_host_metrics': {'task': 'awx.main.tasks.system.cleanup_host_metrics', 'schedule': timedelta(hours=3, minutes=30)},
+    'host_metric_summary_monthly': {'task': 'awx.main.tasks.host_metrics.host_metric_summary_monthly', 'schedule': timedelta(hours=4)},
 }
 
 # Django Caching Configuration
 DJANGO_REDIS_IGNORE_EXCEPTIONS = True
-CACHES = {'default': {'BACKEND': 'django_redis.cache.RedisCache', 'LOCATION': 'unix:/var/run/redis/redis.sock?db=1'}}
+CACHES = {'default': {'BACKEND': 'awx.main.cache.AWXRedisCache', 'LOCATION': 'unix:/var/run/redis/redis.sock?db=1'}}
 
 # Social Auth configuration.
 SOCIAL_AUTH_STRATEGY = 'social_django.strategy.DjangoStrategy'
@@ -1054,4 +1054,12 @@ CLEANUP_HOST_METRICS_LAST_TS = None
 # Host metrics cleanup - minimal interval between two cleanups in days
 CLEANUP_HOST_METRICS_INTERVAL = 30  # days
 # Host metrics cleanup - soft-delete HostMetric records with last_automation < [threshold] (in months)
-CLEANUP_HOST_METRICS_THRESHOLD = 12  # months
+CLEANUP_HOST_METRICS_SOFT_THRESHOLD = 12  # months
+# Host metrics cleanup
+# - delete HostMetric record with deleted=True and last_deleted < [threshold]
+# - also threshold for computing HostMetricSummaryMonthly (command/scheduled task)
+CLEANUP_HOST_METRICS_HARD_THRESHOLD = 36  # months
+
+# Host metric summary monthly task - last time of run
+HOST_METRIC_SUMMARY_TASK_LAST_TS = None
+HOST_METRIC_SUMMARY_TASK_INTERVAL = 7  # days

awx/settings/development.py

@@ -28,8 +28,8 @@ SHELL_PLUS_PRINT_SQL = False
 
 # show colored logs in the dev environment
 # to disable this, set `COLOR_LOGS = False` in awx/settings/local_settings.py
-LOGGING['handlers']['console']['()'] = 'awx.main.utils.handlers.ColorHandler'  # noqa
 COLOR_LOGS = True
+LOGGING['handlers']['console']['()'] = 'awx.main.utils.handlers.ColorHandler'  # noqa
 
 ALLOWED_HOSTS = ['*']
 

awx/templates/rest_framework/login.html

@@ -19,7 +19,7 @@
           <input type="text" name="username" maxlength="100"
               autocapitalize="off"
               autocorrect="off" class="form-control textinput textInput"
-              id="id_username" required autofocus
+              id="id_username"  autocomplete="off" required autofocus
               {% if form.username.value %}value="{{ form.username.value }}"{% endif %}>
             {% if form.username.errors %}
             <p class="text-error">{{ form.username.errors|striptags }}</p>
@@ -31,7 +31,8 @@
         <div class="form-group">
           <label for="id_password">Password:</label>
           <input type="password" name="password" maxlength="100" autocapitalize="off"
-              autocorrect="off" class="form-control textinput textInput" id="id_password" required>
+              autocorrect="off" class="form-control textinput textInput" id="id_password"
+              autocomplete="off" required>
             {% if form.password.errors %}
             <p class="text-error">{{ form.password.errors|striptags }}</p>
             {% endif %}

awx/ui/src/components/AdHocCommands/AdHocCredentialStep.js

@@ -91,7 +91,7 @@ function AdHocCredentialStep({ credentialTypeId }) {
       {meta.touched && meta.error && (
         <CredentialErrorAlert variant="danger" isInline title={meta.error} />
       )}
-      <Form>
+      <Form autoComplete="off">
         <FormGroup
           fieldId="credential"
           label={t`Machine Credential`}

awx/ui/src/components/AdHocCommands/AdHocDetailsStep.js

@@ -50,7 +50,7 @@ function AdHocDetailsStep({ moduleOptions }) {
     : true;
 
   return (
-    <Form>
+    <Form autoComplete="off">
       <FormColumnLayout>
         <FormFullWidthLayout>
           <FormGroup

awx/ui/src/components/AdHocCommands/AdHocExecutionEnvironmentStep.js

@@ -84,7 +84,7 @@ function AdHocExecutionEnvironmentStep({ organizationId }) {
   }
 
   return (
-    <Form>
+    <Form autoComplete="off">
       <FormGroup
         fieldId="execution_enviroment"
         label={t`Execution Environment`}

awx/ui/src/components/LaunchPrompt/LaunchPrompt.js

@@ -77,7 +77,7 @@ function PromptModalForm({
     }
 
     if (launchConfig.ask_labels_on_launch) {
-      const { labelIds } = createNewLabels(
+      const { labelIds } = await createNewLabels(
         values.labels,
         resource.organization
       );

awx/ui/src/components/ResourceAccessList/ResourceAccessList.js

@@ -1,10 +1,10 @@
 import React, { useCallback, useEffect, useState } from 'react';
 import { useLocation } from 'react-router-dom';
 import { t } from '@lingui/macro';
-import { RolesAPI, TeamsAPI, UsersAPI, OrganizationsAPI } from 'api';
+import { RolesAPI, TeamsAPI, UsersAPI } from 'api';
 import { getQSConfig, parseQueryString } from 'util/qs';
 import useRequest, { useDeleteItems } from 'hooks/useRequest';
-import { useUserProfile, useConfig } from 'contexts/Config';
+import { useUserProfile } from 'contexts/Config';
 import AddResourceRole from '../AddRole/AddResourceRole';
 import AlertModal from '../AlertModal';
 import DataListToolbar from '../DataListToolbar';
@@ -25,8 +25,7 @@ const QS_CONFIG = getQSConfig('access', {
 });
 
 function ResourceAccessList({ apiModel, resource }) {
-  const { isSuperUser, isOrgAdmin } = useUserProfile();
-  const { me } = useConfig();
+  const { isSuperUser } = useUserProfile();
   const [submitError, setSubmitError] = useState(null);
   const [deletionRecord, setDeletionRecord] = useState(null);
   const [deletionRole, setDeletionRole] = useState(null);
@@ -34,42 +33,15 @@ function ResourceAccessList({ apiModel, resource }) {
   const [showDeleteModal, setShowDeleteModal] = useState(false);
   const location = useLocation();
 
-  const {
-    isLoading: isFetchingOrgAdmins,
-    error: errorFetchingOrgAdmins,
-    request: fetchOrgAdmins,
-    result: { isCredentialOrgAdmin },
-  } = useRequest(
-    useCallback(async () => {
-      if (
-        isSuperUser ||
-        resource.type !== 'credential' ||
-        !isOrgAdmin ||
-        !resource?.organization
-      ) {
-        return false;
-      }
-      const {
-        data: { count },
-      } = await OrganizationsAPI.readAdmins(resource.organization, {
-        id: me.id,
-      });
-      return { isCredentialOrgAdmin: !!count };
-    }, [me.id, isOrgAdmin, isSuperUser, resource.type, resource.organization]),
-    {
-      isCredentialOrgAdmin: false,
-    }
-  );
-
-  useEffect(() => {
-    fetchOrgAdmins();
-  }, [fetchOrgAdmins]);
-
   let canAddAdditionalControls = false;
   if (isSuperUser) {
     canAddAdditionalControls = true;
   }
-  if (resource.type === 'credential' && isOrgAdmin && isCredentialOrgAdmin) {
+  if (
+    resource.type === 'credential' &&
+    resource?.summary_fields?.user_capabilities?.edit &&
+    resource?.organization
+  ) {
     canAddAdditionalControls = true;
   }
   if (resource.type !== 'credential') {
@@ -195,8 +167,8 @@ function ResourceAccessList({ apiModel, resource }) {
   return (
     <>
       <PaginatedTable
-        error={contentError || errorFetchingOrgAdmins}
-        hasContentLoading={isLoading || isDeleteLoading || isFetchingOrgAdmins}
+        error={contentError}
+        hasContentLoading={isLoading || isDeleteLoading}
         items={accessRecords}
         itemCount={itemCount}
         pluralizedItemName={t`Roles`}

awx/ui/src/components/ResourceAccessList/ResourceAccessList.test.js

@@ -463,7 +463,7 @@ describe('<ResourceAccessList />', () => {
     expect(wrapper.find('ToolbarAddButton').length).toEqual(1);
   });
 
-  test('should not show add button for non system admin & non org admin', async () => {
+  test('should not show add button for a user without edit permissions on the credential', async () => {
     useUserProfile.mockImplementation(() => {
       return {
         isSuperUser: false,
@@ -476,7 +476,21 @@ describe('<ResourceAccessList />', () => {
     let wrapper;
     await act(async () => {
       wrapper = mountWithContexts(
-        <ResourceAccessList resource={credential} apiModel={CredentialsAPI} />,
+        <ResourceAccessList
+          resource={{
+            ...credential,
+            summary_fields: {
+              ...credential.summary_fields,
+              user_capabilities: {
+                edit: false,
+                delete: false,
+                copy: false,
+                use: false,
+              },
+            },
+          }}
+          apiModel={CredentialsAPI}
+        />,
         { context: { router: { credentialHistory } } }
       );
     });

awx/ui/src/components/StatusLabel/StatusLabel.js

@@ -47,7 +47,7 @@ export default function StatusLabel({ status, tooltipContent = '', children }) {
     unreachable: t`Unreachable`,
     running: t`Running`,
     pending: t`Pending`,
-    skipped: t`Skipped'`,
+    skipped: t`Skipped`,
     timedOut: t`Timed out`,
     waiting: t`Waiting`,
     disabled: t`Disabled`,

awx/ui/src/locales/en/messages.po

@@ -8722,8 +8722,8 @@ msgid "Skipped"
 msgstr "Skipped"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
-msgstr "Skipped'"
+msgid "Skipped"
+msgstr "Skipped"
 
 #: components/NotificationList/NotificationList.js:200
 #: screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js:141

awx/ui/src/locales/es/messages.po

@@ -8190,8 +8190,8 @@ msgid "Skipped"
 msgstr "Omitido"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
-msgstr "Omitido'"
+msgid "Skipped"
+msgstr "Omitido"
 
 #: components/NotificationList/NotificationList.js:200
 #: screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js:141

awx/ui/src/locales/fr/messages.po

@@ -8078,7 +8078,7 @@ msgid "Skipped"
 msgstr "Ignoré"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
+msgid "Skipped"
 msgstr "Ignoré"
 
 #: components/NotificationList/NotificationList.js:200

awx/ui/src/locales/ja/messages.po

@@ -8118,8 +8118,8 @@ msgid "Skipped"
 msgstr "スキップ済"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
-msgstr "スキップ済'"
+msgid "Skipped"
+msgstr "スキップ済"
 
 #: components/NotificationList/NotificationList.js:200
 #: screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js:141

awx/ui/src/locales/ko/messages.po

@@ -8072,8 +8072,8 @@ msgid "Skipped"
 msgstr "건너뜀"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
-msgstr "건너뜀'"
+msgid "Skipped"
+msgstr "건너뜀"
 
 #: components/NotificationList/NotificationList.js:200
 #: screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js:141

awx/ui/src/locales/nl/messages.po

@@ -8096,8 +8096,8 @@ msgid "Skipped"
 msgstr "Overgeslagen"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
-msgstr "Overgeslagen'"
+msgid "Skipped"
+msgstr "Overgeslagen"
 
 #: components/NotificationList/NotificationList.js:200
 #: screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js:141

awx/ui/src/locales/zh/messages.po

@@ -8072,8 +8072,8 @@ msgid "Skipped"
 msgstr "跳过"
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
-msgstr "跳过'"
+msgid "Skipped"
+msgstr "跳过"
 
 #: components/NotificationList/NotificationList.js:200
 #: screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js:141

awx/ui/src/locales/zu/messages.po

@@ -8503,7 +8503,7 @@ msgid "Skipped"
 msgstr ""
 
 #: components/StatusLabel/StatusLabel.js:50
-msgid "Skipped'"
+msgid "Skipped"
 msgstr ""
 
 #: components/NotificationList/NotificationList.js:200

awx/ui/src/screens/Credential/shared/CredentialPlugins/CredentialPluginPrompt/MetadataStep.js

@@ -67,7 +67,7 @@ function MetadataStep() {
   return (
     <>
       {fields.length > 0 && (
-        <Form>
+        <Form autoComplete="off">
           <FormFullWidthLayout>
             {fields.map((field) => {
               if (field.type === 'string') {

awx/ui/src/screens/Credential/shared/ExternalTestModal.js

@@ -99,7 +99,7 @@ function ExternalTestModal({
               </Button>,
             ]}
           >
-            <Form>
+            <Form autoComplete="off">
               <FormFullWidthLayout>
                 {credentialType.inputs.metadata.map((field) => {
                   const isRequired = credentialType.inputs?.required.includes(

awx/ui/src/screens/Inventory/shared/ConstructedInventoryHint.js

@@ -122,7 +122,7 @@ function ConstructedInventoryHint() {
       <br />
       <Panel>
         <CardBody>
-          <Form>
+          <Form autoComplete="off">
             <b>{t`Constructed inventory examples`}</b>
             <LimitToIntersectionExample />
             <FilterOnNestedGroupExample />

awx/ui/src/screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js

@@ -178,6 +178,7 @@ function NotificationTemplatesList() {
                 <HeaderCell sortKey="name">{t`Name`}</HeaderCell>
                 <HeaderCell>{t`Status`}</HeaderCell>
                 <HeaderCell sortKey="notification_type">{t`Type`}</HeaderCell>
+                <HeaderCell sortKey="organization">{t`Organization`}</HeaderCell>
                 <HeaderCell>{t`Actions`}</HeaderCell>
               </HeaderRow>
             }

awx/ui/src/screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.test.js

@@ -20,6 +20,10 @@ const mockTemplates = {
         url: '/notification_templates/1',
         type: 'slack',
         summary_fields: {
+          organization: {
+            id: 1,
+            name: 'Foo',
+          },
           recent_notifications: [
             {
               status: 'success',
@@ -36,6 +40,10 @@ const mockTemplates = {
         id: 2,
         url: '/notification_templates/2',
         summary_fields: {
+          organization: {
+            id: 2,
+            name: 'Bar',
+          },
           recent_notifications: [],
           user_capabilities: {
             delete: true,
@@ -48,6 +56,10 @@ const mockTemplates = {
         id: 3,
         url: '/notification_templates/3',
         summary_fields: {
+          organization: {
+            id: 3,
+            name: 'Test',
+          },
           recent_notifications: [
             {
               status: 'failed',

awx/ui/src/screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateListItem.js

@@ -121,6 +121,13 @@ function NotificationTemplateListItem({
           {NOTIFICATION_TYPES[template.notification_type] ||
             template.notification_type}
         </Td>
+        <Td dataLabel={t`Oragnization`}>
+          <Link
+            to={`/organizations/${template.summary_fields.organization.id}/details`}
+          >
+            <b>{template.summary_fields.organization.name}</b>
+          </Link>
+        </Td>
         <ActionsTd dataLabel={t`Actions`}>
           <ActionItem visible tooltip={t`Test notification`}>
             <Button

awx/ui/src/screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateListItem.test.js

@@ -12,6 +12,10 @@ const template = {
   notification_type: 'slack',
   name: 'Test Notification',
   summary_fields: {
+    organization: {
+      id: 1,
+      name: 'Foo',
+    },
     user_capabilities: {
       edit: true,
       copy: true,
@@ -39,7 +43,7 @@ describe('<NotificationTemplateListItem />', () => {
     );
 
     const cells = wrapper.find('Td');
-    expect(cells).toHaveLength(5);
+    expect(cells).toHaveLength(6);
     expect(cells.at(1).text()).toEqual('Test Notification');
     expect(cells.at(2).text()).toEqual('Success');
     expect(cells.at(3).text()).toEqual('Slack');
@@ -133,6 +137,10 @@ describe('<NotificationTemplateListItem />', () => {
             template={{
               ...template,
               summary_fields: {
+                organization: {
+                  id: 3,
+                  name: 'Test',
+                },
                 user_capabilities: {
                   copy: false,
                   edit: false,

awx/ui/src/screens/Setting/Subscription/SubscriptionDetail/SubscriptionDetail.js

@@ -89,7 +89,8 @@ function SubscriptionDetail() {
             />
           )}
           {typeof automatedInstancesCount !== 'undefined' &&
-            automatedInstancesCount !== null && (
+            automatedInstancesCount !== null &&
+            systemConfig?.SUBSCRIPTION_USAGE_MODEL !== '' && (
               <Detail
                 dataCy="subscription-hosts-automated"
                 label={t`Hosts automated`}
@@ -105,11 +106,13 @@ function SubscriptionDetail() {
                 }
               />
             )}
-          <Detail
-            dataCy="subscription-hosts-imported"
-            label={t`Hosts imported`}
-            value={license_info.current_instances}
-          />
+          {systemConfig?.SUBSCRIPTION_USAGE_MODEL !== '' && (
+            <Detail
+              dataCy="subscription-hosts-imported"
+              label={t`Hosts imported`}
+              value={license_info.current_instances}
+            />
+          )}
           {systemConfig?.SUBSCRIPTION_USAGE_MODEL ===
             'unique_managed_hosts' && (
             <Detail
@@ -134,20 +137,23 @@ function SubscriptionDetail() {
               value={license_info.reactivated_instances}
             />
           )}
-          {license_info.instance_count < 9999999 && (
-            <Detail
-              dataCy="subscription-hosts-available"
-              label={t`Hosts available`}
-              value={license_info.available_instances}
-            />
-          )}
-          {license_info.instance_count >= 9999999 && (
-            <Detail
-              dataCy="subscription-unlimited-hosts-available"
-              label={t`Hosts available`}
-              value={t`Unlimited`}
-            />
-          )}
+
+          {systemConfig?.SUBSCRIPTION_USAGE_MODEL !== '' &&
+            license_info.instance_count < 9999999 && (
+              <Detail
+                dataCy="subscription-hosts-available"
+                label={t`Hosts available`}
+                value={license_info.available_instances}
+              />
+            )}
+          {systemConfig?.SUBSCRIPTION_USAGE_MODEL !== '' &&
+            license_info.instance_count >= 9999999 && (
+              <Detail
+                dataCy="subscription-unlimited-hosts-available"
+                label={t`Hosts available`}
+                value={t`Unlimited`}
+              />
+            )}
           <Detail
             dataCy="subscription-type"
             label={t`Subscription type`}

awx/ui/src/screens/Template/WorkflowJobTemplateVisualizer/Modals/NodeModals/NodeTypeStep/NodeTypeStep.js

@@ -164,7 +164,7 @@ function NodeTypeStep({ isIdentifierRequired }) {
           onUpdateNodeResource={nodeResourceHelpers.setValue}
         />
       )}
-      <Form css="margin-top: 20px;">
+      <Form autoComplete="off" css="margin-top: 20px;">
         <FormColumnLayout>
           <FormFullWidthLayout>
             {nodeTypeField.value === 'workflow_approval_template' && (

awx/ui/src/util/getDocsBaseUrl.js

@@ -1,8 +1,17 @@
 export default function getDocsBaseUrl(config) {
   let version = 'latest';
   const licenseType = config?.license_info?.license_type;
+
   if (licenseType && licenseType !== 'open') {
-    version = config?.version ? config.version.split('-')[0] : 'latest';
+    if (config?.version) {
+      if (parseFloat(config?.version.split('-')[0]) >= 4.3) {
+        version = parseFloat(config?.version.split('-')[0]);
+      } else {
+        version = config?.version.split('-')[0];
+      }
+    }
+  } else {
+    version = 'latest';
   }
   return `https://docs.ansible.com/automation-controller/${version}`;
 }

awx_collection/README.md

@@ -68,6 +68,7 @@ Notable releases of the `awx.awx` collection:
  - 7.0.0 is intended to be identical to the content prior to the migration, aside from changes necessary to function as a collection.
  - 11.0.0 has no non-deprecated modules that depend on the deprecated `tower-cli` [PyPI](https://pypi.org/project/ansible-tower-cli/).
  - 19.2.1 large renaming purged "tower" names (like options and module names), adding redirects for old names
+ - X.X.X added support of named URLs to all modules. Anywhere that previously accepted name or id can also support named URLs
  - 0.0.1-devel is the version you should see if installing from source, which is intended for development and expected to be unstable.
 
 The following notes are changes that may require changes to playbooks:

awx_collection/plugins/doc_fragments/auth.py

@@ -50,6 +50,11 @@ options:
     - If value not set, will try environment variable C(CONTROLLER_VERIFY_SSL) and then config files
     type: bool
     aliases: [ tower_verify_ssl ]
+  request_timeout:
+    description:
+    - Specify the timeout Ansible should use in requests to the controller host.
+    - Defaults to 10s, but this is handled by the shared module_utils code
+    type: float
   controller_config_file:
     description:
     - Path to the controller config file.

awx_collection/plugins/doc_fragments/auth_plugin.py

@@ -68,6 +68,14 @@ options:
         why: Collection name change
         alternatives: 'CONTROLLER_VERIFY_SSL'
     aliases: [ validate_certs ]
+  request_timeout:
+    description:
+    - Specify the timeout Ansible should use in requests to the controller host.
+    - Defaults to 10 seconds
+    - This will not work with the export or import modules.
+    type: float
+    env:
+    - name: CONTROLLER_REQUEST_TIMEOUT
 
 notes:
 - If no I(config_file) is provided we will attempt to use the tower-cli library

awx_collection/plugins/module_utils/controller_api.py

@@ -10,7 +10,7 @@ from ansible.module_utils.six import raise_from, string_types
 from ansible.module_utils.six.moves import StringIO
 from ansible.module_utils.six.moves.urllib.error import HTTPError
 from ansible.module_utils.six.moves.http_cookiejar import CookieJar
-from ansible.module_utils.six.moves.urllib.parse import urlparse, urlencode
+from ansible.module_utils.six.moves.urllib.parse import urlparse, urlencode, quote
 from ansible.module_utils.six.moves.configparser import ConfigParser, NoOptionError
 from socket import getaddrinfo, IPPROTO_TCP
 import time
@@ -51,22 +51,27 @@ class ControllerModule(AnsibleModule):
         controller_username=dict(required=False, aliases=['tower_username'], fallback=(env_fallback, ['CONTROLLER_USERNAME', 'TOWER_USERNAME'])),
         controller_password=dict(no_log=True, aliases=['tower_password'], required=False, fallback=(env_fallback, ['CONTROLLER_PASSWORD', 'TOWER_PASSWORD'])),
         validate_certs=dict(type='bool', aliases=['tower_verify_ssl'], required=False, fallback=(env_fallback, ['CONTROLLER_VERIFY_SSL', 'TOWER_VERIFY_SSL'])),
+        request_timeout=dict(type='float', required=False, fallback=(env_fallback, ['CONTROLLER_REQUEST_TIMEOUT'])),
         controller_oauthtoken=dict(
             type='raw', no_log=True, aliases=['tower_oauthtoken'], required=False, fallback=(env_fallback, ['CONTROLLER_OAUTH_TOKEN', 'TOWER_OAUTH_TOKEN'])
         ),
         controller_config_file=dict(type='path', aliases=['tower_config_file'], required=False, default=None),
     )
+    # Associations of these types are ordered and have special consideration in the modified associations function
+    ordered_associations = ['instance_groups', 'galaxy_credentials']
     short_params = {
         'host': 'controller_host',
         'username': 'controller_username',
         'password': 'controller_password',
         'verify_ssl': 'validate_certs',
+        'request_timeout': 'request_timeout',
         'oauth_token': 'controller_oauthtoken',
     }
     host = '127.0.0.1'
     username = None
     password = None
     verify_ssl = True
+    request_timeout = 10
     oauth_token = None
     oauth_token_id = None
     authenticated = False
@@ -302,7 +307,7 @@ class ControllerAPIModule(ControllerModule):
         kwargs['supports_check_mode'] = True
 
         super().__init__(argument_spec=argument_spec, direct_params=direct_params, error_callback=error_callback, warn_callback=warn_callback, **kwargs)
-        self.session = Request(cookies=CookieJar(), validate_certs=self.verify_ssl)
+        self.session = Request(cookies=CookieJar(), timeout=self.request_timeout, validate_certs=self.verify_ssl)
 
         if 'update_secrets' in self.params:
             self.update_secrets = self.params.pop('update_secrets')
@@ -381,29 +386,51 @@ class ControllerAPIModule(ControllerModule):
 
     def get_one(self, endpoint, name_or_id=None, allow_none=True, check_exists=False, **kwargs):
         new_kwargs = kwargs.copy()
-        if name_or_id:
-            name_field = self.get_name_field_from_endpoint(endpoint)
-            new_data = kwargs.get('data', {}).copy()
-            if name_field in new_data:
-                self.fail_json(msg="You can't specify the field {0} in your search data if using the name_or_id field".format(name_field))
+        response = None
 
-            try:
-                new_data['or__id'] = int(name_or_id)
-                new_data['or__{0}'.format(name_field)] = name_or_id
-            except ValueError:
-                # If we get a value error, then we didn't have an integer so we can just pass and fall down to the fail
-                new_data[name_field] = name_or_id
-            new_kwargs['data'] = new_data
+        # A named URL is pretty unique so if we have a ++ in the name then lets start by looking for that
+        # This also needs to go first because if there was data passed in kwargs and we do the next lookup first there may be results
+        if name_or_id is not None and '++' in name_or_id:
+            # Maybe someone gave us a named URL so lets see if we get anything from that.
+            url_quoted_name = quote(name_or_id, safe="+")
+            named_endpoint = '{0}/{1}/'.format(endpoint, url_quoted_name)
+            named_response = self.get_endpoint(named_endpoint)
 
-        response = self.get_endpoint(endpoint, **new_kwargs)
-        if response['status_code'] != 200:
-            fail_msg = "Got a {0} response when trying to get one from {1}".format(response['status_code'], endpoint)
-            if 'detail' in response.get('json', {}):
-                fail_msg += ', detail: {0}'.format(response['json']['detail'])
-            self.fail_json(msg=fail_msg)
+            if named_response['status_code'] == 200 and 'json' in named_response:
+                # We found a named item but we expect to deal with a list view so mock that up
+                response = {
+                    'json': {
+                        'count': 1,
+                        'results': [named_response['json']],
+                    }
+                }
 
-        if 'count' not in response['json'] or 'results' not in response['json']:
-            self.fail_json(msg="The endpoint did not provide count and results")
+        # Since we didn't have a named URL, lets try and find it with a general search
+        if response is None:
+            if name_or_id:
+                name_field = self.get_name_field_from_endpoint(endpoint)
+                new_data = kwargs.get('data', {}).copy()
+                if name_field in new_data:
+                    self.fail_json(msg="You can't specify the field {0} in your search data if using the name_or_id field".format(name_field))
+
+                try:
+                    new_data['or__id'] = int(name_or_id)
+                    new_data['or__{0}'.format(name_field)] = name_or_id
+                except ValueError:
+                    # If we get a value error, then we didn't have an integer so we can just pass and fall down to the fail
+                    new_data[name_field] = name_or_id
+                new_kwargs['data'] = new_data
+
+            response = self.get_endpoint(endpoint, **new_kwargs)
+
+            if response['status_code'] != 200:
+                fail_msg = "Got a {0} response when trying to get one from {1}".format(response['status_code'], endpoint)
+                if 'detail' in response.get('json', {}):
+                    fail_msg += ', detail: {0}'.format(response['json']['detail'])
+                self.fail_json(msg=fail_msg)
+
+            if 'count' not in response['json'] or 'results' not in response['json']:
+                self.fail_json(msg="The endpoint did not provide count and results")
 
         if response['json']['count'] == 0:
             if allow_none:
@@ -421,7 +448,6 @@ class ControllerAPIModule(ControllerModule):
             self.fail_wanted_one(response, endpoint, new_kwargs.get('data'))
 
         if check_exists:
-            name_field = self.get_name_field_from_endpoint(endpoint)
             self.json_output['id'] = response['json']['results'][0]['id']
             self.exit_json(**self.json_output)
 
@@ -477,7 +503,14 @@ class ControllerAPIModule(ControllerModule):
             data = dumps(kwargs.get('data', {}))
 
         try:
-            response = self.session.open(method, url.geturl(), headers=headers, validate_certs=self.verify_ssl, follow_redirects=True, data=data)
+            response = self.session.open(
+                method, url.geturl(),
+                headers=headers,
+                timeout=self.request_timeout,
+                validate_certs=self.verify_ssl,
+                follow_redirects=True,
+                data=data
+            )
         except (SSLValidationError) as ssl_err:
             self.fail_json(msg="Could not establish a secure connection to your host ({1}): {0}.".format(url.netloc, ssl_err))
         except (ConnectionError) as con_err:
@@ -531,13 +564,7 @@ class ControllerAPIModule(ControllerModule):
                 controller_version = response.info().getheader('X-API-Product-Version', None)
 
             parsed_collection_version = Version(self._COLLECTION_VERSION).version
-            if not controller_version:
-                self.warn(
-                    "You are using the {0} version of this collection but connecting to a controller that did not return a version".format(
-                        self._COLLECTION_VERSION
-                    )
-                )
-            else:
+            if controller_version:
                 parsed_controller_version = Version(controller_version).version
                 if controller_type == 'AWX':
                     collection_compare_ver = parsed_collection_version[0]
@@ -595,6 +622,7 @@ class ControllerAPIModule(ControllerModule):
                     'POST',
                     api_token_url,
                     validate_certs=self.verify_ssl,
+                    timeout=self.request_timeout,
                     follow_redirects=True,
                     force_basic_auth=True,
                     url_username=self.username,
@@ -680,17 +708,26 @@ class ControllerAPIModule(ControllerModule):
         response = self.get_all_endpoint(association_endpoint)
         existing_associated_ids = [association['id'] for association in response['json']['results']]
 
-        # Disassociate anything that is in existing_associated_ids but not in new_association_list
-        ids_to_remove = list(set(existing_associated_ids) - set(new_association_list))
-        for an_id in ids_to_remove:
+        # Some associations can be ordered (like galaxy credentials)
+        if association_endpoint.strip('/').split('/')[-1] in self.ordered_associations:
+            if existing_associated_ids == new_association_list:
+                return  # If the current associations EXACTLY match the desired associations then we can return
+            removal_list = existing_associated_ids  # because of ordering, we have to remove everything
+            addition_list = new_association_list  # re-add everything back in-order
+        else:
+            if set(existing_associated_ids) == set(new_association_list):
+                return
+            removal_list = set(existing_associated_ids) - set(new_association_list)
+            addition_list = set(new_association_list) - set(existing_associated_ids)
+
+        for an_id in removal_list:
             response = self.post_endpoint(association_endpoint, **{'data': {'id': int(an_id), 'disassociate': True}})
             if response['status_code'] == 204:
                 self.json_output['changed'] = True
             else:
                 self.fail_json(msg="Failed to disassociate item {0}".format(response['json'].get('detail', response['json'])))
 
-        # Associate anything that is in new_association_list but not in `association`
-        for an_id in list(set(new_association_list) - set(existing_associated_ids)):
+        for an_id in addition_list:
             response = self.post_endpoint(association_endpoint, **{'data': {'id': int(an_id)}})
             if response['status_code'] == 204:
                 self.json_output['changed'] = True
@@ -962,6 +999,7 @@ class ControllerAPIModule(ControllerModule):
                     'DELETE',
                     api_token_url,
                     validate_certs=self.verify_ssl,
+                    timeout=self.request_timeout,
                     follow_redirects=True,
                     force_basic_auth=True,
                     url_username=self.username,

awx_collection/plugins/modules/ad_hoc_command.py

@@ -29,12 +29,12 @@ options:
       choices: [ 'run', 'check' ]
     execution_environment:
       description:
-        - Execution Environment to use for the ad hoc command.
+        - Execution Environment name, ID, or named URL to use for the ad hoc command.
       required: False
       type: str
     inventory:
       description:
-        - Inventory to use for the ad hoc command.
+        - Inventory name, ID, or named URL to use for the ad hoc command.
       required: True
       type: str
     limit:
@@ -43,7 +43,7 @@ options:
       type: str
     credential:
       description:
-        - Credential to use for ad hoc command.
+        - Credential name, ID, or named URL to use for ad hoc command.
       required: True
       type: str
     module_name:

awx_collection/plugins/modules/application.py

@@ -48,7 +48,7 @@ options:
       required: False
     organization:
       description:
-        - Name of organization for application.
+        - Name, ID, or named URL of organization for application.
       type: str
       required: True
     redirect_uris:

awx_collection/plugins/modules/bulk_host_create.py

@@ -48,7 +48,7 @@ options:
           type: str
     inventory:
       description:
-        - Inventory name or ID the hosts should be made a member of.
+        - Inventory name, ID, or named URL the hosts should be made a member of.
       required: True
       type: str
 extends_documentation_fragment: awx.awx.auth

awx_collection/plugins/modules/bulk_job_launch.py

@@ -128,7 +128,7 @@ options:
       type: str
     inventory:
       description:
-        - Inventory name or ID to use for the jobs ran within the bulk job, only used if prompt for inventory is set.
+        - Inventory name, ID, or named URL to use for the jobs ran within the bulk job, only used if prompt for inventory is set.
       type: str
     scm_branch:
       description:

awx_collection/plugins/modules/credential.py

@@ -45,7 +45,7 @@ options:
       type: str
     organization:
       description:
-        - Organization that should own the credential.
+        - Organization name, ID, or named URL that should own the credential.
       type: str
     credential_type:
       description:
@@ -92,11 +92,11 @@ options:
       default: true
     user:
       description:
-        - User that should own this credential.
+        - User name, ID, or named URL that should own this credential.
       type: str
     team:
       description:
-        - Team that should own this credential.
+        - Team name, ID, or named URL that should own this credential.
       type: str
     state:
       description:

awx_collection/plugins/modules/credential_input_source.py

@@ -38,12 +38,12 @@ options:
       type: dict
     target_credential:
       description:
-        - The credential which will have its input defined by this source
+        - The credential name, ID, or named URL which will have its input defined by this source
       required: true
       type: str
     source_credential:
       description:
-        - The credential which is the source of the credential lookup
+        - The credential name, ID, or named URL which is the source of the credential lookup
       type: str
     state:
       description:

awx_collection/plugins/modules/execution_environment.py

@@ -41,11 +41,11 @@ options:
       type: str
     organization:
       description:
-        - The organization the execution environment belongs to.
+        - The organization name, ID, or named URL that the execution environment belongs to.
       type: str
     credential:
       description:
-        - Name of the credential to use for the execution environment.
+        - Name, ID, or named URL of the credential to use for the execution environment.
       type: str
     state:
       description:

awx_collection/plugins/modules/export.py

@@ -28,72 +28,72 @@ options:
       default: 'False'
     organizations:
       description:
-        - organization names to export
+        - organization names, IDs, or named URLs to export
       type: list
       elements: str
     users:
       description:
-        - user names to export
+        - user names, IDs, or named URLs to export
       type: list
       elements: str
     teams:
       description:
-        - team names to export
+        - team names, IDs, or named URLs to export
       type: list
       elements: str
     credential_types:
       description:
-        - credential type names to export
+        - credential type names, IDs, or named URLs to export
       type: list
       elements: str
     credentials:
       description:
-        - credential names to export
+        - credential names, IDs, or named URLs to export
       type: list
       elements: str
     execution_environments:
       description:
-        - execution environment names to export
+        - execution environment names, IDs, or named URLs to export
       type: list
       elements: str
     notification_templates:
       description:
-        - notification template names to export
+        - notification template names, IDs, or named URLs to export
       type: list
       elements: str
     inventory_sources:
       description:
-        - inventory soruces to export
+        - inventory source name, ID, or named URLs to export
       type: list
       elements: str
     inventory:
       description:
-        - inventory names to export
+        - inventory names, IDs, or named URLs to export
       type: list
       elements: str
     projects:
       description:
-        - project names to export
+        - project names, IDs, or named URLs to export
       type: list
       elements: str
     job_templates:
       description:
-        - job template names to export
+        - job template names, IDs, or named URLs to export
       type: list
       elements: str
     workflow_job_templates:
       description:
-        - workflow names to export
+        - workflow names, IDs, or named URLs to export
       type: list
       elements: str
     applications:
       description:
-        - OAuth2 application names to export
+        - OAuth2 application names, IDs, or named URLs to export
       type: list
       elements: str
     schedules:
       description:
-        - schedule names to export
+        - schedule names, IDs, or named URLs to export
       type: list
       elements: str
 requirements:
@@ -154,7 +154,7 @@ def main():
 
     # The exporter code currently works like the following:
     #   Empty string == all assets of that type
-    #   Non-Empty string = just a list of assets of that type (by name or ID)
+    #   Non-Empty string = just a list of assets of that type (by name, ID, or named URL)
     #   Asset type not present or None = skip asset type (unless everything is None, then export all)
     # Here we are going to setup a dict of values to export
     export_args = {}

awx_collection/plugins/modules/group.py

@@ -32,7 +32,7 @@ options:
       type: str
     inventory:
       description:
-        - Inventory the group should be made a member of.
+        - Inventory name, ID, or named URL that the group should be made a member of.
       required: True
       type: str
     variables:
@@ -41,12 +41,12 @@ options:
       type: dict
     hosts:
       description:
-        - List of hosts that should be put in this group.
+        - List of host names, IDs, or named URLs that should be put in this group.
       type: list
       elements: str
     children:
       description:
-        - List of groups that should be nested inside in this group.
+        - List of groups names, IDs, or named URLs that should be nested inside in this group.
       type: list
       elements: str
       aliases:

awx_collection/plugins/modules/host.py

@@ -36,7 +36,7 @@ options:
       type: str
     inventory:
       description:
-        - Inventory the host should be made a member of.
+        - Inventory name, ID, or named URL the host should be made a member of.
       required: True
       type: str
     enabled:

awx_collection/plugins/modules/instance_group.py

@@ -33,7 +33,7 @@ options:
       type: str
     credential:
       description:
-        - Credential to authenticate with Kubernetes or OpenShift.  Must be of type "OpenShift or Kubernetes API Bearer Token".
+        - Credential name, ID, or named URL to authenticate with Kubernetes or OpenShift.  Must be of type "OpenShift or Kubernetes API Bearer Token".
       required: False
       type: str
     is_container_group:
@@ -74,7 +74,7 @@ options:
       type: str
     instances:
       description:
-        - The instances associated with this instance_group
+        - The instance names, IDs, or named URLs associated with this instance_group
       required: False
       type: list
       elements: str

awx_collection/plugins/modules/inventory.py

@@ -44,7 +44,7 @@ options:
       type: str
     organization:
       description:
-        - Organization the inventory belongs to.
+        - Organization name, ID, or named URL the inventory belongs to.
       required: True
       type: str
     variables:
@@ -62,12 +62,12 @@ options:
       type: str
     instance_groups:
       description:
-        - list of Instance Groups for this Organization to run on.
+        - list of Instance Group names, IDs, or named URLs for this Organization to run on.
       type: list
       elements: str
     input_inventories:
       description:
-        - List of Inventories to use as input for Constructed Inventory.
+        - List of Inventory names, IDs, or named URLs to use as input for Constructed Inventory.
       type: list
       elements: str
     prevent_instance_group_fallback:

awx_collection/plugins/modules/inventory_source.py

@@ -36,7 +36,7 @@ options:
       type: str
     inventory:
       description:
-        - Inventory the group should be made a member of.
+        - Inventory name, ID, or named URL the group should be made a member of.
       required: True
       type: str
     source:
@@ -70,11 +70,11 @@ options:
       type: str
     credential:
       description:
-        - Credential to use for the source.
+        - Credential name, ID, or named URL to use for the source.
       type: str
     execution_environment:
       description:
-        - Execution Environment to use for the source.
+        - Execution Environment name, ID, or named URL to use for the source.
       type: str
     custom_virtualenv:
       description:
@@ -107,7 +107,7 @@ options:
       type: int
     source_project:
       description:
-        - Project to use as source with scm option
+        - Project name, ID, or named URL to use as source with scm option
       type: str
     scm_branch:
       description:

awx_collection/plugins/modules/inventory_source_update.py

@@ -35,7 +35,7 @@ options:
       type: str
     organization:
       description:
-        - Name of the inventory source's inventory's organization.
+        - Name, ID, or named URL of the inventory source's inventory's organization.
       type: str
     wait:
       description:

awx_collection/plugins/modules/job_launch.py

@@ -34,17 +34,17 @@ options:
       type: str
     inventory:
       description:
-        - Inventory to use for the job, only used if prompt for inventory is set.
+        - Inventory name, ID, or named URL to use for the job, only used if prompt for inventory is set.
       type: str
     organization:
       description:
-        - Organization the job template exists in.
+        - Organization name, ID, or named URL the job template exists in.
         - Used to help lookup the object, cannot be modified using this module.
         - If not provided, will lookup by name only, which does not work with duplicates.
       type: str
     credentials:
       description:
-        - Credential to use for job, only used if prompt for credential is set.
+        - Credential names, IDs, or named URLs to use for job, only used if prompt for credential is set.
       type: list
       aliases: ['credential']
       elements: str
@@ -88,7 +88,7 @@ options:
       type: dict
     execution_environment:
       description:
-        - Execution environment to use for the job, only used if prompt for execution environment is set.
+        - Execution environment name, ID, or named URL to use for the job, only used if prompt for execution environment is set.
       type: str
     forks:
       description:
@@ -96,7 +96,7 @@ options:
       type: int
     instance_groups:
       description:
-        - Instance groups to use for the job, only used if prompt for instance groups is set.
+        - Instance group names, IDs, or named URLs to use for the job, only used if prompt for instance groups is set.
       type: list
       elements: str
     job_slice_count:

awx_collection/plugins/modules/job_template.py

@@ -49,11 +49,11 @@ options:
       type: str
     inventory:
       description:
-        - Name of the inventory to use for the job template.
+        - Name, ID, or named URL of the inventory to use for the job template.
       type: str
     organization:
       description:
-        - Organization the job template exists in.
+        - Organization name, ID, or named URL the job template exists in.
         - Used to help lookup the object, cannot be modified using this module.
         - The Organization is inferred from the associated project
         - If not provided, will lookup by name only, which does not work with duplicates.
@@ -61,7 +61,7 @@ options:
       type: str
     project:
       description:
-        - Name of the project to use for the job template.
+        - Name, ID, or named URL of the project to use for the job template.
       type: str
     playbook:
       description:
@@ -69,22 +69,22 @@ options:
       type: str
     credential:
       description:
-        - Name of the credential to use for the job template.
+        - Name, ID, or named URL of the credential to use for the job template.
         - Deprecated, use 'credentials'.
       type: str
     credentials:
       description:
-        - List of credentials to use for the job template.
+        - List of credential names, IDs, or named URLs to use for the job template.
       type: list
       elements: str
     vault_credential:
       description:
-        - Name of the vault credential to use for the job template.
+        - Name, ID, or named URL of the vault credential to use for the job template.
         - Deprecated, use 'credentials'.
       type: str
     execution_environment:
       description:
-        - Execution Environment to use for the job template.
+        - Execution Environment name, ID, or named URL to use for the job template.
       type: str
     custom_virtualenv:
       description:
@@ -94,7 +94,7 @@ options:
       type: str
     instance_groups:
       description:
-        - list of Instance Groups for this Organization to run on.
+        - list of Instance Group names, IDs, or named URLs for this Organization to run on.
       type: list
       elements: str
     forks:
@@ -108,7 +108,7 @@ options:
     verbosity:
       description:
         - Control the output level Ansible produces as the playbook runs. 0 - Normal, 1 - Verbose, 2 - More Verbose, 3 - Debug, 4 - Connection Debug.
-      choices: [0, 1, 2, 3, 4]
+      choices: [0, 1, 2, 3, 4, 5]
       type: int
     extra_vars:
       description:
@@ -404,7 +404,7 @@ def main():
         instance_groups=dict(type="list", elements='str'),
         forks=dict(type='int'),
         limit=dict(),
-        verbosity=dict(type='int', choices=[0, 1, 2, 3, 4]),
+        verbosity=dict(type='int', choices=[0, 1, 2, 3, 4, 5]),
         extra_vars=dict(type='dict'),
         job_tags=dict(),
         force_handlers=dict(type='bool', aliases=['force_handlers_enabled']),

awx_collection/plugins/modules/label.py

@@ -34,7 +34,7 @@ options:
       type: str
     organization:
       description:
-        - Organization this label belongs to.
+        - Organization name, ID, or named URL this label belongs to.
       required: True
       type: str
     state:

awx_collection/plugins/modules/notification_template.py

@@ -44,7 +44,7 @@ options:
       type: str
     organization:
       description:
-        - The organization the notification belongs to.
+        - The organization name, ID, or named URL the notification belongs to.
       type: str
     notification_type:
       description:

awx_collection/plugins/modules/organization.py

@@ -36,7 +36,7 @@ options:
       type: str
     default_environment:
       description:
-        - Default Execution Environment to use for jobs owned by the Organization.
+        - Default Execution Environment name, ID, or named URL to use for jobs owned by the Organization.
       type: str
     custom_virtualenv:
       description:
@@ -56,7 +56,7 @@ options:
       type: str
     instance_groups:
       description:
-        - list of Instance Groups for this Organization to run on.
+        - list of Instance Group names, IDs, or named URLs for this Organization to run on.
       type: list
       elements: str
     notification_templates_started:
@@ -81,7 +81,7 @@ options:
       elements: str
     galaxy_credentials:
       description:
-        - list of Ansible Galaxy credentials to associate to the organization
+        - list of Ansible Galaxy credential names, IDs, or named URLs to associate to the organization
       type: list
       elements: str
 extends_documentation_fragment: awx.awx.auth