AAP-36604 (analytics) Thousands of zombie/orphaned Slow/Stuck DB queries in controller querying active host count (#15715)

* lint

* change timeout to 5 minutes

* change timeout to 5 minutes

awx/main/tasks/host_metrics.py

@@ -11,6 +11,8 @@ from awx.main.dispatch.publish import task
 from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly
 from awx.main.tasks.helpers import is_run_threshold_reached
 from awx.conf.license import get_license
+from awx.main.utils.pglock import advisory_lock
+
 
 logger = logging.getLogger('awx.main.tasks.host_metrics')
 
@@ -90,7 +92,10 @@ class HostMetricTask:
 
 
 class HostMetricSummaryMonthlyTask:
+    LOCK_KEY = 'host_metric_summary_monthly'
+    LOCK_SESSION_TIMEOUT = 300000  # 5 minutes.
     """
+    Task runs every four hours, longer lock timeout avoids premature termination due to high db load or other latency.
     This task computes last [threshold] months of HostMetricSummaryMonthly table
     [threshold] is setting CLEANUP_HOST_METRICS_HARD_THRESHOLD
     Each record in the table represents changes in HostMetric table in one month
@@ -115,6 +120,14 @@ class HostMetricSummaryMonthlyTask:
         self.records_to_update = []
 
     def execute(self):
+
+        with advisory_lock(
+            HostMetricSummaryMonthlyTask.LOCK_KEY, lock_session_timeout_milliseconds=HostMetricSummaryMonthlyTask.LOCK_SESSION_TIMEOUT, wait=False
+        ) as acquired:
+            if not acquired:
+                logger.info("Another instance of host_metric_summary_monthly is already running. Exiting.")
+                return
+
             self._load_existing_summaries()
             self._load_hosts_added()
             self._load_hosts_deleted()