External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-03-28 14:25:05 -02:30
Code Issues Packages Projects Releases Wiki Activity

AAP-36604 (analytics) Thousands of zombie/orphaned Slow/Stuck DB queries in controller querying active host count (#15715)

Browse Source 
* lint

* change timeout to 5 minutes

* change timeout to 5 minutes
This commit is contained in:
Andrea Restle-Lay
2024-12-18 17:12:52 -05:00
committed by GitHub
parent 288e8d78d3
commit 1b418f75e6
1 changed files with 31 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
awx/main/tasks/host_metrics.py
View File

@@ -11,6 +11,8 @@ from awx.main.dispatch.publish import task
from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly from awx.main.models.inventory import HostMetric, HostMetricSummaryMonthly
from awx.main.tasks.helpers import is_run_threshold_reached from awx.main.tasks.helpers import is_run_threshold_reached
from awx.conf.license import get_license from awx.conf.license import get_license
from awx.main.utils.pglock import advisory_lock
logger = logging.getLogger('awx.main.tasks.host_metrics') logger = logging.getLogger('awx.main.tasks.host_metrics')
@@ -90,7 +92,10 @@ class HostMetricTask:
class HostMetricSummaryMonthlyTask: class HostMetricSummaryMonthlyTask:
LOCK_KEY = 'host_metric_summary_monthly'
LOCK_SESSION_TIMEOUT = 300000 # 5 minutes.
""" """
Task runs every four hours, longer lock timeout avoids premature termination due to high db load or other latency.
This task computes last [threshold] months of HostMetricSummaryMonthly table This task computes last [threshold] months of HostMetricSummaryMonthly table
[threshold] is setting CLEANUP_HOST_METRICS_HARD_THRESHOLD [threshold] is setting CLEANUP_HOST_METRICS_HARD_THRESHOLD
Each record in the table represents changes in HostMetric table in one month Each record in the table represents changes in HostMetric table in one month
@@ -115,29 +120,37 @@ class HostMetricSummaryMonthlyTask:
self.records_to_update = [] self.records_to_update = []
def execute(self): def execute(self):
self._load_existing_summaries()
self._load_hosts_added()
self._load_hosts_deleted()
# Get first month after last hard delete with advisory_lock(
month = self._get_first_month() HostMetricSummaryMonthlyTask.LOCK_KEY, lock_session_timeout_milliseconds=HostMetricSummaryMonthlyTask.LOCK_SESSION_TIMEOUT, wait=False
license_consumed = self._get_license_consumed_before(month) ) as acquired:
if not acquired:
logger.info("Another instance of host_metric_summary_monthly is already running. Exiting.")
return
# Fill record for each month self._load_existing_summaries()
while month <= datetime.date.today().replace(day=1): self._load_hosts_added()
summary = self._find_or_create_summary(month) self._load_hosts_deleted()
# Update summary and update license_consumed by hosts added/removed this month
self._update_summary(summary, month, license_consumed)
license_consumed = summary.license_consumed
month = month + relativedelta(months=1) # Get first month after last hard delete
month = self._get_first_month()
license_consumed = self._get_license_consumed_before(month)
# Create/Update stats # Fill record for each month
HostMetricSummaryMonthly.objects.bulk_create(self.records_to_create, batch_size=1000) while month <= datetime.date.today().replace(day=1):
HostMetricSummaryMonthly.objects.bulk_update(self.records_to_update, ['license_consumed', 'hosts_added', 'hosts_deleted'], batch_size=1000) summary = self._find_or_create_summary(month)
# Update summary and update license_consumed by hosts added/removed this month
self._update_summary(summary, month, license_consumed)
license_consumed = summary.license_consumed
# Set timestamp of last run month = month + relativedelta(months=1)
settings.HOST_METRIC_SUMMARY_TASK_LAST_TS = now()
# Create/Update stats
HostMetricSummaryMonthly.objects.bulk_create(self.records_to_create, batch_size=1000)
HostMetricSummaryMonthly.objects.bulk_update(self.records_to_update, ['license_consumed', 'hosts_added', 'hosts_deleted'], batch_size=1000)
# Set timestamp of last run
settings.HOST_METRIC_SUMMARY_TASK_LAST_TS = now()
def _get_license_consumed_before(self, month): def _get_license_consumed_before(self, month):
license_consumed = 0 license_consumed = 0