Do each batch of the HostMetric updates in a transaction

It looks like we can't do upserts currently without dropping to raw
SQL, but if we wrap each batch in a transaction, that should insure
that each is updated with the correct count.
This commit is contained in:
Jeff Bradberry
2024-06-04 12:41:45 -04:00
parent d43c91e1a5
commit 4915262af1

View File

@@ -4,11 +4,12 @@ import datetime
from datetime import timezone from datetime import timezone
import logging import logging
from collections import defaultdict from collections import defaultdict
import itertools
import time import time
from django.conf import settings from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ObjectDoesNotExist
from django.db import models, DatabaseError from django.db import models, DatabaseError, transaction
from django.db.models.functions import Cast from django.db.models.functions import Cast
from django.utils.dateparse import parse_datetime from django.utils.dateparse import parse_datetime
from django.utils.text import Truncator from django.utils.text import Truncator
@@ -605,19 +606,23 @@ class JobEvent(BasePlaybookEvent):
def _update_host_metrics(updated_hosts_list): def _update_host_metrics(updated_hosts_list):
from awx.main.models import HostMetric # circular import from awx.main.models import HostMetric # circular import
# bulk-create
current_time = now() current_time = now()
HostMetric.objects.bulk_create(
[HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=100 # FUTURE:
) # - Hand-rolled implementation of itertools.batched(), introduced in Python 3.12. Replace.
# bulk-update # - Ability to do ORM upserts *may* have been introduced in Django 5.0.
batch_start, batch_size = 0, 1000 # See the entry about `create_defaults` in https://docs.djangoproject.com/en/5.0/releases/5.0/#models.
while batch_start <= len(updated_hosts_list): # Hopefully this will be fully ready for batch use by 5.2 LTS.
batched_host_list = updated_hosts_list[batch_start : (batch_start + batch_size)]
HostMetric.objects.filter(hostname__in=batched_host_list).update( args = [iter(updated_hosts_list)] * 500
last_automation=current_time, automated_counter=models.F('automated_counter') + 1, deleted=False for hosts in itertools.zip_longest(*args):
) with transaction.atomic():
batch_start += batch_size HostMetric.objects.bulk_create(
[HostMetric(hostname=hostname, last_automation=current_time) for hostname in hosts if hostname is not None], ignore_conflicts=True
)
HostMetric.objects.filter(hostname__in=hosts).update(
last_automation=current_time, automated_counter=models.F('automated_counter') + 1, deleted=False
)
@property @property
def job_verbosity(self): def job_verbosity(self):