From ba45592d930f0abb2701d3f33f4e11134a92e6ca Mon Sep 17 00:00:00 2001 From: Jim Ladd Date: Wed, 17 Feb 2021 15:48:30 -0800 Subject: [PATCH] create helper method to create partitions * create_partition() --- awx/main/migrations/0124_event_partitions.py | 13 +++----- awx/main/utils/common.py | 34 +++++++++++++------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/awx/main/migrations/0124_event_partitions.py b/awx/main/migrations/0124_event_partitions.py index a4eb7ef55c..ffff8f5397 100644 --- a/awx/main/migrations/0124_event_partitions.py +++ b/awx/main/migrations/0124_event_partitions.py @@ -1,10 +1,12 @@ # Generated by Django 2.2.8 on 2020-02-21 16:31 -from datetime import timedelta +from datetime import datetime from django.db import migrations, models, connection from django.utils.timezone import now +from awx.main.utils.common import create_partition + def migrate_event_data(apps, schema_editor): # see: https://github.com/ansible/awx/issues/9039 @@ -62,13 +64,8 @@ def migrate_event_data(apps, schema_editor): ) # .. as well as initial partition containing all existing events - current_time = now() # only retrieve current time once to avoid a boundary error - end_timestamp = current_time.strftime('%Y-%m-%d %H:%M:%S.000000%z') - cursor.execute( - f'CREATE TABLE {tblname}_old_events ' - f'PARTITION OF {tblname} ' - f'FOR VALUES FROM (\'1980-01-01 00:00:00.000000+00\') to (\'{end_timestamp}\');' - ) + epoch = datetime(1980, 1, 1, 0, 0) + create_partition('old_events', epoch, now()) # copy over all job events into partitioned table # TODO: https://github.com/ansible/awx/issues/9257 diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 543f351d4e..be3732eb93 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -2,6 +2,7 @@ # All Rights Reserved. # Python +from datetime import timedelta import json import yaml import logging @@ -22,6 +23,7 @@ from django.core.exceptions import ObjectDoesNotExist, FieldDoesNotExist from django.utils.dateparse import parse_datetime from django.utils.translation import ugettext_lazy as _ from django.utils.functional import cached_property +from django.db import connection from django.db.models.fields.related import ForeignObjectRel, ManyToManyField from django.db.models.fields.related_descriptors import ForwardManyToOneDescriptor, ManyToManyDescriptor from django.db.models.query import QuerySet @@ -33,6 +35,7 @@ from django.core.cache import cache as django_cache from rest_framework.exceptions import ParseError from django.utils.encoding import smart_str from django.utils.text import slugify +from django.utils.timezone import now from django.apps import apps # AWX @@ -1024,15 +1027,24 @@ def deepmerge(a, b): return b -def cleanup_new_process(func): - """ - Cleanup django connection, cache connection, before executing new thread or processes entry point, func. - """ +def create_partition(partition_label, start, end=None): + """Creates new partition tables for events.""" + if not end: + end = (now() + timedelta(hours=1)) + start_timestamp = start.strftime('%Y-%m-%d %H:00:00.000000%z') + end_timestamp = end.strftime('%Y-%m-%d %H:00:00.000000%z') - @wraps(func) - def wrapper_cleanup_new_process(*args, **kwargs): - django_connection.close() - django_cache.close() - return func(*args, **kwargs) - - return wrapper_cleanup_new_process + with connection.cursor() as cursor: + # Only partitioning main_jobevent on first pass + # + #for tblname in ( + # 'main_jobevent', 'main_inventoryupdateevent', + # 'main_projectupdateevent', 'main_adhoccommandevent', + # 'main_systemjobevent' + #): + for tblname in ('main_jobevent',): + cursor.execute( + f'CREATE TABLE {tblname}_{partition_label} ' + f'PARTITION OF {tblname} ' + f'FOR VALUES FROM (\'{start_timestamp}\') to (\'{end_timestamp}\');' + )