[AAP-78392] Optimize HostManager.active_count() with cache and functional index

active_count() runs a full sequential scan with LOWER()+DISTINCT on main_host for every license check. At customer scale this consumed 74.5 minutes of DB time over 4 hours (47K calls at 93ms avg).

Add a 60-second Redis-backed cache via the existing memoize decorator to reduce call volume by ~99.5%. Add a functional btree index on LOWER(name) to eliminate the sequential scan for the remaining calls.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Dirk Julich
2026-06-16 17:59:58 +02:00
parent 34f34e058b
commit fd1745a375
4 changed files with 76 additions and 0 deletions

View File

@@ -10,6 +10,7 @@ from django.db.models.functions import Lower
from ansible_base.lib.utils.db import advisory_lock from ansible_base.lib.utils.db import advisory_lock
from awx.main.utils.common import memoize
from awx.main.utils.filters import SmartFilter from awx.main.utils.filters import SmartFilter
from awx.main.constants import RECEPTOR_PENDING from awx.main.constants import RECEPTOR_PENDING
@@ -85,6 +86,7 @@ class HostLatestSummaryQuerySet(models.QuerySet):
class HostManager(models.Manager.from_queryset(HostLatestSummaryQuerySet)): class HostManager(models.Manager.from_queryset(HostLatestSummaryQuerySet)):
"""Custom manager class for Hosts model.""" """Custom manager class for Hosts model."""
@memoize(ttl=60, cache_key='host_active_count')
def active_count(self): def active_count(self):
"""Return count of active, unique hosts for licensing. """Return count of active, unique hosts for licensing.
Construction of query involves: Construction of query involves:

View File

@@ -0,0 +1,19 @@
from django.db import migrations, models
from django.db.models.functions import Lower
class Migration(migrations.Migration):
dependencies = [
('main', '0205_add_ordering_to_instancegroup_and_workflow_nodes'),
]
operations = [
migrations.AddIndex(
model_name='host',
index=models.Index(
Lower('name'),
name='main_host_name_lower_idx',
),
),
]

View File

@@ -19,6 +19,7 @@ from django.core.exceptions import ValidationError
from django.urls import resolve from django.urls import resolve
from django.utils.timezone import now from django.utils.timezone import now
from django.db.models import Q, Subquery, OuterRef from django.db.models import Q, Subquery, OuterRef
from django.db.models.functions import Lower
# REST Framework # REST Framework
from rest_framework.exceptions import ParseError from rest_framework.exceptions import ParseError
@@ -523,6 +524,9 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
app_label = 'main' app_label = 'main'
unique_together = (("name", "inventory"),) # FIXME: Add ('instance_id', 'inventory') after migration. unique_together = (("name", "inventory"),) # FIXME: Add ('instance_id', 'inventory') after migration.
ordering = ('name',) ordering = ('name',)
indexes = [
models.Index(Lower('name'), name='main_host_name_lower_idx'),
]
inventory = models.ForeignKey( inventory = models.ForeignKey(
'Inventory', 'Inventory',

View File

@@ -92,6 +92,22 @@ class TestInventoryScript:
@pytest.mark.django_db @pytest.mark.django_db
class TestActiveCount: class TestActiveCount:
@pytest.fixture(autouse=True)
def _bypass_active_count_cache(self):
from django.core.cache import cache
cache.delete('host_active_count')
original_set = cache.set
def skip_host_cache(key, *args, **kwargs):
if key == 'host_active_count':
return
return original_set(key, *args, **kwargs)
with mock.patch.object(cache, 'set', side_effect=skip_host_cache):
yield
cache.delete('host_active_count')
def test_host_active_count(self, organization): def test_host_active_count(self, organization):
inv1 = Inventory.objects.create(name='inv1', organization=organization) inv1 = Inventory.objects.create(name='inv1', organization=organization)
inv2 = Inventory.objects.create(name='inv2', organization=organization) inv2 = Inventory.objects.create(name='inv2', organization=organization)
@@ -141,6 +157,41 @@ class TestActiveCount:
assert Host.objects.active_count() == 2 assert Host.objects.active_count() == 2
@pytest.mark.django_db
class TestActiveCountCache:
@pytest.fixture(autouse=True)
def _clear_active_count_cache(self):
from awx.main.utils.common import memoize_delete
memoize_delete('host_active_count')
yield
memoize_delete('host_active_count')
def test_active_count_cache(self, organization):
inv = Inventory.objects.create(name='inv1', organization=organization)
inv.hosts.create(name='host1')
assert Host.objects.active_count() == 1
inv.hosts.create(name='host2')
assert Host.objects.active_count() == 1 # still cached
from awx.main.utils.common import memoize_delete
memoize_delete('host_active_count')
assert Host.objects.active_count() == 2
def test_active_count_cache_after_delete(self, organization):
inv = Inventory.objects.create(name='inv1', organization=organization)
h = inv.hosts.create(name='host1')
assert Host.objects.active_count() == 1
h.delete()
from awx.main.utils.common import memoize_delete
memoize_delete('host_active_count')
assert Host.objects.active_count() == 0
@pytest.mark.django_db @pytest.mark.django_db
class TestSCMUpdateFeatures: class TestSCMUpdateFeatures:
def test_source_location(self, scm_inventory_source): def test_source_location(self, scm_inventory_source):