From fd1745a375ed5d35f301b82527721ec7656e2e17 Mon Sep 17 00:00:00 2001 From: Dirk Julich Date: Tue, 16 Jun 2026 17:59:58 +0200 Subject: [PATCH] [AAP-78392] Optimize HostManager.active_count() with cache and functional index active_count() runs a full sequential scan with LOWER()+DISTINCT on main_host for every license check. At customer scale this consumed 74.5 minutes of DB time over 4 hours (47K calls at 93ms avg). Add a 60-second Redis-backed cache via the existing memoize decorator to reduce call volume by ~99.5%. Add a functional btree index on LOWER(name) to eliminate the sequential scan for the remaining calls. Co-Authored-By: Claude Opus 4.6 --- awx/main/managers.py | 2 + .../0206_add_host_name_lower_index.py | 19 +++++++ awx/main/models/inventory.py | 4 ++ .../tests/functional/models/test_inventory.py | 51 +++++++++++++++++++ 4 files changed, 76 insertions(+) create mode 100644 awx/main/migrations/0206_add_host_name_lower_index.py diff --git a/awx/main/managers.py b/awx/main/managers.py index 62cbb34721..fb90aa434a 100644 --- a/awx/main/managers.py +++ b/awx/main/managers.py @@ -10,6 +10,7 @@ from django.db.models.functions import Lower from ansible_base.lib.utils.db import advisory_lock +from awx.main.utils.common import memoize from awx.main.utils.filters import SmartFilter from awx.main.constants import RECEPTOR_PENDING @@ -85,6 +86,7 @@ class HostLatestSummaryQuerySet(models.QuerySet): class HostManager(models.Manager.from_queryset(HostLatestSummaryQuerySet)): """Custom manager class for Hosts model.""" + @memoize(ttl=60, cache_key='host_active_count') def active_count(self): """Return count of active, unique hosts for licensing. Construction of query involves: diff --git a/awx/main/migrations/0206_add_host_name_lower_index.py b/awx/main/migrations/0206_add_host_name_lower_index.py new file mode 100644 index 0000000000..405285276b --- /dev/null +++ b/awx/main/migrations/0206_add_host_name_lower_index.py @@ -0,0 +1,19 @@ +from django.db import migrations, models +from django.db.models.functions import Lower + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0205_add_ordering_to_instancegroup_and_workflow_nodes'), + ] + + operations = [ + migrations.AddIndex( + model_name='host', + index=models.Index( + Lower('name'), + name='main_host_name_lower_idx', + ), + ), + ] diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index f323688264..3f41b0aedb 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -19,6 +19,7 @@ from django.core.exceptions import ValidationError from django.urls import resolve from django.utils.timezone import now from django.db.models import Q, Subquery, OuterRef +from django.db.models.functions import Lower # REST Framework from rest_framework.exceptions import ParseError @@ -523,6 +524,9 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin): app_label = 'main' unique_together = (("name", "inventory"),) # FIXME: Add ('instance_id', 'inventory') after migration. ordering = ('name',) + indexes = [ + models.Index(Lower('name'), name='main_host_name_lower_idx'), + ] inventory = models.ForeignKey( 'Inventory', diff --git a/awx/main/tests/functional/models/test_inventory.py b/awx/main/tests/functional/models/test_inventory.py index c725bfbfe5..7e7150250d 100644 --- a/awx/main/tests/functional/models/test_inventory.py +++ b/awx/main/tests/functional/models/test_inventory.py @@ -92,6 +92,22 @@ class TestInventoryScript: @pytest.mark.django_db class TestActiveCount: + @pytest.fixture(autouse=True) + def _bypass_active_count_cache(self): + from django.core.cache import cache + + cache.delete('host_active_count') + original_set = cache.set + + def skip_host_cache(key, *args, **kwargs): + if key == 'host_active_count': + return + return original_set(key, *args, **kwargs) + + with mock.patch.object(cache, 'set', side_effect=skip_host_cache): + yield + cache.delete('host_active_count') + def test_host_active_count(self, organization): inv1 = Inventory.objects.create(name='inv1', organization=organization) inv2 = Inventory.objects.create(name='inv2', organization=organization) @@ -141,6 +157,41 @@ class TestActiveCount: assert Host.objects.active_count() == 2 +@pytest.mark.django_db +class TestActiveCountCache: + @pytest.fixture(autouse=True) + def _clear_active_count_cache(self): + from awx.main.utils.common import memoize_delete + + memoize_delete('host_active_count') + yield + memoize_delete('host_active_count') + + def test_active_count_cache(self, organization): + inv = Inventory.objects.create(name='inv1', organization=organization) + inv.hosts.create(name='host1') + assert Host.objects.active_count() == 1 + + inv.hosts.create(name='host2') + assert Host.objects.active_count() == 1 # still cached + + from awx.main.utils.common import memoize_delete + + memoize_delete('host_active_count') + assert Host.objects.active_count() == 2 + + def test_active_count_cache_after_delete(self, organization): + inv = Inventory.objects.create(name='inv1', organization=organization) + h = inv.hosts.create(name='host1') + assert Host.objects.active_count() == 1 + + h.delete() + from awx.main.utils.common import memoize_delete + + memoize_delete('host_active_count') + assert Host.objects.active_count() == 0 + + @pytest.mark.django_db class TestSCMUpdateFeatures: def test_source_location(self, scm_inventory_source):