From d5e5ea3670b357e35bf3edd5d09b537a43dc192c Mon Sep 17 00:00:00 2001 From: Seth Foster Date: Tue, 9 Jun 2026 12:45:50 -0400 Subject: [PATCH] Lazy-load plugin registries, move DB sync to dispatcher (#16483) Move plugin loading to lazy-on-first-access, DB sync to dispatcher Remove credential type and inventory plugin loading from Django's app.ready() path. In-memory registries (ManagedCredentialType.registry and InventorySourceOptions.injectors) are now populated lazily on first access via LazyLoadDict, a dict subclass that calls a loader function on the first read operation. This ensures web workers, dispatcher workers, and management commands all get the registries populated exactly when needed, without eager loading at startup. The DB sync (CredentialType.setup_tower_managed_defaults) is moved to the dispatcher's startup task, where it only needs to run once per deployment rather than in every Django process. Co-Authored-By: Alan Rominger Co-Authored-By: Claude Opus 4.6 (1M context) --- awx/api/serializers.py | 2 +- awx/main/apps.py | 51 --------------- awx/main/models/credential.py | 10 ++- awx/main/models/inventory.py | 15 ++++- awx/main/tasks/system.py | 17 +++++ .../models/test_context_managers.py | 3 +- awx/main/tests/functional/test_apps.py | 14 ++-- awx/main/utils/common.py | 8 --- awx/main/utils/lazy_registry.py | 64 +++++++++++++++++++ 9 files changed, 115 insertions(+), 69 deletions(-) create mode 100644 awx/main/utils/lazy_registry.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 80594adebe..79d30a2c9f 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -120,7 +120,7 @@ from awx.main.utils.named_url_graph import reset_counters from awx.main.utils.inventory_vars import update_group_variables from awx.main.scheduler.task_manager_models import TaskManagerModels from awx.main.redact import UriCleaner, REPLACE_STR -from awx.main.signals import update_inventory_computed_fields +from awx.main.tasks.system import update_inventory_computed_fields from awx.main.validators import vars_validate_or_raise diff --git a/awx/main/apps.py b/awx/main/apps.py index 2b67de1cf9..c39c882ec7 100644 --- a/awx/main/apps.py +++ b/awx/main/apps.py @@ -1,5 +1,3 @@ -import os - from dispatcherd.config import setup as dispatcher_setup from django.apps import AppConfig @@ -8,14 +6,10 @@ from django.utils.translation import gettext_lazy as _ from django.core.management.base import CommandError from django.db.models.signals import pre_migrate -from awx.main.utils.common import bypass_in_test, load_all_entry_points_for -from awx.main.utils.migration import is_database_synchronized from awx.main.utils.named_url_graph import _customize_graph, generate_graph from awx.main.utils.db import db_requirement_violations from awx.conf import register, fields -from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name - class MainConfig(AppConfig): name = 'awx.main' @@ -52,42 +46,6 @@ class MainConfig(AppConfig): category_slug='named-url', ) - def _load_credential_types_feature(self): - """ - Create CredentialType records for any discovered credentials. - - Note that Django docs advise _against_ interacting with the database using - the ORM models in the ready() path. Specifically, during testing. - However, we explicitly use the @bypass_in_test decorator to avoid calling this - method during testing. - - Django also advises against running pattern because it runs everywhere i.e. - every management command. We use an advisory lock to ensure correctness and - we will deal performance if it becomes an issue. - """ - from awx.main.models.credential import CredentialType - - if is_database_synchronized(): - CredentialType.setup_tower_managed_defaults(app_config=self) - - @bypass_in_test - def load_credential_types_feature(self): - from awx.main.models.credential import load_credentials - - load_credentials() - return self._load_credential_types_feature() - - def load_inventory_plugins(self): - from awx.main.models.inventory import InventorySourceOptions - - is_awx = detect_server_product_name() == 'AWX' - extra_entry_point_groups = () if is_awx else ('inventory.supported',) - entry_points = load_all_entry_points_for(['inventory', *extra_entry_point_groups]) - - for entry_point_name, entry_point in entry_points.items(): - cls = entry_point.load() - InventorySourceOptions.injectors[entry_point_name] = cls - def configure_dispatcherd(self): """This implements the default configuration for dispatcherd @@ -109,14 +67,5 @@ class MainConfig(AppConfig): super().ready() self.configure_dispatcherd() - - """ - Credential loading triggers database operations. There are cases we want to call - awx-manage collectstatic without a database. All management commands invoke the ready() code - path. Using settings.AWX_SKIP_CREDENTIAL_TYPES_DISCOVER _could_ invoke a database operation. - """ - if not os.environ.get('AWX_SKIP_CREDENTIAL_TYPES_DISCOVER', None): - self.load_credential_types_feature() self.load_named_url_feature() - self.load_inventory_plugins() pre_migrate.connect(self.check_db_requirement, sender=self) diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 9959694384..bb8a1e9d21 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -47,6 +47,7 @@ from awx.main.models.rbac import ( ) from awx.main.models import Team, Organization from awx.main.utils import encrypt_field +from awx.main.utils.lazy_registry import LazyLoadDict from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name __all__ = ['Credential', 'CredentialType', 'CredentialInputSource', 'build_safe_env'] @@ -569,7 +570,7 @@ class CredentialTypeHelper: class ManagedCredentialType(SimpleNamespace): - registry = {} + registry = None # initialized as LazyLoadDict after load_credentials is defined class CredentialInputSource(PrimordialModel): @@ -661,6 +662,8 @@ def _is_oidc_namespace_disabled(ns): def load_credentials(): + ManagedCredentialType.registry.clear() + awx_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials')} supported_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials.supported')} plugin_entry_points = awx_entry_points if detect_server_product_name() == 'AWX' else {**awx_entry_points, **supported_entry_points} @@ -692,3 +695,8 @@ def load_credentials(): plugin = ep.load() CredentialType.load_plugin(ns, plugin) + + +# load_credentials writes directly into this dict via registry[ns] = ..., +# LazyLoadDict just ensures it runs once before the first read access +ManagedCredentialType.registry = LazyLoadDict(load_credentials) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 9231d73a38..f323688264 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -27,7 +27,10 @@ from ansible_base.lib.utils.models import prevent_search # AWX from awx.api.versioning import reverse +from awx.main.utils.common import load_all_entry_points_for +from awx.main.utils.lazy_registry import LazyLoadDict from awx.main.utils.plugins import discover_available_cloud_provider_plugin_names, compute_cloud_inventory_sources +from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name from awx.main.consumers import emit_channel_notification from awx.main.fields import ( ImplicitRoleField, @@ -926,12 +929,22 @@ class HostMetricSummaryMonthly(models.Model): indirectly_managed_hosts = models.IntegerField(default=0, help_text=("Manually entered number indirectly managed hosts for a certain month")) +def _load_inventory_plugins(): + is_awx = detect_server_product_name() == 'AWX' + extra_entry_point_groups = () if is_awx else ('inventory.supported',) + all_entry_points = load_all_entry_points_for(['inventory', *extra_entry_point_groups]) + + for entry_point_name, entry_point in all_entry_points.items(): + cls = entry_point.load() + InventorySourceOptions.injectors[entry_point_name] = cls + + class InventorySourceOptions(BaseModel): """ Common fields for InventorySource and InventoryUpdate. """ - injectors = dict() + injectors = LazyLoadDict(_load_inventory_plugins) # From the options of the Django management base command INVENTORY_UPDATE_VERBOSITY_CHOICES = [ diff --git a/awx/main/tasks/system.py b/awx/main/tasks/system.py index 5e1190a41e..58993b0b9b 100644 --- a/awx/main/tasks/system.py +++ b/awx/main/tasks/system.py @@ -69,10 +69,12 @@ from awx.main.models import ( UnifiedJob, convert_jsonfields, ) +from awx.main.models.credential import CredentialType from awx.main.tasks.helpers import is_run_threshold_reached from awx.main.tasks.host_indirect import save_indirect_host_entries from awx.main.tasks.receptor import administrative_workunit_reaper, get_receptor_ctl, worker_cleanup, worker_info, write_receptor_config from awx.main.utils.common import ignore_inventory_computed_fields, ignore_inventory_group_removal +from awx.main.utils.migration import is_database_synchronized from awx.main.utils.reload import stop_local_services logger = logging.getLogger('awx.main.tasks.system') @@ -84,6 +86,16 @@ Try upgrading OpenSSH or providing your private key in an different format. \ ''' +def _sync_credential_types_to_db(): + """Ensure CredentialType DB rows match the installed plugins. + + The in-memory registry is populated lazily on first access via LazyLoadDict. + This function only handles the DB sync step. + """ + if is_database_synchronized(): + CredentialType.setup_tower_managed_defaults() + + def _run_dispatch_startup_common(): """ Execute the common startup initialization steps. @@ -99,6 +111,11 @@ def _run_dispatch_startup_common(): except Exception: logger.exception("Failed to write receptor config, skipping.") + try: + _sync_credential_types_to_db() + except Exception: + logger.exception("Failed to sync credential types to DB, skipping.") + try: convert_jsonfields() except Exception: diff --git a/awx/main/tests/functional/models/test_context_managers.py b/awx/main/tests/functional/models/test_context_managers.py index 7dfc0da117..485dd30f70 100644 --- a/awx/main/tests/functional/models/test_context_managers.py +++ b/awx/main/tests/functional/models/test_context_managers.py @@ -1,7 +1,8 @@ import pytest # AWX context managers for testing -from awx.main.signals import disable_activity_stream, disable_computed_fields, update_inventory_computed_fields +from awx.main.signals import disable_activity_stream, disable_computed_fields +from awx.main.tasks.system import update_inventory_computed_fields # AWX models from awx.main.models.organization import Organization diff --git a/awx/main/tests/functional/test_apps.py b/awx/main/tests/functional/test_apps.py index fbe9a4a370..e4906b4f3e 100644 --- a/awx/main/tests/functional/test_apps.py +++ b/awx/main/tests/functional/test_apps.py @@ -3,6 +3,8 @@ import pytest from django.apps import apps from django.core.management.base import CommandError +from awx.main.tasks.system import _sync_credential_types_to_db + @pytest.fixture def mock_setup_tower_managed_defaults(mocker): @@ -10,19 +12,19 @@ def mock_setup_tower_managed_defaults(mocker): @pytest.mark.django_db -def test_load_credential_types_feature_migrations_ran(mocker, mock_setup_tower_managed_defaults): - mocker.patch('awx.main.apps.is_database_synchronized', return_value=True) +def test_sync_credential_types_migrations_ran(mocker, mock_setup_tower_managed_defaults): + mocker.patch('awx.main.tasks.system.is_database_synchronized', return_value=True) - apps.get_app_config('main')._load_credential_types_feature() + _sync_credential_types_to_db() mock_setup_tower_managed_defaults.assert_called_once() @pytest.mark.django_db -def test_load_credential_types_feature_migrations_not_ran(mocker, mock_setup_tower_managed_defaults): - mocker.patch('awx.main.apps.is_database_synchronized', return_value=False) +def test_sync_credential_types_migrations_not_ran(mocker, mock_setup_tower_managed_defaults): + mocker.patch('awx.main.tasks.system.is_database_synchronized', return_value=False) - apps.get_app_config('main')._load_credential_types_feature() + _sync_credential_types_to_db() mock_setup_tower_managed_defaults.assert_not_called() diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 9226aa05cd..21a27980b2 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -151,14 +151,6 @@ def is_testing(argv=None): return False -def bypass_in_test(func): - def fn(*args, **kwargs): - if not is_testing(): - return func(*args, **kwargs) - - return fn - - class RequireDebugTrueOrTest(logging.Filter): """ Logging filter to output when in DEBUG mode or running tests. diff --git a/awx/main/utils/lazy_registry.py b/awx/main/utils/lazy_registry.py new file mode 100644 index 0000000000..bb90c84df6 --- /dev/null +++ b/awx/main/utils/lazy_registry.py @@ -0,0 +1,64 @@ +class LazyLoadDict(dict): + """A dict subclass that calls a loader function on first read access. + + Writes (e.g. during the loading process itself) go straight through + without triggering the loader. + """ + + def __init__(self, loader): + super().__init__() + self._loader = loader + self._loaded = False + + def _ensure_loaded(self): + if not self._loaded: + self._loaded = True + self._loader() + + def __getitem__(self, key): + self._ensure_loaded() + return super().__getitem__(key) + + def get(self, key, default=None): + self._ensure_loaded() + return super().get(key, default) + + def __contains__(self, key): + self._ensure_loaded() + return super().__contains__(key) + + def __iter__(self): + self._ensure_loaded() + return super().__iter__() + + def __len__(self): + self._ensure_loaded() + return super().__len__() + + def keys(self): + self._ensure_loaded() + return super().keys() + + def values(self): + self._ensure_loaded() + return super().values() + + def items(self): + self._ensure_loaded() + return super().items() + + def __bool__(self): + self._ensure_loaded() + return super().__bool__() + + def __repr__(self): + self._ensure_loaded() + return super().__repr__() + + def copy(self): + self._ensure_loaded() + return super().copy() + + def clear(self): + super().clear() + self._loaded = True