diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 80594adebe..79d30a2c9f 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -120,7 +120,7 @@ from awx.main.utils.named_url_graph import reset_counters from awx.main.utils.inventory_vars import update_group_variables from awx.main.scheduler.task_manager_models import TaskManagerModels from awx.main.redact import UriCleaner, REPLACE_STR -from awx.main.signals import update_inventory_computed_fields +from awx.main.tasks.system import update_inventory_computed_fields from awx.main.validators import vars_validate_or_raise diff --git a/awx/main/apps.py b/awx/main/apps.py index 2b67de1cf9..c39c882ec7 100644 --- a/awx/main/apps.py +++ b/awx/main/apps.py @@ -1,5 +1,3 @@ -import os - from dispatcherd.config import setup as dispatcher_setup from django.apps import AppConfig @@ -8,14 +6,10 @@ from django.utils.translation import gettext_lazy as _ from django.core.management.base import CommandError from django.db.models.signals import pre_migrate -from awx.main.utils.common import bypass_in_test, load_all_entry_points_for -from awx.main.utils.migration import is_database_synchronized from awx.main.utils.named_url_graph import _customize_graph, generate_graph from awx.main.utils.db import db_requirement_violations from awx.conf import register, fields -from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name - class MainConfig(AppConfig): name = 'awx.main' @@ -52,42 +46,6 @@ class MainConfig(AppConfig): category_slug='named-url', ) - def _load_credential_types_feature(self): - """ - Create CredentialType records for any discovered credentials. - - Note that Django docs advise _against_ interacting with the database using - the ORM models in the ready() path. Specifically, during testing. - However, we explicitly use the @bypass_in_test decorator to avoid calling this - method during testing. - - Django also advises against running pattern because it runs everywhere i.e. - every management command. We use an advisory lock to ensure correctness and - we will deal performance if it becomes an issue. - """ - from awx.main.models.credential import CredentialType - - if is_database_synchronized(): - CredentialType.setup_tower_managed_defaults(app_config=self) - - @bypass_in_test - def load_credential_types_feature(self): - from awx.main.models.credential import load_credentials - - load_credentials() - return self._load_credential_types_feature() - - def load_inventory_plugins(self): - from awx.main.models.inventory import InventorySourceOptions - - is_awx = detect_server_product_name() == 'AWX' - extra_entry_point_groups = () if is_awx else ('inventory.supported',) - entry_points = load_all_entry_points_for(['inventory', *extra_entry_point_groups]) - - for entry_point_name, entry_point in entry_points.items(): - cls = entry_point.load() - InventorySourceOptions.injectors[entry_point_name] = cls - def configure_dispatcherd(self): """This implements the default configuration for dispatcherd @@ -109,14 +67,5 @@ class MainConfig(AppConfig): super().ready() self.configure_dispatcherd() - - """ - Credential loading triggers database operations. There are cases we want to call - awx-manage collectstatic without a database. All management commands invoke the ready() code - path. Using settings.AWX_SKIP_CREDENTIAL_TYPES_DISCOVER _could_ invoke a database operation. - """ - if not os.environ.get('AWX_SKIP_CREDENTIAL_TYPES_DISCOVER', None): - self.load_credential_types_feature() self.load_named_url_feature() - self.load_inventory_plugins() pre_migrate.connect(self.check_db_requirement, sender=self) diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 9959694384..bb8a1e9d21 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -47,6 +47,7 @@ from awx.main.models.rbac import ( ) from awx.main.models import Team, Organization from awx.main.utils import encrypt_field +from awx.main.utils.lazy_registry import LazyLoadDict from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name __all__ = ['Credential', 'CredentialType', 'CredentialInputSource', 'build_safe_env'] @@ -569,7 +570,7 @@ class CredentialTypeHelper: class ManagedCredentialType(SimpleNamespace): - registry = {} + registry = None # initialized as LazyLoadDict after load_credentials is defined class CredentialInputSource(PrimordialModel): @@ -661,6 +662,8 @@ def _is_oidc_namespace_disabled(ns): def load_credentials(): + ManagedCredentialType.registry.clear() + awx_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials')} supported_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials.supported')} plugin_entry_points = awx_entry_points if detect_server_product_name() == 'AWX' else {**awx_entry_points, **supported_entry_points} @@ -692,3 +695,8 @@ def load_credentials(): plugin = ep.load() CredentialType.load_plugin(ns, plugin) + + +# load_credentials writes directly into this dict via registry[ns] = ..., +# LazyLoadDict just ensures it runs once before the first read access +ManagedCredentialType.registry = LazyLoadDict(load_credentials) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 9231d73a38..f323688264 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -27,7 +27,10 @@ from ansible_base.lib.utils.models import prevent_search # AWX from awx.api.versioning import reverse +from awx.main.utils.common import load_all_entry_points_for +from awx.main.utils.lazy_registry import LazyLoadDict from awx.main.utils.plugins import discover_available_cloud_provider_plugin_names, compute_cloud_inventory_sources +from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name from awx.main.consumers import emit_channel_notification from awx.main.fields import ( ImplicitRoleField, @@ -926,12 +929,22 @@ class HostMetricSummaryMonthly(models.Model): indirectly_managed_hosts = models.IntegerField(default=0, help_text=("Manually entered number indirectly managed hosts for a certain month")) +def _load_inventory_plugins(): + is_awx = detect_server_product_name() == 'AWX' + extra_entry_point_groups = () if is_awx else ('inventory.supported',) + all_entry_points = load_all_entry_points_for(['inventory', *extra_entry_point_groups]) + + for entry_point_name, entry_point in all_entry_points.items(): + cls = entry_point.load() + InventorySourceOptions.injectors[entry_point_name] = cls + + class InventorySourceOptions(BaseModel): """ Common fields for InventorySource and InventoryUpdate. """ - injectors = dict() + injectors = LazyLoadDict(_load_inventory_plugins) # From the options of the Django management base command INVENTORY_UPDATE_VERBOSITY_CHOICES = [ diff --git a/awx/main/tasks/system.py b/awx/main/tasks/system.py index 5e1190a41e..58993b0b9b 100644 --- a/awx/main/tasks/system.py +++ b/awx/main/tasks/system.py @@ -69,10 +69,12 @@ from awx.main.models import ( UnifiedJob, convert_jsonfields, ) +from awx.main.models.credential import CredentialType from awx.main.tasks.helpers import is_run_threshold_reached from awx.main.tasks.host_indirect import save_indirect_host_entries from awx.main.tasks.receptor import administrative_workunit_reaper, get_receptor_ctl, worker_cleanup, worker_info, write_receptor_config from awx.main.utils.common import ignore_inventory_computed_fields, ignore_inventory_group_removal +from awx.main.utils.migration import is_database_synchronized from awx.main.utils.reload import stop_local_services logger = logging.getLogger('awx.main.tasks.system') @@ -84,6 +86,16 @@ Try upgrading OpenSSH or providing your private key in an different format. \ ''' +def _sync_credential_types_to_db(): + """Ensure CredentialType DB rows match the installed plugins. + + The in-memory registry is populated lazily on first access via LazyLoadDict. + This function only handles the DB sync step. + """ + if is_database_synchronized(): + CredentialType.setup_tower_managed_defaults() + + def _run_dispatch_startup_common(): """ Execute the common startup initialization steps. @@ -99,6 +111,11 @@ def _run_dispatch_startup_common(): except Exception: logger.exception("Failed to write receptor config, skipping.") + try: + _sync_credential_types_to_db() + except Exception: + logger.exception("Failed to sync credential types to DB, skipping.") + try: convert_jsonfields() except Exception: diff --git a/awx/main/tests/functional/models/test_context_managers.py b/awx/main/tests/functional/models/test_context_managers.py index 7dfc0da117..485dd30f70 100644 --- a/awx/main/tests/functional/models/test_context_managers.py +++ b/awx/main/tests/functional/models/test_context_managers.py @@ -1,7 +1,8 @@ import pytest # AWX context managers for testing -from awx.main.signals import disable_activity_stream, disable_computed_fields, update_inventory_computed_fields +from awx.main.signals import disable_activity_stream, disable_computed_fields +from awx.main.tasks.system import update_inventory_computed_fields # AWX models from awx.main.models.organization import Organization diff --git a/awx/main/tests/functional/test_apps.py b/awx/main/tests/functional/test_apps.py index fbe9a4a370..e4906b4f3e 100644 --- a/awx/main/tests/functional/test_apps.py +++ b/awx/main/tests/functional/test_apps.py @@ -3,6 +3,8 @@ import pytest from django.apps import apps from django.core.management.base import CommandError +from awx.main.tasks.system import _sync_credential_types_to_db + @pytest.fixture def mock_setup_tower_managed_defaults(mocker): @@ -10,19 +12,19 @@ def mock_setup_tower_managed_defaults(mocker): @pytest.mark.django_db -def test_load_credential_types_feature_migrations_ran(mocker, mock_setup_tower_managed_defaults): - mocker.patch('awx.main.apps.is_database_synchronized', return_value=True) +def test_sync_credential_types_migrations_ran(mocker, mock_setup_tower_managed_defaults): + mocker.patch('awx.main.tasks.system.is_database_synchronized', return_value=True) - apps.get_app_config('main')._load_credential_types_feature() + _sync_credential_types_to_db() mock_setup_tower_managed_defaults.assert_called_once() @pytest.mark.django_db -def test_load_credential_types_feature_migrations_not_ran(mocker, mock_setup_tower_managed_defaults): - mocker.patch('awx.main.apps.is_database_synchronized', return_value=False) +def test_sync_credential_types_migrations_not_ran(mocker, mock_setup_tower_managed_defaults): + mocker.patch('awx.main.tasks.system.is_database_synchronized', return_value=False) - apps.get_app_config('main')._load_credential_types_feature() + _sync_credential_types_to_db() mock_setup_tower_managed_defaults.assert_not_called() diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 9226aa05cd..21a27980b2 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -151,14 +151,6 @@ def is_testing(argv=None): return False -def bypass_in_test(func): - def fn(*args, **kwargs): - if not is_testing(): - return func(*args, **kwargs) - - return fn - - class RequireDebugTrueOrTest(logging.Filter): """ Logging filter to output when in DEBUG mode or running tests. diff --git a/awx/main/utils/lazy_registry.py b/awx/main/utils/lazy_registry.py new file mode 100644 index 0000000000..bb90c84df6 --- /dev/null +++ b/awx/main/utils/lazy_registry.py @@ -0,0 +1,64 @@ +class LazyLoadDict(dict): + """A dict subclass that calls a loader function on first read access. + + Writes (e.g. during the loading process itself) go straight through + without triggering the loader. + """ + + def __init__(self, loader): + super().__init__() + self._loader = loader + self._loaded = False + + def _ensure_loaded(self): + if not self._loaded: + self._loaded = True + self._loader() + + def __getitem__(self, key): + self._ensure_loaded() + return super().__getitem__(key) + + def get(self, key, default=None): + self._ensure_loaded() + return super().get(key, default) + + def __contains__(self, key): + self._ensure_loaded() + return super().__contains__(key) + + def __iter__(self): + self._ensure_loaded() + return super().__iter__() + + def __len__(self): + self._ensure_loaded() + return super().__len__() + + def keys(self): + self._ensure_loaded() + return super().keys() + + def values(self): + self._ensure_loaded() + return super().values() + + def items(self): + self._ensure_loaded() + return super().items() + + def __bool__(self): + self._ensure_loaded() + return super().__bool__() + + def __repr__(self): + self._ensure_loaded() + return super().__repr__() + + def copy(self): + self._ensure_loaded() + return super().copy() + + def clear(self): + super().clear() + self._loaded = True