From 2ebe03ae2de1f8cd399d0c6a7b81d9f745fa384c Mon Sep 17 00:00:00 2001 From: Pedro Ruivo <1492066+pruivo@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:24:19 +0100 Subject: [PATCH] Ensure cache configuration has correct number of owners Closes #41558 Signed-off-by: Pedro Ruivo <1492066+pruivo@users.noreply.github.com> --- docs/guides/server/caching.adoc | 13 ------ .../impl/embedded/CacheConfigurator.java | 41 +++++++++++++++++-- ...ultCacheEmbeddedConfigProviderFactory.java | 1 + 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/docs/guides/server/caching.adoc b/docs/guides/server/caching.adoc index c24c28ccd66..d5015011d3a 100644 --- a/docs/guides/server/caching.adoc +++ b/docs/guides/server/caching.adoc @@ -170,19 +170,6 @@ apply the upper bound to. For example, to apply an upper-bound of `1000` to the Setting a maximum cache size for `sessions`, `clientSessions`, `offlineSessions` and `offlineClientSessions` is not supported when volatile sessions are enabled. -=== Configuring caches for availability - -Distributed caches replicate cache entries on a subset of nodes in a cluster and assigns entries to fixed owner nodes. - -Each distributed cache, that is a primary source of truth of the data (`authenticationSessions`, `loginFailures` and `actionTokens`) has two owners per default, which means that two nodes have a copy of the specific cache entries. -Non-owner nodes query the owners of a specific cache to obtain data. -When one of the owners becomes unavailable, the data is restored from the remaining owner and rebalanced across the remaining nodes. -When both owner nodes are offline, all data is lost. - -The default number of two owners is the minimum number is necessary to survive one node (owner) failure or a rolling restart in a cluster setup with at least two nodes. -A higher number increases the availability of the data, but at the expense of slower writes as more nodes need to be updated. -Therefore, changing the number of owners for the caches `authenticationSessions`, `loginFailures` and `actionTokens` is not recommended. - === Specify your own cache configuration file To specify your own cache configuration file, enter this command: diff --git a/model/infinispan/src/main/java/org/keycloak/spi/infinispan/impl/embedded/CacheConfigurator.java b/model/infinispan/src/main/java/org/keycloak/spi/infinispan/impl/embedded/CacheConfigurator.java index 311dc96ae29..9ff16ec52c3 100644 --- a/model/infinispan/src/main/java/org/keycloak/spi/infinispan/impl/embedded/CacheConfigurator.java +++ b/model/infinispan/src/main/java/org/keycloak/spi/infinispan/impl/embedded/CacheConfigurator.java @@ -34,15 +34,20 @@ import org.infinispan.transaction.TransactionMode; import org.infinispan.transaction.lookup.EmbeddedTransactionManagerLookup; import org.jboss.logging.Logger; import org.keycloak.Config; -import org.keycloak.connections.infinispan.InfinispanConnectionProvider; +import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.ACTION_TOKEN_CACHE; +import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.ALL_CACHES_NAME; +import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.AUTHENTICATION_SESSIONS_CACHE_NAME; import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.AUTHORIZATION_CACHE_NAME; import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_DEFAULT_MAX; import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.AUTHORIZATION_REVISIONS_CACHE_NAME; import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CLIENT_SESSION_CACHE_NAME; +import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CLUSTERED_CACHE_NAMES; +import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CRL_CACHE_DEFAULT_MAX; import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.CRL_CACHE_NAME; import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.LOCAL_CACHE_NAMES; import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.LOCAL_MAX_COUNT_CACHES; +import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME; import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.OFFLINE_CLIENT_SESSION_CACHE_NAME; import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME; import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.REALM_CACHE_NAME; @@ -102,7 +107,7 @@ public final class CacheConfigurator { */ public static void applyDefaultConfiguration(ConfigurationBuilderHolder holder) { var configs = holder.getNamedConfigurationBuilders(); - for (var name : InfinispanConnectionProvider.ALL_CACHES_NAME) { + for (var name : ALL_CACHES_NAME) { configs.computeIfAbsent(name, cacheName -> DEFAULT_CONFIGS.getOrDefault(cacheName, TO_NULL).get()); } } @@ -157,7 +162,7 @@ public final class CacheConfigurator { */ public static void removeClusteredCaches(ConfigurationBuilderHolder holder) { logger.debug("Removing clustered caches"); - Arrays.stream(InfinispanConnectionProvider.CLUSTERED_CACHE_NAMES).forEach(holder.getNamedConfigurationBuilders()::remove); + Arrays.stream(CLUSTERED_CACHE_NAMES).forEach(holder.getNamedConfigurationBuilders()::remove); } /** @@ -234,6 +239,34 @@ public final class CacheConfigurator { } } + /** + * Configures the caches "actionToken", "authenticationSessions", and "loginFailures" with the minimum number of + * owners to prevent data loss in a single instance crash. + *

+ * The data in those caches only exist in memory, therefore they must have more than one owner configured. + * + * @param holder The {@link ConfigurationBuilderHolder} where the caches are configured. + * @throws IllegalStateException if an Infinispan cache is not defined in the {@code holder}. This could indicate a + * missing or incorrect configuration. + */ + public static void ensureMinimumOwners(ConfigurationBuilderHolder holder) { + for (var name : Arrays.asList( + LOGIN_FAILURE_CACHE_NAME, + AUTHENTICATION_SESSIONS_CACHE_NAME, + ACTION_TOKEN_CACHE)) { + var builder = holder.getNamedConfigurationBuilders().get(name); + if (builder == null) { + throw cacheNotFound(name); + } + var hashConfig = builder.clustering().hash(); + var owners = hashConfig.attributes().attribute(HashConfiguration.NUM_OWNERS).get(); + if (owners < 2) { + logger.infof("Setting num_owners=2 (configured value is %s) for cache '%s' to prevent data loss.", owners, name); + hashConfig.numOwners(2); + } + } + } + // private methods below private static void configureRevisionCache(ConfigurationBuilderHolder holder, String baseCache, String revisionCache, long defaultMaxEntries) { @@ -267,7 +300,7 @@ public final class CacheConfigurator { public static ConfigurationBuilder getCrlCacheConfig() { var builder = createCacheConfigurationBuilder(); - builder.memory().whenFull(EvictionStrategy.REMOVE).maxCount(InfinispanConnectionProvider.CRL_CACHE_DEFAULT_MAX); + builder.memory().whenFull(EvictionStrategy.REMOVE).maxCount(CRL_CACHE_DEFAULT_MAX); return builder; } diff --git a/model/infinispan/src/main/java/org/keycloak/spi/infinispan/impl/embedded/DefaultCacheEmbeddedConfigProviderFactory.java b/model/infinispan/src/main/java/org/keycloak/spi/infinispan/impl/embedded/DefaultCacheEmbeddedConfigProviderFactory.java index c3609235555..927eff851bf 100644 --- a/model/infinispan/src/main/java/org/keycloak/spi/infinispan/impl/embedded/DefaultCacheEmbeddedConfigProviderFactory.java +++ b/model/infinispan/src/main/java/org/keycloak/spi/infinispan/impl/embedded/DefaultCacheEmbeddedConfigProviderFactory.java @@ -210,6 +210,7 @@ public class DefaultCacheEmbeddedConfigProviderFactory implements CacheEmbeddedC CacheConfigurator.checkCachesExist(holder, Arrays.stream(ALL_CACHES_NAME)); CacheConfigurator.configureCacheMaxCount(config, holder, Arrays.stream(CLUSTERED_MAX_COUNT_CACHES)); CacheConfigurator.validateWorkCacheConfiguration(holder); + CacheConfigurator.ensureMinimumOwners(holder); KeycloakModelUtils.runJobInTransaction(factory, session -> JGroupsConfigurator.configureJGroups(config, holder, session)); configureMetrics(config, holder); }