Configure topology information in Infinispan

Closes #41933

Signed-off-by: Pedro Ruivo <1492066+pruivo@users.noreply.github.com>
Co-authored-by: Pedro Ruivo <1492066+pruivo@users.noreply.github.com>

docs/guides/high-availability/single-cluster/concepts.adoc

@@ -164,6 +164,9 @@ nodes/availability-zones in your {kubernetes} cluster.
 See the Operator <@links.operator id="advanced-configuration" anchor="_scheduling" /> details of how to configure custom
 anti-affinity `topologySpreadConstraints` policies.
 
+. The Operator does not configure the site's name (see <@links.server id="caching" anchor="cache-topology" />) in the Pods as its value is not available via the https://kubernetes.io/docs/concepts/workloads/pods/downward-api/[Downward API].
+The machine name option is configured using the `spec.nodeName` from the node where the Pod is scheduled.
+
 == Next steps
 
 Continue reading in the <@links.ha id="single-cluster-building-blocks" /> {section} to find blueprints for the different building blocks.

docs/guides/server/caching.adoc

@@ -203,6 +203,54 @@ For these use case, the CLI options `cache-remote-tls-enabled` disables the encr
 The CLI options `cache-remote-username` and `cache-remote-password` are optional and, if not set, {project_name} will connect to the {jdgserver_name} server without presenting any credentials.
 If the {jdgserver_name} server has authentication enabled, {project_name} will fail to start.
 
+[[cache-topology]]
+== Topology aware data distribution
+
+Configuring {project_name} to be aware of your network topology, increases data availability in the presence of hardware failures, as Infinispan is able to ensure that data is distributed correctly.
+For example, if `num_owners=2` is configured for a cache, it will ensure that the two owners are not stored on the same node when possible.
+
+[NOTE]
+====
+By default, user and client sessions are safely stored in the database, and they are not affected by these settings.
+The remaining distributed caches are affected by this configuration.
+====
+
+The following topology information is available to configure:
+
+Site name::
+If your {project_name} cluster is deployed between different datacenters (stretched cluster), use this option to ensure the data replicas are stored in a different datacenter.
+It prevents data loss if a datacenter goes offline or fails.
++
+Use the SPI option `spi-cache-embedded--default--site-name` (or environment variable `KC_SPI_CACHE_EMBEDDED\__DEFAULT__SITE_NAME`).
+The value itself is not important, but each datacenter must have a unique value.
++
+For example: `--spi-cache-embedded--default--site-name=site-1`
+
+Rack name::
+If your {project_name} cluster is running in different racks on your datacenter, set this option to ensure the data replicas are stored in a different physical rack.
+It prevents data loss if a rack is suddenly disconnected or fails.
++
+Use the SPI option `spi-cache-embedded--default--rack-name` (or environment variable `KC_SPI_CACHE_EMBEDDED\__DEFAULT__RACK_NAME`).
+The value itself is not important, but each rack must have a unique value.
++
+For example: `--spi-cache-embedded--default--rack-name=rack-1`
+
+Machine name::
+If you have multiple {project_name} instances running on the same physical machine (using virtual machines or containers for example), use this option to ensure the data replicas are stored in different physical machines.
+It prevents data loss against a physical machine failure.
++
+Use the SPI option `spi-cache-embedded--default--machine-name` (or environment variable `KC_SPI_CACHE_EMBEDDED\__DEFAULT__MACHINE_NAME`).
+The value itself is not important, but each machine must have a unique value.
++
+For example: `--spi-cache-embedded--default--machine-name=machine-1`
++
+[NOTE]
+====
+The {project_name} Operator automatically configure the machine name based on the Kubernetes node.
+It ensures that if multiple pods are scheduled on the same node, data replicas are still replicated across distinct nodes when possible.
+We recommend to set up anti-affinity rules and/or topology spread constraints to prevent multiple Pods from being scheduled on the same node, further reducing the risk of a single node failure causing data loss.
+====
+
 == Transport stacks
 
 Transport stacks ensure that {project_name} nodes in a cluster communicate in a reliable fashion.

model/infinispan/src/main/java/org/keycloak/spi/infinispan/impl/embedded/DefaultCacheEmbeddedConfigProviderFactory.java

@@ -80,6 +80,8 @@ public class DefaultCacheEmbeddedConfigProviderFactory implements CacheEmbeddedC
     public static final String STACK = "stack";
     public static final String NODE_NAME = "nodeName";
     public static final String SITE_NAME = "siteName";
+    public static final String MACHINE_NAME = "machineName";
+    public static final String RACK_NAME = "rackName";
 
     private volatile ConfigurationBuilderHolder builderHolder;
     private volatile Config.Scope keycloakConfig;
@@ -257,7 +259,19 @@ public class DefaultCacheEmbeddedConfigProviderFactory implements CacheEmbeddedC
                 .add();
         builder.property()
                 .name(SITE_NAME)
-                .helpText("The name of the site where this node runs. Used for server hinting.")
+                .helpText("The name of the site (availability zone) where this instance runs. It can be set if running Keycloak in different availability zones. Infinispan takes into consideration this value to keep the backup data spread between different sites.")
+                .label("name")
+                .type(ProviderConfigProperty.STRING_TYPE)
+                .add();
+        builder.property()
+                .name(MACHINE_NAME)
+                .helpText("The name of the physical machine where this instance runs. It can be set if multiple Keycloak instances are running in the same physical machines. Infinispan takes into consideration this value to keep the backup data spread between different machines.")
+                .label("name")
+                .type(ProviderConfigProperty.STRING_TYPE)
+                .add();
+        builder.property()
+                .name(RACK_NAME)
+                .helpText("The name of the rack where this instance runs. It can be set if multiple Keycloak instances are running in the same physical rack. Infinispan takes into consideration this value to keep the backup data spread between different racks.")
                 .label("name")
                 .type(ProviderConfigProperty.STRING_TYPE)
                 .add();

model/infinispan/src/main/java/org/keycloak/spi/infinispan/impl/embedded/JGroupsConfigurator.java

@@ -20,6 +20,14 @@ package org.keycloak.spi.infinispan.impl.embedded;
 import static org.infinispan.configuration.global.TransportConfiguration.CLUSTER_NAME;
 import static org.infinispan.configuration.global.TransportConfiguration.STACK;
 import static org.keycloak.config.CachingOptions.CACHE_EMBEDDED_PREFIX;
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.JBOSS_NODE_NAME;
+import static org.keycloak.connections.infinispan.InfinispanConnectionProvider.JBOSS_SITE_NAME;
+import static org.keycloak.spi.infinispan.impl.embedded.DefaultCacheEmbeddedConfigProviderFactory.MACHINE_NAME;
+import static org.keycloak.spi.infinispan.impl.embedded.DefaultCacheEmbeddedConfigProviderFactory.NODE_NAME;
+import static org.keycloak.spi.infinispan.impl.embedded.DefaultCacheEmbeddedConfigProviderFactory.PROVIDER_ID;
+import static org.keycloak.spi.infinispan.impl.embedded.DefaultCacheEmbeddedConfigProviderFactory.RACK_NAME;
+import static org.keycloak.spi.infinispan.impl.embedded.DefaultCacheEmbeddedConfigProviderFactory.SITE_NAME;
+import static org.keycloak.spi.infinispan.impl.embedded.DefaultCacheEmbeddedConfigProviderFactory.TRACING;
 
 import java.lang.invoke.MethodHandles;
 import java.security.KeyManagementException;
@@ -31,6 +39,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Consumer;
 
 import javax.net.ssl.KeyManager;
 import javax.net.ssl.SSLContext;
@@ -61,7 +70,6 @@ import org.keycloak.Config;
 import org.keycloak.common.util.Retry;
 import org.keycloak.config.CachingOptions;
 import org.keycloak.config.Option;
-import org.keycloak.connections.infinispan.InfinispanConnectionProvider;
 import org.keycloak.connections.infinispan.InfinispanConnectionSpi;
 import org.keycloak.connections.jpa.JpaConnectionProvider;
 import org.keycloak.connections.jpa.JpaConnectionProviderFactory;
@@ -135,7 +143,7 @@ public final class JGroupsConfigurator {
             transportOf(holder).stack(stack);
         }
         configureTransport(config);
-        boolean tracingEnabled = config.getBoolean(DefaultCacheEmbeddedConfigProviderFactory.TRACING, false);
+        boolean tracingEnabled = config.getBoolean(TRACING, false);
         configureDiscovery(holder, session, tracingEnabled);
         configureTls(holder, session);
         warnDeprecatedStack(holder);
@@ -148,30 +156,29 @@ public final class JGroupsConfigurator {
      * @param holder The {@link ConfigurationBuilderHolder} where the transport is configured.
      */
     public static void configureTopology(Config.Scope config, ConfigurationBuilderHolder holder) {
-        if (System.getProperty(InfinispanConnectionProvider.JBOSS_SITE_NAME) != null) {
+        if (System.getProperty(JBOSS_SITE_NAME) != null) {
             throw new IllegalArgumentException(
                     String.format("System property %s is in use. Use --spi-cache-embedded-%s-site-name config option instead",
-                            InfinispanConnectionProvider.JBOSS_SITE_NAME, DefaultCacheEmbeddedConfigProviderFactory.PROVIDER_ID));
+                            JBOSS_SITE_NAME, PROVIDER_ID));
         }
-        if (System.getProperty(InfinispanConnectionProvider.JBOSS_NODE_NAME) != null) {
+        if (System.getProperty(JBOSS_NODE_NAME) != null) {
             throw new IllegalArgumentException(
                     String.format("System property %s is in use. Use --spi-cache-embedded-%s-node-name config option instead",
-                            InfinispanConnectionProvider.JBOSS_NODE_NAME, DefaultCacheEmbeddedConfigProviderFactory.PROVIDER_ID));
+                            JBOSS_NODE_NAME, PROVIDER_ID));
         }
         var transport = transportOf(holder);
-        var nodeName = config.get(DefaultCacheEmbeddedConfigProviderFactory.NODE_NAME);
-        if (nodeName != null) {
-            transport.nodeName(nodeName);
-        }
         //legacy option, for backwards compatibility --spi-connections-infinispan-quarkus-site-name
         var legacySiteName = Config.scope(InfinispanConnectionSpi.SPI_NAME, "quarkus").get("site-name");
         if (legacySiteName != null) {
-            logger.warn("--spi-connections-infinispan-quarkus-site-name is deprecated and may be removed in the future. Use --spi-cache-embedded-%s-site-name".formatted(DefaultCacheEmbeddedConfigProviderFactory.PROVIDER_ID));
+            logger.warn("--spi-connections-infinispan-quarkus-site-name is deprecated and may be removed in the future. Use --spi-cache-embedded-%s-site-name".formatted(PROVIDER_ID));
         }
-        var siteName = config.get(DefaultCacheEmbeddedConfigProviderFactory.SITE_NAME, legacySiteName);
-        if (siteName != null) {
+        var siteName = config.get(SITE_NAME, legacySiteName);
+        if (siteName != null && !siteName.isEmpty()) {
             transport.siteId(siteName);
         }
+        readConfigAndSet(config, RACK_NAME, transport::rackId);
+        readConfigAndSet(config, MACHINE_NAME, transport::machineId);
+        readConfigAndSet(config, NODE_NAME, transport::nodeName);
     }
 
     static void createJGroupsProperties(ProviderConfigurationBuilder builder) {
@@ -361,6 +368,13 @@ public final class JGroupsConfigurator {
         return "jdbc-ping".equals(stackName) || "jdbc-ping-udp".equals(stackName);
     }
 
+    private static void readConfigAndSet(Config.Scope scope, String configKey, Consumer<String> consumer) {
+        String value = scope.get(configKey);
+        if (value != null && !value.isEmpty()) {
+            consumer.accept(value);
+        }
+    }
+
     private static class JpaFactoryAwareJGroupsChannelConfigurator extends EmbeddedJGroupsChannelConfigurator {
 
         private final JpaConnectionProviderFactory factory;

operator/src/main/java/org/keycloak/operator/controllers/KeycloakDeploymentDependentResource.java

@@ -87,6 +87,7 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
     public static final String HTTP_MANAGEMENT_SCHEME = "http-management-scheme";
 
     public static final String POD_IP = "POD_IP";
+    public static final String HOST_IP_SPI_OPTION = "KC_SPI_CACHE_EMBEDDED_DEFAULT_MACHINE_NAME";
 
     private static final List<String> COPY_ENV = Arrays.asList("HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY");
 
@@ -184,8 +185,8 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
         baseDeployment.getMetadata().getAnnotations().put(Constants.KEYCLOAK_UPDATE_REASON_ANNOTATION, ContextUtils.getUpdateReason(context));
 
         return switch (updateType.get()) {
-            case ROLLING -> handleRollingUpdate(baseDeployment, context, primary);
-            case RECREATE -> handleRecreateUpdate(existingDeployment, baseDeployment, kcContainer, context);
+            case ROLLING -> handleRollingUpdate(baseDeployment);
+            case RECREATE -> handleRecreateUpdate(existingDeployment, baseDeployment, kcContainer);
         };
     }
 
@@ -592,6 +593,12 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
         envVars.add(new EnvVarBuilder().withName(POD_IP).withNewValueFrom().withNewFieldRef()
                 .withFieldPath("status.podIP").withApiVersion("v1").endFieldRef().endValueFrom().build());
 
+        // Both status.hostIP or spec.nodeName would be fine here.
+        // In theory, status.hostIP is a smaller value and, as this value is tagged in all JGroups messages, it should have a lower overhead.
+        // Using spec.nodeName to avoid exposing the IP addresses in the logs.
+        envVars.add(new EnvVarBuilder().withName(HOST_IP_SPI_OPTION).withNewValueFrom().withNewFieldRef()
+                .withFieldPath("spec.nodeName").withApiVersion("v1").endFieldRef().endValueFrom().build());
+
         return envVars;
     }
 
@@ -620,15 +627,14 @@ public class KeycloakDeploymentDependentResource extends CRUDKubernetesDependent
         }));
     }
 
-    private static StatefulSet handleRollingUpdate(StatefulSet desired, Context<Keycloak> context, Keycloak primary) {
+    private static StatefulSet handleRollingUpdate(StatefulSet desired) {
         // return the desired stateful set since Kubernetes does a rolling in-place update by default.
         Log.debug("Performing a rolling update");
         desired.getMetadata().getAnnotations().put(Constants.KEYCLOAK_RECREATE_UPDATE_ANNOTATION, Boolean.FALSE.toString());
         return desired;
     }
 
-    private static StatefulSet handleRecreateUpdate(StatefulSet actual, StatefulSet desired, Container kcContainer,
-            Context<Keycloak> context) {
+    private static StatefulSet handleRecreateUpdate(StatefulSet actual, StatefulSet desired, Container kcContainer) {
         desired.getMetadata().getAnnotations().put(Constants.KEYCLOAK_RECREATE_UPDATE_ANNOTATION, Boolean.TRUE.toString());
 
         if (Optional.ofNullable(actual.getStatus().getReplicas()).orElse(0) == 0) {

operator/src/main/java/org/keycloak/operator/update/impl/BaseUpgradeLogic.java

@@ -148,6 +148,7 @@ abstract class BaseUpdateLogic implements UpdateLogic {
         // The operator only sets value or secrets. Any other combination is from unsupported pod template.
         return container.getEnv().stream()
                 .filter(envVar -> !envVar.getName().equals(KeycloakDeploymentDependentResource.POD_IP))
+                .filter(envVar -> !envVar.getName().equals(KeycloakDeploymentDependentResource.HOST_IP_SPI_OPTION))
                 .collect(Collectors.toMap(EnvVar::getName, Function.identity()));
     }