Test suites config for the new test framework (#41318)

Closes #41316

Signed-off-by: stianst <stianst@gmail.com>

test-framework/README.md

@@ -237,6 +237,33 @@ public void testClientCredentials() throws Exception {
 }
 ```
 
+
+# Test Suites
+
+A `@Suite` can supply configuration to be used when running tests from the suite. For example:
+
+```java
+@Suite
+@SelectClasses(MyTest.class)
+public class MyTestSuite {
+
+    @BeforeSuite
+    public static void beforeSuite() {
+        SuiteSupport.startSuite()
+                .registerServerConfig(MyTestSuiteServerConfig.class)
+                .includedSuppliers("server", "remote");
+    }
+
+    @AfterSuite
+    public static void afterSuite() {
+        SuiteSupport.stopSuite();
+    }
+}
+```
+
+The above example adds some additional Keycloak server configuration, as well as limiting what server suppliers can be used for the suite.
+
+
 # Running tests
 
 Tests can be run from your favourite IDE, or from the command-line using Maven. Simply run the tests and the framework
@@ -336,6 +363,13 @@ Valid values:
 | embedded     | Runs a Keycloak server embedded in the same JVM process                                                |
 | remote       | Connects to a remote Keycloak server. Requires manually configuring the server as needed for the test. |
 
+Configuration:
+
+| Value                                             | Description                                                            |
+|---------------------------------------------------|------------------------------------------------------------------------|
+| `kc.test.server.config` / `KC_TEST_SERVER_CONFIG` | The name of a KeycloakServerConfig class to use when running the tests |
+
+
 ### Database
 
 Option: `kc.test.database` / `KC_TEST_DATABASE`
@@ -370,3 +404,17 @@ Valid values:
 | chrome-headless  | Chrome WebDriver without UI  |
 | firefox          | Firefox WebDriver            |
 | firefox-headless | Firefox WebDriver without UI |
+
+### Supplier configuration
+
+#### Set the supplier
+
+Option: `kc.test.<value type alias>` / `KC_TEST_<value type alias>`
+
+#### Setting included suppliers
+
+Option: `kc.test.<value type alias>.suppliers.included` / `KC_TEST_<value type alias>_SUPPLIERS_INCLUDED`
+
+#### Setting excluded suppliers
+
+Option: `kc.test.<value type alias>.suppliers.excluded` / `KC_TEST_<value type alias>_SUPPLIERS_EXCLUDED`

test-framework/core/src/main/java/org/keycloak/testframework/config/Config.java

@@ -30,6 +30,18 @@ public class Config {
         return config.getOptionalValue("kc.test." + valueTypeAlias.getAlias(valueType), String.class).orElse(null);
     }
 
+    public static String getIncludedSuppliers(Class<?> valueType) {
+        return config.getOptionalValue("kc.test." + valueTypeAlias.getAlias(valueType) + ".suppliers.included", String.class).orElse(null);
+    }
+
+    public static String getExcludedSuppliers(Class<?> valueType) {
+        return config.getOptionalValue("kc.test." + valueTypeAlias.getAlias(valueType) + ".suppliers.excluded", String.class).orElse(null);
+    }
+
+    public static String getSupplierConfig(Class<?> valueType) {
+        return config.getOptionalValue("kc.test." + valueTypeAlias.getAlias(valueType) + ".config", String.class).orElse(null);
+    }
+
     public static <T> T getValueTypeConfig(Class<?> valueType, String name, String defaultValue, Class<T> type) {
         name = getValueTypeFQN(valueType, name);
         Optional<T> optionalValue = config.getOptionalValue(name, type);
@@ -80,7 +92,8 @@ public class Config {
                 .addDefaultSources()
                 .addDefaultInterceptors()
                 .withConverters(new Converter[]{ new CharsetConverter(), new MemorySizeConverter(), new InetSocketAddressConverter() })
-                .withInterceptors(new LogConfigInterceptor());
+                .withInterceptors(new LogConfigInterceptor())
+                .withSources(new SuiteConfigSource());
 
         ConfigSource testEnvConfigSource = initTestEnvConfigSource();
         if (testEnvConfigSource != null) {

test-framework/core/src/main/java/org/keycloak/testframework/config/SuiteConfigSource.java

@@ -0,0 +1,40 @@
+package org.keycloak.testframework.config;
+
+import org.eclipse.microprofile.config.spi.ConfigSource;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+public class SuiteConfigSource implements ConfigSource {
+
+    private static final Map<String, String> SUITE_CONFIG = new HashMap<>();
+
+    public static void set(String key, String value) {
+        SUITE_CONFIG.put(key, value);
+    }
+
+    public static void clear() {
+        SUITE_CONFIG.clear();
+    }
+
+    @Override
+    public Set<String> getPropertyNames() {
+        return SUITE_CONFIG.keySet();
+    }
+
+    @Override
+    public String getValue(String s) {
+        return SUITE_CONFIG.get(s);
+    }
+
+    @Override
+    public String getName() {
+        return "SuiteConfigSource";
+    }
+
+    @Override
+    public int getOrdinal() {
+        return 270;
+    }
+}

test-framework/core/src/main/java/org/keycloak/testframework/injection/Extensions.java

@@ -6,6 +6,7 @@ import org.keycloak.testframework.config.Config;
 
 import java.lang.annotation.Annotation;
 import java.lang.reflect.Field;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -71,7 +72,7 @@ public class Extensions {
             for (var supplier : extension.suppliers()) {
                 Class<?> valueType = supplier.getValueType();
                 String requestedSupplier = Config.getSelectedSupplier(valueType);
-                if (supplier.getAlias().equals(requestedSupplier) || (requestedSupplier == null && !loadedValueTypes.contains(valueType))) {
+                if (isSupplierIncluded(supplier) && (supplier.getAlias().equals(requestedSupplier) || (requestedSupplier == null && !loadedValueTypes.contains(valueType)))) {
                     configureSupplier(supplier);
                     suppliers.add(supplier);
                     loadedValueTypes.add(valueType);
@@ -86,6 +87,22 @@ public class Extensions {
         return suppliers;
     }
 
+    private boolean isSupplierIncluded(Supplier<?, ?> supplier) {
+        String includedSuppliers = Config.getIncludedSuppliers(supplier.getValueType());
+        if (includedSuppliers != null) {
+            if (Arrays.stream(includedSuppliers.split(",")).noneMatch(s -> s.equals(supplier.getAlias()))) {
+                return false;
+            }
+        }
+
+        String excludedSuppliers = Config.getExcludedSuppliers(supplier.getValueType());
+        if (excludedSuppliers != null) {
+            return Arrays.stream(excludedSuppliers.split(",")).noneMatch(s -> s.equals(supplier.getAlias()));
+        }
+
+        return true;
+    }
+
     private List<Class<?>> loadAlwaysEnabledValueTypes(List<TestFrameworkExtension> extensions) {
         return extensions.stream().flatMap(s -> s.alwaysEnabledValueTypes().stream()).toList();
     }

test-framework/core/src/main/java/org/keycloak/testframework/injection/SuiteSupport.java

@@ -0,0 +1,45 @@
+package org.keycloak.testframework.injection;
+
+import org.keycloak.testframework.config.Config;
+import org.keycloak.testframework.config.SuiteConfigSource;
+import org.keycloak.testframework.server.KeycloakServerConfig;
+
+public class SuiteSupport {
+
+    private static SuiteConfig suiteConfig = new SuiteConfig();
+
+    public static SuiteConfig startSuite() {
+        return suiteConfig;
+    }
+
+    public static void stopSuite() {
+        SuiteConfigSource.clear();
+        Config.initConfig();
+        suiteConfig = null;
+    }
+
+    public static class SuiteConfig {
+
+        public SuiteConfig registerServerConfig(Class<? extends KeycloakServerConfig> serverConfig) {
+            SuiteConfigSource.set("kc.test.server.config", serverConfig.getName());
+            return this;
+        }
+
+        public SuiteConfig supplier(String name, String supplier) {
+            SuiteConfigSource.set("kc.test." + name, supplier);
+            return this;
+        }
+
+        public SuiteConfig includedSuppliers(String name, String... suppliers) {
+            SuiteConfigSource.set("kc.test." + name + ".suppliers.included", String.join(",", suppliers));
+            return this;
+        }
+
+        public SuiteConfig excludedSuppliers(String name, String... suppliers) {
+            SuiteConfigSource.set("kc.test." + name + ".suppliers.excluded", String.join(",", suppliers));
+            return this;
+        }
+
+    }
+
+}

test-framework/core/src/main/java/org/keycloak/testframework/injection/SupplierHelpers.java

@@ -16,6 +16,18 @@ public class SupplierHelpers {
         }
     }
 
+    @SuppressWarnings("unchecked")
+    public static <T> T getInstance(String clazzName) {
+        try {
+            Class<T> clazz = (Class<T>) SupplierHelpers.class.getClassLoader().loadClass(clazzName);
+            Constructor<T> declaredConstructor = clazz.getDeclaredConstructor();
+            declaredConstructor.setAccessible(true);
+            return declaredConstructor.newInstance();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     public static <T> T getAnnotationField(Annotation annotation, String name, T defaultValue) {
         T value = getAnnotationField(annotation, name);
         return value != null ? value : defaultValue;

test-framework/core/src/main/java/org/keycloak/testframework/server/AbstractKeycloakServerSupplier.java

@@ -1,10 +1,10 @@
 package org.keycloak.testframework.server;
 
 import org.jboss.logging.Logger;
-import org.keycloak.testframework.injection.AbstractInterceptorHelper;
 import org.keycloak.testframework.annotations.KeycloakIntegrationTest;
 import org.keycloak.testframework.config.Config;
 import org.keycloak.testframework.database.TestDatabase;
+import org.keycloak.testframework.injection.AbstractInterceptorHelper;
 import org.keycloak.testframework.injection.InstanceContext;
 import org.keycloak.testframework.injection.LifeCycle;
 import org.keycloak.testframework.injection.Registry;
@@ -27,6 +27,12 @@ public abstract class AbstractKeycloakServerSupplier implements Supplier<Keycloa
 
         command.log().handlers(KeycloakServerConfigBuilder.LogHandlers.CONSOLE);
 
+        String supplierConfig = Config.getSupplierConfig(KeycloakServer.class);
+        if (supplierConfig != null) {
+            KeycloakServerConfig serverConfigOverride = SupplierHelpers.getInstance(supplierConfig);
+            serverConfigOverride.configure(command);
+        }
+
         command = serverConfig.configure(command);
 
         if (requiresDatabase()) {

tests/base/src/test/java/org/keycloak/tests/suites/VolatileSessionsTestSuite.java

@@ -0,0 +1,35 @@
+package org.keycloak.tests.suites;
+
+import org.junit.platform.suite.api.AfterSuite;
+import org.junit.platform.suite.api.BeforeSuite;
+import org.junit.platform.suite.api.SelectClasses;
+import org.junit.platform.suite.api.Suite;
+import org.keycloak.common.Profile;
+import org.keycloak.testframework.injection.SuiteSupport;
+import org.keycloak.testframework.server.KeycloakServerConfig;
+import org.keycloak.testframework.server.KeycloakServerConfigBuilder;
+import org.keycloak.tests.admin.ClientTest;
+
+@Suite
+@SelectClasses(ClientTest.class)
+public class VolatileSessionsTestSuite {
+
+    @BeforeSuite
+    public static void beforeSuite() {
+        SuiteSupport.startSuite()
+                .registerServerConfig(VolatileSessionsServerConfig.class);
+    }
+
+    @AfterSuite
+    public static void afterSuite() {
+        SuiteSupport.stopSuite();
+    }
+
+    public static class VolatileSessionsServerConfig implements KeycloakServerConfig {
+
+        @Override
+        public KeycloakServerConfigBuilder configure(KeycloakServerConfigBuilder config) {
+            return config.featuresDisabled(Profile.Feature.PERSISTENT_USER_SESSIONS);
+        }
+    }
+}