Run clustering compatibility tests on release/x.y branches

Closes #39966

Signed-off-by: Ryan Emerson <remerson@redhat.com>

.github/scripts/version-compatibility.sh

@@ -0,0 +1,25 @@
+#!/bin/bash -e
+
+if [[ "$RUNNER_DEBUG" == "1" ]]; then
+  set -x
+fi
+
+TARGET_BRANCH="$1"
+REPO="${2:-keycloak}"
+ORG="${3:-keycloak}"
+
+if [[ "${TARGET_BRANCH}" != "release/"* ]]; then
+  exit 0
+fi
+
+ALL_RELEASES=$(gh release list \
+  --repo "${ORG}/${REPO}" \
+  --exclude-drafts \
+  --exclude-pre-releases \
+  --json name \
+  --template '{{range .}}{{.name}}{{"\n"}}{{end}}'
+)
+MAJOR_MINOR=${TARGET_BRANCH#"release/"}
+MAJOR_MINOR_RELEASES=$(echo "${ALL_RELEASES}" | grep "${MAJOR_MINOR}")
+
+echo "${MAJOR_MINOR_RELEASES}" | jq -cnR '[inputs] | map({version: .})'

.github/workflows/ci.yml

@@ -37,6 +37,7 @@ jobs:
       ci-sssd: ${{ steps.conditional.outputs.ci-sssd }}
       ci-webauthn: ${{ steps.conditional.outputs.ci-webauthn }}
       ci-aurora: ${{ steps.auroradb-tests.outputs.run-aurora-tests }}
+      ci-compatibility-matrix: ${{ steps.version-compatibility.outputs.matrix }}
     permissions:
       contents: read
       pull-requests: read
@@ -57,6 +58,20 @@ jobs:
           fi
           echo "run-aurora-tests=$RUN_AURORADB_TESTS" >> $GITHUB_OUTPUT
 
+      - name: Version Compatibility Matrix
+        id: version-compatibility
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            BRANCH="${{ github.base_ref }}"
+          else
+            BRANCH="${{ github.ref_name }}"
+          fi
+          MATRIX_JSON=$(./.github/scripts/version-compatibility.sh "${BRANCH}")
+          echo "${MATRIX_JSON}"
+          echo "matrix=${MATRIX_JSON}" >> $GITHUB_OUTPUT
+
   build:
     name: Build
     if: needs.conditional.outputs.ci == 'true'
@@ -1105,6 +1120,36 @@ jobs:
       - name: Run tests
         run: ./mvnw package -f tests/pom.xml
 
+  mixed-cluster-compatibility-tests:
+    name: Cluster Compatibility Tests
+    if: needs.conditional.outputs.ci-compatibility-matrix != ''
+    runs-on: ubuntu-latest
+    needs:
+      - build
+      - conditional
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.conditional.outputs.ci-compatibility-matrix) }}
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - id: integration-test-setup
+        name: Integration test setup
+        uses: ./.github/actions/integration-test-setup
+
+        # This step is necessary because test/clustering requires building a new Keycloak image built from tar.gz
+        # file that is not part of m2-keycloak.tzts archive
+      - name: Build tar keycloak-quarkus-dist
+        run: ./mvnw package -pl quarkus/server/,quarkus/dist/
+
+      - name: Run tests
+        run: ./mvnw verify -pl tests/clustering
+        env:
+          KC_TEST_SERVER_IMAGES: "quay.io/keycloak/keycloak:${{ matrix.version }},-"
+
   check:
     name: Status Check - Keycloak CI
     if: always()
@@ -1131,6 +1176,7 @@ jobs:
       - external-infinispan-tests
       - test-framework
       - base-new-integration-tests
+      - mixed-cluster-compatibility-tests
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

test-framework/clustering/src/main/java/org/keycloak/testframework/server/ClusteredKeycloakServer.java

@@ -77,12 +77,15 @@ public class ClusteredKeycloakServer implements KeycloakServer {
         if (containers.length != imagePeServer.length) {
             throw new IllegalArgumentException("The number of containers and the number of images must match");
         }
+
         int[] exposedPorts = new int[]{REQUEST_PORT, MANAGEMENT_PORT};
         LazyFuture<String> snapshotImage = null;
         for (int i = 0; i < containers.length; ++i) {
             LazyFuture<String> resolvedImage;
             if (SNAPSHOT_IMAGE.equals(imagePeServer[i])) {
                 if (snapshotImage == null) {
+                    // Required otherwise we will receive an "Incorrect state of migration" error preventing startup
+                    configBuilder.option("spi-datastore--legacy--allow-migrate-existing-database-to-snapshot", "true");
                     snapshotImage = defaultImage();
                 }
                 resolvedImage = snapshotImage;

tests/clustering/README.md

@@ -5,7 +5,10 @@
 KC_TEST_SERVER_IMAGES   -> if empty, uses the built distribution archive from quarkus/dist directory in all containers
 -> if single value, uses that value in all the containers
 -> if comma separated value ("imageA,imageB"), each container will use the image specified from the list. The number of items must match the cluster size.
--> "-" special keyword to use the built distribution archive
+-> "-" special keyword to use the built distribution archive.
+> NOTE: If testing SNAPSHOT versions with "-", it's necessary for it to appear later in the CSV list than
+> non-SNAPSHOT releases in order to avoid "Incorrect state of migration" exceptions.
+
 KC_TEST_SERVER=cluster  -> enables cluster mode (configured by default in clustering module)
 KC_TEST_DATABASE_INTERNAL=true -> configure keycloak with the internal database container IP instead of localhost (configured by default in clustering module)
 

tests/clustering/src/test/resources/keycloak-test.properties

@@ -9,6 +9,7 @@ kc.test.log.filter=true
 kc.test.log.category."org.keycloak.tests".level=INFO
 
 kc.test.log.category."testinfo".level=INFO
+kc.test.log.category."org.keycloak.it".level=INFO
 kc.test.log.category."org.keycloak".level=WARN
 kc.test.log.category."managed.keycloak".level=WARN
 kc.test.log.category."managed.db".level=WARN