mirror of
https://github.com/keycloak/keycloak.git
synced 2026-01-09 15:02:05 -03:30
Add workflows and utils to review stability of testsuite (#40268)
Closes #40267 Signed-off-by: stianst <stianst@gmail.com>
This commit is contained in:
Stian Thorgersen
GitHub
parent
e06748908d
commit
ddfdbfec6a
59
.github/workflows/stability-base-reruns.yml
vendored
Normal file
59
.github/workflows/stability-base-reruns.yml
vendored
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
name: Stability - Base Reruns
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
tests:
|
||||||
|
type: string
|
||||||
|
description: Tests to run
|
||||||
|
required: true
|
||||||
|
count:
|
||||||
|
type: number
|
||||||
|
description: Number of re-runs
|
||||||
|
default: 50
|
||||||
|
|
||||||
|
env:
|
||||||
|
MAVEN_ARGS: "-B -nsu -Daether.connector.http.connectionMaxTtl=25"
|
||||||
|
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
shell: bash
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
|
||||||
|
build:
|
||||||
|
name: Build
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
|
||||||
|
- name: Build Keycloak
|
||||||
|
uses: ./.github/actions/build-keycloak
|
||||||
|
|
||||||
|
base-integration-tests:
|
||||||
|
name: Base IT
|
||||||
|
needs: build
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 360
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
|
||||||
|
- id: integration-test-setup
|
||||||
|
name: Integration test setup
|
||||||
|
uses: ./.github/actions/integration-test-setup
|
||||||
|
|
||||||
|
- name: Run base tests
|
||||||
|
run: |
|
||||||
|
TESTS="${{ inputs.tests }}"
|
||||||
|
COUNT=${{ inputs.count }}
|
||||||
|
echo "Tests: $TESTS, count: $COUNT"
|
||||||
|
FAILURES=0
|
||||||
|
for i in $(seq 1 $COUNT); do
|
||||||
|
echo "========================================================================="
|
||||||
|
echo Run: $i
|
||||||
|
echo "========================================================================="
|
||||||
|
./mvnw test ${{ env.SUREFIRE_RETRY }} -Pauth-server-quarkus -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base 2>&1 | misc/log/trimmer.sh || FAILURES=$(($FAILURES + 1))
|
||||||
|
FAILURES=$(($FAILURES + $?))
|
||||||
|
done
|
||||||
|
echo "Failures: $FAILURES"
|
||||||
|
exit $FAILURES
|
||||||
56
.github/workflows/stability-base.yml
vendored
Normal file
56
.github/workflows/stability-base.yml
vendored
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
name: Stability - Base
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
env:
|
||||||
|
MAVEN_ARGS: "-B -nsu -Daether.connector.http.connectionMaxTtl=25"
|
||||||
|
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
shell: bash
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
|
||||||
|
build:
|
||||||
|
name: Build
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
|
||||||
|
- name: Build Keycloak
|
||||||
|
uses: ./.github/actions/build-keycloak
|
||||||
|
|
||||||
|
base-integration-tests:
|
||||||
|
name: Base IT
|
||||||
|
needs: build
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 100
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
group: [ 1, 2, 3, 4, 5, 6 ]
|
||||||
|
fail-fast: false
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
|
||||||
|
- id: integration-test-setup
|
||||||
|
name: Integration test setup
|
||||||
|
uses: ./.github/actions/integration-test-setup
|
||||||
|
|
||||||
|
- name: Run base tests
|
||||||
|
run: |
|
||||||
|
TESTS=`testsuite/integration-arquillian/tests/base/testsuites/base-suite.sh ${{ matrix.group }}`
|
||||||
|
echo "Tests: $TESTS"
|
||||||
|
./mvnw test ${{ env.SUREFIRE_RETRY }} -Pauth-server-quarkus -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base 2>&1 | misc/log/trimmer.sh
|
||||||
|
|
||||||
|
delete-artifacts:
|
||||||
|
name: Delete artifacts
|
||||||
|
needs: base-integration-tests
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: always()
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
steps:
|
||||||
|
- name: Delete artifacts
|
||||||
|
run:
|
||||||
|
gh api /repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts | jq .artifacts[].id | xargs -I {} gh api -X DELETE /repos/${{ github.repository }}/actions/artifacts/{}
|
||||||
52
.github/workflows/stability-clustering.yml
vendored
Normal file
52
.github/workflows/stability-clustering.yml
vendored
Normal file
@ -0,0 +1,52 @@
|
|||||||
|
name: Stability - Clustering
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
env:
|
||||||
|
MAVEN_ARGS: "-B -nsu -Daether.connector.http.connectionMaxTtl=25"
|
||||||
|
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
shell: bash
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
|
||||||
|
build:
|
||||||
|
name: Build
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
|
||||||
|
- name: Build Keycloak
|
||||||
|
uses: ./.github/actions/build-keycloak
|
||||||
|
|
||||||
|
clustering-integration-tests:
|
||||||
|
name: Clustering IT
|
||||||
|
needs: build
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 35
|
||||||
|
env:
|
||||||
|
MAVEN_OPTS: -Xmx1024m
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
|
||||||
|
- id: integration-test-setup
|
||||||
|
name: Integration test setup
|
||||||
|
uses: ./.github/actions/integration-test-setup
|
||||||
|
|
||||||
|
- name: Run cluster tests
|
||||||
|
run: |
|
||||||
|
./mvnw test ${{ env.SUREFIRE_RETRY }} -Pauth-server-cluster-quarkus,db-postgres "-Dwebdriver.chrome.driver=$CHROMEWEBDRIVER/chromedriver" -Dsession.cache.owners=2 -Dtest=**.cluster.** -pl testsuite/integration-arquillian/tests/base 2>&1 | misc/log/trimmer.sh
|
||||||
|
|
||||||
|
delete-artifacts:
|
||||||
|
name: Delete artifacts
|
||||||
|
needs: clustering-integration-tests
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: always()
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
steps:
|
||||||
|
- name: Delete artifacts
|
||||||
|
run:
|
||||||
|
gh api /repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts | jq .artifacts[].id | xargs -I {} gh api -X DELETE /repos/${{ github.repository }}/actions/artifacts/{}
|
||||||
133
.github/workflows/stability-js-ci.yml
vendored
Normal file
133
.github/workflows/stability-js-ci.yml
vendored
Normal file
@ -0,0 +1,133 @@
|
|||||||
|
name: Stability - Keycloak JavaScript CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
env:
|
||||||
|
MAVEN_ARGS: "-B -nsu -Daether.connector.http.connectionMaxTtl=25"
|
||||||
|
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
shell: bash
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build-keycloak:
|
||||||
|
name: Build Keycloak
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
|
||||||
|
- name: Build Keycloak
|
||||||
|
uses: ./.github/actions/build-keycloak
|
||||||
|
|
||||||
|
- name: Prepare archive for upload
|
||||||
|
run: |
|
||||||
|
mv ./quarkus/dist/target/keycloak-999.0.0-SNAPSHOT.tar.gz ./keycloak-999.0.0-SNAPSHOT.tar.gz
|
||||||
|
|
||||||
|
- name: Upload Keycloak dist
|
||||||
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||||
|
with:
|
||||||
|
name: keycloak
|
||||||
|
path: keycloak-999.0.0-SNAPSHOT.tar.gz
|
||||||
|
|
||||||
|
account-ui-e2e:
|
||||||
|
name: Account UI E2E
|
||||||
|
needs:
|
||||||
|
- build-keycloak
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
WORKSPACE: "@keycloak/keycloak-account-ui"
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
browser: [chromium, firefox]
|
||||||
|
exclude:
|
||||||
|
# Only test with Firefox on scheduled runs
|
||||||
|
- browser: ${{ github.event_name != 'workflow_dispatch' && 'firefox' || '' }}
|
||||||
|
fail-fast: false
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
|
||||||
|
- uses: ./.github/actions/pnpm-setup
|
||||||
|
|
||||||
|
- name: Download Keycloak server
|
||||||
|
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||||
|
with:
|
||||||
|
name: keycloak
|
||||||
|
|
||||||
|
- name: Setup Java
|
||||||
|
uses: ./.github/actions/java-setup
|
||||||
|
|
||||||
|
- name: Start Keycloak server
|
||||||
|
run: |
|
||||||
|
tar xfvz keycloak-999.0.0-SNAPSHOT.tar.gz
|
||||||
|
keycloak-999.0.0-SNAPSHOT/bin/kc.sh start-dev --features=transient-users,oid4vc-vci &> ~/server.log &
|
||||||
|
env:
|
||||||
|
KC_BOOTSTRAP_ADMIN_USERNAME: admin
|
||||||
|
KC_BOOTSTRAP_ADMIN_PASSWORD: admin
|
||||||
|
|
||||||
|
- name: Install Playwright browsers
|
||||||
|
run: pnpm --fail-if-no-match --filter ${{ env.WORKSPACE }} exec playwright install --with-deps
|
||||||
|
working-directory: js
|
||||||
|
|
||||||
|
- name: Run Playwright tests
|
||||||
|
run: pnpm --fail-if-no-match --filter ${{ env.WORKSPACE }} test -- --project=${{ matrix.browser }}
|
||||||
|
working-directory: js
|
||||||
|
|
||||||
|
admin-ui-e2e:
|
||||||
|
name: Admin UI E2E
|
||||||
|
needs:
|
||||||
|
- build-keycloak
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
WORKSPACE: "@keycloak/keycloak-admin-ui"
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
browser: [chromium, firefox]
|
||||||
|
exclude:
|
||||||
|
# Only test with Firefox on scheduled runs
|
||||||
|
- browser: ${{ github.event_name != 'workflow_dispatch' && 'firefox' || '' }}
|
||||||
|
fail-fast: false
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
|
||||||
|
- uses: ./.github/actions/pnpm-setup
|
||||||
|
|
||||||
|
- name: Download Keycloak server
|
||||||
|
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||||
|
with:
|
||||||
|
name: keycloak
|
||||||
|
|
||||||
|
- name: Setup Java
|
||||||
|
uses: ./.github/actions/java-setup
|
||||||
|
|
||||||
|
- name: Start Keycloak server
|
||||||
|
run: |
|
||||||
|
tar xfvz keycloak-999.0.0-SNAPSHOT.tar.gz
|
||||||
|
keycloak-999.0.0-SNAPSHOT/bin/kc.sh start-dev --features=admin-fine-grained-authz:v2,transient-users &> ~/server.log &
|
||||||
|
env:
|
||||||
|
KC_BOOTSTRAP_ADMIN_USERNAME: admin
|
||||||
|
KC_BOOTSTRAP_ADMIN_PASSWORD: admin
|
||||||
|
KC_BOOTSTRAP_ADMIN_CLIENT_ID: temporary-admin-service
|
||||||
|
KC_BOOTSTRAP_ADMIN_CLIENT_SECRET: temporary-admin-service
|
||||||
|
|
||||||
|
- name: Install Playwright browsers
|
||||||
|
run: pnpm --fail-if-no-match --filter ${{ env.WORKSPACE }} exec playwright install --with-deps
|
||||||
|
working-directory: js
|
||||||
|
|
||||||
|
- name: Run Playwright tests
|
||||||
|
run: pnpm --fail-if-no-match --filter ${{ env.WORKSPACE }} test:integration -- --project=${{ matrix.browser }}
|
||||||
|
working-directory: js
|
||||||
|
|
||||||
|
delete-artifacts:
|
||||||
|
name: Delete artifacts
|
||||||
|
needs:
|
||||||
|
- account-ui-e2e
|
||||||
|
- admin-ui-e2e
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: always()
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ github.token }}
|
||||||
|
steps:
|
||||||
|
- name: Delete artifacts
|
||||||
|
run:
|
||||||
|
gh api /repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts | jq .artifacts[].id | xargs -I {} gh api -X DELETE /repos/${{ github.repository }}/actions/artifacts/{}
|
||||||
1
misc/test-stability/.gitignore
vendored
Normal file
1
misc/test-stability/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
|||||||
|
logs*
|
||||||
43
misc/test-stability/README.md
Normal file
43
misc/test-stability/README.md
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
# Test stability utils
|
||||||
|
|
||||||
|
This directory contains a number of utils to schedule many runs on GitHub Actions and parsing the results. This is
|
||||||
|
useful to periodically do a sanity check of the test stability.
|
||||||
|
|
||||||
|
Some dedicated GitHub Actions workflows are used for this purpose, as other workflows include test retries, and will
|
||||||
|
also cancel multiple concurrent runs for the same workflow.
|
||||||
|
|
||||||
|
Available workflows:
|
||||||
|
|
||||||
|
* stability-base.yml
|
||||||
|
* stability-clustering.yml
|
||||||
|
* stability-js-ci.yml
|
||||||
|
|
||||||
|
There is also another special workflow `stability-base-reruns.yml` this can be used to run a single test within the
|
||||||
|
base testsuite many times, which can be useful when trying to fix instability in a specific test.
|
||||||
|
|
||||||
|
The runs should not be scheduled in the main Keycloak organization, but rather in your own fork of Keycloak. This is to
|
||||||
|
prevent impacting other developers and contributors.
|
||||||
|
|
||||||
|
To schedule a run use `schedule-runs.sh`, for example:
|
||||||
|
|
||||||
|
```
|
||||||
|
./schedule-runs.sh -w stability-base.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Once scheduled you need to wait for all runs to complete, you can check the status with `status.sh`, for example:
|
||||||
|
|
||||||
|
```
|
||||||
|
./status.sh -w stability-base.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
After all runs have completed you can download the logs for the failed runs using `download-logs.sh`, for example:
|
||||||
|
|
||||||
|
```
|
||||||
|
./download-logs.sh -w stability-base.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
Final step is to parse the logs to get a report of failed tests using `parse-logs.sh`, for example:
|
||||||
|
|
||||||
|
```
|
||||||
|
./parse-logs.sh logs
|
||||||
|
```
|
||||||
61
misc/test-stability/download-logs.sh
Executable file
61
misc/test-stability/download-logs.sh
Executable file
@ -0,0 +1,61 @@
|
|||||||
|
#!/bin/bash -e
|
||||||
|
|
||||||
|
function help() {
|
||||||
|
echo "Download logs for failed GitHub Actions runs"
|
||||||
|
echo
|
||||||
|
echo "options:"
|
||||||
|
echo "-b Branch to use (defaults to main)"
|
||||||
|
echo "-w Workflow name (required)"
|
||||||
|
echo "-d Date range (defaults to today)"
|
||||||
|
echo "-l Download directory (defaults to logs)"
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
while getopts ":b:d:w:l:" option; do
|
||||||
|
case $option in
|
||||||
|
b)
|
||||||
|
BRANCH=$OPTARG;;
|
||||||
|
w)
|
||||||
|
WORKFLOW=$OPTARG;;
|
||||||
|
d)
|
||||||
|
DATE=$OPTARG;;
|
||||||
|
l)
|
||||||
|
LOGS=$OPTARG;;
|
||||||
|
*)
|
||||||
|
help
|
||||||
|
exit;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$DATE" == "" ]; then
|
||||||
|
DATE=$(date -Idate)
|
||||||
|
fi
|
||||||
|
|
||||||
|
USER=$(gh api user | jq -r '.login')
|
||||||
|
|
||||||
|
if [ "$BRANCH" == "" ]; then
|
||||||
|
BRANCH="main"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$WORKFLOW" == "" ]; then
|
||||||
|
echo -e "Error: Workflow not specified\n" && help && exit 1
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$LOGS" == "" ]; then
|
||||||
|
LOGS="logs"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -d "$LOGS" ]; then
|
||||||
|
mkdir "$LOGS"
|
||||||
|
fi
|
||||||
|
|
||||||
|
for i in $(gh run list -L 100 -R "$USER/keycloak" -w "$WORKFLOW" -s failure --created "$DATE" --json databaseId | jq -r .[].databaseId); do
|
||||||
|
echo -n "($i) "
|
||||||
|
if [ ! -f "$LOGS/$i" ]; then
|
||||||
|
gh run -R "$USER/keycloak" view --log-failed "$i" > "$LOGS/$i"
|
||||||
|
fi
|
||||||
|
if [ ! -f "$LOGS/$i.json" ]; then
|
||||||
|
gh run -R "$USER/keycloak" view --json name,conclusion,databaseId,jobs "$i" > "$LOGS/$i.json"
|
||||||
|
fi
|
||||||
|
done
|
||||||
10
misc/test-stability/parse-logs.sh
Executable file
10
misc/test-stability/parse-logs.sh
Executable file
@ -0,0 +1,10 @@
|
|||||||
|
#!/bin/bash -e
|
||||||
|
|
||||||
|
LOG_DIR="$1"
|
||||||
|
|
||||||
|
if [ "$LOG_DIR" == "" ]; then
|
||||||
|
echo "usage: parse-logs.sh <log directory>"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
java -jar target/test-logs-parser.jar "$1" "$2"
|
||||||
59
misc/test-stability/pom.xml
Normal file
59
misc/test-stability/pom.xml
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>org.keycloak.test.stability</groupId>
|
||||||
|
<artifactId>test-logs-parser</artifactId>
|
||||||
|
<version>0.1-SNAPSHOT</version>
|
||||||
|
|
||||||
|
<name>Test Log Parser</name>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
<maven.compiler.source>17</maven.compiler.source>
|
||||||
|
<maven.compiler.target>17</maven.compiler.target>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.fasterxml.jackson.core</groupId>
|
||||||
|
<artifactId>jackson-databind</artifactId>
|
||||||
|
<version>2.19.0</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<finalName>${project.artifactId}</finalName>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-jar-plugin</artifactId>
|
||||||
|
<version>3.4.2</version>
|
||||||
|
<configuration>
|
||||||
|
<archive>
|
||||||
|
<manifest>
|
||||||
|
<addClasspath>true</addClasspath>
|
||||||
|
<mainClass>org.keycloak.test.logparser.TestReport</mainClass>
|
||||||
|
</manifest>
|
||||||
|
</archive>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-shade-plugin</artifactId>
|
||||||
|
<version>3.6.0</version>
|
||||||
|
<configuration>
|
||||||
|
<dependencyReducedPomLocation>${build.directory}/dependency-reduced-pom.xml</dependencyReducedPomLocation>
|
||||||
|
</configuration>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>shade</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</project>
|
||||||
47
misc/test-stability/schedule-runs.sh
Executable file
47
misc/test-stability/schedule-runs.sh
Executable file
@ -0,0 +1,47 @@
|
|||||||
|
#!/bin/bash -e
|
||||||
|
|
||||||
|
function help() {
|
||||||
|
echo "Schedule many runs for a GitHub Actions workflow"
|
||||||
|
echo
|
||||||
|
echo "options:"
|
||||||
|
echo "-b Branch to use (defaults to main)"
|
||||||
|
echo "-w Workflow name (required)"
|
||||||
|
echo "-n Number of runs (defaults to 100)"
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
while getopts ":b:n:w:" option; do
|
||||||
|
case $option in
|
||||||
|
b)
|
||||||
|
BRANCH=$OPTARG;;
|
||||||
|
w)
|
||||||
|
WORKFLOW=$OPTARG;;
|
||||||
|
n)
|
||||||
|
RUNS=$OPTARG;;
|
||||||
|
*)
|
||||||
|
help
|
||||||
|
exit;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$RUNS" == "" ]; then
|
||||||
|
RUNS=100
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$WORKFLOW" == "" ]; then
|
||||||
|
echo -e "Error: Workflow not specified\n" && help && exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$BRANCH" == "" ]; then
|
||||||
|
BRANCH="main"
|
||||||
|
fi
|
||||||
|
|
||||||
|
USER=$(gh api user | jq -r '.login')
|
||||||
|
|
||||||
|
echo "Scheduling $RUNS run(s) in $USER/keycloak for workflow $WORKFLOW and branch $BRANCH"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
for i in $(seq 1 "$RUNS"); do
|
||||||
|
echo -n "($i) "
|
||||||
|
gh workflow run -R "$USER/keycloak" -r "$BRANCH" "$WORKFLOW"
|
||||||
|
done
|
||||||
@ -0,0 +1,16 @@
|
|||||||
|
package org.keycloak.test.logparser;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public record GitHubRun(String name, String conclusion, String databaseId, List<GitHubRunJob> jobs) {
|
||||||
|
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public record GitHubRunJob(String name, String conclusion, String databaseId, String url, List<GitHubRunJobStep> steps) {}
|
||||||
|
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
|
public record GitHubRunJobStep(String name, String conclusion, String databaseId) {}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,4 @@
|
|||||||
|
package org.keycloak.test.logparser;
|
||||||
|
|
||||||
|
public record LogFailure(String job, String test) {
|
||||||
|
}
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
package org.keycloak.test.logparser;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public interface LogParser {
|
||||||
|
|
||||||
|
boolean supports(List<String> lines);
|
||||||
|
|
||||||
|
List<LogFailure> parseFailures(List<String> lines);
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,40 @@
|
|||||||
|
package org.keycloak.test.logparser;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
public class MarkdownReportGenerator implements ReportGenerator {
|
||||||
|
|
||||||
|
public void printReport(List<GitHubRun> runs, List<TestFailure> failedTests) {
|
||||||
|
printRunSummary(runs);
|
||||||
|
printFailedTests(failedTests);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void printRunSummary(List<GitHubRun> runs) {
|
||||||
|
List<RunFailure> failedRuns = Utils.toRunFailures(runs);
|
||||||
|
|
||||||
|
System.out.println("# Failed steps");
|
||||||
|
System.out.println("| Num | Step | Failures | ");
|
||||||
|
System.out.println("| --- | ---- | -------- | ");
|
||||||
|
|
||||||
|
for (RunFailure run : failedRuns) {
|
||||||
|
String failures = run.details().stream()
|
||||||
|
.map(r -> "[" + r.runId() + "](" + r.url() + ")")
|
||||||
|
.collect(Collectors.joining(" "));
|
||||||
|
System.out.println("| " + run.details().size() + " | " + run.run() + " | " + failures + " |");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void printFailedTests(List<TestFailure> failedTests) {
|
||||||
|
System.out.println("# Failed tests");
|
||||||
|
System.out.println("| Num | Test | Failures | ");
|
||||||
|
System.out.println("| --- | ---- | -------- | ");
|
||||||
|
for (TestFailure failedTest : failedTests) {
|
||||||
|
String failures = failedTest.details().stream()
|
||||||
|
.map(r -> "[" + r.runId() + "](" + r.url() + ")")
|
||||||
|
.collect(Collectors.joining(" "));
|
||||||
|
System.out.println("| " + failedTest.details().size() + " | " + failedTest.test() + " | " + failures + " |");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,34 @@
|
|||||||
|
package org.keycloak.test.logparser;
|
||||||
|
|
||||||
|
import java.util.LinkedList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
public class PlaywrightLogParser implements LogParser {
|
||||||
|
|
||||||
|
private static final Pattern FAILURE = Pattern.compile("([^\\t]*)\\t.*##\\[error].*› (test/[^›]*).*");
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean supports(List<String> lines) {
|
||||||
|
return lines.stream().anyMatch(l -> l.contains("Playwright test"));
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<LogFailure> parseFailures(List<String> lines) {
|
||||||
|
List<LogFailure> logFailures = new LinkedList<>();
|
||||||
|
for (String l : lines) {
|
||||||
|
Matcher m = FAILURE.matcher(l);
|
||||||
|
|
||||||
|
if (m.matches()) {
|
||||||
|
String job = m.group(1);
|
||||||
|
String test = m.group(2);
|
||||||
|
|
||||||
|
if (logFailures.stream().noneMatch(lf -> lf.job().equals(job) && lf.test().equals(test))) {
|
||||||
|
logFailures.add(new LogFailure(job, test));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return logFailures;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,9 @@
|
|||||||
|
package org.keycloak.test.logparser;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public interface ReportGenerator {
|
||||||
|
|
||||||
|
void printReport(List<GitHubRun> runs, List<TestFailure> failedTests);
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,9 @@
|
|||||||
|
package org.keycloak.test.logparser;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public record RunFailure(String run, List<FailedRunDetails> details) {
|
||||||
|
|
||||||
|
public record FailedRunDetails(String runName, String runId, String jobName, String jobId, String url) {}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
package org.keycloak.test.logparser;
|
||||||
|
|
||||||
|
import java.util.LinkedList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
public class SureFireLogParser implements LogParser {
|
||||||
|
|
||||||
|
private static final Pattern FAILURE = Pattern.compile("([^\\t]*)\\t.*\\[ERROR] {3}(.*Test[^ ]*).*");
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean supports(List<String> lines) {
|
||||||
|
return lines.stream().anyMatch(l -> l.contains("org.apache.maven.plugins:maven-surefire-plugin"));
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<LogFailure> parseFailures(List<String> lines) {
|
||||||
|
List<LogFailure> logFailures = new LinkedList<>();
|
||||||
|
for (String l : lines) {
|
||||||
|
Matcher m = FAILURE.matcher(l);
|
||||||
|
if (m.matches()) {
|
||||||
|
logFailures.add(new LogFailure(m.group(1), m.group(2)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return logFailures;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,9 @@
|
|||||||
|
package org.keycloak.test.logparser;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public record TestFailure(String test, List<FailedTestDetails> details) {
|
||||||
|
|
||||||
|
public record FailedTestDetails(String runName, String runId, String jobName, String jobId, String url) {}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,67 @@
|
|||||||
|
package org.keycloak.test.logparser;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.LinkedList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Optional;
|
||||||
|
|
||||||
|
public class TestReport {
|
||||||
|
|
||||||
|
private static final Map<String, ReportGenerator> REPORT_GENERATORS = Map.of(
|
||||||
|
"text", new TextReportGenerator(),
|
||||||
|
"md", new MarkdownReportGenerator()
|
||||||
|
);
|
||||||
|
private static final List<LogParser> LOG_PARSERS = List.of(new SureFireLogParser(), new PlaywrightLogParser());
|
||||||
|
|
||||||
|
public static void main(String[] args) throws IOException {
|
||||||
|
File logDirectory = new File(args[0]);
|
||||||
|
ReportGenerator reportGenerator = REPORT_GENERATORS.get(args.length == 2 && !args[1].isBlank() ? args[1] : "text");
|
||||||
|
|
||||||
|
List<GitHubRun> runs = loadRuns(logDirectory);
|
||||||
|
List<TestFailure> failedTests = loadFailedTests(logDirectory, runs);
|
||||||
|
|
||||||
|
reportGenerator.printReport(runs, failedTests);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<GitHubRun> loadRuns(File logDirectory) throws IOException {
|
||||||
|
ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
List<GitHubRun> runs = new LinkedList<>();
|
||||||
|
for (File logFile : logDirectory.listFiles(f -> f.getName().endsWith(".json"))) {
|
||||||
|
runs.add(objectMapper.readValue(logFile, GitHubRun.class));
|
||||||
|
}
|
||||||
|
return runs;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<TestFailure> loadFailedTests(File logDirectory, List<GitHubRun> runs) throws IOException {
|
||||||
|
List<TestFailure> failedTests = new LinkedList<>();
|
||||||
|
|
||||||
|
for (File logFile : logDirectory.listFiles(f -> !f.getName().endsWith(".json"))) {
|
||||||
|
List<String> lines = Utils.readLines(logFile);
|
||||||
|
Optional<LogParser> logParser = LOG_PARSERS.stream().filter(p -> p.supports(lines)).findFirst();
|
||||||
|
|
||||||
|
if (logParser.isPresent()) {
|
||||||
|
for (LogFailure logFailure : logParser.get().parseFailures(lines)) {
|
||||||
|
TestFailure failedTest = failedTests.stream().filter(f -> f.test().equals(logFailure.test())).findFirst().or(() -> {
|
||||||
|
TestFailure ft = new TestFailure(logFailure.test(), new LinkedList<>());
|
||||||
|
failedTests.add(ft);
|
||||||
|
return Optional.of(ft);
|
||||||
|
}).get();
|
||||||
|
|
||||||
|
String runId = logFile.getName();
|
||||||
|
|
||||||
|
GitHubRun run = runs.stream().filter(r -> r.databaseId().equals(runId)).findFirst().get();
|
||||||
|
GitHubRun.GitHubRunJob job = run.jobs().stream().filter(j -> j.name().equals(logFailure.job())).findFirst().get();
|
||||||
|
|
||||||
|
failedTest.details().add(new TestFailure.FailedTestDetails(run.name(), run.databaseId(), job.name(), job.databaseId(), job.url()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return failedTests;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,43 @@
|
|||||||
|
package org.keycloak.test.logparser;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class TextReportGenerator implements ReportGenerator {
|
||||||
|
|
||||||
|
public void printReport(List<GitHubRun> runs, List<TestFailure> failedTests) {
|
||||||
|
printDivider();
|
||||||
|
printRunSummary(runs);
|
||||||
|
printDivider();
|
||||||
|
printFailedTests(failedTests);
|
||||||
|
printDivider();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void printRunSummary(List<GitHubRun> runs) {
|
||||||
|
List<RunFailure> failedRuns = Utils.toRunFailures(runs);
|
||||||
|
|
||||||
|
System.out.println("Failed steps:");
|
||||||
|
System.out.println();
|
||||||
|
for (RunFailure run : failedRuns) {
|
||||||
|
System.out.println(run.details().size() + "\t" + run.run());
|
||||||
|
for (RunFailure.FailedRunDetails details : run.details()) {
|
||||||
|
System.out.println("\t\t - " + details.url());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void printFailedTests(List<TestFailure> failedTests) {
|
||||||
|
System.out.println("Failed tests:");
|
||||||
|
System.out.println();
|
||||||
|
for (TestFailure failedTest : failedTests) {
|
||||||
|
System.out.println(failedTest.details().size() + "\t" + failedTest.test());
|
||||||
|
for (TestFailure.FailedTestDetails details : failedTest.details()) {
|
||||||
|
System.out.println("\t\t - " + details.url());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void printDivider() {
|
||||||
|
System.out.println("--------------------------------------------------------------------------------");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,39 @@
|
|||||||
|
package org.keycloak.test.logparser;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.util.LinkedList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Optional;
|
||||||
|
|
||||||
|
public class Utils {
|
||||||
|
|
||||||
|
public static List<String> readLines(File file) throws IOException {
|
||||||
|
return Files.lines(file.toPath()).toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<RunFailure> toRunFailures(List<GitHubRun> runs) {
|
||||||
|
List<RunFailure> failedRuns = new LinkedList<>();
|
||||||
|
for (GitHubRun run : runs) {
|
||||||
|
for (GitHubRun.GitHubRunJob job : run.jobs()) {
|
||||||
|
if (job.conclusion().equals("failure")) {
|
||||||
|
for (GitHubRun.GitHubRunJobStep step : job.steps()) {
|
||||||
|
if (step.conclusion().equals("failure")) {
|
||||||
|
String fullName = run.name() + " / " + job.name() + " / " + step.name();
|
||||||
|
RunFailure failedRun = failedRuns.stream().filter(f -> f.run().equals(fullName)).findFirst().or(() -> {
|
||||||
|
RunFailure fr = new RunFailure(fullName, new LinkedList<>());
|
||||||
|
failedRuns.add(fr);
|
||||||
|
return Optional.of(fr);
|
||||||
|
}).get();
|
||||||
|
|
||||||
|
failedRun.details().add(new RunFailure.FailedRunDetails(run.name(), run.databaseId(), job.name(), job.databaseId(), job.url()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return failedRuns;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
44
misc/test-stability/status.sh
Executable file
44
misc/test-stability/status.sh
Executable file
@ -0,0 +1,44 @@
|
|||||||
|
#!/bin/bash -e
|
||||||
|
|
||||||
|
function help() {
|
||||||
|
echo "View status of GitHub Actions runs"
|
||||||
|
echo
|
||||||
|
echo "options:"
|
||||||
|
echo "-b Branch to use (defaults to main)"
|
||||||
|
echo "-w Workflow name (required)"
|
||||||
|
echo "-d Date range (defaults to today)"
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
while getopts ":b:d:w:" option; do
|
||||||
|
case $option in
|
||||||
|
b)
|
||||||
|
BRANCH=$OPTARG;;
|
||||||
|
w)
|
||||||
|
WORKFLOW=$OPTARG;;
|
||||||
|
d)
|
||||||
|
DATE=$OPTARG;;
|
||||||
|
*)
|
||||||
|
help
|
||||||
|
exit;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$DATE" == "" ]; then
|
||||||
|
DATE=$(date -Idate)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$BRANCH" == "" ]; then
|
||||||
|
BRANCH="main"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$WORKFLOW" == "" ]; then
|
||||||
|
echo -e "Error: Workflow not specified\n" && help && exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
USER=$(gh api user | jq -r '.login')
|
||||||
|
|
||||||
|
echo "Status of $WORKFLOW in $USER/keycloak for branch $BRANCH"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
gh api -X GET "/repos/$USER/keycloak/actions/workflows/$WORKFLOW/runs" -F branch="$BRANCH" -F per_page=100 --paginate -F created="$DATE" | jq -r '.workflow_runs[] | [.status, .conclusion] | @csv' | sort | uniq -c
|
||||||
Loading…
x
Reference in New Issue
Block a user