mirror of
https://github.com/keycloak/keycloak.git
synced 2026-01-10 15:32:05 -03:30
[Operator] Network Policy Rules
Closes #35598 Signed-off-by: Pedro Ruivo <pruivo@redhat.com>
This commit is contained in:
parent
481acac41f
commit
3767642f93
@ -145,3 +145,9 @@ In previous versions, clicking on *Sign out all active sessions* in the admin co
|
||||
|
||||
This has been changed. Now all sessions, regular and offline, are removed when signing out of all active sessions.
|
||||
|
||||
= Network Policy support added to the {project_name} Operator
|
||||
|
||||
NOTE: Preview feature.
|
||||
|
||||
To improve the security of your Kubernetes deployment, https://kubernetes.io/docs/concepts/services-networking/network-policies/[Network Policies] can be specified in your {project_name} CR.
|
||||
The {project_name} Operator accepts the ingress rules, which define from where the traffic is allowed to come from, and automatically creates the necessary Network Policies.
|
||||
|
||||
@ -349,7 +349,7 @@ NOTE: The `tracing-jdbc-enabled` is not promoted as a first-class citizen as it
|
||||
|
||||
For more details about tracing, see <@links.observability id="tracing" />.
|
||||
|
||||
=== Network Policies (Experimental)
|
||||
=== Network Policies (Preview)
|
||||
|
||||
NetworkPolicies allow you to specify rules for traffic flow within your cluster, and also between Pods and the outside world.
|
||||
Your cluster must use a network plugin that supports NetworkPolicy enforcement.
|
||||
@ -370,6 +370,73 @@ spec:
|
||||
enabled: true
|
||||
----
|
||||
|
||||
The above example allows traffic from all sources.
|
||||
The Keycloak CR can be extended to include a list of rules for each of the endpoints exposed by {project_name}.
|
||||
These rules specify from where (the source) the traffic is allowed and it's possible to communicate with the {project_name} Pods.
|
||||
|
||||
.Extended Network Policy configuration
|
||||
[source,yaml]
|
||||
----
|
||||
apiVersion: k8s.keycloak.org/v2alpha1
|
||||
kind: Keycloak
|
||||
metadata:
|
||||
name: example-kc
|
||||
spec:
|
||||
networkPolicy:
|
||||
enabled: true
|
||||
http: <list of rules> #<1>
|
||||
https: <list of rules> #<2>
|
||||
management: <list of rules> #<3>
|
||||
----
|
||||
<1> It defines the rules for HTTP endpoint (port 8080 by default).
|
||||
Due to security reasons, the HTTP endpoint is disabled by default.
|
||||
<2> It defines the access rules for HTTPS endpoint (port 8443 by default.
|
||||
<3> It defines the access rules for management endpoint (port 9000 by default).
|
||||
The management endpoint is used by the Kubernetes Probes and to expose the {project_name} metrics.
|
||||
|
||||
The rule syntax is the same as the one used by the Kubernetes Network Policy.
|
||||
It makes it easy to migrate your existing rules into your Keycloak CP.
|
||||
For more information, check the https://kubernetes.io/docs/concepts/services-networking/network-policies/#behavior-of-to-and-from-selectors[rule syntax].
|
||||
|
||||
==== Example with OpenShift
|
||||
|
||||
For a concrete example, let's imagine we have a {project_name} deployment running in a OpenShift cluster.
|
||||
Users have to access {project_name} to login, so {project_name} must be accessible from the Internet.
|
||||
|
||||
To make this example more interesting, let's assume the {project_name} is monitored too.
|
||||
The monitoring is enabled as described in the OpenShift documentation page:
|
||||
https://docs.openshift.com/container-platform/4.12/observability/monitoring/enabling-monitoring-for-user-defined-projects.html[enabling Monitoring for user defined projects].
|
||||
|
||||
Based on those requirements, the Keycloak CR would be like this (most parts are omitted, like DB connection and security):
|
||||
|
||||
.Keycloak CR
|
||||
[source,yaml]
|
||||
----
|
||||
apiVersion: k8s.keycloak.org/v2alpha1
|
||||
kind: Keycloak
|
||||
metadata:
|
||||
name: example-kc
|
||||
spec:
|
||||
ingress:
|
||||
enabled: true #<1>
|
||||
networkPolicy:
|
||||
enabled: true
|
||||
https:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: openshift-ingress #<2>
|
||||
management:
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: openshift-user-workload-monitoring #<3>
|
||||
----
|
||||
<1> Enables Ingress for outside access.
|
||||
<2> The default OpenShift Ingress class pods are running in `openshift-ingress` namespace.
|
||||
We allow traffic from these pods to access the {project_name} HTTPS endpoint.
|
||||
The traffic from outside the OpenShift cluster goes through these pods.
|
||||
<3> Prometheus pods are running in `openshift-user-workload-monitoring`.
|
||||
They need to access {project_name} to scrape the available metrics.
|
||||
|
||||
Check the https://kubernetes.io/docs/concepts/services-networking/network-policies/[Kubernetes Network Policies documentation] for more information about NetworkPolicies.
|
||||
|
||||
</@tmpl.guide>
|
||||
|
||||
@ -16,12 +16,16 @@
|
||||
*/
|
||||
package org.keycloak.operator.controllers;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
import io.fabric8.kubernetes.api.model.IntOrString;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicy;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyBuilder;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyFluent;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyPeer;
|
||||
import io.javaoperatorsdk.operator.api.reconciler.Context;
|
||||
import io.javaoperatorsdk.operator.api.reconciler.dependent.DependentResource;
|
||||
import io.javaoperatorsdk.operator.processing.dependent.kubernetes.CRUDKubernetesDependentResource;
|
||||
@ -60,7 +64,7 @@ public class KeycloakNetworkPolicyDependentResource extends CRUDKubernetesDepend
|
||||
}
|
||||
|
||||
@Override
|
||||
protected NetworkPolicy desired(Keycloak primary, Context<Keycloak> context) {
|
||||
public NetworkPolicy desired(Keycloak primary, Context<Keycloak> context) {
|
||||
var builder = new NetworkPolicyBuilder();
|
||||
addMetadata(builder, primary);
|
||||
|
||||
@ -97,31 +101,32 @@ public class KeycloakNetworkPolicyDependentResource extends CRUDKubernetesDepend
|
||||
.map(HttpSpec::getHttpEnabled)
|
||||
.orElse(false);
|
||||
if (!tlsEnabled || httpEnabled) {
|
||||
var httpIngressBuilder = builder.addNewIngress();
|
||||
httpIngressBuilder.addNewPort()
|
||||
.withPort(new IntOrString(HttpSpec.httpPort(keycloak)))
|
||||
.withProtocol(KEYCLOAK_SERVICE_PROTOCOL)
|
||||
.endPort();
|
||||
httpIngressBuilder.endIngress();
|
||||
addIngress(builder, HttpSpec.httpPort(keycloak), NetworkPolicySpec.httpRules(keycloak));
|
||||
}
|
||||
|
||||
if (tlsEnabled) {
|
||||
var httpsIngressBuilder = builder.addNewIngress();
|
||||
httpsIngressBuilder.addNewPort()
|
||||
.withPort(new IntOrString(HttpSpec.httpsPort(keycloak)))
|
||||
.withProtocol(KEYCLOAK_SERVICE_PROTOCOL)
|
||||
.endPort();
|
||||
httpsIngressBuilder.endIngress();
|
||||
addIngress(builder, HttpSpec.httpsPort(keycloak), NetworkPolicySpec.httpsRules(keycloak));
|
||||
}
|
||||
}
|
||||
|
||||
private static void addManagementPorts(NetworkPolicyFluent<NetworkPolicyBuilder>.SpecNested<NetworkPolicyBuilder> builder, Keycloak keycloak) {
|
||||
var ingressBuilder = builder.addNewIngress();
|
||||
ingressBuilder.addNewPort()
|
||||
.withPort(new IntOrString(HttpManagementSpec.managementPort(keycloak)))
|
||||
addIngress(builder, HttpManagementSpec.managementPort(keycloak), NetworkPolicySpec.managementRules(keycloak));
|
||||
}
|
||||
|
||||
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
|
||||
private static void addIngress(NetworkPolicyFluent<NetworkPolicyBuilder>.SpecNested<NetworkPolicyBuilder> builder,
|
||||
int port,
|
||||
Optional<List<NetworkPolicyPeer>> networkPolicyPeers) {
|
||||
var ingress = builder.addNewIngress();
|
||||
ingress.addNewPort()
|
||||
.withPort(new IntOrString(port))
|
||||
.withProtocol(KEYCLOAK_SERVICE_PROTOCOL)
|
||||
.endPort();
|
||||
ingressBuilder.endIngress();
|
||||
|
||||
networkPolicyPeers
|
||||
.filter(Predicate.not(Collection::isEmpty))
|
||||
.ifPresent(ingress::addAllToFrom);
|
||||
ingress.endIngress();
|
||||
}
|
||||
|
||||
private static void addJGroupsPorts(NetworkPolicyFluent<NetworkPolicyBuilder>.SpecNested<NetworkPolicyBuilder> builder, Keycloak keycloak) {
|
||||
|
||||
@ -17,23 +17,45 @@
|
||||
|
||||
package org.keycloak.operator.crds.v2alpha1.deployment.spec;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyPeer;
|
||||
import io.sundr.builder.annotations.Buildable;
|
||||
import org.keycloak.operator.Constants;
|
||||
import org.keycloak.operator.crds.v2alpha1.CRDUtils;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakSpec;
|
||||
|
||||
@JsonInclude(JsonInclude.Include.NON_NULL)
|
||||
@Buildable(editableEnabled = false, builderPackage = "io.fabric8.kubernetes.api.builder")
|
||||
public class NetworkPolicySpec {
|
||||
|
||||
// Copied from Kubernetes Documentation
|
||||
private static final String RULE_DESCRIPTION = "A list of sources which should be able to access this endpoint. " +
|
||||
"Items in this list are combined using a logical OR operation. " +
|
||||
"If this field is empty or missing, this rule matches all sources (traffic not restricted by source). " +
|
||||
"If this field is present and contains at least one item, this rule allows traffic only if the traffic matches at least one item in the from list.";
|
||||
|
||||
@JsonProperty("enabled")
|
||||
@JsonPropertyDescription("Enables or disable the ingress traffic control.")
|
||||
private boolean networkPolicyEnabled = false;
|
||||
|
||||
@JsonProperty("http")
|
||||
@JsonPropertyDescription(RULE_DESCRIPTION)
|
||||
private List<NetworkPolicyPeer> httpRules;
|
||||
|
||||
@JsonProperty("https")
|
||||
@JsonPropertyDescription(RULE_DESCRIPTION)
|
||||
private List<NetworkPolicyPeer> httpsRules;
|
||||
|
||||
@JsonProperty("management")
|
||||
@JsonPropertyDescription(RULE_DESCRIPTION)
|
||||
private List<NetworkPolicyPeer> managementRules;
|
||||
|
||||
public boolean isNetworkPolicyEnabled() {
|
||||
return networkPolicyEnabled;
|
||||
}
|
||||
@ -42,8 +64,37 @@ public class NetworkPolicySpec {
|
||||
this.networkPolicyEnabled = networkPolicyEnabled;
|
||||
}
|
||||
|
||||
public List<NetworkPolicyPeer> getHttpRules() {
|
||||
return httpRules;
|
||||
}
|
||||
|
||||
public void setHttpRules(List<NetworkPolicyPeer> httpRules) {
|
||||
this.httpRules = httpRules;
|
||||
}
|
||||
|
||||
public List<NetworkPolicyPeer> getHttpsRules() {
|
||||
return httpsRules;
|
||||
}
|
||||
|
||||
public void setHttpsRules(List<NetworkPolicyPeer> httpsRules) {
|
||||
this.httpsRules = httpsRules;
|
||||
}
|
||||
|
||||
public List<NetworkPolicyPeer> getManagementRules() {
|
||||
return managementRules;
|
||||
}
|
||||
|
||||
public void setManagementRules(List<NetworkPolicyPeer> managementRules) {
|
||||
this.managementRules = managementRules;
|
||||
}
|
||||
|
||||
public static Optional<NetworkPolicySpec> networkPolicySpecOf(Keycloak keycloak) {
|
||||
return CRDUtils.keycloakSpecOf(keycloak)
|
||||
.map(KeycloakSpec::getNetworkPolicySpec);
|
||||
}
|
||||
|
||||
public static boolean isNetworkPolicyEnabled(Keycloak keycloak) {
|
||||
return Optional.ofNullable(keycloak.getSpec().getNetworkPolicySpec())
|
||||
return networkPolicySpecOf(keycloak)
|
||||
.map(NetworkPolicySpec::isNetworkPolicyEnabled)
|
||||
.orElse(false);
|
||||
}
|
||||
@ -52,4 +103,19 @@ public class NetworkPolicySpec {
|
||||
return keycloak.getMetadata().getName() + Constants.KEYCLOAK_NETWORK_POLICY_SUFFIX;
|
||||
}
|
||||
|
||||
public static Optional<List<NetworkPolicyPeer>> httpRules(Keycloak keycloak) {
|
||||
return networkPolicySpecOf(keycloak)
|
||||
.map(NetworkPolicySpec::getHttpRules);
|
||||
}
|
||||
|
||||
public static Optional<List<NetworkPolicyPeer>> httpsRules(Keycloak keycloak) {
|
||||
return networkPolicySpecOf(keycloak)
|
||||
.map(NetworkPolicySpec::getHttpsRules);
|
||||
}
|
||||
|
||||
public static Optional<List<NetworkPolicyPeer>> managementRules(Keycloak keycloak) {
|
||||
return networkPolicySpecOf(keycloak)
|
||||
.map(NetworkPolicySpec::getManagementRules);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -74,6 +74,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.keycloak.operator.testsuite.utils.CRAssert.assertKeycloakStatusCondition;
|
||||
import static org.keycloak.operator.testsuite.utils.K8sUtils.deployKeycloak;
|
||||
import static org.keycloak.operator.testsuite.utils.K8sUtils.disableHttps;
|
||||
import static org.keycloak.operator.testsuite.utils.K8sUtils.enableHttp;
|
||||
import static org.keycloak.operator.testsuite.utils.K8sUtils.getResourceFromFile;
|
||||
import static org.keycloak.operator.testsuite.utils.K8sUtils.waitForKeycloakToBeReady;
|
||||
|
||||
@ -177,9 +179,10 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
|
||||
Constants.DEFAULT_DIST_CONFIG_LIST.stream()
|
||||
.filter(oneValueOrSecret -> oneValueOrSecret.getName().equalsIgnoreCase(valueSecretHealthProp.getName()))
|
||||
.findFirst()
|
||||
.get()
|
||||
.getValue()
|
||||
).isEqualTo("true"); // just a sanity check default values did not change
|
||||
.map(ValueOrSecret::getValue)
|
||||
)
|
||||
.isPresent()
|
||||
.contains("true");
|
||||
|
||||
Awaitility.await()
|
||||
.ignoreExceptions()
|
||||
@ -287,8 +290,8 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
|
||||
@Test
|
||||
public void testTlsDisabled() {
|
||||
var kc = getTestKeycloakDeployment(true);
|
||||
kc.getSpec().getHttpSpec().setTlsSecret(null);
|
||||
kc.getSpec().getHttpSpec().setHttpEnabled(true);
|
||||
disableHttps(kc);
|
||||
enableHttp(kc, false);
|
||||
deployKeycloak(k8sclient, kc, true);
|
||||
|
||||
CRAssert.assertKeycloakAccessibleViaService(k8sclient, kc, false, Constants.KEYCLOAK_HTTP_PORT);
|
||||
@ -298,7 +301,7 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
|
||||
@Test
|
||||
public void testHttpEnabledWithTls() {
|
||||
var kc = getTestKeycloakDeployment(true);
|
||||
kc.getSpec().getHttpSpec().setHttpEnabled(true);
|
||||
enableHttp(kc, false);
|
||||
deployKeycloak(k8sclient, kc, true);
|
||||
|
||||
CRAssert.assertKeycloakAccessibleViaService(k8sclient, kc, false, Constants.KEYCLOAK_HTTP_PORT);
|
||||
@ -350,11 +353,9 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
|
||||
|
||||
@Test
|
||||
public void testHttpsPort() {
|
||||
final int httpsPort = 8543;
|
||||
final int httpPort = 8180;
|
||||
var kc = getTestKeycloakDeployment(true);
|
||||
kc.getSpec().getHttpSpec().setHttpsPort(httpsPort);
|
||||
kc.getSpec().getHttpSpec().setHttpPort(httpPort);
|
||||
var httpsPort = K8sUtils.configureHttps(kc, true);
|
||||
enableHttp(kc, true);
|
||||
|
||||
var hostnameSpec = new HostnameSpecBuilder()
|
||||
.withStrict(false)
|
||||
@ -368,13 +369,10 @@ public class KeycloakDeploymentTest extends BaseOperatorTest {
|
||||
|
||||
@Test
|
||||
public void testHttpPort() {
|
||||
final int httpsPort = 8543;
|
||||
final int httpPort = 8180;
|
||||
var kc = getTestKeycloakDeployment(true);
|
||||
kc.getSpec().getHttpSpec().setHttpsPort(httpsPort);
|
||||
kc.getSpec().getHttpSpec().setHttpPort(httpPort);
|
||||
kc.getSpec().getHttpSpec().setTlsSecret(null);
|
||||
kc.getSpec().getHttpSpec().setHttpEnabled(true);
|
||||
K8sUtils.configureHttps(kc, true);
|
||||
disableHttps(kc);
|
||||
var httpPort = enableHttp(kc, true);
|
||||
|
||||
var hostnameSpec = new HostnameSpecBuilder()
|
||||
.withStrict(false)
|
||||
|
||||
@ -42,6 +42,8 @@ import static java.util.concurrent.TimeUnit.MINUTES;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.keycloak.operator.testsuite.utils.K8sUtils.disableHttps;
|
||||
import static org.keycloak.operator.testsuite.utils.K8sUtils.enableHttp;
|
||||
|
||||
@QuarkusTest
|
||||
public class KeycloakIngressTest extends BaseOperatorTest {
|
||||
@ -49,8 +51,8 @@ public class KeycloakIngressTest extends BaseOperatorTest {
|
||||
@Test
|
||||
public void testIngressOnHTTP() {
|
||||
var kc = getTestKeycloakDeployment(false);
|
||||
kc.getSpec().getHttpSpec().setTlsSecret(null);
|
||||
kc.getSpec().getHttpSpec().setHttpEnabled(true);
|
||||
disableHttps(kc);
|
||||
enableHttp(kc, false);
|
||||
var hostnameSpecBuilder = new HostnameSpecBuilder()
|
||||
.withStrict(false)
|
||||
.withStrictBackchannel(false);
|
||||
@ -214,9 +216,7 @@ public class KeycloakIngressTest extends BaseOperatorTest {
|
||||
K8sUtils.deployKeycloak(k8sclient, kc, true);
|
||||
|
||||
Awaitility.await()
|
||||
.untilAsserted(() -> {
|
||||
assertThat(k8sclient.network().v1().ingresses().inNamespace(namespace).list().getItems().size()).isEqualTo(0);
|
||||
});
|
||||
.untilAsserted(() -> assertThat(k8sclient.network().v1().ingresses().inNamespace(namespace).list().getItems()).isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -237,9 +237,9 @@ public class KeycloakIngressTest extends BaseOperatorTest {
|
||||
.inNamespace(namespace)
|
||||
.withName(customIngressCreatedManually.getMetadata().getName());
|
||||
|
||||
Awaitility.await().atMost(1, MINUTES).untilAsserted(() -> {
|
||||
assertThat(k8sclient.network().v1().ingresses().inNamespace(namespace).list().getItems().size()).isEqualTo(1);
|
||||
});
|
||||
Awaitility.await()
|
||||
.atMost(1, MINUTES)
|
||||
.untilAsserted(() -> assertThat(k8sclient.network().v1().ingresses().inNamespace(namespace).list().getItems().size()).isEqualTo(1));
|
||||
|
||||
Log.info("Deploying the Keycloak CR with default Ingress disabled");
|
||||
defaultKeycloakDeployment.getSpec().setIngressSpec(new IngressSpec());
|
||||
@ -256,9 +256,8 @@ public class KeycloakIngressTest extends BaseOperatorTest {
|
||||
Log.info("Destroying the Custom Ingress created manually to avoid errors in others Tests methods");
|
||||
if (customIngressDeployedManuallySelector != null && customIngressDeployedManuallySelector.isReady()) {
|
||||
assertThat(customIngressDeployedManuallySelector.delete()).isNotNull();
|
||||
Awaitility.await().untilAsserted(() -> {
|
||||
assertThat(k8sclient.network().v1().ingresses().inNamespace(namespace).list().getItems().size()).isEqualTo(0);
|
||||
});
|
||||
Awaitility.await()
|
||||
.untilAsserted(() -> assertThat(k8sclient.network().v1().ingresses().inNamespace(namespace).list().getItems()).isEmpty());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -19,37 +19,24 @@ package org.keycloak.operator.testsuite.integration;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import io.fabric8.kubernetes.api.model.NamespaceBuilder;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicy;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyIngressRule;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyPort;
|
||||
import io.quarkus.logging.Log;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyPeer;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyPeerBuilder;
|
||||
import io.quarkus.test.junit.QuarkusTest;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import org.keycloak.operator.Constants;
|
||||
import org.keycloak.operator.Utils;
|
||||
import org.keycloak.operator.controllers.KeycloakController;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.ValueOrSecret;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.spec.HostnameSpecBuilder;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.spec.HttpManagementSpecBuilder;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.spec.IngressSpecBuilder;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.spec.NetworkPolicySpec;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.spec.NetworkPolicySpecBuilder;
|
||||
import org.keycloak.operator.testsuite.utils.CRAssert;
|
||||
import org.keycloak.operator.testsuite.utils.K8sUtils;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@QuarkusTest
|
||||
public class KeycloakNetworkPolicyTest extends BaseOperatorTest {
|
||||
@ -62,89 +49,90 @@ public class KeycloakNetworkPolicyTest extends BaseOperatorTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDefaults() {
|
||||
public void testHttpAndHttps() {
|
||||
var kc = create();
|
||||
K8sUtils.deployKeycloak(k8sclient, kc, true);
|
||||
CRAssert.awaitClusterSize(k8sclient, kc, 2);
|
||||
assertNull(networkPolicy(kc), "Expects no network policies deployed");
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(booleans = {true, false})
|
||||
public void testHttpOnly(boolean randomPort) {
|
||||
var kc = create();
|
||||
enableNetworkPolicy(kc);
|
||||
disableHttps(kc);
|
||||
var httpPort = enableHttp(kc, randomPort);
|
||||
var mngtPort = configureManagement(kc, randomPort);
|
||||
K8sUtils.deployKeycloak(k8sclient, kc, true);
|
||||
CRAssert.awaitClusterSize(k8sclient, kc, 2);
|
||||
|
||||
assertIngressRules(kc, httpPort, -1, mngtPort);
|
||||
|
||||
CRAssert.assertKeycloakAccessibleViaService(k8sclient, kc, false, httpPort);
|
||||
CRAssert.assertManagementInterfaceAccessibleViaService(k8sclient, kc, false, mngtPort);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(booleans = {true, false})
|
||||
public void testHttpsOnly(boolean randomPort) {
|
||||
var kc = create();
|
||||
enableNetworkPolicy(kc);
|
||||
var httpsPort = configureHttps(kc, randomPort);
|
||||
var mngtPort = configureManagement(kc, randomPort);
|
||||
K8sUtils.enableNetworkPolicy(kc);
|
||||
var httpPort = K8sUtils.enableHttp(kc, false);
|
||||
var httpsPort = K8sUtils.configureHttps(kc, false);
|
||||
var mngtPort = K8sUtils.configureManagement(kc, false);
|
||||
|
||||
K8sUtils.deployKeycloak(k8sclient, kc, true);
|
||||
CRAssert.awaitClusterSize(k8sclient, kc, 2);
|
||||
|
||||
assertIngressRules(kc, -1, httpsPort, mngtPort);
|
||||
|
||||
CRAssert.assertKeycloakAccessibleViaService(k8sclient, kc, true, httpsPort);
|
||||
CRAssert.assertManagementInterfaceAccessibleViaService(k8sclient, kc, true, mngtPort);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(booleans = {true, false})
|
||||
public void testHttpAndHttps(boolean randomPort) {
|
||||
var kc = create();
|
||||
enableNetworkPolicy(kc);
|
||||
var httpPort = enableHttp(kc, randomPort);
|
||||
var httpsPort = configureHttps(kc, randomPort);
|
||||
var mngtPort = configureManagement(kc, randomPort);
|
||||
|
||||
K8sUtils.deployKeycloak(k8sclient, kc, true);
|
||||
CRAssert.awaitClusterSize(k8sclient, kc, 2);
|
||||
|
||||
assertIngressRules(kc, httpPort, httpsPort, mngtPort);
|
||||
CRAssert.assertIngressRules(networkPolicy(kc), kc, httpPort, httpsPort, mngtPort);
|
||||
|
||||
CRAssert.assertKeycloakAccessibleViaService(k8sclient, kc, false, httpPort);
|
||||
CRAssert.assertKeycloakAccessibleViaService(k8sclient, kc, true, httpsPort);
|
||||
CRAssert.assertManagementInterfaceAccessibleViaService(k8sclient, kc, true, mngtPort);
|
||||
}
|
||||
|
||||
@ParameterizedTest()
|
||||
@ValueSource(booleans = {true, false})
|
||||
public void testManagementDisabled(boolean legacyOption) {
|
||||
@Test
|
||||
public void testServiceConnectivity() {
|
||||
var kc = create();
|
||||
disableProbes(kc);
|
||||
enableNetworkPolicy(kc);
|
||||
disableManagement(kc, legacyOption);
|
||||
K8sUtils.enableNetworkPolicy(kc);
|
||||
var httpsPort = K8sUtils.configureHttps(kc, false);
|
||||
|
||||
K8sUtils.deployKeycloak(k8sclient, kc, true);
|
||||
CRAssert.awaitClusterSize(k8sclient, kc, 2);
|
||||
var allowNamespace = getNewRandomNamespaceName();
|
||||
var notAllowNamespace = getNewRandomNamespaceName();
|
||||
var anotherNamespace = getNewRandomNamespaceName();
|
||||
var allowLabels = Map.of("allowed", "true");
|
||||
var notAllowLabels = Map.of("allowed", "false");
|
||||
try {
|
||||
k8sclient.resource(new NamespaceBuilder().withNewMetadata().withName(allowNamespace).endMetadata().build()).create();
|
||||
k8sclient.resource(new NamespaceBuilder().withNewMetadata().withName(notAllowNamespace).endMetadata().build()).create();
|
||||
k8sclient.resource(new NamespaceBuilder().withNewMetadata().withName(anotherNamespace).endMetadata().build()).create();
|
||||
|
||||
assertIngressRules(kc, -1, Constants.KEYCLOAK_HTTPS_PORT, -1);
|
||||
CRAssert.assertKeycloakAccessibleViaService(k8sclient, kc, true, Constants.KEYCLOAK_HTTPS_PORT);
|
||||
// allow access from:
|
||||
// * pods from namespace 'allowNamespace' AND with label 'allowLabels'
|
||||
// OR
|
||||
// * pods from namespace 'anotherNamespace' (labels do not matter)
|
||||
kc.getSpec().getNetworkPolicySpec().setHttpsRules(
|
||||
List.of(
|
||||
createRule(allowNamespace, allowLabels),
|
||||
createRule(anotherNamespace, null)
|
||||
)
|
||||
);
|
||||
|
||||
K8sUtils.deployKeycloak(k8sclient, kc, true);
|
||||
CRAssert.awaitClusterSize(k8sclient, kc, 2);
|
||||
|
||||
CRAssert.assertIngressRules(networkPolicy(kc), kc, -1, httpsPort, Constants.KEYCLOAK_MANAGEMENT_PORT);
|
||||
|
||||
// 1st rule have access
|
||||
CRAssert.assertKeycloakAccessibleViaService(k8sclient, kc, allowNamespace, allowLabels, true, httpsPort);
|
||||
// 2nd rule (pod labels do not matter)
|
||||
CRAssert.assertKeycloakAccessibleViaService(k8sclient, kc, anotherNamespace, allowLabels, true, httpsPort);
|
||||
CRAssert.assertKeycloakAccessibleViaService(k8sclient, kc, anotherNamespace, notAllowLabels, true, httpsPort);
|
||||
CRAssert.assertKeycloakAccessibleViaService(k8sclient, kc, anotherNamespace, Map.of(), true, httpsPort);
|
||||
|
||||
// correct namespace but wrong label's value.
|
||||
CRAssert.assertKeycloakServiceBlocked(k8sclient, kc, allowNamespace, notAllowLabels, httpsPort);
|
||||
CRAssert.assertKeycloakServiceBlocked(k8sclient, kc, allowNamespace, Map.of(), httpsPort);
|
||||
|
||||
// wrong namespace but correct label.
|
||||
CRAssert.assertKeycloakServiceBlocked(k8sclient, kc, notAllowNamespace, allowLabels, httpsPort);
|
||||
|
||||
// everything is wrong.
|
||||
CRAssert.assertKeycloakServiceBlocked(k8sclient, kc, notAllowNamespace, notAllowLabels, httpsPort);
|
||||
CRAssert.assertKeycloakServiceBlocked(k8sclient, kc, notAllowNamespace, Map.of(), httpsPort);
|
||||
|
||||
// Pods in the same namespace should not be allowed.
|
||||
CRAssert.assertKeycloakServiceBlocked(k8sclient, kc, namespaceOf(kc), allowLabels, httpsPort);
|
||||
} finally {
|
||||
k8sclient.namespaces().withName(allowNamespace).delete();
|
||||
k8sclient.namespaces().withName(notAllowNamespace).delete();
|
||||
k8sclient.namespaces().withName(anotherNamespace).delete();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testJGroupsConnectivity() {
|
||||
var kc = create();
|
||||
enableNetworkPolicy(kc);
|
||||
K8sUtils.enableNetworkPolicy(kc);
|
||||
|
||||
K8sUtils.deployKeycloak(k8sclient, kc, true);
|
||||
CRAssert.awaitClusterSize(k8sclient, kc, 2);
|
||||
assertIngressRules(kc, -1, Constants.KEYCLOAK_HTTPS_PORT, Constants.KEYCLOAK_MANAGEMENT_PORT);
|
||||
CRAssert.assertIngressRules(networkPolicy(kc), kc, -1, Constants.KEYCLOAK_HTTPS_PORT, Constants.KEYCLOAK_MANAGEMENT_PORT);
|
||||
|
||||
var namespace = namespaceOf(kc);
|
||||
var podIp = k8sclient.pods().inNamespace(namespace).list().getItems().get(0).getStatus().getPodIP();
|
||||
@ -169,11 +157,11 @@ public class KeycloakNetworkPolicyTest extends BaseOperatorTest {
|
||||
@Test
|
||||
public void testUpdate() {
|
||||
var kc = create();
|
||||
enableNetworkPolicy(kc);
|
||||
K8sUtils.enableNetworkPolicy(kc);
|
||||
|
||||
K8sUtils.deployKeycloak(k8sclient, kc, true);
|
||||
CRAssert.awaitClusterSize(k8sclient, kc, 2);
|
||||
assertIngressRules(kc, -1, Constants.KEYCLOAK_HTTPS_PORT, Constants.KEYCLOAK_MANAGEMENT_PORT);
|
||||
CRAssert.assertIngressRules(networkPolicy(kc), kc, -1, Constants.KEYCLOAK_HTTPS_PORT, Constants.KEYCLOAK_MANAGEMENT_PORT);
|
||||
|
||||
// disable should remove the network policy
|
||||
kc.getSpec().getNetworkPolicySpec().setNetworkPolicyEnabled(false);
|
||||
@ -185,140 +173,7 @@ public class KeycloakNetworkPolicyTest extends BaseOperatorTest {
|
||||
kc.getSpec().getNetworkPolicySpec().setNetworkPolicyEnabled(true);
|
||||
K8sUtils.deployKeycloak(k8sclient, kc, true);
|
||||
CRAssert.awaitClusterSize(k8sclient, kc, 2);
|
||||
assertIngressRules(kc, -1, Constants.KEYCLOAK_HTTPS_PORT, Constants.KEYCLOAK_MANAGEMENT_PORT);
|
||||
}
|
||||
|
||||
private static void assertPodSelectorAndPolicy(Keycloak keycloak, NetworkPolicy networkPolicy) {
|
||||
assertNotNull(networkPolicy, "Expects a network policy");
|
||||
assertEquals(Utils.allInstanceLabels(keycloak), networkPolicy.getSpec().getPodSelector().getMatchLabels(), "Expects same pod match labels");
|
||||
assertTrue(networkPolicy.getSpec().getPolicyTypes().contains("Ingress"), "Expect ingress polity type present");
|
||||
}
|
||||
|
||||
private static void assertManagementRulePresent(NetworkPolicy networkPolicy, int mgmtPort) {
|
||||
var rule = findIngressRuleWithPort(networkPolicy, mgmtPort);
|
||||
assertTrue(rule.isPresent(), "Management Ingress Rule is missing");
|
||||
assertTrue(rule.get().getFrom().isEmpty());
|
||||
var ports = portAndProtocol(rule.get());
|
||||
assertEquals(Map.of(mgmtPort, Constants.KEYCLOAK_SERVICE_PROTOCOL), ports);
|
||||
}
|
||||
|
||||
private static void assertApplicationRulePresent(NetworkPolicy networkPolicy, int applicationPort) {
|
||||
var rule = findIngressRuleWithPort(networkPolicy, applicationPort);
|
||||
assertTrue(rule.isPresent(), "Application Ingress Rule is missing");
|
||||
assertTrue(rule.get().getFrom().isEmpty());
|
||||
var ports = portAndProtocol(rule.get());
|
||||
assertEquals(Map.of(applicationPort, Constants.KEYCLOAK_SERVICE_PROTOCOL), ports);
|
||||
}
|
||||
|
||||
private static void assertJGroupsRulePresent(Keycloak keycloak, NetworkPolicy networkPolicy) {
|
||||
var rule = findIngressRuleWithPort(networkPolicy, Constants.KEYCLOAK_JGROUPS_DATA_PORT);
|
||||
assertTrue(rule.isPresent(), "JGroups Ingress Rule is missing");
|
||||
|
||||
var from = rule.get().getFrom();
|
||||
assertEquals(1, from.size(), "Incorrect 'from' list size");
|
||||
assertEquals(Utils.allInstanceLabels(keycloak), from.get(0).getPodSelector().getMatchLabels());
|
||||
|
||||
var ports = portAndProtocol(rule.get());
|
||||
assertEquals(Map.of(
|
||||
Constants.KEYCLOAK_JGROUPS_DATA_PORT, Constants.KEYCLOAK_JGROUPS_PROTOCOL,
|
||||
Constants.KEYCLOAK_JGROUPS_FD_PORT, Constants.KEYCLOAK_JGROUPS_PROTOCOL
|
||||
), ports);
|
||||
}
|
||||
|
||||
private static void assertIngressRules(Keycloak keycloak, int httpPort, int httpsPort, int mgntPort) {
|
||||
var networkPolicy = networkPolicy(keycloak);
|
||||
Log.info(networkPolicy);
|
||||
var expectedNumberOfRules = IntStream.of(httpPort, httpsPort, mgntPort)
|
||||
.filter(value -> value > 0)
|
||||
.count();
|
||||
|
||||
// +1 for JGRP
|
||||
++expectedNumberOfRules;
|
||||
|
||||
long numberOfRules = Optional.ofNullable(networkPolicy.getSpec())
|
||||
.map(io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicySpec::getIngress)
|
||||
.map(List::size)
|
||||
.orElse(0);
|
||||
|
||||
assertEquals(expectedNumberOfRules, numberOfRules);
|
||||
|
||||
// Check selector
|
||||
assertPodSelectorAndPolicy(keycloak, networkPolicy);
|
||||
|
||||
// JGroups is always present
|
||||
assertJGroupsRulePresent(keycloak, networkPolicy);
|
||||
|
||||
if (httpPort > 0) {
|
||||
assertApplicationRulePresent(networkPolicy, httpPort);
|
||||
}
|
||||
if (httpsPort > 0) {
|
||||
assertApplicationRulePresent(networkPolicy, httpsPort);
|
||||
}
|
||||
if (mgntPort > 0) {
|
||||
assertManagementRulePresent(networkPolicy, mgntPort);
|
||||
}
|
||||
}
|
||||
|
||||
private static Map<Integer, String> portAndProtocol(NetworkPolicyIngressRule rule) {
|
||||
return rule.getPorts().stream()
|
||||
.collect(Collectors.toMap(port -> port.getPort().getIntVal(), NetworkPolicyPort::getProtocol));
|
||||
}
|
||||
|
||||
private static Optional<NetworkPolicyIngressRule> findIngressRuleWithPort(NetworkPolicy networkPolicy, int rulePort) {
|
||||
return networkPolicy.getSpec().getIngress().stream()
|
||||
.filter(rule -> rule.getPorts().stream().anyMatch(port -> port.getPort().getIntVal() == rulePort))
|
||||
.findFirst();
|
||||
}
|
||||
|
||||
private static void enableNetworkPolicy(Keycloak keycloak) {
|
||||
var builder = new NetworkPolicySpecBuilder();
|
||||
builder.withNetworkPolicyEnabled(true);
|
||||
keycloak.getSpec().setNetworkPolicySpec(builder.build());
|
||||
}
|
||||
|
||||
private static int configureManagement(Keycloak keycloak, boolean randomPort) {
|
||||
if (!randomPort) {
|
||||
return Constants.KEYCLOAK_MANAGEMENT_PORT;
|
||||
}
|
||||
var port = ThreadLocalRandom.current().nextInt(10_000, 10_100);
|
||||
keycloak.getSpec().setHttpManagementSpec(new HttpManagementSpecBuilder().withPort(port).build());
|
||||
return port;
|
||||
}
|
||||
|
||||
private static int enableHttp(Keycloak keycloak, boolean randomPort) {
|
||||
keycloak.getSpec().getHttpSpec().setHttpEnabled(true);
|
||||
if (randomPort) {
|
||||
var port = ThreadLocalRandom.current().nextInt(10_100, 10_200);
|
||||
keycloak.getSpec().getHttpSpec().setHttpPort(port);
|
||||
return port;
|
||||
}
|
||||
return Constants.KEYCLOAK_HTTP_PORT;
|
||||
}
|
||||
|
||||
private static void disableHttps(Keycloak keycloak) {
|
||||
keycloak.getSpec().getHttpSpec().setTlsSecret(null);
|
||||
}
|
||||
|
||||
private static int configureHttps(Keycloak keycloak, boolean randomPort) {
|
||||
if (randomPort) {
|
||||
var port = ThreadLocalRandom.current().nextInt(10_200, 10_300);
|
||||
keycloak.getSpec().getHttpSpec().setHttpsPort(port);
|
||||
return port;
|
||||
}
|
||||
return Constants.KEYCLOAK_HTTPS_PORT;
|
||||
}
|
||||
|
||||
private static void disableManagement(Keycloak keycloak, boolean legacyOption) {
|
||||
if (legacyOption) {
|
||||
keycloak.getSpec().getAdditionalOptions().add(new ValueOrSecret("legacy-observability-interface", "true"));
|
||||
} else {
|
||||
keycloak.getSpec().getAdditionalOptions().add(new ValueOrSecret("health-enabled", "false"));
|
||||
}
|
||||
// The custom image from GitHub Actions is optimized and does not allow to change the build time attributes
|
||||
// Fallback to the default/nightly image.
|
||||
if (getTestCustomImage() != null) {
|
||||
keycloak.getSpec().setImage(null);
|
||||
}
|
||||
CRAssert.assertIngressRules(networkPolicy(kc), kc, -1, Constants.KEYCLOAK_HTTPS_PORT, Constants.KEYCLOAK_MANAGEMENT_PORT);
|
||||
}
|
||||
|
||||
private static Keycloak create() {
|
||||
@ -334,4 +189,19 @@ public class KeycloakNetworkPolicyTest extends BaseOperatorTest {
|
||||
return kc;
|
||||
}
|
||||
|
||||
private static NetworkPolicyPeer createRule(String namespace, Map<String, String> labels) {
|
||||
var builder = new NetworkPolicyPeerBuilder();
|
||||
if (labels != null) {
|
||||
builder.withNewPodSelector()
|
||||
.withMatchLabels(labels)
|
||||
.endPodSelector();
|
||||
}
|
||||
if (namespace != null) {
|
||||
builder.withNewNamespaceSelector()
|
||||
.addToMatchLabels("kubernetes.io/metadata.name", namespace)
|
||||
.endNamespaceSelector();
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -17,7 +17,13 @@
|
||||
|
||||
package org.keycloak.operator.testsuite.unit;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import io.fabric8.kubernetes.api.model.ResourceRequirements;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyPeer;
|
||||
import io.fabric8.kubernetes.client.utils.Serialization;
|
||||
import org.hamcrest.CoreMatchers;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@ -32,10 +38,6 @@ import org.keycloak.operator.crds.v2alpha1.deployment.spec.TransactionsSpec;
|
||||
import org.keycloak.operator.crds.v2alpha1.realmimport.KeycloakRealmImport;
|
||||
import org.keycloak.operator.testsuite.utils.K8sUtils;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.emptyString;
|
||||
import static org.hamcrest.Matchers.hasEntry;
|
||||
@ -48,6 +50,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
public class CRSerializationTest {
|
||||
|
||||
@ -97,9 +100,6 @@ public class CRSerializationTest {
|
||||
HttpManagementSpec managementSpec = keycloak.getSpec().getHttpManagementSpec();
|
||||
assertNotNull(managementSpec);
|
||||
assertEquals(9003, managementSpec.getPort());
|
||||
|
||||
assertNotNull(keycloak.getSpec().getNetworkPolicySpec());
|
||||
assertTrue(keycloak.getSpec().getNetworkPolicySpec().isNetworkPolicyEnabled());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -242,4 +242,39 @@ public class CRSerializationTest {
|
||||
assertThat(limitMemQuantity.getFormat(), is("Gi"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNetworkPolicy() {
|
||||
var keycloak = Serialization.unmarshal(this.getClass().getResourceAsStream("/test-serialization-keycloak-cr.yml"), Keycloak.class);
|
||||
var networkPolicySpec = keycloak.getSpec().getNetworkPolicySpec();
|
||||
assertNotNull(networkPolicySpec);
|
||||
assertTrue(networkPolicySpec.isNetworkPolicyEnabled());
|
||||
assertNetworkPolicyRules(networkPolicySpec.getHttpRules());
|
||||
assertNetworkPolicyRules(networkPolicySpec.getHttpsRules());
|
||||
assertNetworkPolicyRules(networkPolicySpec.getManagementRules());
|
||||
}
|
||||
|
||||
private static void assertNetworkPolicyRules(Collection<NetworkPolicyPeer> rules) {
|
||||
assertNotNull(rules);
|
||||
assertEquals(3, rules.size());
|
||||
for (var peer : rules) {
|
||||
assertNotNull(peer);
|
||||
if (peer.getPodSelector() != null) {
|
||||
assertEquals("frontend", peer.getPodSelector().getMatchLabels().get("role"));
|
||||
continue;
|
||||
}
|
||||
if (peer.getNamespaceSelector() != null) {
|
||||
assertEquals("myproject", peer.getNamespaceSelector().getMatchLabels().get("project"));
|
||||
continue;
|
||||
}
|
||||
if (peer.getIpBlock() != null) {
|
||||
assertEquals("172.17.0.0/16", peer.getIpBlock().getCidr());
|
||||
var except = peer.getIpBlock().getExcept();
|
||||
assertEquals(1, except.size());
|
||||
assertEquals("172.17.1.0/24", except.get(0));
|
||||
continue;
|
||||
}
|
||||
fail();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -17,29 +17,29 @@
|
||||
|
||||
package org.keycloak.operator.testsuite.unit;
|
||||
|
||||
import io.fabric8.kubernetes.api.model.HasMetadata;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.Ingress;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.Ingress;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.keycloak.operator.controllers.KeycloakIngressDependentResource;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.spec.IngressSpec;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.spec.IngressSpecBuilder;
|
||||
import org.keycloak.operator.testsuite.utils.K8sUtils;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import org.keycloak.operator.testsuite.utils.MockController;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.keycloak.operator.testsuite.utils.K8sUtils.disableHttps;
|
||||
|
||||
public class IngressLogicTest {
|
||||
|
||||
private static final String EXISTING_ANNOTATION_KEY = "annotation";
|
||||
|
||||
static class MockKeycloakIngress {
|
||||
static class MockKeycloakIngress extends MockController<Ingress, KeycloakIngressDependentResource> {
|
||||
|
||||
private static Keycloak getKeycloak(boolean tlsConfigured, IngressSpec ingressSpec, String hostname) {
|
||||
var kc = K8sUtils.getDefaultKeycloakDeployment();
|
||||
@ -48,7 +48,7 @@ public class IngressLogicTest {
|
||||
kc.getSpec().setIngressSpec(ingressSpec);
|
||||
}
|
||||
if (!tlsConfigured) {
|
||||
kc.getSpec().getHttpSpec().setTlsSecret(null);
|
||||
disableHttps(kc);
|
||||
}
|
||||
if (hostname != null) {
|
||||
kc.getSpec().getHostnameSpec().setHostname(hostname);
|
||||
@ -84,40 +84,29 @@ public class IngressLogicTest {
|
||||
}
|
||||
}
|
||||
MockKeycloakIngress mock = new MockKeycloakIngress(tlsConfigured, ingressSpec, hostname);
|
||||
mock.ingressExists = ingressExists;
|
||||
if (ingressExists) {
|
||||
mock.setExists();
|
||||
}
|
||||
return mock;
|
||||
}
|
||||
|
||||
private KeycloakIngressDependentResource keycloakIngressDependentResource = new KeycloakIngressDependentResource();
|
||||
private boolean ingressExists = false;
|
||||
private boolean deleted = false;
|
||||
private Keycloak keycloak;
|
||||
|
||||
public MockKeycloakIngress(boolean tlsConfigured, IngressSpec ingressSpec, String hostname) {
|
||||
this.keycloak = getKeycloak(tlsConfigured, ingressSpec, hostname);
|
||||
super(new KeycloakIngressDependentResource(), getKeycloak(tlsConfigured, ingressSpec, hostname));
|
||||
}
|
||||
|
||||
public MockKeycloakIngress(boolean tlsConfigured, IngressSpec ingressSpec) {
|
||||
this(tlsConfigured, ingressSpec, null);
|
||||
}
|
||||
|
||||
public boolean reconciled() {
|
||||
return getReconciledResource().isPresent();
|
||||
|
||||
@Override
|
||||
protected boolean isEnabled() {
|
||||
return KeycloakIngressDependentResource.isIngressEnabled(keycloak);
|
||||
}
|
||||
|
||||
public boolean deleted() {
|
||||
return deleted;
|
||||
}
|
||||
|
||||
public Optional<HasMetadata> getReconciledResource() {
|
||||
if (!KeycloakIngressDependentResource.isIngressEnabled(keycloak)) {
|
||||
if (ingressExists) {
|
||||
deleted = true;
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
return Optional.of(keycloakIngressDependentResource.desired(keycloak, null));
|
||||
@Override
|
||||
protected Ingress desired() {
|
||||
return dependentResource.desired(keycloak, null);
|
||||
}
|
||||
}
|
||||
|
||||
@ -166,7 +155,7 @@ public class IngressLogicTest {
|
||||
@Test
|
||||
public void testHttpSpecWithTlsSecret() {
|
||||
var kc = MockKeycloakIngress.build(null, false, true, true);
|
||||
Optional<HasMetadata> reconciled = kc.getReconciledResource();
|
||||
Optional<Ingress> reconciled = kc.getReconciledResource();
|
||||
assertTrue(reconciled.isPresent());
|
||||
assertFalse(kc.deleted());
|
||||
assertEquals("HTTPS", reconciled.get().getMetadata().getAnnotations().get("nginx.ingress.kubernetes.io/backend-protocol"));
|
||||
@ -176,7 +165,7 @@ public class IngressLogicTest {
|
||||
@Test
|
||||
public void testHttpSpecWithoutTlsSecret() {
|
||||
var kc = MockKeycloakIngress.build(null, false, true, false);
|
||||
Optional<HasMetadata> reconciled = kc.getReconciledResource();
|
||||
Optional<Ingress> reconciled = kc.getReconciledResource();
|
||||
assertTrue(reconciled.isPresent());
|
||||
assertFalse(kc.deleted());
|
||||
assertEquals("HTTP", reconciled.get().getMetadata().getAnnotations().get("nginx.ingress.kubernetes.io/backend-protocol"));
|
||||
@ -186,7 +175,7 @@ public class IngressLogicTest {
|
||||
@Test
|
||||
public void testCustomAnnotations() {
|
||||
var kc = MockKeycloakIngress.build(null, false, true, true, Map.of("custom", "value"));
|
||||
Optional<HasMetadata> reconciled = kc.getReconciledResource();
|
||||
Optional<Ingress> reconciled = kc.getReconciledResource();
|
||||
assertTrue(reconciled.isPresent());
|
||||
assertFalse(kc.deleted());
|
||||
assertEquals("HTTPS", reconciled.get().getMetadata().getAnnotations().get("nginx.ingress.kubernetes.io/backend-protocol"));
|
||||
@ -198,7 +187,7 @@ public class IngressLogicTest {
|
||||
@Test
|
||||
public void testRemoveCustomAnnotation() {
|
||||
var kc = MockKeycloakIngress.build(null, true, true, true, null);
|
||||
Optional<HasMetadata> reconciled = kc.getReconciledResource();
|
||||
Optional<Ingress> reconciled = kc.getReconciledResource();
|
||||
assertTrue(reconciled.isPresent());
|
||||
assertFalse(kc.deleted());
|
||||
assertEquals("HTTPS", reconciled.get().getMetadata().getAnnotations().get("nginx.ingress.kubernetes.io/backend-protocol"));
|
||||
@ -209,7 +198,7 @@ public class IngressLogicTest {
|
||||
@Test
|
||||
public void testUpdateCustomAnnotation() {
|
||||
var kc = MockKeycloakIngress.build(null, true, true, true, Map.of(EXISTING_ANNOTATION_KEY, "another-value"));
|
||||
Optional<HasMetadata> reconciled = kc.getReconciledResource();
|
||||
Optional<Ingress> reconciled = kc.getReconciledResource();
|
||||
assertTrue(reconciled.isPresent());
|
||||
assertFalse(kc.deleted());
|
||||
assertEquals("HTTPS", reconciled.get().getMetadata().getAnnotations().get("nginx.ingress.kubernetes.io/backend-protocol"));
|
||||
@ -220,24 +209,24 @@ public class IngressLogicTest {
|
||||
@Test
|
||||
public void testIngressSpecDefinedWithoutClassName() {
|
||||
var kc = new MockKeycloakIngress(true, new IngressSpec());
|
||||
Optional<HasMetadata> reconciled = kc.getReconciledResource();
|
||||
Ingress ingress = reconciled.map(Ingress.class::cast).orElseThrow();
|
||||
Optional<Ingress> reconciled = kc.getReconciledResource();
|
||||
Ingress ingress = reconciled.orElseThrow();
|
||||
assertNull(ingress.getSpec().getIngressClassName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIngressSpecDefinedWithClassName() {
|
||||
var kc = new MockKeycloakIngress(true, new IngressSpecBuilder().withIngressClassName("my-class").build());
|
||||
Optional<HasMetadata> reconciled = kc.getReconciledResource();
|
||||
Ingress ingress = reconciled.map(Ingress.class::cast).orElseThrow();
|
||||
Optional<Ingress> reconciled = kc.getReconciledResource();
|
||||
Ingress ingress = reconciled.orElseThrow();
|
||||
assertEquals("my-class", ingress.getSpec().getIngressClassName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHostnameSanitizing() {
|
||||
var kc = MockKeycloakIngress.build("https://my-other.example.com:443/my-path");
|
||||
Optional<HasMetadata> reconciled = kc.getReconciledResource();
|
||||
Ingress ingress = reconciled.map(Ingress.class::cast).orElseThrow();
|
||||
Optional<Ingress> reconciled = kc.getReconciledResource();
|
||||
Ingress ingress = reconciled.orElseThrow();
|
||||
assertEquals("my-other.example.com", ingress.getSpec().getRules().get(0).getHost());
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,225 @@
|
||||
/*
|
||||
* Copyright 2024 Red Hat, Inc. and/or its affiliates
|
||||
* and other contributors as indicated by the @author tags.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.keycloak.operator.testsuite.unit;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicy;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyPeer;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyPeerBuilder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import org.keycloak.operator.Constants;
|
||||
import org.keycloak.operator.controllers.KeycloakNetworkPolicyDependentResource;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.ValueOrSecret;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.spec.NetworkPolicySpec;
|
||||
import org.keycloak.operator.testsuite.utils.CRAssert;
|
||||
import org.keycloak.operator.testsuite.utils.K8sUtils;
|
||||
import org.keycloak.operator.testsuite.utils.MockController;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
public class NetworkPolicyLogicTest {
|
||||
|
||||
private static class MockKeycloakNetworkPolicy extends MockController<NetworkPolicy, KeycloakNetworkPolicyDependentResource> {
|
||||
|
||||
MockKeycloakNetworkPolicy(Keycloak keycloak) {
|
||||
super(new KeycloakNetworkPolicyDependentResource(), keycloak);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean isEnabled() {
|
||||
return NetworkPolicySpec.isNetworkPolicyEnabled(keycloak);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected NetworkPolicy desired() {
|
||||
return dependentResource.desired(keycloak, null);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDefaults() {
|
||||
var keycloak = K8sUtils.getDefaultKeycloakDeployment();
|
||||
var controller = new MockKeycloakNetworkPolicy(keycloak);
|
||||
assertFalse(controller.isEnabled());
|
||||
assertFalse(controller.reconciled());
|
||||
assertFalse(controller.deleted());
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(booleans = {true, false})
|
||||
public void testHttpOnly(boolean randomPort) {
|
||||
var kc = K8sUtils.getDefaultKeycloakDeployment();
|
||||
K8sUtils.enableNetworkPolicy(kc);
|
||||
K8sUtils.disableHttps(kc);
|
||||
var httpPort = K8sUtils.enableHttp(kc, randomPort);
|
||||
var mngtPort = K8sUtils.configureManagement(kc, randomPort);
|
||||
kc.getSpec().getNetworkPolicySpec().setHttpRules(List.of(namespaceSelectorWithMatchLabel("http", "true")));
|
||||
kc.getSpec().getNetworkPolicySpec().setManagementRules(List.of(podSelectorWithMatchExpression("monitoring", "from", "1", "and", "2")));
|
||||
var networkPolicy = assertEnabledAndGet(kc);
|
||||
CRAssert.assertIngressRules(networkPolicy, kc, httpPort, -1, mngtPort);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(booleans = {true, false})
|
||||
public void testHttpsOnly(boolean randomPort) {
|
||||
var kc = K8sUtils.getDefaultKeycloakDeployment();
|
||||
K8sUtils.enableNetworkPolicy(kc);
|
||||
var httpsPort = K8sUtils.configureHttps(kc, randomPort);
|
||||
var mngtPort = K8sUtils.configureManagement(kc, randomPort);
|
||||
kc.getSpec().getNetworkPolicySpec().setHttpsRules(List.of(podSelectorWithMatchLabel("https", "yes!")));
|
||||
kc.getSpec().getNetworkPolicySpec().setManagementRules(List.of(namespaceSelectorWithMatchExpressions("namespace", "in", "somewhere")));
|
||||
var networkPolicy = assertEnabledAndGet(kc);
|
||||
CRAssert.assertIngressRules(networkPolicy, kc, -1, httpsPort, mngtPort);
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(booleans = {true, false})
|
||||
public void testHttpAndHttps(boolean randomPort) {
|
||||
var kc = K8sUtils.getDefaultKeycloakDeployment();
|
||||
K8sUtils.enableNetworkPolicy(kc);
|
||||
var httpPort = K8sUtils.enableHttp(kc, randomPort);
|
||||
var httpsPort = K8sUtils.configureHttps(kc, randomPort);
|
||||
var mngtPort = K8sUtils.configureManagement(kc, randomPort);
|
||||
|
||||
kc.getSpec().getNetworkPolicySpec().setHttpRules(List.of(
|
||||
ipBlock("127.0.0.1"),
|
||||
namespaceSelectorWithMatchExpressions("name", "in", "local", "local-2")
|
||||
));
|
||||
|
||||
kc.getSpec().getNetworkPolicySpec().setHttpsRules(List.of(
|
||||
podSelectorWithMatchExpression("app", "equals", "keycloak"),
|
||||
ipBlock("10.0.0.0/8")
|
||||
));
|
||||
|
||||
kc.getSpec().getNetworkPolicySpec().setManagementRules(List.of(
|
||||
namespaceSelectorWithMatchExpressions("monitoring", "contains", "always", "on")
|
||||
));
|
||||
|
||||
var networkPolicy = assertEnabledAndGet(kc);
|
||||
|
||||
CRAssert.assertIngressRules(networkPolicy, kc, httpPort, httpsPort, mngtPort);
|
||||
}
|
||||
|
||||
@ParameterizedTest()
|
||||
@ValueSource(booleans = {true, false})
|
||||
public void testManagementDisabled(boolean legacyOption) {
|
||||
var kc = K8sUtils.getDefaultKeycloakDeployment();
|
||||
K8sUtils.enableNetworkPolicy(kc);
|
||||
disableManagement(kc, legacyOption);
|
||||
kc.getSpec().getNetworkPolicySpec().setHttpsRules(List.of(
|
||||
ipBlock("127.0.0.1/15", "127.0.0.1/18", "127.0.0.1/19"),
|
||||
podSelectorWithMatchLabel("app", "keycloak"),
|
||||
namespaceSelectorWithMatchLabel("kubernetes.io/name", "keycloak")
|
||||
));
|
||||
var networkPolicy = assertEnabledAndGet(kc);
|
||||
CRAssert.assertIngressRules(networkPolicy, kc, -1, Constants.KEYCLOAK_HTTPS_PORT, -1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUpdate() {
|
||||
var kc = K8sUtils.getDefaultKeycloakDeployment();
|
||||
K8sUtils.enableNetworkPolicy(kc);
|
||||
|
||||
var controller = new MockKeycloakNetworkPolicy(kc);
|
||||
|
||||
assertTrue(controller.isEnabled());
|
||||
assertTrue(controller.reconciled());
|
||||
assertFalse(controller.deleted());
|
||||
|
||||
kc.getSpec().getNetworkPolicySpec().setNetworkPolicyEnabled(false);
|
||||
|
||||
assertFalse(controller.isEnabled());
|
||||
assertFalse(controller.reconciled());
|
||||
assertTrue(controller.deleted());
|
||||
}
|
||||
|
||||
private static NetworkPolicy assertEnabledAndGet(Keycloak keycloak) {
|
||||
var controller = new MockKeycloakNetworkPolicy(keycloak);
|
||||
|
||||
assertTrue(controller.isEnabled());
|
||||
assertTrue(controller.reconciled());
|
||||
assertFalse(controller.deleted());
|
||||
|
||||
var networkPolicy = controller.getReconciledResource();
|
||||
assertTrue(networkPolicy.isPresent());
|
||||
return networkPolicy.get();
|
||||
}
|
||||
|
||||
private static void disableManagement(Keycloak keycloak, boolean legacyOption) {
|
||||
if (legacyOption) {
|
||||
keycloak.getSpec().getAdditionalOptions().add(new ValueOrSecret("legacy-observability-interface", "true"));
|
||||
} else {
|
||||
keycloak.getSpec().getAdditionalOptions().add(new ValueOrSecret("health-enabled", "false"));
|
||||
}
|
||||
}
|
||||
|
||||
private static NetworkPolicyPeer podSelectorWithMatchLabel(String label, String value) {
|
||||
var builder = new NetworkPolicyPeerBuilder();
|
||||
builder.withNewPodSelector()
|
||||
.addToMatchLabels(label, value)
|
||||
.endPodSelector();
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private static NetworkPolicyPeer podSelectorWithMatchExpression(String key, String operator, String... values) {
|
||||
var builder = new NetworkPolicyPeerBuilder();
|
||||
var selector = builder.withNewPodSelector();
|
||||
selector.addNewMatchExpression()
|
||||
.withKey(key)
|
||||
.withOperator(operator)
|
||||
.addToValues(values)
|
||||
.endMatchExpression();
|
||||
selector.endPodSelector();
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private static NetworkPolicyPeer namespaceSelectorWithMatchLabel(String label, String value) {
|
||||
var builder = new NetworkPolicyPeerBuilder();
|
||||
builder.withNewPodSelector()
|
||||
.addToMatchLabels(label, value)
|
||||
.endPodSelector();
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private static NetworkPolicyPeer namespaceSelectorWithMatchExpressions(String key, String operator, String... values) {
|
||||
var builder = new NetworkPolicyPeerBuilder();
|
||||
var selector = builder.withNewNamespaceSelector();
|
||||
selector.addNewMatchExpression()
|
||||
.withKey(key)
|
||||
.withOperator(operator)
|
||||
.addToValues(values)
|
||||
.endMatchExpression();
|
||||
selector.endNamespaceSelector();
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private static NetworkPolicyPeer ipBlock(String cidr, String... except) {
|
||||
var builder = new NetworkPolicyPeerBuilder();
|
||||
var selector = builder.withNewIpBlock();
|
||||
selector.withCidr(cidr)
|
||||
.withExcept(except);
|
||||
selector.endIpBlock();
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
}
|
||||
@ -17,35 +17,41 @@
|
||||
|
||||
package org.keycloak.operator.testsuite.utils;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import io.fabric8.kubernetes.api.model.PodBuilder;
|
||||
import io.fabric8.kubernetes.api.model.Service;
|
||||
import io.fabric8.kubernetes.api.model.ServicePort;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicy;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyIngressRule;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyPeer;
|
||||
import io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicyPort;
|
||||
import io.fabric8.kubernetes.client.KubernetesClient;
|
||||
import io.fabric8.kubernetes.client.KubernetesClientException;
|
||||
import io.fabric8.kubernetes.client.dsl.ExecWatch;
|
||||
import io.fabric8.kubernetes.client.utils.Serialization;
|
||||
import io.quarkus.logging.Log;
|
||||
import org.assertj.core.api.ObjectAssert;
|
||||
import org.awaitility.Awaitility;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.keycloak.operator.Constants;
|
||||
import org.keycloak.operator.Utils;
|
||||
import org.keycloak.operator.controllers.KeycloakServiceDependentResource;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakStatus;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakStatusCondition;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.spec.NetworkPolicySpec;
|
||||
import org.keycloak.operator.crds.v2alpha1.realmimport.KeycloakRealmImport;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* @author Vaclav Muzikar <vmuzikar@redhat.com>
|
||||
@ -112,44 +118,30 @@ public final class CRAssert {
|
||||
.await()
|
||||
.pollInterval(1, TimeUnit.SECONDS)
|
||||
.timeout(Duration.ofMinutes(5))
|
||||
.untilAsserted(() -> {
|
||||
client.pods()
|
||||
.inNamespace(namespaceOf(keycloak))
|
||||
.withLabels(org.keycloak.operator.Utils.allInstanceLabels(keycloak))
|
||||
.resources()
|
||||
.forEach(pod -> {
|
||||
var logs = pod.getLog();
|
||||
var matcher = CLUSTER_SIZE_PATTERN.matcher(logs);
|
||||
int size = 0;
|
||||
// We want the last view change.
|
||||
// The other alternative is to reverse the string.
|
||||
while (matcher.find()) {
|
||||
size = Integer.parseInt(matcher.group(1));
|
||||
}
|
||||
Assertions.assertEquals(expectedSize, size, "Wrong cluster size in pod " + pod);
|
||||
});
|
||||
});
|
||||
.untilAsserted(() -> client.pods()
|
||||
.inNamespace(namespaceOf(keycloak))
|
||||
.withLabels(Utils.allInstanceLabels(keycloak))
|
||||
.resources()
|
||||
.forEach(pod -> {
|
||||
var logs = pod.getLog();
|
||||
var matcher = CLUSTER_SIZE_PATTERN.matcher(logs);
|
||||
int size = 0;
|
||||
// We want the last view change.
|
||||
// The other alternative is to reverse the string.
|
||||
while (matcher.find()) {
|
||||
size = Integer.parseInt(matcher.group(1));
|
||||
}
|
||||
Assertions.assertEquals(expectedSize, size, "Wrong cluster size in pod " + pod);
|
||||
}));
|
||||
}
|
||||
|
||||
public static void assertKeycloakAccessibleViaService(KubernetesClient client, Keycloak keycloak, boolean https, int port) {
|
||||
Awaitility.await()
|
||||
.ignoreExceptions()
|
||||
.untilAsserted(() -> {
|
||||
String protocol = https ? "https" : "http";
|
||||
var namespace = namespaceOf(keycloak);
|
||||
assertKeycloakAccessibleViaService(client, keycloak, null, Map.of(), https, port);
|
||||
}
|
||||
|
||||
String serviceName = KeycloakServiceDependentResource.getServiceName(keycloak);
|
||||
assertThat(client.resources(Service.class).withName(serviceName).require().getSpec().getPorts()
|
||||
.stream().map(ServicePort::getName).anyMatch(protocol::equals)).isTrue();
|
||||
|
||||
String url = protocol + "://" + serviceName + "." + namespace + ":" + port + "/admin/master/console/";
|
||||
Log.info("Checking url: " + url);
|
||||
|
||||
var curlOutput = K8sUtils.inClusterCurl(client, namespace, url);
|
||||
Log.info("Curl Output: " + curlOutput);
|
||||
|
||||
assertEquals("200", curlOutput);
|
||||
});
|
||||
public static void assertKeycloakAccessibleViaService(KubernetesClient client, Keycloak keycloak, String podNamespace, Map<String, String> labels, boolean https, int port) {
|
||||
var protocol = https ? "https" : "http";
|
||||
assertServiceAccessible(client, keycloak, podNamespace, labels, protocol, protocol, port, "/admin/master/console/");
|
||||
}
|
||||
|
||||
public static void assertManagementInterfaceAccessibleViaService(KubernetesClient client, Keycloak kc, boolean https) {
|
||||
@ -157,82 +149,139 @@ public final class CRAssert {
|
||||
}
|
||||
|
||||
public static void assertManagementInterfaceAccessibleViaService(KubernetesClient client, Keycloak keycloak, boolean https, int port) {
|
||||
assertManagementInterfaceAccessibleViaService(client, keycloak, null, Map.of(), https, port);
|
||||
}
|
||||
|
||||
public static void assertManagementInterfaceAccessibleViaService(KubernetesClient client, Keycloak keycloak, String podNamespace, Map<String, String> labels, boolean https, int port) {
|
||||
assertServiceAccessible(client, keycloak, podNamespace, labels, https ? "https" : "http", Constants.KEYCLOAK_MANAGEMENT_PORT_NAME, port, null);
|
||||
}
|
||||
|
||||
private static void assertServiceAccessible(KubernetesClient client, Keycloak keycloak, String podNamespace, Map<String, String> labels, String protocol, String portName, int port, String path) {
|
||||
Awaitility.await()
|
||||
.timeout(30, TimeUnit.SECONDS)
|
||||
.ignoreExceptions()
|
||||
.untilAsserted(() -> {
|
||||
String serviceName = KeycloakServiceDependentResource.getServiceName(keycloak);
|
||||
var serviceName = KeycloakServiceDependentResource.getServiceName(keycloak);
|
||||
var namespace = namespaceOf(keycloak);
|
||||
assertThat(client.resources(Service.class).withName(serviceName).require().getSpec().getPorts()
|
||||
.stream().map(ServicePort::getName).anyMatch(Constants.KEYCLOAK_MANAGEMENT_PORT_NAME::equals)).isTrue();
|
||||
.stream().map(ServicePort::getName).anyMatch(portName::equals)).isTrue();
|
||||
|
||||
String protocol = https ? "https" : "http";
|
||||
String url = protocol + "://" + serviceName + "." + namespace + ":" + port;
|
||||
var url = protocol + "://" + serviceName + "." + namespace + ":" + port;
|
||||
if (path != null) {
|
||||
url += path;
|
||||
}
|
||||
Log.info("Checking url: " + url);
|
||||
|
||||
var curlOutput = K8sUtils.inClusterCurl(client, namespace, url);
|
||||
var curlOutput = K8sUtils.inClusterCurl(client, podNamespace == null ? namespace : podNamespace, labels == null ? Map.of() : labels, url);
|
||||
Log.info("Curl Output: " + curlOutput);
|
||||
|
||||
assertEquals("200", curlOutput);
|
||||
});
|
||||
}
|
||||
|
||||
public static void assertKeycloakServiceBlocked(KubernetesClient client, Keycloak keycloak, String podNamespace, Map<String, String> labels, int port) {
|
||||
var serviceName = KeycloakServiceDependentResource.getServiceName(keycloak);
|
||||
var namespace = namespaceOf(keycloak);
|
||||
assertConnection(client, "%s.%s".formatted(serviceName, namespace), port, podNamespace, labels, false);
|
||||
}
|
||||
|
||||
public static void assertJGroupsConnection(KubernetesClient client, String podIp, String namespace, Map<String, String> labels, boolean connects) {
|
||||
// Send a bogus command to JGroups port
|
||||
// relevant exit codes:
|
||||
assertConnection(client, podIp ,7800, namespace, labels, connects);
|
||||
}
|
||||
|
||||
public static void assertConnection(KubernetesClient client, String hostname, int port, String namespace, Map<String, String> labels, boolean connects) {
|
||||
// Send a bogus command to the port
|
||||
var result = K8sUtils.inClusterCurl(client, namespace, labels, "--telnet-option",
|
||||
"'BOGUS=1'",
|
||||
"--connect-timeout",
|
||||
"2",
|
||||
"-s",
|
||||
"telnet://%s:%s".formatted(hostname, port));
|
||||
// Relevant exit codes:
|
||||
// 28-Operation timeout.
|
||||
// 48-Unknown option specified to libcurl.
|
||||
int expectedExitCode = connects ? 48 : 28;
|
||||
int exitCode;
|
||||
try {
|
||||
var builder = new PodBuilder();
|
||||
builder.withNewMetadata()
|
||||
.withName("curl-telnet-" + UUID.randomUUID())
|
||||
.withNamespace(namespace)
|
||||
.withLabels(labels)
|
||||
.endMetadata();
|
||||
builder.withNewSpec()
|
||||
.addNewContainer()
|
||||
.withImage("curlimages/curl:8.1.2")
|
||||
.withCommand("sh")
|
||||
.withName("curl")
|
||||
.withStdin()
|
||||
.endContainer()
|
||||
.endSpec();
|
||||
|
||||
var curlPod = builder.build();
|
||||
try {
|
||||
client.resource(curlPod).create();
|
||||
} catch (KubernetesClientException e) {
|
||||
if (e.getCode() != HttpURLConnection.HTTP_CONFLICT) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
var args = new String[]{
|
||||
"curl",
|
||||
"--telnet-option",
|
||||
"'BOGUS=1'",
|
||||
"--connect-timeout",
|
||||
"2",
|
||||
"-s",
|
||||
"telnet://%s:7800".formatted(podIp)
|
||||
};
|
||||
|
||||
Log.infof("Run telnet: %s", String.join(" ", args));
|
||||
|
||||
try (ExecWatch watch = client.pods().resource(curlPod).withReadyWaitTimeout(60000)
|
||||
.writingOutput(new ByteArrayOutputStream())
|
||||
.exec(args)) {
|
||||
exitCode = watch.exitCode().get(15, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
} catch (Exception ex) {
|
||||
throw KubernetesClientException.launderThrowable(ex);
|
||||
}
|
||||
assertEquals(expectedExitCode, exitCode);
|
||||
// 48-Unknown option specified to libcurl (BOGUS=1 is not a valid option, but the connection is successful).
|
||||
assertEquals(connects ? 48 : 28, result.exitCode());
|
||||
}
|
||||
|
||||
private static String namespaceOf(Keycloak keycloak) {
|
||||
return keycloak.getMetadata().getNamespace();
|
||||
}
|
||||
|
||||
public static void assertIngressRules(NetworkPolicy networkPolicy, Keycloak keycloak, int httpPort, int httpsPort, int mgntPort) {
|
||||
Log.info(networkPolicy);
|
||||
var expectedNumberOfRules = IntStream.of(httpPort, httpsPort, mgntPort)
|
||||
.filter(value -> value > 0)
|
||||
.count();
|
||||
|
||||
// +1 for JGRP
|
||||
++expectedNumberOfRules;
|
||||
|
||||
long numberOfRules = Optional.ofNullable(networkPolicy.getSpec())
|
||||
.map(io.fabric8.kubernetes.api.model.networking.v1.NetworkPolicySpec::getIngress)
|
||||
.map(List::size)
|
||||
.orElse(0);
|
||||
|
||||
assertEquals(expectedNumberOfRules, numberOfRules);
|
||||
|
||||
// Check selector
|
||||
assertPodSelectorAndPolicy(keycloak, networkPolicy);
|
||||
|
||||
// JGroups is always present
|
||||
assertJGroupsRulePresent(keycloak, networkPolicy);
|
||||
|
||||
if (httpPort > 0) {
|
||||
assertKeycloakEndpointRulePresent("HTTP", networkPolicy, NetworkPolicySpec.httpRules(keycloak).orElse(null), httpPort);
|
||||
}
|
||||
if (httpsPort > 0) {
|
||||
assertKeycloakEndpointRulePresent("HTTPS", networkPolicy, NetworkPolicySpec.httpsRules(keycloak).orElse(null), httpsPort);
|
||||
}
|
||||
if (mgntPort > 0) {
|
||||
assertKeycloakEndpointRulePresent("Management", networkPolicy, NetworkPolicySpec.managementRules(keycloak).orElse(null), mgntPort);
|
||||
}
|
||||
}
|
||||
|
||||
private static void assertPodSelectorAndPolicy(Keycloak keycloak, NetworkPolicy networkPolicy) {
|
||||
assertNotNull(networkPolicy, "Expects a network policy");
|
||||
assertEquals(Utils.allInstanceLabels(keycloak), networkPolicy.getSpec().getPodSelector().getMatchLabels(), "Expects same pod match labels");
|
||||
assertTrue(networkPolicy.getSpec().getPolicyTypes().contains("Ingress"), "Expect ingress polity type present");
|
||||
}
|
||||
|
||||
private static void assertKeycloakEndpointRulePresent(String name, NetworkPolicy networkPolicy, List<NetworkPolicyPeer> from, int mgmtPort) {
|
||||
var rule = findIngressRuleWithPort(networkPolicy, mgmtPort);
|
||||
assertTrue(rule.isPresent(), name + " Ingress Rule is missing");
|
||||
if (from == null || from.isEmpty()) {
|
||||
assertTrue(rule.get().getFrom().isEmpty());
|
||||
} else {
|
||||
assertEquals(from, rule.get().getFrom());
|
||||
}
|
||||
var ports = portAndProtocol(rule.get());
|
||||
assertEquals(Map.of(mgmtPort, Constants.KEYCLOAK_SERVICE_PROTOCOL), ports);
|
||||
}
|
||||
|
||||
private static void assertJGroupsRulePresent(Keycloak keycloak, NetworkPolicy networkPolicy) {
|
||||
var rule = findIngressRuleWithPort(networkPolicy, Constants.KEYCLOAK_JGROUPS_DATA_PORT);
|
||||
assertTrue(rule.isPresent(), "JGroups Ingress Rule is missing");
|
||||
|
||||
var from = rule.get().getFrom();
|
||||
assertEquals(1, from.size(), "Incorrect 'from' list size");
|
||||
assertEquals(Utils.allInstanceLabels(keycloak), from.get(0).getPodSelector().getMatchLabels());
|
||||
|
||||
var ports = portAndProtocol(rule.get());
|
||||
assertEquals(Map.of(
|
||||
Constants.KEYCLOAK_JGROUPS_DATA_PORT, Constants.KEYCLOAK_JGROUPS_PROTOCOL,
|
||||
Constants.KEYCLOAK_JGROUPS_FD_PORT, Constants.KEYCLOAK_JGROUPS_PROTOCOL
|
||||
), ports);
|
||||
}
|
||||
|
||||
private static Map<Integer, String> portAndProtocol(NetworkPolicyIngressRule rule) {
|
||||
return rule.getPorts().stream()
|
||||
.collect(Collectors.toMap(port -> port.getPort().getIntVal(), NetworkPolicyPort::getProtocol));
|
||||
}
|
||||
|
||||
private static Optional<NetworkPolicyIngressRule> findIngressRuleWithPort(NetworkPolicy networkPolicy, int rulePort) {
|
||||
return networkPolicy.getSpec().getIngress().stream()
|
||||
.filter(rule -> rule.getPorts().stream().anyMatch(port -> port.getPort().getIntVal() == rulePort))
|
||||
.findFirst();
|
||||
}
|
||||
}
|
||||
|
||||
@ -28,15 +28,21 @@ import io.fabric8.kubernetes.client.utils.Serialization;
|
||||
import io.quarkus.logging.Log;
|
||||
|
||||
import org.awaitility.Awaitility;
|
||||
import org.keycloak.operator.Constants;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.KeycloakStatusCondition;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.spec.HttpManagementSpecBuilder;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.spec.NetworkPolicySpecBuilder;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
@ -109,19 +115,32 @@ public final class K8sUtils {
|
||||
}
|
||||
|
||||
public static String inClusterCurl(KubernetesClient k8sclient, String namespace, String url) {
|
||||
return inClusterCurl(k8sclient, namespace, "--insecure", "-s", "-o", "/dev/null", "-w", "%{http_code}", url);
|
||||
return inClusterCurl(k8sclient, namespace, Map.of(), url);
|
||||
}
|
||||
|
||||
public static String inClusterCurl(KubernetesClient k8sclient, String namespace, String... args) {
|
||||
var podName = "curl-pod";
|
||||
public static String inClusterCurl(KubernetesClient k8sclient, String namespace, Map<String, String> labels, String url) {
|
||||
return inClusterCurl(k8sclient, namespace, labels, "--insecure", "-s", "-o", "/dev/null", "-w", "%{http_code}", url).stdout();
|
||||
}
|
||||
|
||||
public static String inClusterCurl(KubernetesClient k8sClient, String namespace, String... args) {
|
||||
return inClusterCurl(k8sClient, namespace, Map.of(), args).stdout();
|
||||
}
|
||||
|
||||
public static CurlResult inClusterCurl(KubernetesClient k8sClient, String namespace, Map<String, String> labels, String... args) {
|
||||
Log.infof("Starting cURL in namespace '%s' with labels '%s'", namespace, labels);
|
||||
var podName = "curl-pod-" + UUID.randomUUID();
|
||||
try {
|
||||
var builder = new PodBuilder();
|
||||
builder.withNewMetadata().withName(podName).endMetadata();
|
||||
builder.withNewMetadata()
|
||||
.withName(podName)
|
||||
.withNamespace(namespace)
|
||||
.withLabels(labels)
|
||||
.endMetadata();
|
||||
createCurlContainer(builder);
|
||||
var curlPod = builder.build();
|
||||
|
||||
try {
|
||||
k8sclient.resource(curlPod).create();
|
||||
k8sClient.resource(curlPod).create();
|
||||
} catch (KubernetesClientException e) {
|
||||
if (e.getCode() != HttpURLConnection.HTTP_CONFLICT) {
|
||||
throw e;
|
||||
@ -130,13 +149,14 @@ public final class K8sUtils {
|
||||
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
|
||||
try (ExecWatch watch = k8sclient.pods().resource(curlPod).withReadyWaitTimeout(60000)
|
||||
try (ExecWatch watch = k8sClient.pods().resource(curlPod).withReadyWaitTimeout(60000)
|
||||
.writingOutput(output)
|
||||
.exec(Stream.concat(Stream.of("curl"), Stream.of(args)).toArray(String[]::new))) {
|
||||
watch.exitCode().get(15, TimeUnit.SECONDS);
|
||||
var exitCode = watch.exitCode().get(15, TimeUnit.SECONDS);
|
||||
return new CurlResult(exitCode, output.toString(StandardCharsets.UTF_8));
|
||||
} finally {
|
||||
k8sClient.resource(curlPod).delete();
|
||||
}
|
||||
|
||||
return output.toString(StandardCharsets.UTF_8);
|
||||
} catch (Exception ex) {
|
||||
throw KubernetesClientException.launderThrowable(ex);
|
||||
}
|
||||
@ -152,4 +172,44 @@ public final class K8sUtils {
|
||||
.endContainer()
|
||||
.endSpec();
|
||||
}
|
||||
|
||||
public static void enableNetworkPolicy(Keycloak keycloak) {
|
||||
var builder = new NetworkPolicySpecBuilder();
|
||||
builder.withNetworkPolicyEnabled(true);
|
||||
keycloak.getSpec().setNetworkPolicySpec(builder.build());
|
||||
}
|
||||
|
||||
public static int configureManagement(Keycloak keycloak, boolean randomPort) {
|
||||
if (!randomPort) {
|
||||
return Constants.KEYCLOAK_MANAGEMENT_PORT;
|
||||
}
|
||||
var port = ThreadLocalRandom.current().nextInt(10_000, 10_100);
|
||||
keycloak.getSpec().setHttpManagementSpec(new HttpManagementSpecBuilder().withPort(port).build());
|
||||
return port;
|
||||
}
|
||||
|
||||
public static int enableHttp(Keycloak keycloak, boolean randomPort) {
|
||||
keycloak.getSpec().getHttpSpec().setHttpEnabled(true);
|
||||
if (randomPort) {
|
||||
var port = ThreadLocalRandom.current().nextInt(10_100, 10_200);
|
||||
keycloak.getSpec().getHttpSpec().setHttpPort(port);
|
||||
return port;
|
||||
}
|
||||
return Constants.KEYCLOAK_HTTP_PORT;
|
||||
}
|
||||
|
||||
public static int configureHttps(Keycloak keycloak, boolean randomPort) {
|
||||
if (!randomPort) {
|
||||
return Constants.KEYCLOAK_HTTPS_PORT;
|
||||
}
|
||||
var port = ThreadLocalRandom.current().nextInt(10_200, 10_300);
|
||||
keycloak.getSpec().getHttpSpec().setHttpsPort(port);
|
||||
return port;
|
||||
}
|
||||
|
||||
public static void disableHttps(Keycloak keycloak) {
|
||||
keycloak.getSpec().getHttpSpec().setTlsSecret(null);
|
||||
}
|
||||
|
||||
public record CurlResult(int exitCode, String stdout) {}
|
||||
}
|
||||
|
||||
@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Copyright 2024 Red Hat, Inc. and/or its affiliates
|
||||
* and other contributors as indicated by the @author tags.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.keycloak.operator.testsuite.utils;
|
||||
|
||||
import java.util.Optional;
|
||||
|
||||
import io.fabric8.kubernetes.api.model.HasMetadata;
|
||||
import io.javaoperatorsdk.operator.processing.dependent.kubernetes.KubernetesDependentResource;
|
||||
import org.keycloak.operator.crds.v2alpha1.deployment.Keycloak;
|
||||
|
||||
public abstract class MockController<T extends HasMetadata, R extends KubernetesDependentResource<T, Keycloak>> {
|
||||
|
||||
protected final R dependentResource;
|
||||
protected final Keycloak keycloak;
|
||||
private Status status;
|
||||
|
||||
protected MockController(R dependentResource, Keycloak keycloak) {
|
||||
this.dependentResource = dependentResource;
|
||||
this.keycloak = keycloak;
|
||||
this.status = Status.NEW;
|
||||
}
|
||||
|
||||
public boolean reconciled() {
|
||||
return getReconciledResource().isPresent();
|
||||
}
|
||||
|
||||
public boolean deleted() {
|
||||
return status == Status.DELETED;
|
||||
}
|
||||
|
||||
public void setExists() {
|
||||
status = Status.EXISTS;
|
||||
}
|
||||
|
||||
public Optional<T> getReconciledResource() {
|
||||
if (isEnabled()) {
|
||||
status = Status.EXISTS;
|
||||
return Optional.of(desired());
|
||||
}
|
||||
if (status == Status.EXISTS) {
|
||||
status = Status.DELETED;
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
protected abstract boolean isEnabled();
|
||||
|
||||
protected abstract T desired();
|
||||
|
||||
private enum Status {
|
||||
NEW,
|
||||
EXISTS,
|
||||
DELETED,
|
||||
}
|
||||
}
|
||||
@ -34,6 +34,39 @@ spec:
|
||||
anotherAnnotation: anotherValue
|
||||
networkPolicy:
|
||||
enabled: true
|
||||
http:
|
||||
- ipBlock:
|
||||
cidr: 172.17.0.0/16
|
||||
except:
|
||||
- 172.17.1.0/24
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
project: myproject
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
role: frontend
|
||||
https:
|
||||
- ipBlock:
|
||||
cidr: 172.17.0.0/16
|
||||
except:
|
||||
- 172.17.1.0/24
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
project: myproject
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
role: frontend
|
||||
management:
|
||||
- ipBlock:
|
||||
cidr: 172.17.0.0/16
|
||||
except:
|
||||
- 172.17.1.0/24
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
project: myproject
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
role: frontend
|
||||
http:
|
||||
httpEnabled: true
|
||||
httpPort: 123
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user