http proxy support in JWT realm (#127337)

http proxy support in JWT realm

docs/changelog/127337.yaml

@@ -0,0 +1,6 @@
+pr: 127337
+summary: Http proxy support in JWT realm
+area: Authentication
+type: enhancement
+issues:
+ - 114956

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java

@@ -29,6 +29,7 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static org.elasticsearch.xpack.core.security.authc.support.SecuritySettingsUtil.verifyNonNullNotEmpty;
+import static org.elasticsearch.xpack.core.security.authc.support.SecuritySettingsUtil.verifyProxySettings;
 
 /**
  * Settings unique to each JWT realm.
@@ -193,7 +194,10 @@ public class JwtRealmSettings {
                 HTTP_CONNECTION_READ_TIMEOUT,
                 HTTP_SOCKET_TIMEOUT,
                 HTTP_MAX_CONNECTIONS,
-                HTTP_MAX_ENDPOINT_CONNECTIONS
+                HTTP_MAX_ENDPOINT_CONNECTIONS,
+                HTTP_PROXY_SCHEME,
+                HTTP_PROXY_HOST,
+                HTTP_PROXY_PORT
             )
         );
         // Standard TLS connection settings for outgoing connections to get JWT issuer jwkset_path
@@ -481,6 +485,49 @@ public class JwtRealmSettings {
         key -> Setting.intSetting(key, DEFAULT_HTTP_MAX_ENDPOINT_CONNECTIONS, MIN_HTTP_MAX_ENDPOINT_CONNECTIONS, Setting.Property.NodeScope)
     );
 
+    public static final Setting.AffixSetting<String> HTTP_PROXY_HOST = Setting.affixKeySetting(
+        RealmSettings.realmSettingPrefix(TYPE),
+        "http.proxy.host",
+        key -> Setting.simpleString(key, new Setting.Validator<>() {
+            @Override
+            public void validate(String value) {
+                // There is no point in validating the hostname in itself without the scheme and port
+            }
+
+            @Override
+            public void validate(String value, Map<Setting<?>, Object> settings) {
+                verifyProxySettings(key, value, settings, HTTP_PROXY_HOST, HTTP_PROXY_SCHEME, HTTP_PROXY_PORT);
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                final String namespace = HTTP_PROXY_HOST.getNamespace(HTTP_PROXY_HOST.getConcreteSetting(key));
+                final List<Setting<?>> settings = List.of(
+                    HTTP_PROXY_PORT.getConcreteSettingForNamespace(namespace),
+                    HTTP_PROXY_SCHEME.getConcreteSettingForNamespace(namespace)
+                );
+                return settings.iterator();
+            }
+        }, Setting.Property.NodeScope)
+    );
+    public static final Setting.AffixSetting<Integer> HTTP_PROXY_PORT = Setting.affixKeySetting(
+        RealmSettings.realmSettingPrefix(TYPE),
+        "http.proxy.port",
+        key -> Setting.intSetting(key, 80, 1, 65535, Setting.Property.NodeScope),
+        () -> HTTP_PROXY_HOST
+    );
+    public static final Setting.AffixSetting<String> HTTP_PROXY_SCHEME = Setting.affixKeySetting(
+        RealmSettings.realmSettingPrefix(TYPE),
+        "http.proxy.scheme",
+        key -> Setting.simpleString(
+            key,
+            "http",
+            // TODO allow HTTPS once https://github.com/elastic/elasticsearch/issues/100264 is fixed
+            value -> verifyNonNullNotEmpty(key, value, List.of("http")),
+            Setting.Property.NodeScope
+        )
+    );
+
     // SSL Configuration settings
 
     public static final Collection<Setting.AffixSetting<?>> SSL_CONFIGURATION_SETTINGS = SSLConfigurationSettings.getRealmSettings(TYPE);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/oidc/OpenIdConnectRealmSettings.java

@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.xpack.core.security.authc.oidc;
 
-import org.apache.http.HttpHost;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.set.Sets;
@@ -14,6 +13,7 @@ import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.ClaimSetting;
 import org.elasticsearch.xpack.core.security.authc.support.DelegatedAuthorizationSettings;
+import org.elasticsearch.xpack.core.security.authc.support.SecuritySettingsUtil;
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;
 
 import java.net.URI;
@@ -234,32 +234,7 @@ public class OpenIdConnectRealmSettings {
 
             @Override
             public void validate(String value, Map<Setting<?>, Object> settings) {
-                final String namespace = HTTP_PROXY_HOST.getNamespace(HTTP_PROXY_HOST.getConcreteSetting(key));
+                SecuritySettingsUtil.verifyProxySettings(key, value, settings, HTTP_PROXY_HOST, HTTP_PROXY_SCHEME, HTTP_PROXY_PORT);
-                final Setting<Integer> portSetting = HTTP_PROXY_PORT.getConcreteSettingForNamespace(namespace);
-                final Integer port = (Integer) settings.get(portSetting);
-                final Setting<String> schemeSetting = HTTP_PROXY_SCHEME.getConcreteSettingForNamespace(namespace);
-                final String scheme = (String) settings.get(schemeSetting);
-                try {
-                    new HttpHost(value, port, scheme);
-                } catch (Exception e) {
-                    throw new IllegalArgumentException(
-                        "HTTP host for hostname ["
-                            + value
-                            + "] (from ["
-                            + key
-                            + "]),"
-                            + " port ["
-                            + port
-                            + "] (from ["
-                            + portSetting.getKey()
-                            + "]) and "
-                            + "scheme ["
-                            + scheme
-                            + "] (from (["
-                            + schemeSetting.getKey()
-                            + "]) is invalid"
-                    );
-                }
             }
 
             @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/SecuritySettingsUtil.java

@@ -7,8 +7,12 @@
 
 package org.elasticsearch.xpack.core.security.authc.support;
 
+import org.apache.http.HttpHost;
+import org.elasticsearch.common.settings.Setting;
+
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Utilities for validating security settings.
@@ -85,6 +89,45 @@ public final class SecuritySettingsUtil {
         }
     }
 
+    public static void verifyProxySettings(
+        String key,
+        String hostValue,
+        Map<Setting<?>, Object> settings,
+        Setting.AffixSetting<String> hostKey,
+        Setting.AffixSetting<String> schemeKey,
+        Setting.AffixSetting<Integer> portKey
+    ) {
+        final String namespace = hostKey.getNamespace(hostKey.getConcreteSetting(key));
+
+        final Setting<Integer> portSetting = portKey.getConcreteSettingForNamespace(namespace);
+        final Integer port = (Integer) settings.get(portSetting);
+
+        final Setting<String> schemeSetting = schemeKey.getConcreteSettingForNamespace(namespace);
+        final String scheme = (String) settings.get(schemeSetting);
+
+        try {
+            new HttpHost(hostValue, port, scheme);
+        } catch (Exception e) {
+            throw new IllegalArgumentException(
+                "HTTP host for hostname ["
+                    + hostValue
+                    + "] (from ["
+                    + key
+                    + "]),"
+                    + " port ["
+                    + port
+                    + "] (from ["
+                    + portSetting.getKey()
+                    + "]) and "
+                    + "scheme ["
+                    + scheme
+                    + "] (from (["
+                    + schemeSetting.getKey()
+                    + "]) is invalid"
+            );
+        }
+    }
+
     private SecuritySettingsUtil() {
         throw new IllegalAccessError("not allowed!");
     }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java

@@ -15,6 +15,7 @@ import com.nimbusds.jwt.JWT;
 import com.nimbusds.jwt.SignedJWT;
 
 import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
 import org.apache.http.StatusLine;
 import org.apache.http.client.config.RequestConfig;
@@ -27,6 +28,7 @@ import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
 import org.apache.http.impl.nio.client.HttpAsyncClients;
 import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager;
 import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor;
+import org.apache.http.nio.conn.NoopIOSessionStrategy;
 import org.apache.http.nio.conn.SchemeIOSessionStrategy;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
 import org.apache.http.nio.reactor.ConnectingIOReactor;
@@ -74,6 +76,10 @@ import java.util.function.Supplier;
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.SSLContext;
 
+import static org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings.HTTP_PROXY_HOST;
+import static org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings.HTTP_PROXY_PORT;
+import static org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings.HTTP_PROXY_SCHEME;
+
 /**
  * Utilities for JWT realm.
  */
@@ -271,6 +277,7 @@ public class JwtUtil {
                 final SSLContext clientContext = sslService.sslContext(sslConfiguration);
                 final HostnameVerifier verifier = SSLService.getHostnameVerifier(sslConfiguration);
                 final Registry<SchemeIOSessionStrategy> registry = RegistryBuilder.<SchemeIOSessionStrategy>create()
+                    .register("http", NoopIOSessionStrategy.INSTANCE)
                     .register("https", new SSLIOSessionStrategy(clientContext, verifier))
                     .build();
                 final PoolingNHttpClientConnectionManager connectionManager = new PoolingNHttpClientConnectionManager(ioReactor, registry);
@@ -286,6 +293,15 @@ public class JwtUtil {
                 final HttpAsyncClientBuilder httpAsyncClientBuilder = HttpAsyncClients.custom()
                     .setConnectionManager(connectionManager)
                     .setDefaultRequestConfig(requestConfig);
+                if (realmConfig.hasSetting(HTTP_PROXY_HOST)) {
+                    httpAsyncClientBuilder.setProxy(
+                        new HttpHost(
+                            realmConfig.getSetting(HTTP_PROXY_HOST),
+                            realmConfig.getSetting(HTTP_PROXY_PORT),
+                            realmConfig.getSetting(HTTP_PROXY_SCHEME)
+                        )
+                    );
+                }
                 final CloseableHttpAsyncClient httpAsyncClient = httpAsyncClientBuilder.build();
                 httpAsyncClient.start();
                 return httpAsyncClient;

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSettingsTests.java

@@ -23,11 +23,14 @@ import java.util.List;
 import java.util.Locale;
 
 import static org.elasticsearch.common.Strings.capitalize;
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.startsWith;
 
 /**
  * JWT realm settings unit tests. These are low-level tests against ES settings parsers.
@@ -588,4 +591,59 @@ public class JwtRealmSettingsTests extends JwtTestCase {
 
         assertThat(e.getMessage(), containsString("required claim [" + fullSettingKey + "] cannot be empty"));
     }
+
+    public void testInvalidProxySchemeThrowsError() {
+        final String scheme = randomBoolean() ? "https" : randomAlphaOfLengthBetween(3, 8);
+        final String realmName = randomAlphaOfLengthBetween(3, 8);
+        final String proxySchemeSettingKey = RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.HTTP_PROXY_SCHEME);
+        final Settings settings = Settings.builder().put(proxySchemeSettingKey, scheme).build();
+
+        final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, randomInt());
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> realmConfig.getSetting(JwtRealmSettings.HTTP_PROXY_SCHEME)
+        );
+
+        assertThat(
+            e.getMessage(),
+            equalTo(Strings.format("Invalid value [%s] for [%s]. Allowed values are [http].", scheme, proxySchemeSettingKey))
+        );
+    }
+
+    public void testInvalidProxyHostThrowsError() {
+        final int proxyPort = randomIntBetween(1, 65535);
+        final String realmName = randomAlphaOfLengthBetween(3, 8);
+        final String proxyPortSettingKey = RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.HTTP_PROXY_PORT);
+        final String proxyHostSettingKey = RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.HTTP_PROXY_HOST);
+        final Settings settings = Settings.builder().put(proxyHostSettingKey, "not a url").put(proxyPortSettingKey, proxyPort).build();
+
+        final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, randomInt());
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> realmConfig.getSetting(JwtRealmSettings.HTTP_PROXY_HOST)
+        );
+
+        assertThat(
+            e.getMessage(),
+            allOf(startsWith(Strings.format("HTTP host for hostname [not a url] (from [%s])", proxyHostSettingKey)), endsWith("is invalid"))
+        );
+    }
+
+    public void testInvalidProxyPortThrowsError() {
+        final int proxyPort = randomFrom(randomIntBetween(Integer.MIN_VALUE, -1), randomIntBetween(65536, Integer.MAX_VALUE));
+        final String realmName = randomAlphaOfLengthBetween(3, 8);
+        final String proxyPortSettingKey = RealmSettings.getFullSettingKey(realmName, JwtRealmSettings.HTTP_PROXY_PORT);
+        final Settings settings = Settings.builder().put(proxyPortSettingKey, proxyPort).build();
+
+        final RealmConfig realmConfig = buildRealmConfig(JwtRealmSettings.TYPE, realmName, settings, randomInt());
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> realmConfig.getSetting(JwtRealmSettings.HTTP_PROXY_PORT)
+        );
+
+        assertThat(
+            e.getMessage(),
+            startsWith(Strings.format("Failed to parse value [%d] for setting [%s]", proxyPort, proxyPortSettingKey))
+        );
+    }
 }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtTestCase.java

@@ -98,12 +98,14 @@ public abstract class JwtTestCase extends ESTestCase {
         final boolean includePublicKey = includeRsa || includeEc;
         final boolean includeHmac = randomBoolean() || (includePublicKey == false); // one of HMAC/RSA/EC must be true
         final boolean populateUserMetadata = randomBoolean();
+        final boolean useJwksEndpoint = randomBoolean();
+        final boolean useProxy = useJwksEndpoint && randomBoolean();
         final Path jwtSetPathObj = PathUtils.get(pathHome);
-        final String jwkSetPath = randomBoolean()
+        final String jwkSetPath = useJwksEndpoint
             ? "https://op.example.com/jwkset.json"
             : Files.createTempFile(jwtSetPathObj, "jwkset.", ".json").toString();
 
-        if (jwkSetPath.equals("https://op.example.com/jwkset.json") == false) {
+        if (useJwksEndpoint == false) {
             Files.writeString(PathUtils.get(jwkSetPath), "Non-empty JWK Set Path contents");
         }
         final ClientAuthenticationType clientAuthenticationType = randomFrom(ClientAuthenticationType.values());
@@ -195,6 +197,16 @@ public abstract class JwtTestCase extends ESTestCase {
             .put(RealmSettings.getFullSettingKey(name, SSLConfigurationSettings.TRUSTSTORE_ALGORITHM.realm(JwtRealmSettings.TYPE)), "PKIX")
             .put(RealmSettings.getFullSettingKey(name, SSLConfigurationSettings.CERT_AUTH_PATH.realm(JwtRealmSettings.TYPE)), "ca2.pem");
 
+        if (useProxy) {
+            if (randomBoolean()) {
+                // Scheme is optional, and defaults to HTTP
+                settingsBuilder.put(RealmSettings.getFullSettingKey(name, JwtRealmSettings.HTTP_PROXY_SCHEME), "http");
+            }
+
+            settingsBuilder.put(RealmSettings.getFullSettingKey(name, JwtRealmSettings.HTTP_PROXY_HOST), "localhost/proxy")
+                .put(RealmSettings.getFullSettingKey(name, JwtRealmSettings.HTTP_PROXY_PORT), randomIntBetween(1, 65535));
+        }
+
         final MockSecureSettings secureSettings = new MockSecureSettings();
         if (includeHmac) {
             if (randomBoolean()) {

x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtWithOidcAuthIT.java

@@ -47,13 +47,17 @@ import static org.hamcrest.Matchers.hasKey;
  */
 public class JwtWithOidcAuthIT extends C2IdOpTestCase {
 
-    // configured in the Elasticearch node test fixture
+    // configured in the Elasticsearch node test fixture
     private static final List<String> ALLOWED_AUDIENCES = List.of("elasticsearch-jwt1", "elasticsearch-jwt2");
-    private static final String JWT_REALM_NAME = "op-jwt";
+    private static final String JWT_FILE_REALM_NAME = "op-jwt";
+    private static final String JWT_PROXY_REALM_NAME = "op-jwt-proxy";
 
     // Constants for role mapping
-    private static final String ROLE_NAME = "jwt_role";
+    private static final String FILE_ROLE_NAME = "jwt_role";
-    private static final String SHARED_SECRET = "jwt-realm-shared-secret";
+    private static final String FILE_SHARED_SECRET = "jwt-realm-shared-secret";
+
+    private static final String PROXY_ROLE_NAME = "jwt_proxy_role";
+    private static final String PROXY_SHARED_SECRET = "jwt-proxy-realm-shared-secret";
 
     // Randomised values
     private static String clientId;
@@ -79,10 +83,10 @@ public class JwtWithOidcAuthIT extends C2IdOpTestCase {
     }
 
     @Before
-    public void setupRoleMapping() throws Exception {
+    public void setupRoleMappings() throws Exception {
         try (var restClient = getElasticsearchClient()) {
             var client = new TestSecurityClient(restClient);
-            final String mappingJson = Strings.format("""
+            String mappingJson = Strings.format("""
                 {
                   "roles": [ "%s" ],
                   "enabled": true,
@@ -93,8 +97,22 @@ public class JwtWithOidcAuthIT extends C2IdOpTestCase {
                     ]
                   }
                 }
-                """, ROLE_NAME, JWT_REALM_NAME, TEST_SUBJECT_ID);
+                """, FILE_ROLE_NAME, JWT_FILE_REALM_NAME, TEST_SUBJECT_ID);
-            client.putRoleMapping(getTestName(), mappingJson);
+            client.putRoleMapping(FILE_ROLE_NAME, mappingJson);
+
+            mappingJson = Strings.format("""
+                {
+                  "roles": [ "%s" ],
+                  "enabled": true,
+                  "rules": {
+                    "all": [
+                      { "field": { "realm.name": "%s" } },
+                      { "field": { "metadata.jwt_claim_sub": "%s" } }
+                    ]
+                  }
+                }
+                """, PROXY_ROLE_NAME, JWT_PROXY_REALM_NAME, TEST_SUBJECT_ID);
+            client.putRoleMapping(PROXY_ROLE_NAME, mappingJson);
         }
     }
 
@@ -127,15 +145,21 @@ public class JwtWithOidcAuthIT extends C2IdOpTestCase {
         assertThat("Hash value of URI [" + implicitFlowURI + "] should be a JWT with an id Token", hashParams, hasKey("id_token"));
         String idJwt = hashParams.get("id_token");
 
-        final Map<String, Object> authenticateResponse = authenticateWithJwtAndSharedSecret(idJwt, SHARED_SECRET);
+        final Map<String, Object> authenticateResponse = authenticateWithJwtAndSharedSecret(idJwt, FILE_SHARED_SECRET);
         assertThat(authenticateResponse, Matchers.hasEntry(User.Fields.USERNAME.getPreferredName(), TEST_SUBJECT_ID));
         assertThat(authenticateResponse, Matchers.hasKey(User.Fields.ROLES.getPreferredName()));
-        assertThat((List<?>) authenticateResponse.get(User.Fields.ROLES.getPreferredName()), contains(ROLE_NAME));
+        assertThat((List<?>) authenticateResponse.get(User.Fields.ROLES.getPreferredName()), contains(FILE_ROLE_NAME));
+
+        // test that the proxy realm successfully loads the JWKS
+        final Map<String, Object> proxyAuthenticateResponse = authenticateWithJwtAndSharedSecret(idJwt, PROXY_SHARED_SECRET);
+        assertThat(proxyAuthenticateResponse, Matchers.hasEntry(User.Fields.USERNAME.getPreferredName(), TEST_SUBJECT_ID));
+        assertThat(proxyAuthenticateResponse, Matchers.hasKey(User.Fields.ROLES.getPreferredName()));
+        assertThat((List<?>) proxyAuthenticateResponse.get(User.Fields.ROLES.getPreferredName()), contains(PROXY_ROLE_NAME));
 
         // Use an incorrect shared secret and check it fails
         ResponseException ex = expectThrows(
             ResponseException.class,
-            () -> authenticateWithJwtAndSharedSecret(idJwt, "not-" + SHARED_SECRET)
+            () -> authenticateWithJwtAndSharedSecret(idJwt, "not-" + FILE_SHARED_SECRET)
         );
         assertThat(ex.getResponse(), TestMatchers.hasStatusCode(RestStatus.UNAUTHORIZED));
 
@@ -144,7 +168,7 @@ public class JwtWithOidcAuthIT extends C2IdOpTestCase {
         assertThat(dot, greaterThan(0));
         // change the first character of the payload section of the encoded JWT
         final String corruptToken = idJwt.substring(0, dot) + "." + transformChar(idJwt.charAt(dot + 1)) + idJwt.substring(dot + 2);
-        ex = expectThrows(ResponseException.class, () -> authenticateWithJwtAndSharedSecret(corruptToken, SHARED_SECRET));
+        ex = expectThrows(ResponseException.class, () -> authenticateWithJwtAndSharedSecret(corruptToken, FILE_SHARED_SECRET));
         assertThat(ex.getResponse(), TestMatchers.hasStatusCode(RestStatus.UNAUTHORIZED));
     }
 

x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/C2IdOpTestCase.java

@@ -75,7 +75,7 @@ public abstract class C2IdOpTestCase extends ESRestTestCase {
 
     private static final String CLIENT_SECRET = "b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2";
 
-    private static Network network = Network.newNetwork();
+    private static final Network network = Network.newNetwork();
     protected static OidcProviderTestContainer c2id = new OidcProviderTestContainer(network);
     protected static HttpProxyTestContainer proxy = new HttpProxyTestContainer(network);
 
@@ -165,6 +165,17 @@ public abstract class C2IdOpTestCase extends ESRestTestCase {
         .setting("xpack.security.authc.realms.jwt.op-jwt.claims.principal", "sub")
         .setting("xpack.security.authc.realms.jwt.op-jwt.claims.groups", "groups")
         .setting("xpack.security.authc.realms.jwt.op-jwt.client_authentication.type", "shared_secret")
+        .setting("xpack.security.authc.realms.jwt.op-jwt-proxy.order", "8")
+        .setting("xpack.security.authc.realms.jwt.op-jwt-proxy.allowed_issuer", () -> c2id.getC2IssuerUrl())
+        .setting("xpack.security.authc.realms.jwt.op-jwt-proxy.allowed_audiences", "elasticsearch-jwt1,elasticsearch-jwt2")
+        .setting("xpack.security.authc.realms.jwt.op-jwt-proxy.pkc_jwkset_path", () -> c2id.getC2IDSslUrl() + "/jwks.json")
+        .setting("xpack.security.authc.realms.jwt.op-jwt-proxy.claims.principal", "sub")
+        .setting("xpack.security.authc.realms.jwt.op-jwt-proxy.claims.groups", "groups")
+        .setting("xpack.security.authc.realms.jwt.op-jwt-proxy.client_authentication.type", "shared_secret")
+        .setting("xpack.security.authc.realms.jwt.op-jwt-proxy.http.proxy.scheme", "http")
+        .setting("xpack.security.authc.realms.jwt.op-jwt-proxy.http.proxy.host", "localhost")
+        .setting("xpack.security.authc.realms.jwt.op-jwt-proxy.http.proxy.port", () -> proxy.getTlsPort().toString())
+        .setting("xpack.security.authc.realms.jwt.op-jwt-proxy.ssl.keystore.path", "testnode.jks")
         .keystore("bootstrap.password", "x-pack-test-password")
         .keystore("xpack.security.http.ssl.keystore.secure_password", "testnode")
         .keystore("xpack.security.authc.realms.oidc.c2id.rp.client_secret", CLIENT_SECRET)
@@ -173,6 +184,8 @@ public abstract class C2IdOpTestCase extends ESRestTestCase {
         .keystore("xpack.security.authc.realms.oidc.c2id-post.rp.client_secret", CLIENT_SECRET)
         .keystore("xpack.security.authc.realms.oidc.c2id-jwt.rp.client_secret", CLIENT_SECRET)
         .keystore("xpack.security.authc.realms.jwt.op-jwt.client_authentication.shared_secret", "jwt-realm-shared-secret")
+        .keystore("xpack.security.authc.realms.jwt.op-jwt-proxy.client_authentication.shared_secret", "jwt-proxy-realm-shared-secret")
+        .keystore("xpack.security.authc.realms.jwt.op-jwt-proxy.ssl.keystore.secure_password", "testnode")
         .configFile("testnode.jks", Resource.fromClasspath("ssl/testnode.jks"))
         .configFile("op-jwks.json", Resource.fromClasspath("op-jwks.json"))
         .user("x_pack_rest_user", "x-pack-test-password", "superuser", false)

x-pack/test/idp-fixture/build.gradle

@@ -19,4 +19,6 @@ dependencies {
   testImplementation project(':test:framework')
   api project(':test:fixtures:testcontainer-utils')
   api "junit:junit:${versions.junit}"
+
+  runtimeOnly "net.java.dev.jna:jna:${versions.jna}"
 }

x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/HttpProxyTestContainer.java

@@ -13,8 +13,8 @@ import org.testcontainers.images.builder.ImageFromDockerfile;
 
 public final class HttpProxyTestContainer extends DockerEnvironmentAwareTestContainer {
 
-    public static final String DOCKER_BASE_IMAGE = "nginx:latest";
     private static final Integer PORT = 8888;
+    private static final Integer TLS_PORT = 8889;
 
     /**
      * for packer caching only
@@ -25,15 +25,18 @@ public final class HttpProxyTestContainer extends DockerEnvironmentAwareTestCont
 
     public HttpProxyTestContainer(Network network) {
         super(
-            new ImageFromDockerfile("es-http-proxy-fixture").withDockerfileFromBuilder(
+            new ImageFromDockerfile("es-http-proxy-fixture").withFileFromClasspath("Dockerfile", "nginx/Dockerfile")
-                builder -> builder.from(DOCKER_BASE_IMAGE).copy("oidc/nginx.conf", "/etc/nginx/nginx.conf").build()
+                .withFileFromClasspath("nginx/nginx.conf", "/nginx/nginx.conf")
-            ).withFileFromClasspath("oidc/nginx.conf", "/oidc/nginx.conf")
         );
-        addExposedPort(PORT);
+        addExposedPorts(PORT, TLS_PORT);
         withNetwork(network);
     }
 
     public Integer getProxyPort() {
         return getMappedPort(PORT);
     }
+
+    public Integer getTlsPort() {
+        return getMappedPort(TLS_PORT);
+    }
 }

x-pack/test/idp-fixture/src/main/java/org/elasticsearch/test/fixtures/idp/OidcProviderTestContainer.java

@@ -15,6 +15,7 @@ import org.testcontainers.images.builder.Transferable;
 public final class OidcProviderTestContainer extends DockerEnvironmentAwareTestContainer {
 
     private static final int PORT = 8080;
+    private static final int SSL_PORT = 8443;
 
     /**
      * for packer caching only
@@ -26,13 +27,14 @@ public final class OidcProviderTestContainer extends DockerEnvironmentAwareTestC
     public OidcProviderTestContainer(Network network) {
         super(
             new ImageFromDockerfile("es-oidc-provider-fixture").withFileFromClasspath("oidc/setup.sh", "/oidc/setup.sh")
+                .withFileFromClasspath("oidc/testnode.jks", "/oidc/testnode.jks")
                 // we cannot make use of docker file builder
                 // as it does not support multi-stage builds
                 .withFileFromClasspath("Dockerfile", "oidc/Dockerfile")
         );
         withNetworkAliases("oidc-provider");
         withNetwork(network);
-        addExposedPort(PORT);
+        addExposedPorts(PORT, SSL_PORT);
     }
 
     @Override
@@ -47,7 +49,6 @@ public final class OidcProviderTestContainer extends DockerEnvironmentAwareTestC
                     + getMappedPort(PORT)
                     + "/c2id-login/\n"
                     + "op.reg.apiAccessTokenSHA256=d1c4fa70d9ee708d13cfa01daa0e060a05a2075a53c5cc1ad79e460e96ab5363\n"
-                    + "jose.jwkSer=RnVsbCBrZXk6CnsKICAia2V5cyI6IFsKICAgIHsKICAgICAgInAiOiAiLXhhN2d2aW5tY3N3QXU3Vm1mV2loZ2o3U3gzUzhmd2dFSTdMZEVveW5FU1RzcElaeUY5aHc0NVhQZmI5VHlpbzZsOHZTS0F5RmU4T2lOalpkNE1Ra0ttYlJzTmxxR1Y5VlBoWF84UG1JSm5mcGVhb3E5YnZfU0k1blZHUl9zYUUzZE9sTEE2VWpaS0lsRVBNb0ZuRlZCMUFaUU9qQlhRRzZPTDg2eDZ2NHMwIiwKICAgICAgImt0eSI6ICJSU0EiLAogICAgICAicSI6ICJ2Q3pDQUlpdHV0MGx1V0djQloyLUFabURLc1RxNkkxcUp0RmlEYkIyZFBNQVlBNldOWTdaWEZoVWxsSjJrT2ZELWdlYjlkYkN2ODBxNEwyajVZSjZoOTBUc1NRWWVHRlljN1lZMGdCMU5VR3l5cXctb29QN0EtYlJmMGI3b3I4ajZJb0hzQTZKa2JranN6c3otbkJ2U2RmUURlZkRNSVc3Ni1ZWjN0c2hsY2MiLAogICAgICAiZCI6ICJtbFBOcm1zVVM5UmJtX1I5SElyeHdmeFYzZnJ2QzlaQktFZzRzc1ZZaThfY09lSjV2U1hyQV9laEtwa2g4QVhYaUdWUGpQbVlyd29xQzFVUksxUkZmLVg0dG10emV2OUVHaU12Z0JCaEF5RkdTSUd0VUNla2x4Q2dhb3BpMXdZSU1Bd0M0STZwMUtaZURxTVNCWVZGeHA5ZWlJZ2pwb05JbV9lR3hXUUs5VHNnYmk5T3lyc1VqaE9KLVczN2JVMEJWUU56UXpxODhCcGxmNzM3VmV1dy1FeDZaMk1iWXR3SWdfZ0JVb0JEZ0NrZkhoOVE4MElYcEZRV0x1RzgwenFrdkVwTHZ0RWxLbDRvQ3BHVnBjcmFUOFNsOGpYc3FDT1k0dnVRT19LRVUzS2VPNUNJbHd4eEhJYXZjQTE5cHFpSWJ5cm1LbThxS0ZEWHluUFJMSGFNZ1EiLAogICAgICAiZSI6ICJBUUFCIiwKICAgICAgImtpZCI6ICJyc2EzODRfMjA0OCIsCiAgICAgICJxaSI6ICJzMldTamVrVDl3S2JPbk9neGNoaDJPY3VubzE2Y20wS281Z3hoUWJTdVMyMldfUjJBR2ZVdkRieGF0cTRLakQ3THo3X1k2TjdTUkwzUVpudVhoZ1djeXgyNGhrUGppQUZLNmlkYVZKQzJqQmgycEZTUDVTNXZxZ0lsME12eWY4NjlwdkN4S0NzaGRKMGdlRWhveE93VkRPYXJqdTl2Zm9IQV90LWJoRlZrUnciLAogICAgICAiZHAiOiAiQlJhQTFqYVRydG9mTHZBSUJBYW1OSEVhSm51RU9zTVJJMFRCZXFuR1BNUm0tY2RjSG1OUVo5WUtqb2JpdXlmbnhGZ0piVDlSeElBRG0ySkpoZEp5RTN4Y1dTSzhmSjBSM1Jick1aT1dwako0QmJTVzFtU1VtRnlKTGxib3puRFhZR2RaZ1hzS0o1UkFrRUNQZFBCY3YwZVlkbk9NYWhfZndfaFZoNjRuZ2tFIiwKICAgICAgImFsZyI6ICJSU0EzODQiLAogICAgICAiZHEiOiAiUFJoVERKVlR3cDNXaDZfWFZrTjIwMUlpTWhxcElrUDN1UTYyUlRlTDNrQ2ZXSkNqMkZPLTRxcVRIQk0tQjZJWUVPLXpoVWZyQnhiMzJ1djNjS2JDWGFZN3BJSFJxQlFEQWQ2WGhHYzlwc0xqNThXd3VGY2RncERJYUFpRjNyc3NUMjJ4UFVvYkJFTVdBalV3bFJrNEtNTjItMnpLQk5FR3lIcDIzOUpKdnpVIiwKICAgICAgIm4iOiAidUpDWDVDbEZpM0JnTXBvOWhRSVZ2SDh0Vi1jLTVFdG5OeUZxVm91R3NlNWwyUG92MWJGb0tsRllsU25YTzNWUE9KRWR3azNDdl9VT0UtQzlqZERYRHpvS3Z4RURaTVM1TDZWMFpIVEJoNndIOV9iN3JHSlBxLV9RdlNkejczSzZxbHpGaUtQamRvdTF6VlFYTmZfblBZbnRnQkdNRUtBc1pRNGp0cWJCdE5lV0h0MF9UM001cEktTV9KNGVlRWpCTW95TkZuU2ExTEZDVmZRNl9YVnpjelp1TlRGMlh6UmdRWkFmcmJGRXZ6eXR1TzVMZTNTTXFrUUFJeDhFQmkwYXVlRUNqNEQ4cDNVNXFVRG92NEF2VnRJbUZlbFJvb1pBMHJtVW1KRHJ4WExrVkhuVUpzaUF6ZW9TLTNBSnV1bHJkMGpuNjJ5VjZHV2dFWklZMVNlZVd3IgogICAgfQogIF0KfQo\n"
                     + "op.authz.alwaysPromptForConsent=true\n"
                     + "op.authz.alwaysPromptForAuth=true"
             ),
@@ -63,4 +64,7 @@ public final class OidcProviderTestContainer extends DockerEnvironmentAwareTestC
         return getC2OPUrl() + "/c2id";
     }
 
+    public String getC2IDSslUrl() {
+        return "https://127.0.0.1:" + getMappedPort(SSL_PORT) + "/c2id";
+    }
 }

x-pack/test/idp-fixture/src/main/resources/nginx/Dockerfile

@@ -0,0 +1,37 @@
+# We need to do SSL tunnelling, but NGINX doesn't support this out of the box - there's a module for this, but the only
+#  way to install it is to compile NGINX from source, so here we go
+
+FROM nginx:1.27.1
+
+RUN set -x \
+    && apt-get update \
+    # libs and tools that we'll need to build NGINX
+    && apt-get install --no-install-recommends --no-install-suggests -y  \
+      zlib1g-dev libpcre2-dev libssl-dev wget git gcc patch make  \
+    && wget http://nginx.org/download/nginx-1.27.1.tar.gz \
+    && git clone --depth 1 --branch v0.0.7 --single-branch https://github.com/chobits/ngx_http_proxy_connect_module.git \
+    && tar -xzvf nginx-1.27.1.tar.gz \
+    && cd nginx-1.27.1 \
+    # patch the CONNECT module in
+    && patch -p1 < ../ngx_http_proxy_connect_module/patch/proxy_connect_rewrite_102101.patch \
+    && ./configure \
+      --sbin-path=/usr/local/bin/nginx \
+      --conf-path=/etc/nginx/nginx.conf \
+      --pid-path=/etc/nginx/nginx.pid \
+      --with-http_ssl_module \
+      --http-log-path=/dev/stdout \
+      --error-log-path=/dev/stderr \
+      --add-module=../ngx_http_proxy_connect_module \
+    && make && make install \
+    && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/*
+
+COPY ./nginx/nginx.conf /etc/nginx/nginx.conf
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
+EXPOSE 8888
+EXPOSE 8889
+
+STOPSIGNAL SIGQUIT
+
+CMD ["nginx", "-g", "daemon off;"]

x-pack/test/idp-fixture/src/main/resources/nginx/nginx.conf

@@ -14,5 +14,26 @@ http {
           proxy_pass http://$ophost;
       }
   }
+
+  server {
+    set $ophost "oidc-provider:8443";
+
+    listen 8889;
+
+    # dns resolver used by forward proxying
+    resolver                       127.0.0.11;
+
+    # forward proxy for CONNECT request
+    proxy_connect;
+    proxy_connect_allow            1-65535;
+    proxy_connect_connect_timeout  10s;
+    proxy_connect_data_timeout     10s;
+    proxy_connect_address          $ophost;
+
+    location / {
+        resolver 127.0.0.11;
+        proxy_pass https://$ophost;
+    }
+  }
 }
 

x-pack/test/idp-fixture/src/main/resources/oidc/Dockerfile

@@ -1,5 +1,5 @@
-FROM c2id/c2id-server-demo:12.18 AS c2id
+FROM c2id/c2id-server-demo:16.1.1 AS c2id
-FROM openjdk:11.0.16-jre
+FROM openjdk:21-jdk-buster
 
 # Using this to launch a fake server on container start; see `setup.sh`
 RUN apt-get update -qqy && apt-get install -qqy python3
@@ -7,7 +7,28 @@ RUN apt-get update -qqy && apt-get install -qqy python3
 COPY --from=c2id /c2id-server /c2id-server
 COPY --from=c2id /etc/c2id /etc/c2id
 COPY ./oidc/setup.sh /fixture/
+COPY ./oidc/testnode.jks /c2id-server/tomcat/conf/keystore.jks
+
+RUN sed -i '/<!-- A "Connector" using the shared thread pool-->/ i\
+    <Connector port="8443" \
+               protocol="org.apache.coyote.http11.Http11NioProtocol" \
+               SSLEnabled="true" \
+               maxThreads="150" \
+               scheme="https" \
+               secure="true" \
+               clientAuth="false" \
+               sslProtocol="TLS" \
+               sslEnabledProtocols="TLSv1.3"> \
+        <SSLHostConfig> \
+            <Certificate \
+                certificateKeystoreFile="/c2id-server/tomcat/conf/keystore.jks" \
+                certificateKeystorePassword="testnode" \
+                type="RSA" /> \
+        </SSLHostConfig> \
+    </Connector>' \
+    /c2id-server/tomcat/conf/server.xml
 
 ENV CATALINA_OPTS="-DsystemPropertiesURL=file:///config/c2id/override.properties"
 EXPOSE 8080
+EXPOSE 8443
 CMD ["/bin/bash", "/fixture/setup.sh"]

x-pack/test/idp-fixture/src/main/resources/oidc/setup.sh

@@ -7,12 +7,15 @@
 
 #!/bin/bash
 
-# HACK: we start serving on 8080 so that we can progress to the postProcessFixture step. That's the step during which
+# HACK: we start serving on 8080 & 8443 so that we can progress to the postProcessFixture step. That's the step during which
 # we have access to the ephemeral port of the container, which we need to properly configure the issuer field in c2id
 # config
 python3 -m http.server 8080 &
 PY_PID=$!
 
+python3 -m http.server 8443 &
+PY_PID_2=$!
+
 until [ -f /config/c2id/override.properties ]
 do
   echo "Waiting for properties file"
@@ -21,5 +24,5 @@ done
 echo "Properties file available. Starting server..."
 
 # now that the properties file is configured and available, stop our fake server and launch the real thing
-kill -SIGKILL $PY_PID
+kill -SIGKILL $PY_PID $PY_PID_2
 bash /c2id-server/tomcat/bin/catalina.sh run