Support configurable principal claim in JWT Realm Tokens

docs/changelog/86533.yaml

@@ -0,0 +1,5 @@
+pr: 86533
+summary: Support configurable claims in JWT Realm Tokens
+area: Authentication
+type: enhancement
+issues: []

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java

@@ -24,7 +24,7 @@
 import java.util.stream.Stream;
 
 /**
- * Settings for JWT realms.
+ * Settings unique to each JWT realm.
  */
 public class JwtRealmSettings {
 
@@ -49,7 +49,7 @@ public enum ClientAuthenticationType {
         NONE("none"),
         SHARED_SECRET("shared_secret");
 
-        private String value;
+        private final String value;
 
         ClientAuthenticationType(String value) {
             this.value = value;
@@ -75,7 +75,7 @@ public static ClientAuthenticationType parse(String value, String settingKey) {
                     + "]"
             );
         }
-    };
+    }
 
     // Default values and min/max constraints
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmsServiceSettings.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.core.security.authc.jwt;
+
+import org.elasticsearch.common.settings.Setting;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Function;
+
+/**
+ * Settings used by JwtRealmsService for common handling of JWT realm instances.
+ */
+public class JwtRealmsServiceSettings {
+
+    public static final List<String> DEFAULT_PRINCIPAL_CLAIMS = List.of("sub", "oid", "client_id", "appid", "azp", "email");
+
+    public static final Setting<List<String>> PRINCIPAL_CLAIMS_SETTING = Setting.listSetting(
+        "xpack.security.authc.jwt.principal_claims",
+        DEFAULT_PRINCIPAL_CLAIMS,
+        Function.identity(),
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * Get all settings shared by all JWT Realms.
+     * @return All settings shared by all JWT Realms.
+     */
+    public static Collection<Setting<?>> getSettings() {
+        return List.of(PRINCIPAL_CLAIMS_SETTING);
+    }
+
+    private JwtRealmsServiceSettings() {}
+}

x-pack/plugin/security/qa/jwt-realm/build.gradle

@@ -37,6 +37,8 @@ testClusters.matching { it.name == 'javaRestTest' }.configureEach {
   setting 'xpack.security.http.ssl.certificate_authorities', 'ca.crt'
   setting 'xpack.security.http.ssl.client_authentication', 'optional'
 
+  setting 'xpack.security.authc.jwt.principal_claims', 'sub,oid,client_id,azp,appid,email'
+
   setting 'xpack.security.authc.realms.file.admin_file.order', '0'
 
   // These realm settings are generated by JwtRealmGenerateTests

x-pack/plugin/security/qa/jwt-realm/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRestIT.java

@@ -502,7 +502,8 @@ private JWTClaimsSet buildJwtForRealm2(String principal, Instant issueTime) {
         final String audience = "es0" + randomIntBetween(1, 3);
         final JWTClaimsSet claimsSet = buildJwt(
             Map.ofEntries(Map.entry("iss", "my-issuer"), Map.entry("aud", audience), Map.entry("email", emailAddress)),
-            issueTime
+            issueTime,
+            false
         );
         return claimsSet;
     }
@@ -563,9 +564,15 @@ private SignedJWT signHmacJwt(JWTClaimsSet claimsSet, String hmacPassphrase) thr
 
     // JWT construction
     private JWTClaimsSet buildJwt(Map<String, Object> claims, Instant issueTime) {
+        return buildJwt(claims, issueTime, true);
+    }
+
+    private JWTClaimsSet buildJwt(Map<String, Object> claims, Instant issueTime, boolean includeSub) {
         final JWTClaimsSet.Builder builder = new JWTClaimsSet.Builder();
         builder.issuer(randomAlphaOfLengthBetween(4, 24));
-        builder.subject(randomAlphaOfLengthBetween(4, 24));
+        if (includeSub) {
+            builder.subject(randomAlphaOfLengthBetween(4, 24));
+        }
         builder.audience(randomList(1, 6, () -> randomAlphaOfLengthBetween(4, 12)));
         if (randomBoolean()) {
             builder.jwtID(UUIDs.randomBase64UUID(random()));

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -151,6 +151,7 @@
 import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.RealmSettings;
+import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmsServiceSettings;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField;
@@ -640,6 +641,7 @@ Collection<Object> createComponents(
         Map<String, Realm.Factory> realmFactories = new HashMap<>(
             InternalRealms.getFactories(
                 threadPool,
+                settings,
                 resourceWatcherService,
                 getSslService(),
                 nativeUsersStore,
@@ -1056,6 +1058,7 @@ public static List<Setting<?>> getSettings(List<SecurityExtension> securityExten
         // authentication and authorization settings
         AnonymousUser.addSettings(settingsList);
         settingsList.addAll(InternalRealmsSettings.getSettings());
+        settingsList.addAll(JwtRealmsServiceSettings.getSettings());
         ReservedRealm.addSettings(settingsList);
         AuthenticationService.addSettings(settingsList);
         AuthorizationService.addSettings(settingsList);

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/InternalRealms.java

@@ -32,7 +32,7 @@
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.security.authc.file.FileRealm;
-import org.elasticsearch.xpack.security.authc.jwt.JwtRealm;
+import org.elasticsearch.xpack.security.authc.jwt.JwtRealmsService;
 import org.elasticsearch.xpack.security.authc.kerberos.KerberosRealm;
 import org.elasticsearch.xpack.security.authc.ldap.LdapRealm;
 import org.elasticsearch.xpack.security.authc.oidc.OpenIdConnectRealm;
@@ -130,13 +130,14 @@ static LicensedFeature.Persistent getLicensedFeature(String type) {
      */
     public static Map<String, Realm.Factory> getFactories(
         ThreadPool threadPool,
+        Settings settings,
         ResourceWatcherService resourceWatcherService,
         SSLService sslService,
         NativeUsersStore nativeUsersStore,
         NativeRoleMappingStore nativeRoleMappingStore,
         SecurityIndexManager securityIndex
     ) {
-
+        final JwtRealmsService jwtRealmsService = new JwtRealmsService(settings); // parse shared settings needed by all JwtRealm instances
         return Map.of(
             // file realm
             FileRealmSettings.TYPE,
@@ -168,7 +169,7 @@ public static Map<String, Realm.Factory> getFactories(
             config -> new OpenIdConnectRealm(config, sslService, nativeRoleMappingStore, resourceWatcherService),
             // JWT realm
             JwtRealmSettings.TYPE,
-            config -> new JwtRealm(config, sslService, nativeRoleMappingStore)
+            config -> jwtRealmsService.createJwtRealm(config, sslService, nativeRoleMappingStore)
         );
     }
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtAuthenticationToken.java

@@ -9,33 +9,48 @@
 import com.nimbusds.jwt.JWTClaimsSet;
 import com.nimbusds.jwt.SignedJWT;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
+import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmsServiceSettings;
 
 import java.text.ParseException;
 import java.util.List;
+import java.util.Map;
 import java.util.TreeSet;
+import java.util.stream.Collectors;
 
 /**
  * An {@link AuthenticationToken} to hold JWT authentication related content.
  */
 public class JwtAuthenticationToken implements AuthenticationToken {
+    private static final Logger LOGGER = LogManager.getLogger(JwtAuthenticationToken.class);
 
     // Stored members
     protected SecureString endUserSignedJwt; // required
     protected SecureString clientAuthenticationSharedSecret; // optional, nullable
-    protected String principal; // "iss/aud/sub"
+    protected String principal; // Defaults to "iss/aud/sub", with an ordered "aud" list
 
     /**
      * Store a mandatory JWT and optional Shared Secret. Parse the JWT, and extract the header, claims set, and signature.
-     * Throws IllegalArgumentException if bearerString is missing, or if JWT parsing fails.
+     * Compute a token principal, for use as a realm order cache key. For OIDC ID Tokens, cache key is iss/aud/sub.
+     * For other JWTs, {@link JwtRealmsServiceSettings#PRINCIPAL_CLAIMS_SETTING} supports alternative claims for sub.
+     * Throws IllegalArgumentException if principalClaimNames is empty, JWT is missing, or if JWT parsing fails.
+     * @param principalClaimNames Ordered list of string claims to use for principalClaimValue. The first one found is used (ex: sub).
      * @param endUserSignedJwt Base64Url-encoded JWT for End-user authentication. Required by all JWT realms.
      * @param clientAuthenticationSharedSecret URL-safe Shared Secret for Client authentication. Required by some JWT realms.
      */
-    public JwtAuthenticationToken(final SecureString endUserSignedJwt, @Nullable final SecureString clientAuthenticationSharedSecret) {
-        if (endUserSignedJwt.isEmpty()) {
+    public JwtAuthenticationToken(
+        final List<String> principalClaimNames,
+        final SecureString endUserSignedJwt,
+        @Nullable final SecureString clientAuthenticationSharedSecret
+    ) {
+        if (principalClaimNames.isEmpty()) {
+            throw new IllegalArgumentException("JWT token principal claim names list must be non-empty");
+        } else if (endUserSignedJwt.isEmpty()) {
             throw new IllegalArgumentException("JWT bearer token must be non-empty");
         } else if ((clientAuthenticationSharedSecret != null) && (clientAuthenticationSharedSecret.isEmpty())) {
             throw new IllegalArgumentException("Client shared secret must be non-empty");
@@ -49,18 +64,61 @@ public JwtAuthenticationToken(final SecureString endUserSignedJwt, @Nullable fin
         } catch (ParseException e) {
             throw new IllegalArgumentException("Failed to parse JWT bearer token", e);
         }
+
+        // get and validate iss and aud claims
         final String issuer = jwtClaimsSet.getIssuer();
         final List<String> audiences = jwtClaimsSet.getAudience();
-        final String subject = jwtClaimsSet.getSubject();
-
         if (Strings.hasText(issuer) == false) {
             throw new IllegalArgumentException("Issuer claim 'iss' is missing.");
         } else if ((audiences == null) || (audiences.isEmpty())) {
             throw new IllegalArgumentException("Audiences claim 'aud' is missing.");
-        } else if (Strings.hasText(subject) == false) {
-            throw new IllegalArgumentException("Subject claim 'sub' is missing.");
         }
-        this.principal = issuer + "/" + String.join(",", new TreeSet<>(audiences)) + "/" + subject;
+
+        // get and validate sub claim, or the first configured backup claim (if sub is absent)
+        final String principalClaimValue = this.resolvePrincipalClaimName(jwtClaimsSet, principalClaimNames);
+        this.principal = issuer + "/" + String.join(",", new TreeSet<>(audiences)) + "/" + principalClaimValue;
+    }
+
+    private String resolvePrincipalClaimName(final JWTClaimsSet jwtClaimsSet, final List<String> principalClaimNames) {
+        for (final String principalClaimName : principalClaimNames) {
+            final Object claimValue = jwtClaimsSet.getClaim(principalClaimName);
+            if (claimValue instanceof String principalClaimValue) {
+                // found an allowed string claim name
+                if (principalClaimValue.isEmpty()) {
+                    throw new IllegalArgumentException(
+                        "Allowed principal claim name '"
+                            + principalClaimName
+                            + "' exists but cannot be used because the value of that claim is an empty string"
+                    );
+                }
+                LOGGER.trace("Found allowed principal claim name [{}] with value [{}]", principalClaimName, principalClaimValue);
+                return principalClaimValue;
+            } else if (claimValue != null) {
+                throw new IllegalArgumentException(
+                    "Allowed principal claim name '"
+                        + principalClaimName
+                        + "' exists but cannot be used because the value of that claim must be a string, but instead it was a ["
+                        + claimValue.getClass().getSimpleName()
+                        + "]"
+                );
+            }
+        }
+
+        // at this point, none of the principalClaimNames were found
+        // throw an exception with a detailed log message about which string claims were available in the JWT
+        final String allClaimNamesWithStringValues = jwtClaimsSet.getClaims()
+            .entrySet()
+            .stream()
+            .filter(e -> e.getValue() instanceof String)
+            .map(Map.Entry::getKey)
+            .collect(Collectors.joining(","));
+        throw new IllegalArgumentException(
+            "None of these configured principal claim names were found in the JWT Claims Set ["
+                + String.join(",", principalClaimNames)
+                + "] - available claims in the JWT with potential compatible string values are ["
+                + allClaimNamesWithStringValues
+                + "]"
+        );
     }
 
     @Override

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java

@@ -73,6 +73,7 @@ boolean isEmpty() {
     public static final String HEADER_END_USER_AUTHENTICATION_SCHEME = "Bearer";
     public static final String HEADER_SHARED_SECRET_AUTHENTICATION_SCHEME = "SharedSecret";
 
+    private final JwtRealmsService jwtRealmsService;
     final UserRoleMapper userRoleMapper;
     final String allowedIssuer;
     final List<String> allowedAudiences;
@@ -93,9 +94,14 @@ boolean isEmpty() {
     final CacheIteratorHelper<BytesKey, ExpiringUser> jwtCacheHelper;
     DelegatedAuthorizationSupport delegatedAuthorizationSupport = null;
 
-    public JwtRealm(final RealmConfig realmConfig, final SSLService sslService, final UserRoleMapper userRoleMapper)
-        throws SettingsException {
+    JwtRealm(
+        final RealmConfig realmConfig,
+        final JwtRealmsService jwtRealmsService,
+        final SSLService sslService,
+        final UserRoleMapper userRoleMapper
+    ) throws SettingsException {
         super(realmConfig);
+        this.jwtRealmsService = jwtRealmsService; // common configuration settings shared by all JwtRealm instances
         this.userRoleMapper = userRoleMapper;
         this.userRoleMapper.refreshRealmOnChange(this);
         this.allowedIssuer = realmConfig.getSetting(JwtRealmSettings.ALLOWED_ISSUER);
@@ -328,23 +334,10 @@ public void expireAll() {
     @Override
     public AuthenticationToken token(final ThreadContext threadContext) {
         this.ensureInitialized();
-        final SecureString authenticationParameterValue = JwtUtil.getHeaderValue(
-            threadContext,
-            JwtRealm.HEADER_END_USER_AUTHENTICATION,
-            JwtRealm.HEADER_END_USER_AUTHENTICATION_SCHEME,
-            false
-        );
-        if (authenticationParameterValue == null) {
-            return null;
-        }
-        // Get all other possible parameters. A different JWT realm may do the actual authentication.
-        final SecureString clientAuthenticationSharedSecretValue = JwtUtil.getHeaderValue(
-            threadContext,
-            JwtRealm.HEADER_CLIENT_AUTHENTICATION,
-            JwtRealm.HEADER_SHARED_SECRET_AUTHENTICATION_SCHEME,
-            true
-        );
-        return new JwtAuthenticationToken(authenticationParameterValue, clientAuthenticationSharedSecretValue);
+        // Token parsing is common code for all realms
+        // First JWT realm will parse in a way that is compatible with all JWT realms,
+        // taking into consideration each JWT realm might have a different principal claim name
+        return this.jwtRealmsService.token(threadContext);
     }
 
     @Override