Allow to use custom TrustManagerFactory for JdkSslServerContent and OpenSslServerContext

Motivation:

When using client auth it is sometimes needed to use a custom TrustManagerFactory.

Modifications:

Allow to pass in TrustManagerFactory

Result:

It's now possible to use custom TrustManagerFactories for JdkSslServerContext and OpenSslServerContext

handler/src/main/java/io/netty/handler/ssl/OpenSslServerContext.java

@@ -67,7 +67,8 @@ public final class OpenSslServerContext extends OpenSslContext {
      *                    {@code null} if it's not password-protected.
      */
     public OpenSslServerContext(File certChainFile, File keyFile, String keyPassword) throws SSLException {
-        this(certChainFile, keyFile, keyPassword, null, OpenSslDefaultApplicationProtocolNegotiator.INSTANCE, 0, 0);
+        this(certChainFile, keyFile, keyPassword, null, null,
+            OpenSslDefaultApplicationProtocolNegotiator.INSTANCE, 0, 0);
     }
 
     /**
@@ -89,7 +90,8 @@ public final class OpenSslServerContext extends OpenSslContext {
             File certChainFile, File keyFile, String keyPassword,
             Iterable<String> ciphers, ApplicationProtocolConfig apn,
             long sessionCacheSize, long sessionTimeout) throws SSLException {
-        this(certChainFile, keyFile, keyPassword, ciphers, toNegotiator(apn, false), sessionCacheSize, sessionTimeout);
+        this(certChainFile, keyFile, keyPassword, null, ciphers,
+            toNegotiator(apn, false), sessionCacheSize, sessionTimeout);
     }
 
     /**
@@ -117,7 +119,7 @@ public final class OpenSslServerContext extends OpenSslContext {
             Iterable<String> ciphers, Iterable<String> nextProtocols,
             long sessionCacheSize, long sessionTimeout) throws SSLException {
         this(certChainFile, keyFile, keyPassword, ciphers,
-             toApplicationProtocolConfig(nextProtocols), sessionCacheSize, sessionTimeout);
+            toApplicationProtocolConfig(nextProtocols), sessionCacheSize, sessionTimeout);
     }
 
     /**
@@ -136,12 +138,12 @@ public final class OpenSslServerContext extends OpenSslContext {
      *                       {@code 0} to use the default value.
      */
     public OpenSslServerContext(
-            File certChainFile, File keyFile, String keyPassword,
+            File certChainFile, File keyFile, String keyPassword, TrustManagerFactory trustManagerFactory,
             Iterable<String> ciphers, OpenSslApplicationProtocolNegotiator apn,
             long sessionCacheSize, long sessionTimeout) throws SSLException {
 
-        super(ciphers, apn, sessionCacheSize, sessionTimeout, SSL.SSL_MODE_SERVER);
+         super(ciphers, apn, sessionCacheSize, sessionTimeout, SSL.SSL_MODE_SERVER);
-        OpenSsl.ensureAvailability();
+         OpenSsl.ensureAvailability();
 
         checkNotNull(certChainFile, "certChainFile");
         if (!certChainFile.isFile()) {
@@ -218,11 +220,16 @@ public final class OpenSslServerContext extends OpenSslContext {
 
                     ks.setKeyEntry("key", key, keyPasswordChars, certChain.toArray(new Certificate[certChain.size()]));
 
-                    // This mimics the behavior of using SSLContext.init(...);
+                    if (trustManagerFactory == null) {
-                    TrustManagerFactory factory = TrustManagerFactory.getInstance(
+                        // Mimic the way SSLContext.getInstance(KeyManager[], null, null) works
-                            TrustManagerFactory.getDefaultAlgorithm());
+                        trustManagerFactory = TrustManagerFactory.getInstance(
-                    factory.init((KeyStore) null);
+                                TrustManagerFactory.getDefaultAlgorithm());
-                    final X509TrustManager manager = chooseTrustManager(factory.getTrustManagers());
+                        trustManagerFactory.init((KeyStore) null);
+                    } else {
+                        trustManagerFactory.init(ks);
+                    }
+
+                    final X509TrustManager manager = chooseTrustManager(trustManagerFactory.getTrustManagers());
                     SSLContext.setCertVerifyCallback(ctx, new CertificateVerifier() {
                         @Override
                         public boolean verify(long ssl, byte[][] chain, String auth) {

handler/src/main/java/io/netty/handler/ssl/SslContext.java

@@ -16,9 +16,7 @@
 
 package io.netty.handler.ssl;
 
-import io.netty.buffer.ByteBuf;
 import io.netty.buffer.ByteBufAllocator;
-import io.netty.buffer.ByteBufInputStream;
 import io.netty.channel.ChannelInitializer;
 import io.netty.channel.ChannelPipeline;
 import io.netty.handler.ssl.ApplicationProtocolConfig.Protocol;
@@ -38,17 +36,13 @@ import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLException;
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.TrustManagerFactory;
-import javax.security.auth.x500.X500Principal;
 import java.io.File;
 import java.io.IOException;
 import java.security.InvalidAlgorithmParameterException;
 import java.security.InvalidKeyException;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
 import java.security.cert.CertificateException;
 import java.security.cert.CertificateFactory;
-import java.security.cert.X509Certificate;
 import java.security.spec.InvalidKeySpecException;
 import java.security.spec.PKCS8EncodedKeySpec;
 import java.util.List;
@@ -246,10 +240,42 @@ public abstract class SslContext {
             File certChainFile, File keyFile, String keyPassword,
             Iterable<String> ciphers, Iterable<String> nextProtocols,
             long sessionCacheSize, long sessionTimeout) throws SSLException {
+        return newServerContext(provider, certChainFile, keyFile, keyPassword,
+                ciphers, IdentityCipherSuiteFilter.INSTANCE,
+                toApplicationProtocolConfig(nextProtocols), sessionCacheSize, sessionTimeout);
+    }
+
+    /**
+     * Creates a new server-side {@link SslContext}.
+     *
+     * @param provider the {@link SslContext} implementation to use.
+     *                 {@code null} to use the current default one.
+     * @param certChainFile an X.509 certificate chain file in PEM format
+     * @param keyFile a PKCS#8 private key file in PEM format
+     * @param keyPassword the password of the {@code keyFile}.
+     *                    {@code null} if it's not password-protected.
+     * @param trustManagerFactory the {@link TrustManagerFactory} that provides the {@link javax.net.ssl.TrustManager}s
+     *                            that verifies the certificates sent from servers.
+     *                            {@code null} to use the default.
+     * @param ciphers the cipher suites to enable, in the order of preference.
+     *                {@code null} to use the default cipher suites.
+     * @param nextProtocols the application layer protocols to accept, in the order of preference.
+     *                      {@code null} to disable TLS NPN/ALPN extension.
+     * @param sessionCacheSize the size of the cache used for storing SSL session objects.
+     *                         {@code 0} to use the default value.
+     * @param sessionTimeout the timeout for the cached SSL session objects, in seconds.
+     *                       {@code 0} to use the default value.
+     * @return a new server-side {@link SslContext}
+     */
+    public static SslContext newServerContext(
+            SslProvider provider,
+            File certChainFile, File keyFile, String keyPassword, TrustManagerFactory trustManagerFactory,
+            Iterable<String> ciphers, Iterable<String> nextProtocols,
+            long sessionCacheSize, long sessionTimeout) throws SSLException {
 
         return newServerContext(
-                provider, certChainFile, keyFile, keyPassword,
+                provider, null, trustManagerFactory, certChainFile, keyFile, keyPassword,
-                ciphers, IdentityCipherSuiteFilter.INSTANCE,
+                null, ciphers, IdentityCipherSuiteFilter.INSTANCE,
                 toApplicationProtocolConfig(nextProtocols), sessionCacheSize, sessionTimeout);
     }
 
@@ -329,9 +355,6 @@ public abstract class SslContext {
                     trustCertChainFile, trustManagerFactory, keyCertChainFile, keyFile, keyPassword,
                     keyManagerFactory, ciphers, cipherFilter, apn, sessionCacheSize, sessionTimeout);
         case OPENSSL:
-            if (trustCertChainFile != null) {
-                throw new UnsupportedOperationException("OpenSSL provider does not support mutual authentication");
-            }
             return new OpenSslServerContext(
                     keyCertChainFile, keyFile, keyPassword,
                     ciphers, apn, sessionCacheSize, sessionTimeout);