Ensure X509KeyManager methods are called on the correct time when using server-side and support more methods of ExtendedSSLSession. (#8283)

Motivation:

Before when on server-side we just called the X509KeyManager methods when handshake() was called the first time which is not quite correct as we may not have received the full SSL hello / handshake and so could not extra for example the SNI hostname that was requested.
OpenSSL exposes the SSL_CTX_set_cert_cb function which allows to set a callback which is executed at the correct moment, so we should use it. This also allows us to support more methods of ExtendedSSLSession easily.

Modifications:

- Make use of new methods exposed by netty-tcnative since https://github.com/netty/netty-tcnative/pull/388 to ensure we select the key material at the correct time.
- Implement more methods of ExtendedOpenSslSession
- Add unit tests to ensure we are able to retrieve various things on server-side in the X509KeyManager and so verify it is called at the correct time.
- Simplify code by using new netty-tcnative methods.

Result:

More correct implementation for server-side usage and more complete implemented of ExtendedSSLSession.

handler/src/main/java/io/netty/handler/ssl/ExtendedOpenSslSession.java

@@ -15,8 +15,6 @@
  */
 package io.netty.handler.ssl;
 
-import io.netty.util.internal.EmptyArrays;
-
 import javax.net.ssl.ExtendedSSLSession;
 import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLPeerUnverifiedException;
@@ -60,128 +58,122 @@ abstract class ExtendedOpenSslSession extends ExtendedSSLSession implements Open
     }
 
     @Override
-    public void handshakeFinished() throws SSLException {
+    public final void handshakeFinished() throws SSLException {
         wrapped.handshakeFinished();
     }
 
     @Override
-    public void tryExpandApplicationBufferSize(int packetLengthDataOnly) {
+    public final void tryExpandApplicationBufferSize(int packetLengthDataOnly) {
         wrapped.tryExpandApplicationBufferSize(packetLengthDataOnly);
     }
 
     @Override
-    public String[] getLocalSupportedSignatureAlgorithms() {
+    public final String[] getLocalSupportedSignatureAlgorithms() {
         return LOCAL_SUPPORTED_SIGNATURE_ALGORITHMS.clone();
     }
 
     @Override
-    public String[] getPeerSupportedSignatureAlgorithms() {
+    public final byte[] getId() {
-        // Always return empty for now.
-        return EmptyArrays.EMPTY_STRINGS;
-    }
-
-    @Override
-    public byte[] getId() {
         return wrapped.getId();
     }
 
     @Override
-    public SSLSessionContext getSessionContext() {
+    public final SSLSessionContext getSessionContext() {
         return wrapped.getSessionContext();
     }
 
     @Override
-    public long getCreationTime() {
+    public final long getCreationTime() {
         return wrapped.getCreationTime();
     }
 
     @Override
-    public long getLastAccessedTime() {
+    public final long getLastAccessedTime() {
         return wrapped.getLastAccessedTime();
     }
 
     @Override
-    public void invalidate() {
+    public final void invalidate() {
         wrapped.invalidate();
     }
 
     @Override
-    public boolean isValid() {
+    public final boolean isValid() {
         return wrapped.isValid();
     }
 
     @Override
-    public void putValue(String s, Object o) {
+    public final void putValue(String s, Object o) {
         wrapped.putValue(s, o);
     }
 
     @Override
-    public Object getValue(String s) {
+    public final Object getValue(String s) {
         return wrapped.getValue(s);
     }
 
     @Override
-    public void removeValue(String s) {
+    public final void removeValue(String s) {
         wrapped.removeValue(s);
     }
 
     @Override
-    public String[] getValueNames() {
+    public final String[] getValueNames() {
         return wrapped.getValueNames();
     }
 
     @Override
-    public Certificate[] getPeerCertificates() throws SSLPeerUnverifiedException {
+    public final Certificate[] getPeerCertificates() throws SSLPeerUnverifiedException {
         return wrapped.getPeerCertificates();
     }
 
     @Override
-    public Certificate[] getLocalCertificates() {
+    public final Certificate[] getLocalCertificates() {
         return wrapped.getLocalCertificates();
     }
 
     @Override
-    public X509Certificate[] getPeerCertificateChain() throws SSLPeerUnverifiedException {
+    public final X509Certificate[] getPeerCertificateChain() throws SSLPeerUnverifiedException {
         return wrapped.getPeerCertificateChain();
     }
 
     @Override
-    public Principal getPeerPrincipal() throws SSLPeerUnverifiedException {
+    public final Principal getPeerPrincipal() throws SSLPeerUnverifiedException {
         return wrapped.getPeerPrincipal();
     }
 
     @Override
-    public Principal getLocalPrincipal() {
+    public final Principal getLocalPrincipal() {
         return wrapped.getLocalPrincipal();
     }
 
     @Override
-    public String getCipherSuite() {
+    public final String getCipherSuite() {
         return wrapped.getCipherSuite();
     }
 
     @Override
-    public String getProtocol() {
+    public final String getProtocol() {
         return wrapped.getProtocol();
     }
 
     @Override
-    public String getPeerHost() {
+    public final String getPeerHost() {
         return wrapped.getPeerHost();
     }
 
     @Override
-    public int getPeerPort() {
+    public final int getPeerPort() {
         return wrapped.getPeerPort();
     }
 
     @Override
-    public int getPacketBufferSize() {
+    public final int getPacketBufferSize() {
         return wrapped.getPacketBufferSize();
     }
 
     @Override
-    public int getApplicationBufferSize() {
+    public final int getApplicationBufferSize() {
         return wrapped.getApplicationBufferSize();
     }
 }

handler/src/main/java/io/netty/handler/ssl/Java8SslUtils.java

@@ -62,6 +62,13 @@ final class Java8SslUtils {
         return sniServerNames;
     }
 
+    static List getSniHostName(byte[] hostname) {
+        if (hostname == null || hostname.length == 0) {
+            return Collections.emptyList();
+        }
+        return Collections.singletonList(new SNIHostName(hostname));
+    }
+
     static boolean getUseCipherSuitesOrder(SSLParameters sslParameters) {
         return sslParameters.getUseCipherSuitesOrder();
     }

handler/src/main/java/io/netty/handler/ssl/OpenSslClientContext.java

@@ -203,9 +203,4 @@ public final class OpenSslClientContext extends OpenSslContext {
     public OpenSslSessionContext sessionContext() {
         return sessionContext;
     }
-
-    @Override
-    OpenSslKeyMaterialManager keyMaterialManager() {
-        return null;
-    }
 }

handler/src/main/java/io/netty/handler/ssl/OpenSslKeyMaterialManager.java

@@ -74,36 +74,25 @@ final class OpenSslKeyMaterialManager {
             if (type != null) {
                 String alias = chooseServerAlias(engine, type);
                 if (alias != null && aliases.add(alias)) {
-                    OpenSslKeyMaterial keyMaterial = null;
+                    setKeyMaterial(engine, alias);
-                    try {
-                        keyMaterial = provider.chooseKeyMaterial(engine.alloc, alias);
-                        if (keyMaterial != null) {
-                            SSL.setKeyMaterialServerSide(
-                                    ssl, keyMaterial.certificateChainAddress(), keyMaterial.privateKeyAddress());
-                        }
-                    } catch (SSLException e) {
-                        throw e;
-                    } catch (Exception e) {
-                        throw new SSLException(e);
-                    } finally {
-                        if (keyMaterial != null) {
-                            keyMaterial.release();
-                        }
-                    }
                 }
             }
         }
     }
 
-    void setKeyMaterialClientSide(ReferenceCountedOpenSslEngine engine, long certOut, long keyOut, String[] keyTypes,
+    void setKeyMaterialClientSide(ReferenceCountedOpenSslEngine engine, String[] keyTypes,
                                   X500Principal[] issuer) throws SSLException {
         String alias = chooseClientAlias(engine, keyTypes, issuer);
+        setKeyMaterial(engine, alias);
+    }
+
+    private void setKeyMaterial(ReferenceCountedOpenSslEngine engine, String alias) throws SSLException {
         OpenSslKeyMaterial keyMaterial = null;
         try {
             keyMaterial = provider.chooseKeyMaterial(engine.alloc, alias);
             if (keyMaterial != null) {
-                SSL.setKeyMaterialClientSide(engine.sslPointer(), certOut, keyOut,
+                SSL.setKeyMaterial(engine.sslPointer(),
-                        keyMaterial.certificateChainAddress(), keyMaterial.privateKeyAddress());
+                                   keyMaterial.certificateChainAddress(), keyMaterial.privateKeyAddress());
             }
         } catch (SSLException e) {
             throw e;
@@ -115,7 +104,6 @@ final class OpenSslKeyMaterialManager {
             }
         }
     }
-
     private String chooseClientAlias(ReferenceCountedOpenSslEngine engine,
                                        String[] keyTypes, X500Principal[] issuer) {
         X509KeyManager manager = provider.keyManager();

handler/src/main/java/io/netty/handler/ssl/OpenSslServerContext.java

@@ -15,7 +15,6 @@
  */
 package io.netty.handler.ssl;
 
-import io.netty.handler.ssl.ReferenceCountedOpenSslServerContext.ServerContext;
 import io.netty.internal.tcnative.SSL;
 
 import java.io.File;
@@ -37,7 +36,6 @@ import static io.netty.handler.ssl.ReferenceCountedOpenSslServerContext.newSessi
  */
 public final class OpenSslServerContext extends OpenSslContext {
     private final OpenSslServerSessionContext sessionContext;
-    private final OpenSslKeyMaterialManager keyMaterialManager;
 
     /**
      * Creates a new instance.
@@ -349,10 +347,8 @@ public final class OpenSslServerContext extends OpenSslContext {
         // Create a new SSL_CTX and configure it.
         boolean success = false;
         try {
-            ServerContext context = newSessionContext(this, ctx, engineMap, trustCertCollection, trustManagerFactory,
+            sessionContext = newSessionContext(this, ctx, engineMap, trustCertCollection, trustManagerFactory,
-                                                      keyCertChain, key, keyPassword, keyManagerFactory);
+                                               keyCertChain, key, keyPassword, keyManagerFactory);
-            sessionContext = context.sessionContext;
-            keyMaterialManager = context.keyMaterialManager;
             success = true;
         } finally {
             if (!success) {
@@ -365,9 +361,4 @@ public final class OpenSslServerContext extends OpenSslContext {
     public OpenSslServerSessionContext sessionContext() {
         return sessionContext;
     }
-
-    @Override
-    OpenSslKeyMaterialManager keyMaterialManager() {
-        return keyMaterialManager;
-    }
 }

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslClientContext.java

@@ -15,15 +15,16 @@
  */
 package io.netty.handler.ssl;
 
+import io.netty.internal.tcnative.CertificateCallback;
 import io.netty.util.internal.logging.InternalLogger;
 import io.netty.util.internal.logging.InternalLoggerFactory;
-import io.netty.internal.tcnative.CertificateRequestedCallback;
 import io.netty.internal.tcnative.SSL;
 import io.netty.internal.tcnative.SSLContext;
 
 import java.security.KeyStore;
 import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -68,11 +69,6 @@ public final class ReferenceCountedOpenSslClientContext extends ReferenceCounted
         }
     }
 
-    @Override
-    OpenSslKeyMaterialManager keyMaterialManager() {
-        return null;
-    }
-
     @Override
     public OpenSslSessionContext sessionContext() {
         return sessionContext;
@@ -118,7 +114,7 @@ public final class ReferenceCountedOpenSslClientContext extends ReferenceCounted
 
                     if (keyMaterialProvider != null) {
                         OpenSslKeyMaterialManager materialManager = new OpenSslKeyMaterialManager(keyMaterialProvider);
-                        SSLContext.setCertRequestedCallback(ctx, new OpenSslCertificateRequestedCallback(
+                        SSLContext.setCertificateCallback(ctx, new OpenSslClientCertificateCallback(
                                 engineMap, materialManager));
                     }
                 }
@@ -238,18 +234,17 @@ public final class ReferenceCountedOpenSslClientContext extends ReferenceCounted
         }
     }
 
-    private static final class OpenSslCertificateRequestedCallback implements CertificateRequestedCallback {
+    private static final class OpenSslClientCertificateCallback implements CertificateCallback {
         private final OpenSslEngineMap engineMap;
         private final OpenSslKeyMaterialManager keyManagerHolder;
 
-        OpenSslCertificateRequestedCallback(OpenSslEngineMap engineMap, OpenSslKeyMaterialManager keyManagerHolder) {
+        OpenSslClientCertificateCallback(OpenSslEngineMap engineMap, OpenSslKeyMaterialManager keyManagerHolder) {
             this.engineMap = engineMap;
             this.keyManagerHolder = keyManagerHolder;
         }
 
         @Override
-        public void requested(
+        public void handle(long ssl, byte[] keyTypeBytes, byte[][] asn1DerEncodedPrincipals) throws Exception {
-                long ssl, long certOut, long keyOut, byte[] keyTypeBytes, byte[][] asn1DerEncodedPrincipals) {
             final ReferenceCountedOpenSslEngine engine = engineMap.get(ssl);
             try {
                 final Set<String> keyTypesSet = supportedClientKeyTypes(keyTypeBytes);
@@ -263,7 +258,7 @@ public final class ReferenceCountedOpenSslClientContext extends ReferenceCounted
                         issuers[i] = new X500Principal(asn1DerEncodedPrincipals[i]);
                     }
                 }
-                keyManagerHolder.setKeyMaterialClientSide(engine, certOut, keyOut, keyTypes, issuers);
+                keyManagerHolder.setKeyMaterialClientSide(engine, keyTypes, issuers);
             } catch (Throwable cause) {
                 logger.debug("request of key failed", cause);
                 SSLHandshakeException e = new SSLHandshakeException("General OpenSslEngine problem");
@@ -281,6 +276,9 @@ public final class ReferenceCountedOpenSslClientContext extends ReferenceCounted
          *         {@code X509ExtendedKeyManager.chooseEngineClientAlias}.
          */
         private static Set<String> supportedClientKeyTypes(byte[] clientCertificateTypes) {
+            if (clientCertificateTypes == null) {
+                return Collections.emptySet();
+            }
             Set<String> result = new HashSet<String>(clientCertificateTypes.length);
             for (byte keyTypeCode : clientCertificateTypes) {
                 String keyType = clientKeyType(keyTypeCode);
@@ -296,15 +294,15 @@ public final class ReferenceCountedOpenSslClientContext extends ReferenceCounted
         private static String clientKeyType(byte clientCertificateType) {
             // See also http://www.ietf.org/assignments/tls-parameters/tls-parameters.xml
             switch (clientCertificateType) {
-                case CertificateRequestedCallback.TLS_CT_RSA_SIGN:
+                case CertificateCallback.TLS_CT_RSA_SIGN:
                     return OpenSslKeyMaterialManager.KEY_TYPE_RSA; // RFC rsa_sign
-                case CertificateRequestedCallback.TLS_CT_RSA_FIXED_DH:
+                case CertificateCallback.TLS_CT_RSA_FIXED_DH:
                     return OpenSslKeyMaterialManager.KEY_TYPE_DH_RSA; // RFC rsa_fixed_dh
-                case CertificateRequestedCallback.TLS_CT_ECDSA_SIGN:
+                case CertificateCallback.TLS_CT_ECDSA_SIGN:
                     return OpenSslKeyMaterialManager.KEY_TYPE_EC; // RFC ecdsa_sign
-                case CertificateRequestedCallback.TLS_CT_RSA_FIXED_ECDH:
+                case CertificateCallback.TLS_CT_RSA_FIXED_ECDH:
                     return OpenSslKeyMaterialManager.KEY_TYPE_EC_RSA; // RFC rsa_fixed_ecdh
-                case CertificateRequestedCallback.TLS_CT_ECDSA_FIXED_ECDH:
+                case CertificateCallback.TLS_CT_ECDSA_FIXED_ECDH:
                     return OpenSslKeyMaterialManager.KEY_TYPE_EC_EC; // RFC ecdsa_fixed_ecdh
                 default:
                     return null;

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslContext.java

@@ -225,7 +225,9 @@ public abstract class ReferenceCountedOpenSslContext extends SslContext implemen
         boolean success = false;
         try {
             try {
-                ctx = SSLContext.make(SSL.SSL_PROTOCOL_ALL, mode);
+                int opts = SSL.SSL_PROTOCOL_SSLV3 | SSL.SSL_PROTOCOL_TLSV1 |
+                           SSL.SSL_PROTOCOL_TLSV1_1 | SSL.SSL_PROTOCOL_TLSV1_2;
+                ctx = SSLContext.make(opts, mode);
             } catch (Exception e) {
                 throw new SSLException("failed to create an SSL_CTX", e);
             }
@@ -366,8 +368,6 @@ public abstract class ReferenceCountedOpenSslContext extends SslContext implemen
         return new ReferenceCountedOpenSslEngine(this, alloc, peerHost, peerPort, jdkCompatibilityMode, true);
     }
 
-    abstract OpenSslKeyMaterialManager keyMaterialManager();
-
     /**
      * Returns a new server-side {@link SSLEngine} with the current configuration.
      */

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngine.java

@@ -20,6 +20,7 @@ import io.netty.buffer.ByteBufAllocator;
 import io.netty.internal.tcnative.Buffer;
 import io.netty.internal.tcnative.SSL;
 import io.netty.util.AbstractReferenceCounted;
+import io.netty.util.CharsetUtil;
 import io.netty.util.ReferenceCounted;
 import io.netty.util.ResourceLeakDetector;
 import io.netty.util.ResourceLeakDetectorFactory;
@@ -142,7 +143,6 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
     // OpenSSL state
     private long ssl;
     private long networkBIO;
-    private boolean certificateSet;
 
     private enum HandshakeState {
         /**
@@ -217,7 +217,6 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
     private final Certificate[] localCerts;
     private final ByteBuffer[] singleSrcBuffer = new ByteBuffer[1];
     private final ByteBuffer[] singleDstBuffer = new ByteBuffer[1];
-    private final OpenSslKeyMaterialManager keyMaterialManager;
     private final boolean enableOcsp;
     private int maxWrapOverhead;
     private int maxWrapBufferSize;
@@ -238,18 +237,69 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
      *                             wrap or unwrap call.
      * @param leakDetection {@code true} to enable leak detection of this object.
      */
-    ReferenceCountedOpenSslEngine(ReferenceCountedOpenSslContext context, ByteBufAllocator alloc, String peerHost,
+    ReferenceCountedOpenSslEngine(ReferenceCountedOpenSslContext context, final ByteBufAllocator alloc, String peerHost,
                                   int peerPort, boolean jdkCompatibilityMode, boolean leakDetection) {
         super(peerHost, peerPort);
         OpenSsl.ensureAvailability();
         this.alloc = checkNotNull(alloc, "alloc");
         apn = (OpenSslApplicationProtocolNegotiator) context.applicationProtocolNegotiator();
         clientMode = context.isClient();
-        if (PlatformDependent.javaVersion() >= 7 && context.isClient()) {
+        if (PlatformDependent.javaVersion() >= 7) {
             session = new ExtendedOpenSslSession(new DefaultOpenSslSession(context.sessionContext())) {
+                private String[] peerSupportedSignatureAlgorithms;
+                private List requestedServerNames;
+
                 @Override
                 public List getRequestedServerNames() {
-                    return Java8SslUtils.getSniHostNames(sniHostNames);
+                    if (clientMode) {
+                        return Java8SslUtils.getSniHostNames(sniHostNames);
+                    } else {
+                        synchronized (ReferenceCountedOpenSslEngine.this) {
+                            if (requestedServerNames == null) {
+                                if (isDestroyed()) {
+                                    requestedServerNames = Collections.emptyList();
+                                } else {
+                                    String name = SSL.getSniHostname(ssl);
+                                    if (name == null) {
+                                        requestedServerNames = Collections.emptyList();
+                                    } else {
+                                        // Convert to bytes as we do not want to do any strict validation of the
+                                        // SNIHostName while creating it.
+                                        requestedServerNames =
+                                                Java8SslUtils.getSniHostName(
+                                                        SSL.getSniHostname(ssl).getBytes(CharsetUtil.UTF_8));
+                                    }
+                                }
+                            }
+                            return requestedServerNames;
+                        }
+                    }
+                }
+
+                @Override
+                public String[] getPeerSupportedSignatureAlgorithms() {
+                    synchronized (ReferenceCountedOpenSslEngine.this) {
+                        if (peerSupportedSignatureAlgorithms == null) {
+                            if (isDestroyed()) {
+                                peerSupportedSignatureAlgorithms = EmptyArrays.EMPTY_STRINGS;
+                            } else {
+                                String[] algs = SSL.getSigAlgs(ssl);
+                                if (algs == null) {
+                                    peerSupportedSignatureAlgorithms = EmptyArrays.EMPTY_STRINGS;
+                                } else {
+                                    List<String> algorithmList = new ArrayList<String>(algs.length);
+                                    for (String alg: algs) {
+                                        String converted = SignatureAlgorithmConverter.toJavaName(alg);
+                                        if (converted != null) {
+                                            algorithmList.add(converted);
+                                        }
+                                    }
+                                    peerSupportedSignatureAlgorithms = algorithmList.toArray(new String[0]);
+                                }
+                            }
+                        }
+                        return peerSupportedSignatureAlgorithms.clone();
+                    }
                 }
 
                 @Override
@@ -271,7 +321,6 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
         }
         engineMap = context.engineMap;
         localCerts = context.keyCertChain;
-        keyMaterialManager = context.keyMaterialManager();
         enableOcsp = context.enableOcsp;
         this.jdkCompatibilityMode = jdkCompatibilityMode;
         Lock readerLock = context.ctxLock.readLock();
@@ -1593,11 +1642,6 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
             lastAccessed = System.currentTimeMillis();
         }
 
-        if (!certificateSet && keyMaterialManager != null) {
-            certificateSet = true;
-            keyMaterialManager.setKeyMaterialServerSide(this);
-        }
-
         int code = SSL.doHandshake(ssl);
         if (code <= 0) {
             // Check if we have a pending exception that was created during the handshake and if so throw it after

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslServerContext.java

@@ -16,6 +16,7 @@
 package io.netty.handler.ssl;
 
 import io.netty.buffer.ByteBufAllocator;
+import io.netty.internal.tcnative.CertificateCallback;
 import io.netty.internal.tcnative.SSL;
 import io.netty.internal.tcnative.SSLContext;
 import io.netty.internal.tcnative.SniHostNameMatcher;
@@ -29,6 +30,7 @@ import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.TrustManagerFactory;
 import javax.net.ssl.X509ExtendedTrustManager;
 import javax.net.ssl.X509TrustManager;
@@ -48,7 +50,6 @@ public final class ReferenceCountedOpenSslServerContext extends ReferenceCounted
             InternalLoggerFactory.getInstance(ReferenceCountedOpenSslServerContext.class);
     private static final byte[] ID = {'n', 'e', 't', 't', 'y'};
     private final OpenSslServerSessionContext sessionContext;
-    private final OpenSslKeyMaterialManager keyMaterialManager;
 
     ReferenceCountedOpenSslServerContext(
             X509Certificate[] trustCertCollection, TrustManagerFactory trustManagerFactory,
@@ -72,10 +73,8 @@ public final class ReferenceCountedOpenSslServerContext extends ReferenceCounted
         // Create a new SSL_CTX and configure it.
         boolean success = false;
         try {
-            ServerContext context = newSessionContext(this, ctx, engineMap, trustCertCollection, trustManagerFactory,
+            sessionContext = newSessionContext(this, ctx, engineMap, trustCertCollection, trustManagerFactory,
                                                       keyCertChain, key, keyPassword, keyManagerFactory);
-            sessionContext = context.sessionContext;
-            keyMaterialManager = context.keyMaterialManager;
             success = true;
         } finally {
             if (!success) {
@@ -89,23 +88,13 @@ public final class ReferenceCountedOpenSslServerContext extends ReferenceCounted
         return sessionContext;
     }
 
-    @Override
+    static OpenSslServerSessionContext newSessionContext(ReferenceCountedOpenSslContext thiz, long ctx,
-    OpenSslKeyMaterialManager keyMaterialManager() {
+                                                         OpenSslEngineMap engineMap,
-        return keyMaterialManager;
+                                                         X509Certificate[] trustCertCollection,
-    }
+                                                         TrustManagerFactory trustManagerFactory,
-
+                                                         X509Certificate[] keyCertChain, PrivateKey key,
-    static final class ServerContext {
+                                                         String keyPassword, KeyManagerFactory keyManagerFactory)
-        OpenSslServerSessionContext sessionContext;
-        OpenSslKeyMaterialManager keyMaterialManager;
-    }
-
-    static ServerContext newSessionContext(ReferenceCountedOpenSslContext thiz, long ctx, OpenSslEngineMap engineMap,
-                                           X509Certificate[] trustCertCollection,
-                                           TrustManagerFactory trustManagerFactory,
-                                           X509Certificate[] keyCertChain, PrivateKey key,
-                                           String keyPassword, KeyManagerFactory keyManagerFactory)
             throws SSLException {
-        ServerContext result = new ServerContext();
         OpenSslKeyMaterialProvider keyMaterialProvider = null;
         try {
             try {
@@ -134,7 +123,8 @@ public final class ReferenceCountedOpenSslServerContext extends ReferenceCounted
                     }
                     keyMaterialProvider = providerFor(keyManagerFactory, keyPassword);
 
-                    result.keyMaterialManager = new OpenSslKeyMaterialManager(keyMaterialProvider);
+                    SSLContext.setCertificateCallback(ctx, new OpenSslServerCertificateCallback(
+                            engineMap, new OpenSslKeyMaterialManager(keyMaterialProvider)));
                 }
             } catch (Exception e) {
                 throw new SSLException("failed to set certificate and key", e);
@@ -159,8 +149,8 @@ public final class ReferenceCountedOpenSslServerContext extends ReferenceCounted
 
                 // Use this to prevent an error when running on java < 7
                 if (useExtendedTrustManager(manager)) {
-                    SSLContext.setCertVerifyCallback(ctx,
+                    SSLContext.setCertVerifyCallback(ctx, new ExtendedTrustManagerVerifyCallback(
-                            new ExtendedTrustManagerVerifyCallback(engineMap, (X509ExtendedTrustManager) manager));
+                            engineMap, (X509ExtendedTrustManager) manager));
                 } else {
                     SSLContext.setCertVerifyCallback(ctx, new TrustManagerVerifyCallback(engineMap, manager));
                 }
@@ -191,12 +181,12 @@ public final class ReferenceCountedOpenSslServerContext extends ReferenceCounted
                 throw new SSLException("unable to setup trustmanager", e);
             }
 
-            result.sessionContext = new OpenSslServerSessionContext(thiz, keyMaterialProvider);
+            OpenSslServerSessionContext sessionContext = new OpenSslServerSessionContext(thiz, keyMaterialProvider);
-            result.sessionContext.setSessionIdContext(ID);
+            sessionContext.setSessionIdContext(ID);
 
             keyMaterialProvider = null;
 
-            return result;
+            return sessionContext;
         } finally {
             if (keyMaterialProvider != null) {
                 keyMaterialProvider.destroy();
@@ -204,6 +194,31 @@ public final class ReferenceCountedOpenSslServerContext extends ReferenceCounted
         }
     }
 
+    private static final class OpenSslServerCertificateCallback implements CertificateCallback {
+        private final OpenSslEngineMap engineMap;
+        private final OpenSslKeyMaterialManager keyManagerHolder;
+
+        OpenSslServerCertificateCallback(OpenSslEngineMap engineMap, OpenSslKeyMaterialManager keyManagerHolder) {
+            this.engineMap = engineMap;
+            this.keyManagerHolder = keyManagerHolder;
+        }
+
+        @Override
+        public void handle(long ssl, byte[] keyTypeBytes, byte[][] asn1DerEncodedPrincipals) throws Exception {
+            final ReferenceCountedOpenSslEngine engine = engineMap.get(ssl);
+            try {
+                // For now we just ignore the asn1DerEncodedPrincipals as this is kind of inline with what the
+                // OpenJDK SSLEngineImpl does.
+                keyManagerHolder.setKeyMaterialServerSide(engine);
+            } catch (Throwable cause) {
+                logger.debug("Failed to set the server-side key material", cause);
+                SSLHandshakeException e = new SSLHandshakeException("General OpenSslEngine problem");
+                e.initCause(cause);
+                engine.handshakeException = e;
+            }
+        }
+    }
+
     private static final class TrustManagerVerifyCallback extends AbstractCertificateVerifier {
         private final X509TrustManager manager;
 

handler/src/main/java/io/netty/handler/ssl/SignatureAlgorithmConverter.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright 2018 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.ssl;
+
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Converts OpenSSL signature Algorithm names to
+ * <a href="https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#Signature">
+ *     Java signature Algorithm names</a>.
+ */
+final class SignatureAlgorithmConverter {
+
+    private SignatureAlgorithmConverter() { }
+
+    // OpenSSL has 3 different formats it uses at the moment we will match against all of these.
+    // For example:
+    //              ecdsa-with-SHA384
+    //              hmacWithSHA384
+    //              dsa_with_SHA224
+    //
+    // For more details see https://github.com/openssl/openssl/blob/OpenSSL_1_0_2p/crypto/objects/obj_dat.h
+    private static final Pattern PATTERN = Pattern.compile(
+            "((^[a-zA-Z].+)With(.+)Encryption$)|((^[a-zA-Z].+)(_with_|-with-)(.+$))");
+
+    /**
+     * Converts an OpenSSL algorithm name to a Java algorithm name and return it,
+     * or return {@code null} if the conversation failed because the format is not known.
+     */
+    static String toJavaName(String opensslName) {
+        if (opensslName == null) {
+            return null;
+        }
+        Matcher matcher = PATTERN.matcher(opensslName);
+        if (matcher.matches()) {
+            String group2 = matcher.group(2);
+            if (group2 != null) {
+                return group2.toUpperCase(Locale.ROOT) + "with" + matcher.group(3).toUpperCase(Locale.ROOT);
+            }
+            if (matcher.group(4) != null) {
+                return matcher.group(7).toUpperCase(Locale.ROOT) + "with" + matcher.group(5).toUpperCase(Locale.ROOT);
+            }
+        }
+        return null;
+    }
+}

handler/src/test/java/io/netty/handler/ssl/OpenSslTestUtils.java

@@ -24,4 +24,8 @@ final class OpenSslTestUtils {
     static void checkShouldUseKeyManagerFactory() {
         assumeTrue(OpenSsl.supportsKeyManagerFactory() && OpenSsl.useKeyManagerFactory());
     }
+
+    static boolean isBoringSSL() {
+        return "BoringSSL".equals(OpenSsl.versionString());
+    }
 }

handler/src/test/java/io/netty/handler/ssl/SignatureAlgorithmConverterTest.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2018 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.ssl;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class SignatureAlgorithmConverterTest {
+
+    @Test
+    public void testWithEncryption() {
+        assertEquals("SHA512withRSA", SignatureAlgorithmConverter.toJavaName("sha512WithRSAEncryption"));
+    }
+
+    @Test
+    public void testWithDash() {
+        assertEquals("SHA256withECDSA", SignatureAlgorithmConverter.toJavaName("ecdsa-with-SHA256"));
+    }
+
+    @Test
+    public void testWithUnderscore() {
+        assertEquals("SHA256withDSA", SignatureAlgorithmConverter.toJavaName("dsa_with_SHA256"));
+    }
+
+    @Test
+    public void testInvalid() {
+        assertNull(SignatureAlgorithmConverter.toJavaName("ThisIsSomethingInvalid"));
+    }
+}

handler/src/test/java/io/netty/handler/ssl/SniClientJava8TestUtil.java

@@ -64,6 +64,7 @@ import java.security.UnrecoverableKeyException;
 import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
@@ -166,11 +167,11 @@ final class SniClientJava8TestUtil {
         }
     }
 
-    static void assertSSLSession(SSLSession session, String name) {
+    static void assertSSLSession(boolean clientSide, SSLSession session, String name) {
-        assertSSLSession(session, new SNIHostName(name));
+        assertSSLSession(clientSide, session, new SNIHostName(name));
     }
 
-    private static void assertSSLSession(SSLSession session, SNIServerName name) {
+    private static void assertSSLSession(boolean clientSide, SSLSession session, SNIServerName name) {
         Assert.assertNotNull(session);
         if (session instanceof ExtendedSSLSession) {
             ExtendedSSLSession extendedSSLSession = (ExtendedSSLSession) session;
@@ -178,6 +179,17 @@ final class SniClientJava8TestUtil {
             Assert.assertEquals(1, names.size());
             Assert.assertEquals(name, names.get(0));
             Assert.assertTrue(extendedSSLSession.getLocalSupportedSignatureAlgorithms().length > 0);
+            if (clientSide) {
+                Assert.assertEquals(0, extendedSSLSession.getPeerSupportedSignatureAlgorithms().length);
+            } else {
+                if (session instanceof OpenSslSession && OpenSslTestUtils.isBoringSSL()) {
+                    // BoringSSL does not support SSL_get_sigalgs(...)
+                    // https://boringssl.googlesource.com/boringssl/+/ba16a1e405c617f4179bd780ad15522fb25b0a65%5E%21/
+                    Assert.assertEquals(0, extendedSSLSession.getPeerSupportedSignatureAlgorithms().length);
+                } else {
+                    Assert.assertTrue(extendedSSLSession.getPeerSupportedSignatureAlgorithms().length > 0);
+                }
+            }
         }
     }
 
@@ -227,7 +239,7 @@ final class SniClientJava8TestUtil {
                 @Override
                 public void checkServerTrusted(X509Certificate[] x509Certificates, String s, SSLEngine sslEngine)
                         throws CertificateException {
-                    assertSSLSession(sslEngine.getHandshakeSession(), name);
+                    assertSSLSession(sslEngine.getUseClientMode(), sslEngine.getHandshakeSession(), name);
                 }
 
                 @Override
@@ -323,7 +335,7 @@ final class SniClientJava8TestUtil {
                                                                       SSLEngine sslEngine) {
 
                                     SSLSession session = sslEngine.getHandshakeSession();
-                                    assertSSLSession(session, name);
+                                    assertSSLSession(sslEngine.getUseClientMode(), session, name);
                                     return ((X509ExtendedKeyManager) km)
                                             .chooseEngineServerAlias(s, principals, sslEngine);
                                 }

handler/src/test/java/io/netty/handler/ssl/SniClientTest.java

@@ -155,7 +155,8 @@ public class SniClientTest {
             Assert.assertNull(handler.engine().getHandshakeSession());
 
             if (PlatformDependent.javaVersion() >= 8) {
-                SniClientJava8TestUtil.assertSSLSession(handler.engine().getSession(), sniHostName);
+                SniClientJava8TestUtil.assertSSLSession(
+                        handler.engine().getUseClientMode(), handler.engine().getSession(), sniHostName);
             }
         } finally {
             if (cc != null) {

handler/src/test/java/io/netty/handler/ssl/SslErrorTest.java

@@ -204,7 +204,7 @@ public class SslErrorTest {
                                                 verifyException(unwrappedCause, "expired", promise);
                                             } else if (reason == CertPathValidatorException.BasicReason.NOT_YET_VALID) {
                                                 // BoringSSL uses "expired" in this case while others use "bad"
-                                                if ("BoringSSL".equals(OpenSsl.versionString())) {
+                                                if (OpenSslTestUtils.isBoringSSL()) {
                                                     verifyException(unwrappedCause, "expired", promise);
                                                 } else {
                                                     verifyException(unwrappedCause, "bad", promise);
@@ -216,7 +216,7 @@ public class SslErrorTest {
                                             verifyException(unwrappedCause, "expired", promise);
                                         } else if (exception instanceof CertificateNotYetValidException) {
                                             // BoringSSL uses "expired" in this case while others use "bad"
-                                            if ("BoringSSL".equals(OpenSsl.versionString())) {
+                                            if (OpenSslTestUtils.isBoringSSL()) {
                                                 verifyException(unwrappedCause, "expired", promise);
                                             } else {
                                                 verifyException(unwrappedCause, "bad", promise);

pom.xml

@@ -241,7 +241,7 @@
     <!-- Fedora-"like" systems. This is currently only used for the netty-tcnative dependency -->
     <os.detection.classifierWithLikes>fedora</os.detection.classifierWithLikes>
     <tcnative.artifactId>netty-tcnative</tcnative.artifactId>
-    <tcnative.version>2.0.15.Final</tcnative.version>
+    <tcnative.version>2.0.17.Final</tcnative.version>
     <tcnative.classifier>${os.detected.classifier}</tcnative.classifier>
     <conscrypt.groupId>org.conscrypt</conscrypt.groupId>
     <conscrypt.artifactId>conscrypt-openjdk-uber</conscrypt.artifactId>