We should fail fast if the given PrivateKey or X509Certificate chain is not supported by the used SslProvider. (#9009)

Motivation:

Some SslProvider do support different types of keys and chains. We should fail fast if we can not support the type.

Related to https://github.com/netty/netty-tcnative/issues/455.

Modifications:

- Try to parse key / chain first and if if this fails throw and SslException
- Add tests.

Result:

Fail fast.

handler/src/main/java/io/netty/handler/ssl/OpenSslClientContext.java

@@ -52,6 +52,7 @@ final class OpenSslClientContext extends OpenSslContext {
           ClientAuth.NONE, protocols, false, enableOcsp);
         boolean success = false;
         try {
+            OpenSslKeyMaterialProvider.validateKeyMaterialSupported(keyCertChain, key, keyPassword);
             sessionContext = newSessionContext(this, ctx, engineMap, trustCertCollection, trustManagerFactory,
               keyCertChain, key, keyPassword, keyManagerFactory);
             success = true;

handler/src/main/java/io/netty/handler/ssl/OpenSslKeyMaterialProvider.java

@@ -16,8 +16,10 @@
 package io.netty.handler.ssl;
 
 import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.UnpooledByteBufAllocator;
 import io.netty.internal.tcnative.SSL;
 
+import javax.net.ssl.SSLException;
 import javax.net.ssl.X509KeyManager;
 import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
@@ -37,6 +39,58 @@ class OpenSslKeyMaterialProvider {
         this.password = password;
     }
 
+    static void validateKeyMaterialSupported(X509Certificate[] keyCertChain, PrivateKey key, String keyPassword)
+            throws SSLException {
+        validateSupported(keyCertChain);
+        validateSupported(key, keyPassword);
+    }
+
+    private static void validateSupported(PrivateKey key, String password) throws SSLException {
+        if (key == null) {
+            return;
+        }
+
+        long pkeyBio = 0;
+        long pkey = 0;
+
+        try {
+            pkeyBio = toBIO(UnpooledByteBufAllocator.DEFAULT, key);
+            pkey = SSL.parsePrivateKey(pkeyBio, password);
+        } catch (Exception e) {
+            throw new SSLException("PrivateKey type not supported " + key.getFormat(), e);
+        } finally {
+            SSL.freeBIO(pkeyBio);
+            if (pkey != 0) {
+                SSL.freePrivateKey(pkey);
+            }
+        }
+    }
+
+    private static void validateSupported(X509Certificate[] certificates) throws SSLException {
+        if (certificates == null || certificates.length == 0) {
+            return;
+        }
+
+        long chainBio = 0;
+        long chain = 0;
+        PemEncoded encoded = null;
+        try {
+            encoded = PemX509Certificate.toPEM(UnpooledByteBufAllocator.DEFAULT, true, certificates);
+            chainBio = toBIO(UnpooledByteBufAllocator.DEFAULT, encoded.retain());
+            chain = SSL.parseX509Chain(chainBio);
+        } catch (Exception e) {
+            throw new SSLException("Certificate type not supported", e);
+        } finally {
+            SSL.freeBIO(chainBio);
+            if (chain != 0) {
+                SSL.freeX509Chain(chain);
+            }
+            if (encoded != null) {
+                encoded.release();
+            }
+        }
+    }
+
     /**
      * Returns the underlying {@link X509KeyManager} that is used.
      */

handler/src/main/java/io/netty/handler/ssl/OpenSslServerContext.java

@@ -55,6 +55,7 @@ final class OpenSslServerContext extends OpenSslContext {
         // Create a new SSL_CTX and configure it.
         boolean success = false;
         try {
+            OpenSslKeyMaterialProvider.validateKeyMaterialSupported(keyCertChain, key, keyPassword);
             sessionContext = newSessionContext(this, ctx, engineMap, trustCertCollection, trustManagerFactory,
               keyCertChain, key, keyPassword, keyManagerFactory);
             success = true;

handler/src/test/java/io/netty/handler/ssl/SslContextBuilderTest.java

@@ -17,11 +17,13 @@ package io.netty.handler.ssl;
 
 import io.netty.buffer.UnpooledByteBufAllocator;
 import io.netty.handler.ssl.util.SelfSignedCertificate;
+import io.netty.util.CharsetUtil;
 import org.junit.Assume;
 import org.junit.Test;
 
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLException;
+import java.io.ByteArrayInputStream;
 import java.util.Collections;
 
 import static org.junit.Assert.*;
@@ -72,6 +74,52 @@ public class SslContextBuilderTest {
         testServerContext(SslProvider.OPENSSL);
     }
 
+    @Test(expected = SSLException.class)
+    public void testUnsupportedPrivateKeyFailsFastForServer() throws Exception {
+        Assume.assumeTrue(OpenSsl.isBoringSSL());
+        testUnsupportedPrivateKeyFailsFast(true);
+    }
+
+    @Test(expected = SSLException.class)
+    public void testUnsupportedPrivateKeyFailsFastForClient() throws Exception {
+        Assume.assumeTrue(OpenSsl.isBoringSSL());
+        testUnsupportedPrivateKeyFailsFast(false);
+    }
+    private static void testUnsupportedPrivateKeyFailsFast(boolean server) throws Exception {
+        Assume.assumeTrue(OpenSsl.isBoringSSL());
+        String cert = "-----BEGIN CERTIFICATE-----\n" +
+                "MIICODCCAY2gAwIBAgIEXKTrajAKBggqhkjOPQQDBDBUMQswCQYDVQQGEwJVUzEM\n" +
+                "MAoGA1UECAwDTi9hMQwwCgYDVQQHDANOL2ExDDAKBgNVBAoMA04vYTEMMAoGA1UE\n" +
+                "CwwDTi9hMQ0wCwYDVQQDDARUZXN0MB4XDTE5MDQwMzE3MjA0MloXDTIwMDQwMjE3\n" +
+                "MjA0MlowVDELMAkGA1UEBhMCVVMxDDAKBgNVBAgMA04vYTEMMAoGA1UEBwwDTi9h\n" +
+                "MQwwCgYDVQQKDANOL2ExDDAKBgNVBAsMA04vYTENMAsGA1UEAwwEVGVzdDCBpzAQ\n" +
+                "BgcqhkjOPQIBBgUrgQQAJwOBkgAEBPYWoTjlS2pCMGEM2P8qZnmURWA5e7XxPfIh\n" +
+                "HA876sjmgjJluPgT0OkweuxI4Y/XjzcPnnEBONgzAV1X93UmXdtRiIau/zvsAeFb\n" +
+                "j/q+6sfj1jdnUk6QsMx22kAwplXHmdz1z5ShXQ7mDZPxDbhCPEAUXzIzOqvWIZyA\n" +
+                "HgFxZXmQKEhExA8nxgSIvzQ3ucMwMAoGCCqGSM49BAMEA4GYADCBlAJIAdPD6jaN\n" +
+                "vGxkxcsIbcHn2gSfP1F1G8iNJYrXIN91KbQm8OEp4wxqnBwX8gb/3rmSoEhIU/te\n" +
+                "CcHuFs0guBjfgRWtJ/eDnKB/AkgDbkqrB5wqJFBmVd/rJ5QdwUVNuGP/vDjFVlb6\n" +
+                "Esny6//gTL7jYubLUKHOPIMftCZ2Jn4b+5l0kAs62HD5XkZLPDTwRbf7VCE=\n" +
+                "-----END CERTIFICATE-----";
+        String key = "-----BEGIN PRIVATE KEY-----\n" +
+                "MIIBCQIBADAQBgcqhkjOPQIBBgUrgQQAJwSB8TCB7gIBAQRIALNClTXqQWWlYDHw\n" +
+                "LjNxXpLk17iPepkmablhbxmYX/8CNzoz1o2gcUidoIO2DM9hm7adI/W31EOmSiUJ\n" +
+                "+UsC/ZH3i2qr0wn+oAcGBSuBBAAnoYGVA4GSAAQE9hahOOVLakIwYQzY/ypmeZRF\n" +
+                "YDl7tfE98iEcDzvqyOaCMmW4+BPQ6TB67Ejhj9ePNw+ecQE42DMBXVf3dSZd21GI\n" +
+                "hq7/O+wB4VuP+r7qx+PWN2dSTpCwzHbaQDCmVceZ3PXPlKFdDuYNk/ENuEI8QBRf\n" +
+                "MjM6q9YhnIAeAXFleZAoSETEDyfGBIi/NDe5wzA=\n" +
+                "-----END PRIVATE KEY-----";
+        if (server) {
+            SslContextBuilder.forServer(new ByteArrayInputStream(cert.getBytes(CharsetUtil.US_ASCII)),
+                    new ByteArrayInputStream(key.getBytes(CharsetUtil.US_ASCII)), null)
+                    .sslProvider(SslProvider.OPENSSL).build();
+        } else {
+            SslContextBuilder.forClient().keyManager(new ByteArrayInputStream(cert.getBytes(CharsetUtil.US_ASCII)),
+                new ByteArrayInputStream(key.getBytes(CharsetUtil.US_ASCII)), null)
+                    .sslProvider(SslProvider.OPENSSL).build();
+        }
+    }
+
     @Test(expected = IllegalArgumentException.class)
     public void testInvalidCipherJdk() throws Exception {
         Assume.assumeTrue(OpenSsl.isAvailable());