Add support for SNIMatcher when using SslProvider.OPENSSL* and Java8+

Motivation:

Java8 adds support for SNIMatcher to reject SNI when the hostname not matches what is expected. We not supported doing this when using SslProvider.OPENSSL*.

Modifications:

- Add support for SNIMatcher when using SslProvider.OPENSSL*
- Add unit tests

Result:

SNIMatcher now support with our own SSLEngine as well.

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngine.java

@@ -43,7 +43,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
-
+import javax.net.ssl.SNIHostName;
 import javax.net.ssl.SNIMatcher;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
@@ -190,6 +190,9 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
     private Object algorithmConstraints;
     private List<String> sniHostNames;
 
+    // Mark as volatile as accessed by checkSniHostnameMatch(...)
+    private volatile Collection<SNIMatcher> matchers;
+
     // SSL Engine status variables
     private boolean isInboundDone;
     private boolean outboundClosed;
@@ -1597,6 +1600,8 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
                     Java8SslParametersUtils.setUseCipherSuitesOrder(
                             sslParameters, (SSL.getOptions(ssl) & SSL.SSL_OP_CIPHER_SERVER_PREFERENCE) != 0);
                 }
+
+                sslParameters.setSNIMatchers(matchers);
             }
         }
         return sslParameters;
@@ -1611,11 +1616,6 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
             }
 
             if (version >= 8) {
-                Collection<SNIMatcher> matchers = sslParameters.getSNIMatchers();
-                if (matchers != null && !matchers.isEmpty()) {
-                    throw new IllegalArgumentException("SNIMatchers are not supported.");
-                }
-
                 if (!isDestroyed()) {
                     if (clientMode) {
                         final List<String> sniHostNames = Java8SslParametersUtils.getSniHostNames(sslParameters);
@@ -1630,6 +1630,7 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
                         SSL.clearOptions(ssl, SSL.SSL_OP_CIPHER_SERVER_PREFERENCE);
                     }
                 }
+                matchers = sslParameters.getSNIMatchers();
             }
 
             final String endPointIdentificationAlgorithm = sslParameters.getEndpointIdentificationAlgorithm();
@@ -1658,6 +1659,21 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
                 pendingBytes + (long) MAX_TLS_RECORD_OVERHEAD_LENGTH * numComponents);
     }
 
+    final boolean checkSniHostnameMatch(String hostname) {
+        Collection<SNIMatcher> matchers = this.matchers;
+        if (matchers != null && !matchers.isEmpty()) {
+            SNIHostName name = new SNIHostName(hostname);
+            for (SNIMatcher matcher: matchers) {
+                // type 0 is for hostname
+                if (matcher.getType() == 0 && matcher.matches(name)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+        return true;
+    }
+
     private final class OpenSslSession implements SSLSession, ApplicationProtocolAccessor {
         private final OpenSslSessionContext sessionContext;
 

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslServerContext.java

@@ -17,6 +17,10 @@ package io.netty.handler.ssl;
 
 import io.netty.internal.tcnative.SSL;
 import io.netty.internal.tcnative.SSLContext;
+import io.netty.internal.tcnative.SniHostNameMatcher;
+import io.netty.util.internal.PlatformDependent;
+import io.netty.util.internal.logging.InternalLogger;
+import io.netty.util.internal.logging.InternalLoggerFactory;
 
 import java.security.KeyStore;
 import java.security.PrivateKey;
@@ -40,6 +44,8 @@ import static io.netty.util.internal.ObjectUtil.checkNotNull;
  * {@link ReferenceCountedOpenSslEngine} is called which uses this class's JNI resources the JVM may crash.
  */
 public final class ReferenceCountedOpenSslServerContext extends ReferenceCountedOpenSslContext {
+    private static final InternalLogger logger =
+            InternalLoggerFactory.getInstance(ReferenceCountedOpenSslServerContext.class);
     private static final byte[] ID = {'n', 'e', 't', 't', 'y'};
     private final OpenSslServerSessionContext sessionContext;
     private final OpenSslKeyMaterialManager keyMaterialManager;
@@ -170,6 +176,14 @@ public final class ReferenceCountedOpenSslServerContext extends ReferenceCounted
             } catch (Exception e) {
                 throw new SSLException("unable to setup trustmanager", e);
             }
+
+            if (PlatformDependent.javaVersion() >= 8) {
+                // Only do on Java8+ as SNIMatcher is not supported in earlier releases.
+                // IMPORTANT: The callbacks set for hostname matching must be static to prevent memory leak as
+                //            otherwise the context can never be collected. This is because the JNI code holds
+                //            a global reference to the matcher.
+                SSLContext.setSniHostnameMatcher(ctx, new OpenSslSniHostnameMatcher(engineMap));
+            }
         }
 
         result.sessionContext = new OpenSslServerSessionContext(thiz);
@@ -206,4 +220,22 @@ public final class ReferenceCountedOpenSslServerContext extends ReferenceCounted
             manager.checkClientTrusted(peerCerts, auth, engine);
         }
     }
+
+    private static final class OpenSslSniHostnameMatcher implements SniHostNameMatcher {
+        private final OpenSslEngineMap engineMap;
+
+        OpenSslSniHostnameMatcher(OpenSslEngineMap engineMap) {
+            this.engineMap = engineMap;
+        }
+
+        @Override
+        public boolean match(long ssl, String hostname) {
+            ReferenceCountedOpenSslEngine engine = engineMap.get(ssl);
+            if (engine != null) {
+                return engine.checkSniHostnameMatch(hostname);
+            }
+            logger.warn("No ReferenceCountedOpenSslEngine found for SSL pointer: {}", ssl);
+            return false;
+        }
+    }
 }

handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java

@@ -587,8 +587,8 @@ public class OpenSslEngineTest extends SSLEngineTest {
         assertFalse(src.hasRemaining());
     }
 
-    @Test(expected = IllegalArgumentException.class)
+    @Test
-    public void testSNIMatchersThrows() throws Exception {
+    public void testSNIMatchersDoesNotThrow() throws Exception {
         assumeTrue(PlatformDependent.javaVersion() >= 8);
         SelfSignedCertificate ssc = new SelfSignedCertificate();
         serverSslCtx = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey())

handler/src/test/java/io/netty/handler/ssl/SniClientJava8TestUtil.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright 2017 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.ssl;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.DefaultEventLoopGroup;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.local.LocalAddress;
+import io.netty.channel.local.LocalChannel;
+import io.netty.channel.local.LocalServerChannel;
+import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import io.netty.handler.ssl.util.SelfSignedCertificate;
+import io.netty.util.concurrent.Promise;
+import io.netty.util.internal.ThrowableUtil;
+
+import javax.net.ssl.SNIMatcher;
+import javax.net.ssl.SNIServerName;
+import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLParameters;
+import java.util.Collections;
+
+/**
+ * In extra class to be able to run tests with java7 without trying to load classes that not exists in java7.
+ */
+final class SniClientJava8TestUtil {
+
+    private SniClientJava8TestUtil() { }
+
+    static void testSniClient(SslProvider sslClientProvider, SslProvider sslServerProvider, final boolean match)
+            throws Exception {
+        final String sniHost = "sni.netty.io";
+        LocalAddress address = new LocalAddress("test");
+        EventLoopGroup group = new DefaultEventLoopGroup(1);
+        Channel sc = null;
+        Channel cc = null;
+        try {
+            SelfSignedCertificate cert = new SelfSignedCertificate();
+            final SslContext sslServerContext = SslContextBuilder.forServer(cert.key(), cert.cert())
+                    .sslProvider(sslServerProvider).build();
+            final Promise<Void> promise = group.next().newPromise();
+            ServerBootstrap sb = new ServerBootstrap();
+            sc = sb.group(group).channel(LocalServerChannel.class).childHandler(new ChannelInitializer<Channel>() {
+                @Override
+                protected void initChannel(Channel ch) throws Exception {
+                    SslHandler handler = sslServerContext.newHandler(ch.alloc());
+                    SSLParameters parameters = handler.engine().getSSLParameters();
+                    SNIMatcher matcher = new SNIMatcher(0) {
+                        @Override
+                        public boolean matches(SNIServerName sniServerName) {
+                            return match;
+                        }
+                    };
+                    parameters.setSNIMatchers(Collections.singleton(matcher));
+                    handler.engine().setSSLParameters(parameters);
+
+                    ch.pipeline().addFirst(handler);
+                    ch.pipeline().addLast(new ChannelInboundHandlerAdapter() {
+                        @Override
+                        public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
+                            if (evt instanceof SslHandshakeCompletionEvent) {
+                                SslHandshakeCompletionEvent event = (SslHandshakeCompletionEvent) evt;
+                                if (match) {
+                                    if (event.isSuccess()) {
+                                        promise.setSuccess(null);
+                                    } else {
+                                        promise.setFailure(event.cause());
+                                    }
+                                } else {
+                                    if (event.isSuccess()) {
+                                        promise.setFailure(new AssertionError("expected SSLException"));
+                                    } else {
+                                        Throwable cause = event.cause();
+                                        if (cause instanceof SSLException) {
+                                            promise.setSuccess(null);
+                                        } else {
+                                            promise.setFailure(
+                                                    new AssertionError("cause not of type SSLException: "
+                                                            + ThrowableUtil.stackTraceToString(cause)));
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    });
+                }
+            }).bind(address).syncUninterruptibly().channel();
+
+            SslContext sslContext = SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE)
+                    .sslProvider(sslClientProvider).build();
+
+            SslHandler sslHandler = new SslHandler(
+                    sslContext.newEngine(ByteBufAllocator.DEFAULT, sniHost, -1));
+            Bootstrap cb = new Bootstrap();
+            cc = cb.group(group).channel(LocalChannel.class).handler(sslHandler)
+                    .connect(address).syncUninterruptibly().channel();
+
+            promise.syncUninterruptibly();
+            sslHandler.handshakeFuture().syncUninterruptibly();
+        } finally {
+            if (cc != null) {
+                cc.close().syncUninterruptibly();
+            }
+            if (sc != null) {
+                sc.close().syncUninterruptibly();
+            }
+            group.shutdownGracefully();
+        }
+    }
+}

handler/src/test/java/io/netty/handler/ssl/SniClientTest.java

@@ -29,10 +29,13 @@ import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
 import io.netty.handler.ssl.util.SelfSignedCertificate;
 import io.netty.util.Mapping;
 import io.netty.util.concurrent.Promise;
+import io.netty.util.internal.PlatformDependent;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Test;
 
+import java.nio.channels.ClosedChannelException;
+
 public class SniClientTest {
 
     @Test(timeout = 30000)
@@ -58,6 +61,60 @@ public class SniClientTest {
         testSniClient(SslProvider.OPENSSL, SslProvider.JDK);
     }
 
+    @Test(timeout = 30000)
+    public void testSniSNIMatcherMatchesClientJdkSslServerJdkSsl() throws Exception {
+        Assume.assumeTrue(PlatformDependent.javaVersion() >= 8);
+        SniClientJava8TestUtil.testSniClient(SslProvider.JDK, SslProvider.JDK, true);
+    }
+
+    @Test(timeout = 30000, expected = ClosedChannelException.class)
+    public void testSniSNIMatcherDoesNotMatchClientJdkSslServerJdkSsl() throws Exception {
+        Assume.assumeTrue(PlatformDependent.javaVersion() >= 8);
+        SniClientJava8TestUtil.testSniClient(SslProvider.JDK, SslProvider.JDK, false);
+    }
+
+    @Test(timeout = 30000)
+    public void testSniSNIMatcherMatchesClientOpenSslServerOpenSsl() throws Exception {
+        Assume.assumeTrue(PlatformDependent.javaVersion() >= 8);
+        Assume.assumeTrue(OpenSsl.isAvailable());
+        SniClientJava8TestUtil.testSniClient(SslProvider.OPENSSL, SslProvider.OPENSSL, true);
+    }
+
+    @Test(timeout = 30000, expected = ClosedChannelException.class)
+    public void testSniSNIMatcherDoesNotMatchClientOpenSslServerOpenSsl() throws Exception {
+        Assume.assumeTrue(PlatformDependent.javaVersion() >= 8);
+        Assume.assumeTrue(OpenSsl.isAvailable());
+        SniClientJava8TestUtil.testSniClient(SslProvider.OPENSSL, SslProvider.OPENSSL, false);
+    }
+
+    @Test(timeout = 30000)
+    public void testSniSNIMatcherMatchesClientJdkSslServerOpenSsl() throws Exception {
+        Assume.assumeTrue(PlatformDependent.javaVersion() >= 8);
+        Assume.assumeTrue(OpenSsl.isAvailable());
+        SniClientJava8TestUtil.testSniClient(SslProvider.JDK, SslProvider.OPENSSL, true);
+    }
+
+    @Test(timeout = 30000, expected = ClosedChannelException.class)
+    public void testSniSNIMatcherDoesNotMatchClientJdkSslServerOpenSsl() throws Exception {
+        Assume.assumeTrue(PlatformDependent.javaVersion() >= 8);
+        Assume.assumeTrue(OpenSsl.isAvailable());
+        SniClientJava8TestUtil.testSniClient(SslProvider.JDK, SslProvider.OPENSSL, false);
+    }
+
+    @Test(timeout = 30000)
+    public void testSniSNIMatcherMatchesClientOpenSslServerJdkSsl() throws Exception {
+        Assume.assumeTrue(PlatformDependent.javaVersion() >= 8);
+        Assume.assumeTrue(OpenSsl.isAvailable());
+        SniClientJava8TestUtil.testSniClient(SslProvider.OPENSSL, SslProvider.JDK, true);
+    }
+
+    @Test(timeout = 30000, expected = ClosedChannelException.class)
+    public void testSniSNIMatcherDoesNotMatchClientOpenSslServerJdkSsl() throws Exception {
+        Assume.assumeTrue(PlatformDependent.javaVersion() >= 8);
+        Assume.assumeTrue(OpenSsl.isAvailable());
+        SniClientJava8TestUtil.testSniClient(SslProvider.OPENSSL, SslProvider.JDK, false);
+    }
+
     private static void testSniClient(SslProvider sslClientProvider, SslProvider sslServerProvider) throws Exception {
         final String sniHost = "sni.netty.io";
         LocalAddress address = new LocalAddress("test");