[#4746] Support SNI when using OpenSSL

Motivation:

When using SslProvider.OPENSSL we currently not handle SNI on the client side.

Modifications:

Correctly enable SNI when using clientMode and peerHost != null.

Result:

SNI works even with SslProvider.OPENSSL.

handler/src/main/java/io/netty/handler/ssl/OpenSslEngine.java

@@ -222,6 +222,12 @@ public final class OpenSslEngine extends SSLEngine {
         // Set the client auth mode, this needs to be done via setClientAuth(...) method so we actually call the
         // needed JNI methods.
         setClientAuth(clientMode ? ClientAuth.NONE : checkNotNull(clientAuth, "clientAuth"));
+
+        // Use SNI if peerHost was specified
+        // See https://github.com/netty/netty/issues/4746
+        if (clientMode && peerHost != null) {
+            SSL.setTlsExtHostName(ssl, peerHost);
+        }
     }
 
     @Override

handler/src/test/java/io/netty/handler/ssl/SniClientTest.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2016 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.ssl;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.bootstrap.ServerBootstrap;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.EventLoopGroup;
+import io.netty.channel.local.LocalAddress;
+import io.netty.channel.local.LocalChannel;
+import io.netty.channel.local.LocalEventLoopGroup;
+import io.netty.channel.local.LocalServerChannel;
+import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import io.netty.handler.ssl.util.SelfSignedCertificate;
+import io.netty.util.DomainNameMapping;
+import io.netty.util.concurrent.Promise;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+
+public class SniClientTest {
+
+    @Test
+    public void testSniClientJdkSslServerJdkSsl() throws Exception {
+        testSniClient(SslProvider.JDK, SslProvider.JDK);
+    }
+
+    @Test
+    public void testSniClientOpenSslServerOpenSsl() throws Exception {
+        Assume.assumeTrue(OpenSsl.isAvailable());
+        testSniClient(SslProvider.OPENSSL, SslProvider.OPENSSL);
+    }
+
+    @Test
+    public void testSniClientJdkSslServerOpenSsl() throws Exception {
+        Assume.assumeTrue(OpenSsl.isAvailable());
+        testSniClient(SslProvider.JDK, SslProvider.OPENSSL);
+    }
+
+    @Test
+    public void testSniClientOpenSslServerJdkSsl() throws Exception {
+        Assume.assumeTrue(OpenSsl.isAvailable());
+        testSniClient(SslProvider.OPENSSL, SslProvider.JDK);
+    }
+
+    @SuppressWarnings("deprecation")
+    private static void testSniClient(SslProvider sslClientProvider, SslProvider sslServerProvider) throws Exception {
+        final String sniHost = "sni.netty.io";
+        LocalAddress address = new LocalAddress("test");
+        EventLoopGroup group = new LocalEventLoopGroup(1);
+        Channel sc = null;
+        Channel cc = null;
+        try {
+            SelfSignedCertificate cert = new SelfSignedCertificate();
+            final SslContext sslServerContext = SslContextBuilder.forServer(cert.key(), cert.cert())
+                    .sslProvider(sslServerProvider).build();
+
+            final Promise<String> promise = group.next().newPromise();
+            ServerBootstrap sb = new ServerBootstrap();
+            sc = sb.group(group).channel(LocalServerChannel.class).childHandler(new ChannelInitializer<Channel>() {
+                @Override
+                protected void initChannel(Channel ch) throws Exception {
+                    ch.pipeline().addFirst(new SniHandler(new DomainNameMapping<SslContext>(sslServerContext) {
+                        @Override
+                        public SslContext map(String input) {
+                            promise.setSuccess(input);
+                            return sslServerContext;
+                        }
+                    }));
+                }
+            }).bind(address).syncUninterruptibly().channel();
+
+            SslContext sslContext = SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE)
+                    .sslProvider(sslClientProvider).build();
+            Bootstrap cb = new Bootstrap();
+            cc = cb.group(group).channel(LocalChannel.class).handler(new SslHandler(
+                    sslContext.newEngine(ByteBufAllocator.DEFAULT, sniHost, -1)))
+                    .connect(address).syncUninterruptibly().channel();
+            Assert.assertEquals(sniHost, promise.syncUninterruptibly().getNow());
+        } finally {
+            if (cc != null) {
+                cc.close();
+            }
+            if (sc != null) {
+                sc.close();
+            }
+            group.shutdownGracefully();
+        }
+    }
+}

pom.xml

@@ -216,7 +216,7 @@
     <!-- Fedora-"like" systems. This is currently only used for the netty-tcnative dependency -->
     <os.detection.classifierWithLikes>fedora</os.detection.classifierWithLikes>
     <tcnative.artifactId>netty-tcnative</tcnative.artifactId>
-    <tcnative.version>1.1.33.Fork11</tcnative.version>
+    <tcnative.version>1.1.33.Fork12</tcnative.version>
     <tcnative.classifier>${os.detected.classifier}</tcnative.classifier>
     <epoll.classifier>${os.detected.name}-${os.detected.arch}</epoll.classifier>
   </properties>