Set SNI servernames in OpenSSL engine when created in client mode (#8178)

Motivation:

When using the JDK SSL provider in client mode, the SNI host names (called serverNames in SslEngineImpl) is set to the peerHost (if available) that is used to initialize the SSL Engine:

http://hg.openjdk.java.net/jdk8/jdk8/jdk/file/687fd7c7986d/src/share/classes/sun/security/ssl/SSLEngineImpl.java#l377

This allows one to call SslEngine.getSSLParameters() and inspect what is the SNI name to be sent. The same should be done in the OpenSSL provider as well. Currently even though the the SNI name is sent by the OpenSSL provider during handshake when the peerHost is specified, it is missing from the parameters.

Modification:

Set the sniHostNames field when SNI is to be used. Also verifies the peer is actually a hostname before setting it as the SNI name, which is consistent with JDK SSL provider's behavior.

Result:

SslEngine using the OpenSSL provider created in client mode with peerHost will initialize sniHostNames with the peerHost.

Calling SslEngine.getSSLParameters().getServerNames() will return a list that contains that name.

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngine.java

@@ -20,7 +20,6 @@ import io.netty.buffer.ByteBufAllocator;
 import io.netty.internal.tcnative.Buffer;
 import io.netty.internal.tcnative.SSL;
 import io.netty.util.AbstractReferenceCounted;
-import io.netty.util.CharsetUtil;
 import io.netty.util.ReferenceCounted;
 import io.netty.util.ResourceLeakDetector;
 import io.netty.util.ResourceLeakDetectorFactory;
@@ -40,6 +39,7 @@ import java.security.cert.Certificate;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -272,10 +272,11 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
                     setEnabledProtocols(context.protocols);
                 }
 
-                // Use SNI if peerHost was specified
+                // Use SNI if peerHost was specified and a valid hostname
                 // See https://github.com/netty/netty/issues/4746
-                if (clientMode && peerHost != null) {
+                if (clientMode && SslUtils.isValidHostNameForSNI(peerHost)) {
                     SSL.setTlsExtHostName(ssl, peerHost);
+                    sniHostNames = Collections.singletonList(peerHost);
                 }
 
                 if (enableOcsp) {

handler/src/main/java/io/netty/handler/ssl/SslUtils.java

@@ -21,6 +21,7 @@ import io.netty.buffer.ByteBufUtil;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.base64.Base64;
 import io.netty.handler.codec.base64.Base64Dialect;
+import io.netty.util.NetUtil;
 
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -349,6 +350,17 @@ final class SslUtils {
         return dst;
     }
 
+    /**
+     * Validate that the given hostname can be used in SNI extension.
+     */
+    static boolean isValidHostNameForSNI(String hostname) {
+        return hostname != null &&
+               hostname.indexOf('.') > 0 &&
+               !hostname.endsWith(".") &&
+               !NetUtil.isValidIpV4Address(hostname) &&
+               !NetUtil.isValidIpV6Address(hostname);
+    }
+
     private SslUtils() {
     }
 }

handler/src/test/java/io/netty/handler/ssl/SSLEngineTest.java

@@ -71,6 +71,7 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
 import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SNIHostName;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
 import javax.net.ssl.SSLException;
@@ -834,6 +835,10 @@ public abstract class SSLEngineTest {
                 InetSocketAddress remoteAddress = (InetSocketAddress) serverChannel.localAddress();
                 SslHandler sslHandler = clientSslCtx.newHandler(ch.alloc(), expectedHost, 0);
                 SSLParameters parameters = sslHandler.engine().getSSLParameters();
+                if (SslUtils.isValidHostNameForSNI(expectedHost)) {
+                    assertEquals(1, parameters.getServerNames().size());
+                    assertEquals(new SNIHostName(expectedHost), parameters.getServerNames().get(0));
+                }
                 parameters.setEndpointIdentificationAlgorithm("HTTPS");
                 sslHandler.engine().setSSLParameters(parameters);
                 p.addLast(sslHandler);