[#3784] Support hostname verification when using OpenSSLEngine

Motivation:

At the moment hostname verification is not supported with OpenSSLEngine.

Modifications:

- Allow to create OpenSslEngine with peerHost and peerPort informations.
- Respect endPointIdentificationAlgorithm and algorithmConstraints when set and get SSLParamaters.

Result:

hostname verification is supported now.

handler/src/main/java/io/netty/handler/ssl/OpenSslContext.java

@@ -288,18 +288,18 @@ public abstract class OpenSslContext extends SslContext {
 
     @Override
     public final SSLEngine newEngine(ByteBufAllocator alloc, String peerHost, int peerPort) {
-        throw new UnsupportedOperationException();
+        final OpenSslEngine engine = new OpenSslEngine(ctx, alloc, isClient(), sessionContext(), apn, engineMap,
+                rejectRemoteInitiatedRenegotiation, peerHost, peerPort);
+        engineMap.add(engine);
+        return engine;
     }
 
     /**
-     * Returns a new server-side {@link javax.net.ssl.SSLEngine} with the current configuration.
+     * Returns a new server-side {@link SSLEngine} with the current configuration.
      */
     @Override
     public final SSLEngine newEngine(ByteBufAllocator alloc) {
-        final OpenSslEngine engine = new OpenSslEngine(
-                ctx, alloc, isClient(), sessionContext(), apn, engineMap, rejectRemoteInitiatedRenegotiation);
-        engineMap.add(engine);
-        return engine;
+        return newEngine(alloc, null, -1);
     }
 
     /**

handler/src/main/java/io/netty/handler/ssl/OpenSslEngine.java

@@ -30,6 +30,7 @@ import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
 import javax.net.ssl.SSLException;
 import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.SSLParameters;
 import javax.net.ssl.SSLPeerUnverifiedException;
 import javax.net.ssl.SSLSession;
 import javax.net.ssl.SSLSessionBindingEvent;
@@ -149,6 +150,10 @@ public final class OpenSslEngine extends SSLEngine {
     private volatile Certificate[] peerCerts;
     private volatile ClientAuthMode clientAuth = ClientAuthMode.NONE;
 
+    private volatile String endPointIdentificationAlgorithm;
+    // Store as object as AlgorithmConstraints only exists since java 7.
+    private volatile Object algorithmConstraints;
+
     // SSL Engine status variables
     private boolean isInboundDone;
     private boolean isOutboundDone;
@@ -190,6 +195,14 @@ public final class OpenSslEngine extends SSLEngine {
                   boolean clientMode, OpenSslSessionContext sessionContext,
                   OpenSslApplicationProtocolNegotiator apn, OpenSslEngineMap engineMap,
                   boolean rejectRemoteInitiatedRenegation) {
+        this(sslCtx, alloc, clientMode, sessionContext, apn, engineMap, rejectRemoteInitiatedRenegation, null, -1);
+    }
+
+    OpenSslEngine(long sslCtx, ByteBufAllocator alloc,
+                  boolean clientMode, OpenSslSessionContext sessionContext,
+                  OpenSslApplicationProtocolNegotiator apn, OpenSslEngineMap engineMap,
+                  boolean rejectRemoteInitiatedRenegation, String peerHost, int peerPort) {
+        super(peerHost, peerPort);
         OpenSsl.ensureAvailability();
         if (sslCtx == 0) {
             throw new NullPointerException("sslCtx");
@@ -1220,6 +1233,27 @@ public final class OpenSslEngine extends SSLEngine {
         return false;
     }
 
+    @Override
+    public SSLParameters getSSLParameters() {
+        SSLParameters sslParameters = super.getSSLParameters();
+
+        if (PlatformDependent.javaVersion() >= 7) {
+            sslParameters.setEndpointIdentificationAlgorithm(endPointIdentificationAlgorithm);
+            SslParametersUtils.setAlgorithmConstraints(sslParameters, algorithmConstraints);
+        }
+        return sslParameters;
+    }
+
+    @Override
+    public void setSSLParameters(SSLParameters sslParameters) {
+        super.setSSLParameters(sslParameters);
+
+        if (PlatformDependent.javaVersion() >= 7) {
+            endPointIdentificationAlgorithm = sslParameters.getEndpointIdentificationAlgorithm();
+            algorithmConstraints = sslParameters.getAlgorithmConstraints();
+        }
+    }
+
     @Override
     @SuppressWarnings("FinalizeDeclaration")
     protected void finalize() throws Throwable {
@@ -1465,12 +1499,12 @@ public final class OpenSslEngine extends SSLEngine {
 
         @Override
         public String getPeerHost() {
-            return null;
+            return OpenSslEngine.this.getPeerHost();
         }
 
         @Override
         public int getPeerPort() {
-            return 0;
+            return OpenSslEngine.this.getPeerPort();
         }
 
         @Override

handler/src/main/java/io/netty/handler/ssl/SslParametersUtils.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2014 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.ssl;
+
+import javax.net.ssl.SSLParameters;
+import java.security.AlgorithmConstraints;
+
+final class SslParametersUtils {
+
+    private SslParametersUtils() {
+        // Utility
+    }
+
+    /**
+     * Utility method that is used by {@link OpenSslEngine} and so allow use not not have any reference to
+     * {@link AlgorithmConstraints} in the code. This helps us to not get into trouble when using it in java
+     * version < 7 and especially when using on android.
+     */
+    static void setAlgorithmConstraints(SSLParameters sslParameters, Object algorithmConstraints) {
+        sslParameters.setAlgorithmConstraints((AlgorithmConstraints) algorithmConstraints);
+    }
+}

pom.xml

@@ -1037,6 +1037,8 @@
             <!-- SSLSession implementation -->
             <ignore>javax.net.ssl.SSLEngine</ignore>
             <ignore>javax.net.ssl.X509ExtendedTrustManager</ignore>
+            <ignore>javax.net.ssl.SSLParameters</ignore>
+            <ignore>java.security.AlgorithmConstraints</ignore>
 
             <ignore>java.util.concurrent.ConcurrentLinkedDeque</ignore>
           </ignores>