Correctly set the alert type depending of the CertificateException

Motivation:

Depending on the actual CertificateException we should set the correct alert type so it will be sent back to the remote peer and so make it easier for them to fix it.

Modification:

Correctly set the alert and not always just use a general alert.

Result:

It's easier for the remote peer to fix the problems.

handler/src/main/java/io/netty/handler/ssl/OpenSslCertificateException.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright 2016 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.ssl;
+
+import org.apache.tomcat.jni.CertificateVerifier;
+
+import java.security.cert.CertificateException;
+
+/**
+ * A special {@link CertificateException} which allows to specify which error code is included in the
+ * SSL Record. This only work when {@link SslProvider#OPENSSL} is used.
+ */
+public final class OpenSslCertificateException extends CertificateException {
+    private static final long serialVersionUID = 5542675253797129798L;
+
+    private final int errorCode;
+
+    /**
+     * Construct a new exception with the
+     * <a href="https://www.openssl.org/docs/manmaster/apps/verify.html">error code</a>.
+     */
+    public OpenSslCertificateException(int errorCode) {
+        this((String) null, errorCode);
+    }
+
+    /**
+     * Construct a new exception with the msg and
+     * <a href="https://www.openssl.org/docs/manmaster/apps/verify.html">error code</a> .
+     */
+    public OpenSslCertificateException(String msg, int errorCode) {
+        super(msg);
+        this.errorCode = checkErrorCode(errorCode);
+    }
+
+    /**
+     * Construct a new exception with the msg, cause and
+     * <a href="https://www.openssl.org/docs/manmaster/apps/verify.html">error code</a> .
+     */
+    public OpenSslCertificateException(String message, Throwable cause, int errorCode) {
+        super(message, cause);
+        this.errorCode = checkErrorCode(errorCode);
+    }
+
+    /**
+     * Construct a new exception with the cause and
+     * <a href="https://www.openssl.org/docs/manmaster/apps/verify.html">error code</a> .
+     */
+    public OpenSslCertificateException(Throwable cause, int errorCode) {
+        this(null, cause, errorCode);
+    }
+
+    /**
+     * Return the <a href="https://www.openssl.org/docs/manmaster/apps/verify.html">error code</a> to use.
+     */
+    public int errorCode() {
+        return errorCode;
+    }
+
+    private static int checkErrorCode(int errorCode) {
+        if (errorCode < CertificateVerifier.X509_V_OK || errorCode > CertificateVerifier.X509_V_ERR_DANE_NO_MATCH) {
+            throw new IllegalArgumentException("errorCode must be " + CertificateVerifier.X509_V_OK + " => "
+                    + CertificateVerifier.X509_V_ERR_DANE_NO_MATCH +
+                    ". See https://www.openssl.org/docs/manmaster/apps/verify.html .");
+        }
+        return errorCode;
+    }
+}

handler/src/main/java/io/netty/handler/ssl/OpenSslContext.java

@@ -37,6 +37,9 @@ import javax.net.ssl.X509ExtendedTrustManager;
 import javax.net.ssl.X509TrustManager;
 import java.security.PrivateKey;
 import java.security.cert.Certificate;
+import java.security.cert.CertificateExpiredException;
+import java.security.cert.CertificateNotYetValidException;
+import java.security.cert.CertificateRevokedException;
 import java.security.cert.X509Certificate;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -449,19 +452,32 @@ public abstract class OpenSslContext extends SslContext {
 
     abstract class AbstractCertificateVerifier implements CertificateVerifier {
         @Override
-        public final boolean verify(long ssl, byte[][] chain, String auth) {
+        public final int verify(long ssl, byte[][] chain, String auth) {
             X509Certificate[] peerCerts = certificates(chain);
             final OpenSslEngine engine = engineMap.remove(ssl);
             try {
                 verify(engine, peerCerts, auth);
-                return true;
+                return CertificateVerifier.X509_V_OK;
             } catch (Throwable cause) {
                 logger.debug("verification of certificate failed", cause);
                 SSLHandshakeException e = new SSLHandshakeException("General OpenSslEngine problem");
                 e.initCause(cause);
                 engine.handshakeException = e;
+
+                if (cause instanceof OpenSslCertificateException) {
+                    return ((OpenSslCertificateException) cause).errorCode();
+                }
+                if (cause instanceof CertificateExpiredException) {
+                    return CertificateVerifier.X509_V_ERR_CERT_HAS_EXPIRED;
+                }
+                if (cause instanceof CertificateNotYetValidException) {
+                    return CertificateVerifier.X509_V_ERR_CERT_NOT_YET_VALID;
+                }
+                if (PlatformDependent.javaVersion() >= 7 && cause instanceof CertificateRevokedException) {
+                    return CertificateVerifier.X509_V_ERR_CERT_REVOKED;
+                }
+                return CertificateVerifier.X509_V_ERR_UNSPECIFIED;
             }
-            return false;
         }
 
         abstract void verify(OpenSslEngine engine, X509Certificate[] peerCerts, String auth) throws Exception;

pom.xml

@@ -223,7 +223,7 @@
     <!-- Fedora-"like" systems. This is currently only used for the netty-tcnative dependency -->
     <os.detection.classifierWithLikes>fedora</os.detection.classifierWithLikes>
     <tcnative.artifactId>netty-tcnative</tcnative.artifactId>
-    <tcnative.version>1.1.33.Fork13</tcnative.version>
+    <tcnative.version>1.1.33.Fork14</tcnative.version>
     <tcnative.classifier>${os.detected.classifier}</tcnative.classifier>
     <epoll.classifier>${os.detected.name}-${os.detected.arch}</epoll.classifier>
   </properties>
@@ -676,6 +676,7 @@
             <ignore>javax.net.ssl.X509ExtendedTrustManager</ignore>
             <ignore>javax.net.ssl.SSLParameters</ignore>
             <ignore>java.security.AlgorithmConstraints</ignore>
+            <ignore>java.security.cert.CertificateRevokedException</ignore>
 
             <ignore>java.util.concurrent.ConcurrentLinkedDeque</ignore>
           </ignores>