Consolidate creation of SslHandshakeException when caused by a callback that is used in the native SSL implementation. (#8979)

Motivation:

We have multiple places where we store the exception that was produced by a callback in ReferenceCountedOpenSslEngine, and so have a lot of code-duplication.

Modifications:

- Consolidate code into a package-private method that is called from the callbacks if needed

Result:

Less code-duplication and cleaner code.

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslClientContext.java

@@ -33,7 +33,6 @@ import java.util.Set;
 
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLException;
-import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.TrustManagerFactory;
 import javax.net.ssl.X509ExtendedTrustManager;
 import javax.net.ssl.X509TrustManager;
@@ -276,10 +275,7 @@ public final class ReferenceCountedOpenSslClientContext extends ReferenceCounted
                 keyManagerHolder.setKeyMaterialClientSide(engine, keyTypes, issuers);
             } catch (Throwable cause) {
                 logger.debug("request of key failed", cause);
-                SSLHandshakeException e = new SSLHandshakeException("General OpenSslEngine problem");
+                engine.initHandshakeException(cause);
-                e.initCause(cause);
-                assert engine.handshakeException == null;
-                engine.handshakeException = e;
             }
         }
 

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslContext.java

@@ -662,10 +662,7 @@ public abstract class ReferenceCountedOpenSslContext extends SslContext implemen
                 return CertificateVerifier.X509_V_OK;
             } catch (Throwable cause) {
                 logger.debug("verification of certificate failed", cause);
-                SSLHandshakeException e = new SSLHandshakeException("General OpenSslEngine problem");
+                engine.initHandshakeException(cause);
-                e.initCause(cause);
-                assert engine.handshakeException == null;
-                engine.handshakeException = e;
 
                 // Try to extract the correct error code that should be used.
                 if (cause instanceof OpenSslCertificateException) {

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngine.java

@@ -224,10 +224,7 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
     private final boolean enableOcsp;
     private int maxWrapOverhead;
     private int maxWrapBufferSize;
-
+    private Throwable handshakeException;
-    // This is package-private as we set it from OpenSslContext if an exception is thrown during
-    // the verification step.
-    SSLHandshakeException handshakeException;
 
     /**
      * Create a new instance.
@@ -1690,11 +1687,25 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc
             return NEED_WRAP;
         }
 
-        SSLHandshakeException exception = handshakeException;
+        Throwable exception = handshakeException;
         assert exception != null;
         handshakeException = null;
         shutdown();
-        throw exception;
+        if (exception instanceof SSLHandshakeException) {
+            throw (SSLHandshakeException) exception;
+        }
+        SSLHandshakeException e = new SSLHandshakeException("General OpenSslEngine problem");
+        e.initCause(exception);
+        throw e;
+    }
+
+    /**
+     * Should be called if the handshake will be failed due a callback that throws an exception.
+     * This cause will then be used to give more details as part of the {@link SSLHandshakeException}.
+     */
+    final void initHandshakeException(Throwable cause) {
+        assert handshakeException == null;
+        handshakeException = cause;
     }
 
     private SSLEngineResult.HandshakeStatus handshake() throws SSLException {

handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslServerContext.java

@@ -29,7 +29,6 @@ import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLException;
-import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.TrustManagerFactory;
 import javax.net.ssl.X509ExtendedTrustManager;
 import javax.net.ssl.X509TrustManager;
@@ -208,10 +207,7 @@ public final class ReferenceCountedOpenSslServerContext extends ReferenceCounted
                 keyManagerHolder.setKeyMaterialServerSide(engine);
             } catch (Throwable cause) {
                 logger.debug("Failed to set the server-side key material", cause);
-                SSLHandshakeException e = new SSLHandshakeException("General OpenSslEngine problem");
+                engine.initHandshakeException(cause);
-                e.initCause(cause);
-                assert engine.handshakeException == null;
-                engine.handshakeException = e;
             }
         }
     }