Fix memory leak in OpenSslEngine

Motivation:

When a remote peer did open a connection and only do the handshake without sending any data and then directly close the connection we did not call shutdown() in the OpenSslEngine. This leads to a native memory leak. Beside this it also was not fireed when a OpenSslEngine was created but never used.

Modifications:

- Make sure shutdown() is called in all cases when closeInbound() is called
- Call shutdown() also in the finalize() method to ensure we release native memory when the OpenSslEngine is GC'ed

Result:

No more memory leak when using OpenSslEngine

handler/src/main/java/io/netty/handler/ssl/OpenSslEngine.java

@@ -554,15 +554,11 @@ public final class OpenSslEngine extends SSLEngine {
         isInboundDone = true;
         engineClosed = true;
 
-        if (accepted != 0) {
-            if (!receivedShutdown) {
-                shutdown();
-                throw new SSLException(
-                        "Inbound closed before receiving peer's close_notify: possible truncation attack?");
-            }
-        } else {
-            // engine closing before initial handshake
-            shutdown();
+        shutdown();
+
+        if (accepted != 0 && !receivedShutdown) {
+            throw new SSLException(
+                    "Inbound closed before receiving peer's close_notify: possible truncation attack?");
         }
     }
 
@@ -746,10 +742,9 @@ public final class OpenSslEngine extends SSLEngine {
 
     @Override
     public synchronized void beginHandshake() throws SSLException {
-        if (engineClosed) {
+        if (engineClosed || destroyed != 0) {
             throw ENGINE_CLOSED;
         }
-
         switch (accepted) {
             case 0:
                 SSL.doHandshake(ssl);
@@ -772,7 +767,7 @@ public final class OpenSslEngine extends SSLEngine {
     }
 
     private synchronized void beginHandshakeImplicitly() throws SSLException {
-        if (engineClosed) {
+        if (engineClosed || destroyed != 0) {
             throw ENGINE_CLOSED;
         }
 
@@ -882,4 +877,11 @@ public final class OpenSslEngine extends SSLEngine {
     public boolean getEnableSessionCreation() {
         return false;
     }
+
+    @Override
+    protected void finalize() throws Throwable {
+        super.finalize();
+        // Call shutdown as the user may have created the OpenSslEngine and not used it at all.
+        shutdown();
+    }
 }