From 6e5ab465d9f469f1abec3e2be5f198b3cf98d345 Mon Sep 17 00:00:00 2001
From: Norman Maurer <norman_maurer@apple.com>
Date: Thu, 11 Dec 2014 10:36:56 +0100
Subject: [PATCH] Fix memory leak in OpenSslEngine

Motivation:

When a remote peer did open a connection and only do the handshake without sending any data and then directly close the connection we did not call shutdown() in the OpenSslEngine. This leads to a native memory leak. Beside this it also was not fireed when a OpenSslEngine was created but never used.

Modifications:

- Make sure shutdown() is called in all cases when closeInbound() is called
- Call shutdown() also in the finalize() method to ensure we release native memory when the OpenSslEngine is GC'ed

Result:

No more memory leak when using OpenSslEngine
---
 .../io/netty/handler/ssl/OpenSslEngine.java   | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/handler/src/main/java/io/netty/handler/ssl/OpenSslEngine.java b/handler/src/main/java/io/netty/handler/ssl/OpenSslEngine.java
index a753c0ba6b..4e15508630 100644
--- a/handler/src/main/java/io/netty/handler/ssl/OpenSslEngine.java
+++ b/handler/src/main/java/io/netty/handler/ssl/OpenSslEngine.java
@@ -554,15 +554,11 @@ public final class OpenSslEngine extends SSLEngine {
         isInboundDone = true;
         engineClosed = true;
 
-        if (accepted != 0) {
-            if (!receivedShutdown) {
-                shutdown();
-                throw new SSLException(
-                        "Inbound closed before receiving peer's close_notify: possible truncation attack?");
-            }
-        } else {
-            // engine closing before initial handshake
-            shutdown();
+        shutdown();
+
+        if (accepted != 0 && !receivedShutdown) {
+            throw new SSLException(
+                    "Inbound closed before receiving peer's close_notify: possible truncation attack?");
         }
     }
 
@@ -746,10 +742,9 @@ public final class OpenSslEngine extends SSLEngine {
 
     @Override
     public synchronized void beginHandshake() throws SSLException {
-        if (engineClosed) {
+        if (engineClosed || destroyed != 0) {
             throw ENGINE_CLOSED;
         }
-
         switch (accepted) {
             case 0:
                 SSL.doHandshake(ssl);
@@ -772,7 +767,7 @@ public final class OpenSslEngine extends SSLEngine {
     }
 
     private synchronized void beginHandshakeImplicitly() throws SSLException {
-        if (engineClosed) {
+        if (engineClosed || destroyed != 0) {
             throw ENGINE_CLOSED;
         }
 
@@ -882,4 +877,11 @@ public final class OpenSslEngine extends SSLEngine {
     public boolean getEnableSessionCreation() {
         return false;
     }
+
+    @Override
+    protected void finalize() throws Throwable {
+        super.finalize();
+        // Call shutdown as the user may have created the OpenSslEngine and not used it at all.
+        shutdown();
+    }
 }