From 2b16d9f67084350ce936815639e770a0b3bca3d0 Mon Sep 17 00:00:00 2001
From: Trustin Lee <t@motd.kr>
Date: Tue, 5 Aug 2014 17:35:11 -0700
Subject: [PATCH] Fix the dead lock while failing SSL pending writes on a
 closed channel

Related issue: #2093

Motivation:

When a channel with SslHandler is closed with pending writes (either
unencrypted or encrypted), SslHandler.channelClosed() attempts to
acquire a lock and fail the futures of the pending writes.

If a user added a listener to any of the failed futures and the listener
attempts to write something, it will make SslHandler.handleDownstream()
invoked, which also attempts to acquire the same lock.

Because the lock is non-reentrant, even if these two lock acquisitions
are taking place in the same thread, the second attempt will block for
ever.

Modification:

Do not fail the futures while holding a lock.  Notify them after
releasing the lock.

Result:

One less dead lock
---
 .../jboss/netty/handler/ssl/SslHandler.java   | 21 ++++++++++++-------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/jboss/netty/handler/ssl/SslHandler.java b/src/main/java/org/jboss/netty/handler/ssl/SslHandler.java
index c1a8861ca4..05b3e81b83 100644
--- a/src/main/java/org/jboss/netty/handler/ssl/SslHandler.java
+++ b/src/main/java/org/jboss/netty/handler/ssl/SslHandler.java
@@ -1735,17 +1735,17 @@ public class SslHandler extends FrameDecoder
                     return;
                 }
 
-                Throwable cause = null;
+                List<ChannelFuture> futures = null;
                 try {
                     for (;;) {
                         PendingWrite pw = pendingUnencryptedWrites.poll();
                         if (pw == null) {
                             break;
                         }
-                        if (cause == null) {
-                            cause = new ClosedChannelException();
+                        if (futures == null) {
+                            futures = new ArrayList<ChannelFuture>();
                         }
-                        pw.future.setFailure(cause);
+                        futures.add(pw.future);
                     }
 
                     for (;;) {
@@ -1753,16 +1753,21 @@ public class SslHandler extends FrameDecoder
                         if (ev == null) {
                             break;
                         }
-                        if (cause == null) {
-                            cause = new ClosedChannelException();
+                        if (futures != null) {
+                            futures = new ArrayList<ChannelFuture>();
                         }
-                        ev.getFuture().setFailure(cause);
+                        futures.add(ev.getFuture());
                     }
                 } finally {
                     pendingUnencryptedWritesLock.unlock();
                 }
 
-                if (cause != null) {
+                if (futures != null) {
+                    final ClosedChannelException cause = new ClosedChannelException();
+                    final int size = futures.size();
+                    for (int i = 0; i < size; i ++) {
+                        futures.get(i).setFailure(cause);
+                    }
                     fireExceptionCaught(ctx, cause);
                 }
             }