From 2fa9400a59d0563a66908aba55c41e7285a04994 Mon Sep 17 00:00:00 2001
From: Trustin Lee <t@motd.kr>
Date: Tue, 10 Jun 2014 17:55:19 +0900
Subject: [PATCH] Fix a bug where SslHandler does not handle SSLv2Hello
 correctly

Motivation:

When a SSLv2Hello message is received, SSLEngine expects the application buffer size to be more than 30KB which is larger than what SslBufferPool can provide.  SSLEngine will always return with BUFFER_OVERFLOW status, blocking the SSL session from continuing the handshake.

Modifications:

When SSLEngine.getSession().getApplicationBufferSize() returns a value larger than what SslBufferPool provides, allocate a temporary heap buffer.

Result:

SSLv2Hello is handled correctly.
---
 .../jboss/netty/handler/ssl/SslHandler.java   | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/jboss/netty/handler/ssl/SslHandler.java b/src/main/java/org/jboss/netty/handler/ssl/SslHandler.java
index 83dfeb999b..c3bb18d4cb 100644
--- a/src/main/java/org/jboss/netty/handler/ssl/SslHandler.java
+++ b/src/main/java/org/jboss/netty/handler/ssl/SslHandler.java
@@ -1268,8 +1268,18 @@ public class SslHandler extends FrameDecoder
                     // always contain at least one record in decode().  Therefore, if SSLEngine.unwrap() returns
                     // BUFFER_OVERFLOW, it is always resolved by retrying after emptying the application buffer.
                     for (;;) {
+                        final int outAppBufSize = engine.getSession().getApplicationBufferSize();
+                        final ByteBuffer outAppBuf;
+                        if (nioOutAppBuf.capacity() < outAppBufSize) {
+                            // SSLEngine wants a buffer larger than what the pool can provide.
+                            // Allocate a temporary heap buffer.
+                            outAppBuf = ByteBuffer.allocate(outAppBufSize);
+                        } else {
+                            outAppBuf = nioOutAppBuf;
+                        }
+
                         try {
-                            result = engine.unwrap(nioInNetBuf, nioOutAppBuf);
+                            result = engine.unwrap(nioInNetBuf, outAppBuf);
                             switch (result.getStatus()) {
                                 case CLOSED:
                                     // notify about the CLOSED state of the SSLEngine. See #137
@@ -1283,21 +1293,21 @@ public class SslHandler extends FrameDecoder
 
                             break;
                         } finally {
-                            nioOutAppBuf.flip();
+                            outAppBuf.flip();
 
                             // Sync the offset of the inbound buffer.
                             nettyInNetBuf.readerIndex(
                                     nettyInNetBufStartOffset + nioInNetBuf.position() - nioInNetBufStartOffset);
 
                             // Copy the unwrapped data into a smaller buffer.
-                            if (nioOutAppBuf.hasRemaining()) {
+                            if (outAppBuf.hasRemaining()) {
                                 if (nettyOutAppBuf == null) {
                                     ChannelBufferFactory factory = ctx.getChannel().getConfig().getBufferFactory();
                                     nettyOutAppBuf = factory.getBuffer(initialNettyOutAppBufCapacity);
                                 }
-                                nettyOutAppBuf.writeBytes(nioOutAppBuf);
+                                nettyOutAppBuf.writeBytes(outAppBuf);
                             }
-                            nioOutAppBuf.clear();
+                            outAppBuf.clear();
                         }
                     }