From ed1071d327cdd77bf520befda7f62081374e77bc Mon Sep 17 00:00:00 2001 From: Norman Maurer Date: Sat, 25 Mar 2017 17:14:53 -0700 Subject: [PATCH] Limit the maximum size of the allocated outbound buffer to MAX_ENCRYPTED_PACKET_LENGTH Motivation: We should limit the size of the allocated outbound buffer to MAX_ENCRYPTED_PACKET_LENGTH to ensure we not cause an OOME when the user tries to encrypt a very big buffer. Modifications: Limit the size of the allocated outbound buffer to MAX_ENCRYPTED_PACKET_LENGTH Result: Fixes [#6564] --- .../netty/handler/ssl/ReferenceCountedOpenSslEngine.java | 3 ++- .../test/java/io/netty/handler/ssl/OpenSslEngineTest.java | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngine.java b/handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngine.java index 7a6833ec5d..9db63c9b0a 100644 --- a/handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngine.java +++ b/handler/src/main/java/io/netty/handler/ssl/ReferenceCountedOpenSslEngine.java @@ -1611,7 +1611,8 @@ public class ReferenceCountedOpenSslEngine extends SSLEngine implements Referenc } static int calculateOutNetBufSize(int pendingBytes, int numComponents) { - return (int) min(Integer.MAX_VALUE, pendingBytes + (long) MAX_TLS_RECORD_OVERHEAD_LENGTH * numComponents); + return (int) min(MAX_ENCRYPTED_PACKET_LENGTH, + pendingBytes + (long) MAX_TLS_RECORD_OVERHEAD_LENGTH * numComponents); } private final class OpenSslSession implements SSLSession, ApplicationProtocolAccessor { diff --git a/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java b/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java index 36ce1f63f0..33797a34df 100644 --- a/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java +++ b/handler/src/test/java/io/netty/handler/ssl/OpenSslEngineTest.java @@ -318,7 +318,7 @@ public class OpenSslEngineTest extends SSLEngineTest { @Test public void testCalculateOutNetBufSizeOverflow() { - assertEquals(MAX_VALUE, + assertEquals(MAX_ENCRYPTED_PACKET_LENGTH, ReferenceCountedOpenSslEngine.calculateOutNetBufSize(MAX_VALUE, 1)); } @@ -328,6 +328,12 @@ public class OpenSslEngineTest extends SSLEngineTest { ReferenceCountedOpenSslEngine.calculateOutNetBufSize(0, 1)); } + @Test + public void testCalculateOutNetBufSizeMaxEncryptedPacketLength() { + assertEquals(MAX_ENCRYPTED_PACKET_LENGTH, + ReferenceCountedOpenSslEngine.calculateOutNetBufSize(MAX_ENCRYPTED_PACKET_LENGTH + 1, 2)); + } + @Override protected void mySetupMutualAuthServerInitSslHandler(SslHandler handler) { ReferenceCountedOpenSslEngine engine = (ReferenceCountedOpenSslEngine) handler.engine();