andreacavalli/netty5
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

SslHandler wrap memory leak

Browse Source 
Motivation:
The SslHandler wrap method requires that a direct buffer be passed to the SSLEngine.wrap() call. If the ByteBuf parameter does not have an underlying direct buffer then one is allocated in this method, but it is not released.

Modifications:
- Release the direct ByteBuffer only accessible in the scope of SslHandler.wrap

Result:
Memory leak in SslHandler.wrap is fixed.
This commit is contained in:
Scott Mitchell 2014-10-27 13:47:54 -04:00 committed by Norman Maurer
parent 7fbd66f814
commit 06ea226a28
1 changed files with 28 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
handler/src/main/java/io/netty/handler/ssl/SslHandler.java
View File

@ -436,6 +436,7 @@ public class SslHandler extends ByteToMessageDecoder implements ChannelOutboundH
private void wrap(ChannelHandlerContext ctx, boolean inUnwrap) throws SSLException {
ByteBuf out = null;
ChannelPromise promise = null;
ByteBufAllocator alloc = ctx.alloc();
try {
for (;;) {
Object msg = pendingUnencryptedWrites.current();
@ -453,7 +454,7 @@ public class SslHandler extends ByteToMessageDecoder implements ChannelOutboundH
out = allocateOutNetBuf(ctx, buf.readableBytes());
}
SSLEngineResult result = wrap(engine, buf, out);
SSLEngineResult result = wrap(alloc, engine, buf, out);
if (!buf.isReadable()) {
promise = pendingUnencryptedWrites.remove();
@ -519,12 +520,13 @@ public class SslHandler extends ByteToMessageDecoder implements ChannelOutboundH
private void wrapNonAppData(ChannelHandlerContext ctx, boolean inUnwrap) throws SSLException {
ByteBuf out = null;
ByteBufAllocator alloc = ctx.alloc();
try {
for (;;) {
if (out == null) {
out = allocateOutNetBuf(ctx, 0);
}
SSLEngineResult result = wrap(engine, Unpooled.EMPTY_BUFFER, out);
SSLEngineResult result = wrap(alloc, engine, Unpooled.EMPTY_BUFFER, out);
if (result.bytesProduced() > 0) {
ctx.write(out);
@ -574,26 +576,37 @@ public class SslHandler extends ByteToMessageDecoder implements ChannelOutboundH
}
}
private SSLEngineResult wrap(SSLEngine engine, ByteBuf in, ByteBuf out) throws SSLException {
ByteBuffer in0 = in.nioBuffer();
if (!in0.isDirect()) {
ByteBuffer newIn0 = ByteBuffer.allocateDirect(in0.remaining());
newIn0.put(in0).flip();
in0 = newIn0;
}
private SSLEngineResult wrap(ByteBufAllocator alloc, SSLEngine engine, ByteBuf in, ByteBuf out)
throws SSLException {
ByteBuf newDirectIn = null;
try {
final ByteBuffer in0;
if (in.isDirect()) {
in0 = in.nioBuffer();
} else {
int readableBytes = in.readableBytes();
newDirectIn = alloc.directBuffer(readableBytes);
newDirectIn.writeBytes(in, in.readerIndex(), readableBytes);
in0 = newDirectIn.internalNioBuffer(0, readableBytes);
}
for (;;) {
ByteBuffer out0 = out.nioBuffer(out.writerIndex(), out.writableBytes());
SSLEngineResult result = engine.wrap(in0, out0);
in.skipBytes(result.bytesConsumed());
out.writerIndex(out.writerIndex() + result.bytesProduced());
for (;;) {
ByteBuffer out0 = out.nioBuffer(out.writerIndex(), out.writableBytes());
SSLEngineResult result = engine.wrap(in0, out0);
in.skipBytes(result.bytesConsumed());
out.writerIndex(out.writerIndex() + result.bytesProduced());
switch (result.getStatus()) {
switch (result.getStatus()) {
case BUFFER_OVERFLOW:
out.ensureWritable(maxPacketBufferSize);
break;
default:
return result;
}
}
} finally {
if (newDirectIn != null) {
newDirectIn.release();
}
}
}