[#3968] Disallow pass-through of non ByteBufs in SslHandler

Motivation:

We pass-through non ByteBuf when SslHandler.write(...) is called which can lead to have unencrypted data to be send (like for example if a FileRegion is written).

Modifications:

- Fail ChannelPromise with UnsupportedMessageException if a non ByteBuf is written.

Result:

Only allow ByteBuf to be written when using SslHandler.

handler/src/main/java/io/netty/handler/ssl/SslHandler.java

@@ -32,6 +32,7 @@ import io.netty.channel.ChannelPromise;
 import io.netty.channel.ChannelPromiseNotifier;
 import io.netty.channel.PendingWriteQueue;
 import io.netty.handler.codec.ByteToMessageDecoder;
+import io.netty.handler.codec.UnsupportedMessageTypeException;
 import io.netty.util.concurrent.DefaultPromise;
 import io.netty.util.concurrent.EventExecutor;
 import io.netty.util.concurrent.Future;
@@ -423,6 +424,10 @@ public class SslHandler extends ByteToMessageDecoder {
 
     @Override
     public void write(final ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
+        if (!(msg instanceof ByteBuf)) {
+            promise.setFailure(new UnsupportedMessageTypeException(msg, ByteBuf.class));
+            return;
+        }
         pendingUnencryptedWrites.add(msg, promise);
     }
 
@@ -461,11 +466,6 @@ public class SslHandler extends ByteToMessageDecoder {
                     break;
                 }
 
-                if (!(msg instanceof ByteBuf)) {
-                    pendingUnencryptedWrites.removeAndWrite();
-                    continue;
-                }
-
                 ByteBuf buf = (ByteBuf) msg;
                 if (out == null) {
                     out = allocateOutNetBuf(ctx, buf.readableBytes());

handler/src/test/java/io/netty/handler/ssl/SslHandlerTest.java

@@ -19,6 +19,7 @@ package io.netty.handler.ssl;
 import io.netty.buffer.Unpooled;
 import io.netty.channel.embedded.EmbeddedChannel;
 import io.netty.handler.codec.DecoderException;
+import io.netty.handler.codec.UnsupportedMessageTypeException;
 import org.junit.Test;
 
 import javax.net.ssl.SSLContext;
@@ -53,21 +54,13 @@ public class SslHandlerTest {
         }
     }
 
-    @Test
-    public void testNonByteBufPassthrough() throws Exception {
+    @Test(expected = UnsupportedMessageTypeException.class)
+    public void testNonByteBufNotPassThrough() throws Exception {
         SSLEngine engine = SSLContext.getDefault().createSSLEngine();
         engine.setUseClientMode(false);
 
         EmbeddedChannel ch = new EmbeddedChannel(new SslHandler(engine));
 
-        Object msg1 = new Object();
-        ch.writeOutbound(msg1);
-        assertThat(ch.readOutbound(), is(sameInstance(msg1)));
-
-        Object msg2 = new Object();
-        ch.writeInbound(msg2);
-        assertThat(ch.readInbound(), is(sameInstance(msg2)));
-
-        ch.finish();
+        ch.writeOutbound(new Object());
     }
 }