Close connection for CorruptedFrameException (#8705)

Motivation:

The CorruptedFrameException from the finish() method of the Utf8Validator gets propagated to other handlers while the connection is still open.

Modification:

Override exceptionCaught method of the Utf8FrameValidator and close the connection if it is a CorruptedFrameException.

Result:

The CorruptedFrameException gets propagated to other handlers only after properly closing the connection.

codec-http/src/main/java/io/netty/handler/codec/http/websocketx/Utf8FrameValidator.java

@@ -47,7 +47,7 @@ public class Utf8FrameValidator extends ChannelInboundHandlerAdapter {
                     if ((frame instanceof TextWebSocketFrame) ||
                             (utf8Validator != null && utf8Validator.isChecking())) {
                         // Check UTF-8 correctness for this payload
-                        checkUTF8String(ctx, frame.content());
+                        checkUTF8String(frame.content());
 
                         // This does a second check to make sure UTF-8
                         // correctness for entire text message
@@ -60,12 +60,12 @@ public class Utf8FrameValidator extends ChannelInboundHandlerAdapter {
                 if (fragmentedFramesCount == 0) {
                     // First text or binary frame for a fragmented set
                     if (frame instanceof TextWebSocketFrame) {
-                        checkUTF8String(ctx, frame.content());
+                        checkUTF8String(frame.content());
                     }
                 } else {
                     // Subsequent frames - only check if init frame is text
                     if (utf8Validator != null && utf8Validator.isChecking()) {
-                        checkUTF8String(ctx, frame.content());
+                        checkUTF8String(frame.content());
                     }
                 }
 
@@ -77,17 +77,18 @@ public class Utf8FrameValidator extends ChannelInboundHandlerAdapter {
         super.channelRead(ctx, msg);
     }
 
-    private void checkUTF8String(ChannelHandlerContext ctx, ByteBuf buffer) {
-        try {
-            if (utf8Validator == null) {
-                utf8Validator = new Utf8Validator();
-            }
-            utf8Validator.check(buffer);
-        } catch (CorruptedFrameException ex) {
-            if (ctx.channel().isActive()) {
-                ctx.writeAndFlush(Unpooled.EMPTY_BUFFER).addListener(ChannelFutureListener.CLOSE);
-            }
+    private void checkUTF8String(ByteBuf buffer) {
+        if (utf8Validator == null) {
+            utf8Validator = new Utf8Validator();
         }
+        utf8Validator.check(buffer);
     }
 
+    @Override
+    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
+        if (cause instanceof CorruptedFrameException && ctx.channel().isOpen()) {
+            ctx.writeAndFlush(Unpooled.EMPTY_BUFFER).addListener(ChannelFutureListener.CLOSE);
+        }
+        super.exceptionCaught(ctx, cause);
+    }
 }

codec-http/src/test/java/io/netty/handler/codec/http/websocketx/WebSocketUtf8FrameValidatorTest.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2019 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.codec.http.websocketx;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.embedded.EmbeddedChannel;
+import io.netty.handler.codec.CorruptedFrameException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class WebSocketUtf8FrameValidatorTest {
+
+    @Test
+    public void testCorruptedFrameExceptionInFinish() {
+        assertCorruptedFrameExceptionHandling(new byte[]{-50});
+    }
+
+    @Test
+    public void testCorruptedFrameExceptionInCheck() {
+        assertCorruptedFrameExceptionHandling(new byte[]{-8, -120, -128, -128, -128});
+    }
+
+    private void assertCorruptedFrameExceptionHandling(byte[] data) {
+        EmbeddedChannel channel = new EmbeddedChannel(new Utf8FrameValidator());
+        try {
+            channel.writeInbound(new TextWebSocketFrame(Unpooled.copiedBuffer(data)));
+            Assert.fail();
+        } catch (CorruptedFrameException e) {
+            // expected exception
+        }
+        Assert.assertTrue(channel.finish());
+        ByteBuf buf = channel.readOutbound();
+        Assert.assertNotNull(buf);
+        try {
+            Assert.assertFalse(buf.isReadable());
+        } finally {
+            buf.release();
+        }
+        Assert.assertNull(channel.readOutbound());
+    }
+}