[#1798] Correctly handle skipping of too big frames

codec/src/main/java/io/netty/handler/codec/LengthFieldBasedFrameDecoder.java

@@ -369,8 +369,8 @@ public class LengthFieldBasedFrameDecoder extends ByteToMessageDecoder {
             in.skipBytes(localBytesToDiscard);
             bytesToDiscard -= localBytesToDiscard;
             this.bytesToDiscard = bytesToDiscard;
+
             failIfNecessary(false);
-            return null;
         }
 
         if (in.readableBytes() < lengthFieldEndOffset) {
@@ -396,11 +396,18 @@ public class LengthFieldBasedFrameDecoder extends ByteToMessageDecoder {
         }
 
         if (frameLength > maxFrameLength) {
-            // Enter the discard mode and discard everything received so far.
-            discardingTooLongFrame = true;
+            long discard = frameLength - in.readableBytes();
             tooLongFrameLength = frameLength;
-            bytesToDiscard = frameLength - in.readableBytes();
-            in.skipBytes(in.readableBytes());
+
+            if (discard < 0) {
+                // buffer contains more bytes then the frameLength so we can discard all now
+                in.skipBytes((int) frameLength);
+            } else {
+                // Enter the discard mode and discard everything received so far.
+                discardingTooLongFrame = true;
+                bytesToDiscard = discard;
+                in.skipBytes(in.readableBytes());
+            }
             failIfNecessary(true);
             return null;
         }

codec/src/test/java/io/netty/handler/codec/LengthFieldBasedFrameDecoderTest.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2013 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package io.netty.handler.codec;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.embedded.EmbeddedChannel;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class LengthFieldBasedFrameDecoderTest {
+
+    @Test
+    public void testDiscardTooLongFrame1() {
+        ByteBuf buf = Unpooled.buffer();
+        buf.writeInt(32);
+        for (int i = 0; i < 32; i++) {
+            buf.writeByte(i);
+        }
+        buf.writeInt(1);
+        buf.writeByte('a');
+        EmbeddedChannel channel = new EmbeddedChannel(new LengthFieldBasedFrameDecoder(16, 0, 4));
+        try {
+            channel.writeInbound(buf);
+            Assert.fail();
+        } catch (TooLongFrameException e) {
+            // expected
+        }
+        Assert.assertTrue(channel.finish());
+
+        ByteBuf b = (ByteBuf) channel.readInbound();
+        Assert.assertEquals(5, b.readableBytes());
+        Assert.assertEquals(1, b.readInt());
+        Assert.assertEquals('a', b.readByte());
+        b.release();
+
+        Assert.assertNull(channel.readInbound());
+    }
+
+    @Test
+    public void testDiscardTooLongFrame2() {
+        ByteBuf buf = Unpooled.buffer();
+        buf.writeInt(32);
+        for (int i = 0; i < 32; i++) {
+            buf.writeByte(i);
+        }
+        buf.writeInt(1);
+        buf.writeByte('a');
+        EmbeddedChannel channel = new EmbeddedChannel(new LengthFieldBasedFrameDecoder(16, 0, 4));
+        try {
+            channel.writeInbound(buf.readSlice(14).retain());
+            Assert.fail();
+        } catch (TooLongFrameException e) {
+            // expected
+        }
+        Assert.assertTrue(channel.writeInbound(buf.readSlice(buf.readableBytes()).retain()));
+
+        Assert.assertTrue(channel.finish());
+
+        ByteBuf b = (ByteBuf) channel.readInbound();
+        Assert.assertEquals(5, b.readableBytes());
+        Assert.assertEquals(1, b.readInt());
+        Assert.assertEquals('a', b.readByte());
+        b.release();
+
+        Assert.assertNull(channel.readInbound());
+    }
+}