JdkZlibDecoder may corrupt data when GZIP footer is fragmented (#11521)

Motivation:

Due a bug in the statemachine we produced an decoding error when the GZIP footer was fragmented in some cases

Modifications:

- Fix statemachine
- Add testcase

Result:

Correctly decode GZIP in all cases

codec/src/main/java/io/netty/handler/codec/compression/JdkZlibDecoder.java

@@ -204,12 +204,15 @@ public class JdkZlibDecoder extends ZlibDecoder {
             if (gzipState != GzipState.HEADER_END) {
                 if (gzipState == GzipState.FOOTER_START) {
                     if (!handleGzipFooter(in)) {
+                        // Either there was not enough data or the input is finished.
                         return;
                     }
-                } else {
+                    // If we consumed the footer we will start with the header again.
-                    if (!readGZIPHeader(in)) {
+                    assert gzipState == GzipState.HEADER_START;
-                        return;
+                }
-                    }
+                if (!readGZIPHeader(in)) {
+                    // There was not enough data readable to read the GZIP header.
+                    return;
                 }
                 // Some bytes may have been consumed, and so we must re-set the number of readable bytes.
                 readableBytes = in.readableBytes();

codec/src/test/java/io/netty/handler/codec/compression/JdkZlibTest.java

@@ -20,16 +20,20 @@ import io.netty.buffer.Unpooled;
 import io.netty.channel.embedded.EmbeddedChannel;
 import io.netty.util.CharsetUtil;
 import io.netty.util.ReferenceCountUtil;
+import io.netty.util.internal.PlatformDependent;
 import org.apache.commons.compress.utils.IOUtils;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.function.Executable;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Queue;
+import java.util.zip.GZIPOutputStream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -129,4 +133,35 @@ public class JdkZlibTest extends ZlibTest {
             chDecoderGZip.close();
         }
     }
+
+    @Test
+    public void testDecodeWithHeaderFollowingFooter() throws Exception {
+        byte[] bytes = new byte[1024];
+        PlatformDependent.threadLocalRandom().nextBytes(bytes);
+        ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+        GZIPOutputStream out = new GZIPOutputStream(bytesOut);
+        out.write(bytes);
+        out.close();
+
+        byte[] compressed = bytesOut.toByteArray();
+        ByteBuf buffer = Unpooled.buffer().writeBytes(compressed).writeBytes(compressed);
+        EmbeddedChannel channel = new EmbeddedChannel(new JdkZlibDecoder(ZlibWrapper.GZIP, true));
+        // Write it into the Channel in a way that we were able to decompress the first data completely but not the
+        // whole footer.
+        assertTrue(channel.writeInbound(buffer.readRetainedSlice(compressed.length - 1)));
+        assertTrue(channel.writeInbound(buffer));
+        assertTrue(channel.finish());
+
+        ByteBuf uncompressedBuffer = Unpooled.wrappedBuffer(bytes);
+        ByteBuf read = channel.readInbound();
+        assertEquals(uncompressedBuffer, read);
+        read.release();
+
+        read = channel.readInbound();
+        assertEquals(uncompressedBuffer, read);
+        read.release();
+
+        assertNull(channel.readInbound());
+        uncompressedBuffer.release();
+    }
 }