JdkZlibDecoder may corrupt data when GZIP footer is fragmented (#11521)

Motivation:

Due a bug in the statemachine we produced an decoding error when the GZIP footer was fragmented in some cases

Modifications:

- Fix statemachine
- Add testcase

Result:

Correctly decode GZIP in all cases

codec/src/main/java/io/netty/handler/codec/compression/JdkZlibDecoder.java

@@ -205,12 +205,15 @@ public class JdkZlibDecoder extends ZlibDecoder {
             if (gzipState != GzipState.HEADER_END) {
                 if (gzipState == GzipState.FOOTER_START) {
                     if (!handleGzipFooter(in)) {
+                        // Either there was not enough data or the input is finished.
                         return;
                     }
-                } else {
+                    // If we consumed the footer we will start with the header again.
-                    if (!readGZIPHeader(in)) {
+                    assert gzipState == GzipState.HEADER_START;
-                        return;
+                }
-                    }
+                if (!readGZIPHeader(in)) {
+                    // There was not enough data readable to read the GZIP header.
+                    return;
                 }
                 // Some bytes may have been consumed, and so we must re-set the number of readable bytes.
                 readableBytes = in.readableBytes();

codec/src/test/java/io/netty/handler/codec/compression/JdkZlibTest.java

@@ -597,6 +597,37 @@ public class JdkZlibTest {
         }
     }
 
+    @Test
+    public void testDecodeWithHeaderFollowingFooter() throws Exception {
+        byte[] bytes = new byte[1024];
+        ThreadLocalRandom.current().nextBytes(bytes);
+        ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+        GZIPOutputStream out = new GZIPOutputStream(bytesOut);
+        out.write(bytes);
+        out.close();
+
+        byte[] compressed = bytesOut.toByteArray();
+        ByteBuf buffer = Unpooled.buffer().writeBytes(compressed).writeBytes(compressed);
+        EmbeddedChannel channel = new EmbeddedChannel(new JdkZlibDecoder(ZlibWrapper.GZIP, true));
+        // Write it into the Channel in a way that we were able to decompress the first data completely but not the
+        // whole footer.
+        assertTrue(channel.writeInbound(buffer.readRetainedSlice(compressed.length - 1)));
+        assertTrue(channel.writeInbound(buffer));
+        assertTrue(channel.finish());
+
+        ByteBuf uncompressedBuffer = Unpooled.wrappedBuffer(bytes);
+        ByteBuf read = channel.readInbound();
+        assertEquals(uncompressedBuffer, read);
+        read.release();
+
+        read = channel.readInbound();
+        assertEquals(uncompressedBuffer, read);
+        read.release();
+
+        assertNull(channel.readInbound());
+        uncompressedBuffer.release();
+    }
+
     private static byte[] gzip(byte[] bytes) throws IOException {
         ByteArrayOutputStream out = new ByteArrayOutputStream();
         GZIPOutputStream stream = new GZIPOutputStream(out);