From 9cab17effe8bdf4a3db65f8bec310ae98cff3973 Mon Sep 17 00:00:00 2001 From: Trustin Lee Date: Wed, 21 Oct 2009 09:08:38 +0000 Subject: [PATCH] Preliminary GZIP implementation (no CRC32 and ISIZE checksum yet) --- .../codec/compression/ZlibDecoder.java | 4 +- .../codec/compression/ZlibEncoder.java | 2 +- .../netty/util/internal/jzlib/CRC32.java | 23 ++ .../netty/util/internal/jzlib/Deflate.java | 90 +++++-- .../netty/util/internal/jzlib/Inflate.java | 253 +++++++++++++++++- .../netty/util/internal/jzlib/ZStream.java | 9 +- 6 files changed, 354 insertions(+), 27 deletions(-) create mode 100644 src/main/java/org/jboss/netty/util/internal/jzlib/CRC32.java diff --git a/src/main/java/org/jboss/netty/handler/codec/compression/ZlibDecoder.java b/src/main/java/org/jboss/netty/handler/codec/compression/ZlibDecoder.java index 3dbe358cbb..cd133107ca 100644 --- a/src/main/java/org/jboss/netty/handler/codec/compression/ZlibDecoder.java +++ b/src/main/java/org/jboss/netty/handler/codec/compression/ZlibDecoder.java @@ -47,7 +47,7 @@ public class ZlibDecoder extends OneToOneDecoder { */ public ZlibDecoder() { synchronized (z) { - int resultCode = z.inflateInit(); + int resultCode = z.inflateInit(JZlib.W_ZLIB); if (resultCode != JZlib.Z_OK) { ZlibUtil.fail(z, "initialization failure", resultCode); } @@ -66,7 +66,7 @@ public class ZlibDecoder extends OneToOneDecoder { synchronized (z) { int resultCode; - resultCode = z.inflateInit(); + resultCode = z.inflateInit(JZlib.W_ZLIB); if (resultCode != JZlib.Z_OK) { ZlibUtil.fail(z, "initialization failure", resultCode); } else { diff --git a/src/main/java/org/jboss/netty/handler/codec/compression/ZlibEncoder.java b/src/main/java/org/jboss/netty/handler/codec/compression/ZlibEncoder.java index 6129402512..9f7c452135 100644 --- a/src/main/java/org/jboss/netty/handler/codec/compression/ZlibEncoder.java +++ b/src/main/java/org/jboss/netty/handler/codec/compression/ZlibEncoder.java @@ -223,7 +223,7 @@ public class ZlibEncoder extends OneToOneEncoder { z.avail_in = 0; // Configure output. - byte[] out = new byte[8]; // Minimum room for ADLER32 + ZLIB header + byte[] out = new byte[32]; // room for ADLER32 + ZLIB / CRC32 + GZIP header z.next_out = out; z.next_out_index = 0; z.avail_out = out.length; diff --git a/src/main/java/org/jboss/netty/util/internal/jzlib/CRC32.java b/src/main/java/org/jboss/netty/util/internal/jzlib/CRC32.java new file mode 100644 index 0000000000..6a17dc7f8f --- /dev/null +++ b/src/main/java/org/jboss/netty/util/internal/jzlib/CRC32.java @@ -0,0 +1,23 @@ +/* + * Copyright 2009 Red Hat, Inc. + * + * Red Hat licenses this file to you under the Apache License, version 2.0 + * (the "License"); you may not use this file except in compliance with the + * License. You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.jboss.netty.util.internal.jzlib; + +final class CRC32 { + static long crc32(long crc32, byte[] buf, int index, int len) { + // TODO implement me + return crc32; + } +} diff --git a/src/main/java/org/jboss/netty/util/internal/jzlib/Deflate.java b/src/main/java/org/jboss/netty/util/internal/jzlib/Deflate.java index b381c658e4..cb927a6481 100644 --- a/src/main/java/org/jboss/netty/util/internal/jzlib/Deflate.java +++ b/src/main/java/org/jboss/netty/util/internal/jzlib/Deflate.java @@ -1378,6 +1378,7 @@ final class Deflate { wroteTrailer = false; status = wrapperType == WrapperType.NONE? BUSY_STATE : INIT_STATE; strm.adler = Adler32.adler32(0, null, 0, 0); + strm.crc32 = CRC32.crc32(0, null, 0, 0); last_flush = JZlib.Z_NO_FLUSH; @@ -1490,27 +1491,62 @@ final class Deflate { // Write the zlib header if (status == INIT_STATE) { - int header = JZlib.Z_DEFLATED + (w_bits - 8 << 4) << 8; - int level_flags = (level - 1 & 0xff) >> 1; + switch (wrapperType) { + case ZLIB: + int header = JZlib.Z_DEFLATED + (w_bits - 8 << 4) << 8; + int level_flags = (level - 1 & 0xff) >> 1; - if (level_flags > 3) { - level_flags = 3; + if (level_flags > 3) { + level_flags = 3; + } + header |= level_flags << 6; + if (strstart != 0) { + header |= JZlib.PRESET_DICT; + } + header += 31 - header % 31; + + putShortMSB(header); + + // Save the adler32 of the preset dictionary: + if (strstart != 0) { + putShortMSB((int) (strm.adler >>> 16)); + putShortMSB((int) (strm.adler & 0xffff)); + } + strm.adler = Adler32.adler32(0, null, 0, 0); + break; + case GZIP: + // Identification + put_byte((byte) 0x1f); + put_byte((byte) 0x8b); + // Compression method: DEFLATE + put_byte((byte) 8); + // Flags + put_byte((byte) 0); + // MTIME + put_byte((byte) 0); + put_byte((byte) 0); + put_byte((byte) 0); + put_byte((byte) 0); + // XFL + switch (config_table[level].func) { + case FAST: + put_byte((byte) 4); + break; + case SLOW: + put_byte((byte) 2); + break; + default: + put_byte((byte) 0); + break; + } + // OS + put_byte((byte) 255); + + strm.crc32 = CRC32.crc32(0, null, 0, 0); + break; } - header |= level_flags << 6; - if (strstart != 0) { - header |= JZlib.PRESET_DICT; - } - header += 31 - header % 31; status = BUSY_STATE; - putShortMSB(header); - - // Save the adler32 of the preset dictionary: - if (strstart != 0) { - putShortMSB((int) (strm.adler >>> 16)); - putShortMSB((int) (strm.adler & 0xffff)); - } - strm.adler = Adler32.adler32(0, null, 0, 0); } // Flush as much pending output as possible @@ -1605,11 +1641,23 @@ final class Deflate { return JZlib.Z_STREAM_END; } - // Write the zlib trailer (adler32) - putShortMSB((int) (strm.adler >>> 16)); - putShortMSB((int) (strm.adler & 0xffff)); - strm.flush_pending(); + switch (wrapperType) { + case ZLIB: + // Write the zlib trailer (adler32) + putShortMSB((int) (strm.adler >>> 16)); + putShortMSB((int) (strm.adler & 0xffff)); + break; + case GZIP: + // Write the gzip trailer (crc32) + putShortMSB((int) (strm.crc32 >>> 16)); + putShortMSB((int) (strm.crc32 & 0xffff)); + // FIXME implement me + putShortMSB(0); // ISIZE 1 + putShortMSB(0); // ISIZE 2 + break; + } + strm.flush_pending(); // If avail_out is zero, the application will call deflate again // to flush the rest. wroteTrailer = true; // write the trailer only once! diff --git a/src/main/java/org/jboss/netty/util/internal/jzlib/Inflate.java b/src/main/java/org/jboss/netty/util/internal/jzlib/Inflate.java index 2bf1231e80..5cf84be84a 100644 --- a/src/main/java/org/jboss/netty/util/internal/jzlib/Inflate.java +++ b/src/main/java/org/jboss/netty/util/internal/jzlib/Inflate.java @@ -67,6 +67,26 @@ final class Inflate { private static final int DONE = 12; // finished check, done private static final int BAD = 13; // got an error--stay here + private static final int GZIP_CHECK8 = 14; + private static final int GZIP_CHECK7 = 15; + private static final int GZIP_CHECK6 = 16; + private static final int GZIP_CHECK5 = 17; + private static final int GZIP_CHECK4 = 18; + private static final int GZIP_CHECK3 = 19; + private static final int GZIP_CHECK2 = 20; + private static final int GZIP_CHECK1 = 21; + + private static final int GZIP_ID1 = 22; + private static final int GZIP_ID2 = 23; + private static final int GZIP_CM = 24; + private static final int GZIP_FLG = 25; + private static final int GZIP_MTIME_XFL_OS = 26; + private static final int GZIP_XLEN = 27; + private static final int GZIP_FEXTRA = 28; + private static final int GZIP_FNAME = 29; + private static final int GZIP_FCOMMENT = 30; + private static final int GZIP_FHCRC = 31; + private int mode; // current inflate mode // mode dependent information private int method; // if FLAGS, method byte @@ -79,6 +99,9 @@ final class Inflate { private WrapperType wrapperType; private int wbits; // log2(window size) (8..15, defaults to 15) private InfBlocks blocks; // current inflate_blocks state + private int gzipFlag; + private int gzipBytesToRead; + private int gzipXLen; private int inflateReset(ZStream z) { if (z == null || z.istate == null) { @@ -87,7 +110,17 @@ final class Inflate { z.total_in = z.total_out = 0; z.msg = null; - z.istate.mode = z.istate.wrapperType == WrapperType.NONE? BLOCKS : METHOD; + switch (wrapperType) { + case NONE: + z.istate.mode = BLOCKS; + break; + case ZLIB: + z.istate.mode = METHOD; + break; + case GZIP: + z.istate.mode = GZIP_ID1; + break; + } z.istate.blocks.reset(z, null); return JZlib.Z_OK; } @@ -254,8 +287,12 @@ final class Inflate { if (z.istate.wrapperType == WrapperType.NONE) { z.istate.mode = DONE; break; + } else if (z.istate.wrapperType == WrapperType.ZLIB) { + z.istate.mode = CHECK4; + } else { + z.istate.mode = GZIP_CHECK8; + break; } - z.istate.mode = CHECK4; case CHECK4: if (z.avail_in == 0) { @@ -312,6 +349,218 @@ final class Inflate { return JZlib.Z_STREAM_END; case BAD: return JZlib.Z_DATA_ERROR; + case GZIP_ID1: + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + + if ((z.next_in[z.next_in_index ++] & 0xff) != 31) { + z.istate.mode = BAD; + z.msg = "not a gzip stream"; + z.istate.marker = 5; // can't try inflateSync + break; + } + z.istate.mode = GZIP_ID2; + case GZIP_ID2: + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + + if ((z.next_in[z.next_in_index ++] & 0xff) != 139) { + z.istate.mode = BAD; + z.msg = "not a gzip stream"; + z.istate.marker = 5; // can't try inflateSync + break; + } + z.istate.mode = GZIP_CM; + case GZIP_CM: + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + + if ((z.next_in[z.next_in_index ++] & 0xff) != JZlib.Z_DEFLATED) { + z.istate.mode = BAD; + z.msg = "unknown compression method"; + z.istate.marker = 5; // can't try inflateSync + break; + } + z.istate.mode = GZIP_FLG; + case GZIP_FLG: + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + gzipFlag = z.next_in[z.next_in_index ++] & 0xff; + gzipBytesToRead = 6; + z.istate.mode = GZIP_MTIME_XFL_OS; + case GZIP_MTIME_XFL_OS: + while (gzipBytesToRead > 0) { + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + z.next_in_index ++; + gzipBytesToRead --; + } + z.istate.mode = GZIP_XLEN; + gzipXLen = 0; + gzipBytesToRead = 2; + case GZIP_XLEN: + if ((gzipFlag & 4) != 0) { + while (gzipBytesToRead > 0) { + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + gzipXLen = gzipXLen << 8 | z.next_in[z.next_in_index ++] & 0xff; + gzipBytesToRead --; + } + gzipBytesToRead = gzipXLen; + z.istate.mode = GZIP_FEXTRA; + } else { + z.istate.mode = GZIP_FNAME; + break; + } + case GZIP_FEXTRA: + while (gzipBytesToRead > 0) { + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + z.next_in_index ++; + gzipBytesToRead --; + } + z.istate.mode = GZIP_FNAME; + case GZIP_FNAME: + if ((gzipFlag & 8) != 0) { + do { + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + } while (z.next_in[z.next_in_index ++] == 0); + } + z.istate.mode = GZIP_FCOMMENT; + case GZIP_FCOMMENT: + if ((gzipFlag & 16) != 0) { + do { + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + } while (z.next_in[z.next_in_index ++] == 0); + } + gzipBytesToRead = 2; + z.istate.mode = GZIP_FHCRC; + case GZIP_FHCRC: + if ((gzipFlag & 2) != 0) { + while (gzipBytesToRead > 0) { + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + z.next_in_index ++; + gzipBytesToRead --; + } + } + z.istate.mode = BLOCKS; + break; + case GZIP_CHECK8: + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + z.istate.need = (z.next_in[z.next_in_index ++] & 0xff) << 24 & 0xff000000L; + z.istate.mode = GZIP_CHECK7; + case GZIP_CHECK7: + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + z.istate.need = (z.next_in[z.next_in_index ++] & 0xff) << 24 & 0xff000000L; + z.istate.mode = GZIP_CHECK6; + case GZIP_CHECK6: + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + z.istate.need = (z.next_in[z.next_in_index ++] & 0xff) << 24 & 0xff000000L; + z.istate.mode = GZIP_CHECK5; + case GZIP_CHECK5: + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + z.istate.need = (z.next_in[z.next_in_index ++] & 0xff) << 24 & 0xff000000L; + z.istate.mode = GZIP_CHECK4; + case GZIP_CHECK4: + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + z.istate.need = (z.next_in[z.next_in_index ++] & 0xff) << 24 & 0xff000000L; + z.istate.mode = GZIP_CHECK3; + case GZIP_CHECK3: + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + z.istate.need = (z.next_in[z.next_in_index ++] & 0xff) << 24 & 0xff000000L; + z.istate.mode = GZIP_CHECK2; + case GZIP_CHECK2: + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + z.istate.need = (z.next_in[z.next_in_index ++] & 0xff) << 24 & 0xff000000L; + z.istate.mode = GZIP_CHECK1; + case GZIP_CHECK1: + if (z.avail_in == 0) { + return r; + } + r = f; + z.avail_in --; + z.total_in ++; + z.istate.need = (z.next_in[z.next_in_index ++] & 0xff) << 24 & 0xff000000L; + z.istate.mode = DONE; + break; default: return JZlib.Z_STREAM_ERROR; } diff --git a/src/main/java/org/jboss/netty/util/internal/jzlib/ZStream.java b/src/main/java/org/jboss/netty/util/internal/jzlib/ZStream.java index 054037a354..f347c18278 100644 --- a/src/main/java/org/jboss/netty/util/internal/jzlib/ZStream.java +++ b/src/main/java/org/jboss/netty/util/internal/jzlib/ZStream.java @@ -65,6 +65,7 @@ public final class ZStream { Inflate istate; int data_type; // best guess about the data type: ascii or binary long adler; + long crc32; public int inflateInit() { return inflateInit(JZlib.DEF_WBITS); @@ -214,9 +215,15 @@ public final class ZStream { avail_in -= len; - if (dstate.wrapperType == WrapperType.ZLIB) { + switch (dstate.wrapperType) { + case ZLIB: adler = Adler32.adler32(adler, next_in, next_in_index, len); + break; + case GZIP: + crc32 = CRC32.crc32(crc32, next_in, next_in_index, len); + break; } + System.arraycopy(next_in, next_in_index, buf, start, len); next_in_index += len; total_in += len;