Implemented FastLZ compression codec
Motivation: FastLZ compression codec provides sending and receiving data encoded by fast FastLZ algorithm using block mode. Modifications: - Added part of `jfastlz` library which implements FastLZ algorithm. See FastLz class. - Implemented FastLzFramedEncoder which extends MessageToByteEncoder and provides compression of outgoing messages. - Implemented FastLzFramedDecoder which extends ByteToMessageDecoder and provides uncompression of incoming messages. - Added integration tests for `FastLzFramedEncoder/Decoder`. Result: Full FastLZ compression codec which can compress/uncompress data using FastLZ algorithm.
This commit is contained in:
Idel Pivnitskiy
Trustin Lee
parent
60764200d7
commit
a0c466a276
|
|
@ -114,6 +114,14 @@ decoding data in LZF format, written by Tatu Saloranta. It can be obtained at:
|
|||
* HOMEPAGE:
|
||||
* https://github.com/ning/compress
|
||||
|
||||
This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression
|
||||
and decompression library written by William Kinney. It can be obtained at:
|
||||
|
||||
* LICENSE:
|
||||
* license/LICENSE.jfastlz.txt (MIT License)
|
||||
* HOMEPAGE:
|
||||
* https://code.google.com/p/jfastlz/
|
||||
|
||||
This product optionally depends on 'Protocol Buffers', Google's data
|
||||
interchange format, which can be obtained at:
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,575 @@
|
|||
/*
|
||||
* Copyright 2014 The Netty Project
|
||||
*
|
||||
* The Netty Project licenses this file to you under the Apache License,
|
||||
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at:
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package io.netty.handler.codec.compression;
|
||||
|
||||
/**
|
||||
* Core of FastLZ compression algorithm.
|
||||
*
|
||||
* This class provides methods for compression and decompression of buffers and saves
|
||||
* constants which use by {@link FastLzFramedEncoder} and {@link FastLzFramedDecoder}.
|
||||
*
|
||||
* This is refactored code of <a href="https://code.google.com/p/jfastlz/">jfastlz</a>
|
||||
* library written by William Kinney.
|
||||
*/
|
||||
final class FastLz {
|
||||
|
||||
private static final int MAX_DISTANCE = 8191;
|
||||
private static final int MAX_FARDISTANCE = 65535 + MAX_DISTANCE - 1;
|
||||
|
||||
private static final int HASH_LOG = 13;
|
||||
private static final int HASH_SIZE = 1 << HASH_LOG; // 8192
|
||||
private static final int HASH_MASK = HASH_SIZE - 1;
|
||||
|
||||
private static final int MAX_COPY = 32;
|
||||
private static final int MAX_LEN = 256 + 8;
|
||||
|
||||
private static final int MIN_RECOMENDED_LENGTH_FOR_LEVEL_2 = 1024 * 64;
|
||||
|
||||
static final int MAGIC_NUMBER = 'F' << 16 | 'L' << 8 | 'Z';
|
||||
|
||||
static final byte BLOCK_TYPE_NON_COMPRESSED = 0x00;
|
||||
static final byte BLOCK_TYPE_COMPRESSED = 0x01;
|
||||
static final byte BLOCK_WITHOUT_CHECKSUM = 0x00;
|
||||
static final byte BLOCK_WITH_CHECKSUM = 0x10;
|
||||
|
||||
static final int OPTIONS_OFFSET = 3;
|
||||
static final int CHECKSUM_OFFSET = 4;
|
||||
|
||||
static final int MAX_CHUNK_LENGTH = 0xFFFF;
|
||||
|
||||
/**
|
||||
* Do not call {@link #compress(byte[], int, int, byte[], int, int)} for input buffers
|
||||
* which length less than this value.
|
||||
*/
|
||||
static final int MIN_LENGTH_TO_COMPRESSION = 32;
|
||||
|
||||
/**
|
||||
* In this case {@link #compress(byte[], int, int, byte[], int, int)} will choose level
|
||||
* automatically depending on the length of the input buffer. If length less than
|
||||
* {@link #MIN_RECOMENDED_LENGTH_FOR_LEVEL_2} {@link #LEVEL_1} will be choosen,
|
||||
* otherwise {@link #LEVEL_2}.
|
||||
*/
|
||||
static final int LEVEL_AUTO = 0;
|
||||
|
||||
/**
|
||||
* Level 1 is the fastest compression and generally useful for short data.
|
||||
*/
|
||||
static final int LEVEL_1 = 1;
|
||||
|
||||
/**
|
||||
* Level 2 is slightly slower but it gives better compression ratio.
|
||||
*/
|
||||
static final int LEVEL_2 = 2;
|
||||
|
||||
/**
|
||||
* The output buffer must be at least 6% larger than the input buffer and can not be smaller than 66 bytes.
|
||||
* @param inputLength length of input buffer
|
||||
* @return Maximum output buffer length
|
||||
*/
|
||||
static int calculateOutputBufferLength(int inputLength) {
|
||||
final int outputLength = (int) (inputLength * 1.06);
|
||||
return Math.max(outputLength, 66);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress a block of data in the input buffer and returns the size of compressed block.
|
||||
* The size of input buffer is specified by length. The minimum input buffer size is 32.
|
||||
*
|
||||
* If the input is not compressible, the return value might be larger than length (input buffer size).
|
||||
*/
|
||||
static int compress(final byte[] input, final int inOffset, final int inLength,
|
||||
final byte[] output, final int outOffset, final int proposedLevel) {
|
||||
final int level;
|
||||
if (proposedLevel == LEVEL_AUTO) {
|
||||
level = inLength < MIN_RECOMENDED_LENGTH_FOR_LEVEL_2 ? LEVEL_1 : LEVEL_2;
|
||||
} else {
|
||||
level = proposedLevel;
|
||||
}
|
||||
|
||||
int ip = 0;
|
||||
int ipBound = ip + inLength - 2;
|
||||
int ipLimit = ip + inLength - 12;
|
||||
|
||||
int op = 0;
|
||||
|
||||
// const flzuint8* htab[HASH_SIZE];
|
||||
int[] htab = new int[HASH_SIZE];
|
||||
// const flzuint8** hslot;
|
||||
int hslot;
|
||||
// flzuint32 hval;
|
||||
// int OK b/c address starting from 0
|
||||
int hval;
|
||||
// flzuint32 copy;
|
||||
// int OK b/c address starting from 0
|
||||
int copy;
|
||||
|
||||
/* sanity check */
|
||||
if (inLength < 4) {
|
||||
if (inLength != 0) {
|
||||
// *op++ = length-1;
|
||||
output[outOffset + op++] = (byte) (inLength - 1);
|
||||
ipBound++;
|
||||
while (ip <= ipBound) {
|
||||
output[outOffset + op++] = input[inOffset + ip++];
|
||||
}
|
||||
return inLength + 1;
|
||||
}
|
||||
// else
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* initializes hash table */
|
||||
// for (hslot = htab; hslot < htab + HASH_SIZE; hslot++)
|
||||
for (hslot = 0; hslot < HASH_SIZE; hslot++) {
|
||||
//*hslot = ip;
|
||||
htab[hslot] = ip;
|
||||
}
|
||||
|
||||
/* we start with literal copy */
|
||||
copy = 2;
|
||||
output[outOffset + op++] = MAX_COPY - 1;
|
||||
output[outOffset + op++] = input[inOffset + ip++];
|
||||
output[outOffset + op++] = input[inOffset + ip++];
|
||||
|
||||
/* main loop */
|
||||
while (ip < ipLimit) {
|
||||
int ref = 0;
|
||||
|
||||
long distance = 0;
|
||||
|
||||
/* minimum match length */
|
||||
// flzuint32 len = 3;
|
||||
// int OK b/c len is 0 and octal based
|
||||
int len = 3;
|
||||
|
||||
/* comparison starting-point */
|
||||
int anchor = ip;
|
||||
|
||||
boolean matchLabel = false;
|
||||
|
||||
/* check for a run */
|
||||
if (level == LEVEL_2) {
|
||||
//if(ip[0] == ip[-1] && FASTLZ_READU16(ip-1)==FASTLZ_READU16(ip+1))
|
||||
if (input[inOffset + ip] == input[inOffset + ip - 1] &&
|
||||
readU16(input, inOffset + ip - 1) == readU16(input, inOffset + ip + 1)) {
|
||||
distance = 1;
|
||||
ip += 3;
|
||||
ref = anchor - 1 + 3;
|
||||
|
||||
/*
|
||||
* goto match;
|
||||
*/
|
||||
matchLabel = true;
|
||||
}
|
||||
}
|
||||
if (!matchLabel) {
|
||||
/* find potential match */
|
||||
// HASH_FUNCTION(hval,ip);
|
||||
hval = hashFunction(input, inOffset + ip);
|
||||
// hslot = htab + hval;
|
||||
hslot = hval;
|
||||
// ref = htab[hval];
|
||||
ref = htab[hval];
|
||||
|
||||
/* calculate distance to the match */
|
||||
distance = anchor - ref;
|
||||
|
||||
/* update hash table */
|
||||
//*hslot = anchor;
|
||||
htab[hslot] = anchor;
|
||||
|
||||
/* is this a match? check the first 3 bytes */
|
||||
if (distance == 0
|
||||
|| (level == LEVEL_1 ? distance >= MAX_DISTANCE : distance >= MAX_FARDISTANCE)
|
||||
|| input[inOffset + ref++] != input[inOffset + ip++]
|
||||
|| input[inOffset + ref++] != input[inOffset + ip++]
|
||||
|| input[inOffset + ref++] != input[inOffset + ip++]) {
|
||||
/*
|
||||
* goto literal;
|
||||
*/
|
||||
output[outOffset + op++] = input[inOffset + anchor++];
|
||||
ip = anchor;
|
||||
copy++;
|
||||
if (copy == MAX_COPY) {
|
||||
copy = 0;
|
||||
output[outOffset + op++] = MAX_COPY - 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (level == LEVEL_2) {
|
||||
/* far, needs at least 5-byte match */
|
||||
if (distance >= MAX_DISTANCE) {
|
||||
if (input[inOffset + ip++] != input[inOffset + ref++]
|
||||
|| input[inOffset + ip++] != input[inOffset + ref++]) {
|
||||
/*
|
||||
* goto literal;
|
||||
*/
|
||||
output[outOffset + op++] = input[inOffset + anchor++];
|
||||
ip = anchor;
|
||||
copy++;
|
||||
if (copy == MAX_COPY) {
|
||||
copy = 0;
|
||||
output[outOffset + op++] = MAX_COPY - 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
len += 2;
|
||||
}
|
||||
}
|
||||
} // end if(!matchLabel)
|
||||
/*
|
||||
* match:
|
||||
*/
|
||||
/* last matched byte */
|
||||
ip = anchor + len;
|
||||
|
||||
/* distance is biased */
|
||||
distance--;
|
||||
|
||||
if (distance == 0) {
|
||||
/* zero distance means a run */
|
||||
//flzuint8 x = ip[-1];
|
||||
byte x = input[inOffset + ip - 1];
|
||||
while (ip < ipBound) {
|
||||
if (input[inOffset + ref++] != x) {
|
||||
break;
|
||||
} else {
|
||||
ip++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (;;) {
|
||||
/* safe because the outer check against ip limit */
|
||||
if (input[inOffset + ref++] != input[inOffset + ip++]) {
|
||||
break;
|
||||
}
|
||||
if (input[inOffset + ref++] != input[inOffset + ip++]) {
|
||||
break;
|
||||
}
|
||||
if (input[inOffset + ref++] != input[inOffset + ip++]) {
|
||||
break;
|
||||
}
|
||||
if (input[inOffset + ref++] != input[inOffset + ip++]) {
|
||||
break;
|
||||
}
|
||||
if (input[inOffset + ref++] != input[inOffset + ip++]) {
|
||||
break;
|
||||
}
|
||||
if (input[inOffset + ref++] != input[inOffset + ip++]) {
|
||||
break;
|
||||
}
|
||||
if (input[inOffset + ref++] != input[inOffset + ip++]) {
|
||||
break;
|
||||
}
|
||||
if (input[inOffset + ref++] != input[inOffset + ip++]) {
|
||||
break;
|
||||
}
|
||||
while (ip < ipBound) {
|
||||
if (input[inOffset + ref++] != input[inOffset + ip++]) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* if we have copied something, adjust the copy count */
|
||||
if (copy != 0) {
|
||||
/* copy is biased, '0' means 1 byte copy */
|
||||
// *(op-copy-1) = copy-1;
|
||||
output[outOffset + op - copy - 1] = (byte) (copy - 1);
|
||||
} else {
|
||||
/* back, to overwrite the copy count */
|
||||
op--;
|
||||
}
|
||||
|
||||
/* reset literal counter */
|
||||
copy = 0;
|
||||
|
||||
/* length is biased, '1' means a match of 3 bytes */
|
||||
ip -= 3;
|
||||
len = ip - anchor;
|
||||
|
||||
/* encode the match */
|
||||
if (level == LEVEL_2) {
|
||||
if (distance < MAX_DISTANCE) {
|
||||
if (len < 7) {
|
||||
output[outOffset + op++] = (byte) ((len << 5) + (distance >>> 8));
|
||||
output[outOffset + op++] = (byte) (distance & 255);
|
||||
} else {
|
||||
output[outOffset + op++] = (byte) ((7 << 5) + (distance >>> 8));
|
||||
for (len -= 7; len >= 255; len -= 255) {
|
||||
output[outOffset + op++] = (byte) 255;
|
||||
}
|
||||
output[outOffset + op++] = (byte) len;
|
||||
output[outOffset + op++] = (byte) (distance & 255);
|
||||
}
|
||||
} else {
|
||||
/* far away, but not yet in the another galaxy... */
|
||||
if (len < 7) {
|
||||
distance -= MAX_DISTANCE;
|
||||
output[outOffset + op++] = (byte) ((len << 5) + 31);
|
||||
output[outOffset + op++] = (byte) 255;
|
||||
output[outOffset + op++] = (byte) (distance >>> 8);
|
||||
output[outOffset + op++] = (byte) (distance & 255);
|
||||
} else {
|
||||
distance -= MAX_DISTANCE;
|
||||
output[outOffset + op++] = (byte) ((7 << 5) + 31);
|
||||
for (len -= 7; len >= 255; len -= 255) {
|
||||
output[outOffset + op++] = (byte) 255;
|
||||
}
|
||||
output[outOffset + op++] = (byte) len;
|
||||
output[outOffset + op++] = (byte) 255;
|
||||
output[outOffset + op++] = (byte) (distance >>> 8);
|
||||
output[outOffset + op++] = (byte) (distance & 255);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (len > MAX_LEN - 2) {
|
||||
while (len > MAX_LEN - 2) {
|
||||
output[outOffset + op++] = (byte) ((7 << 5) + (distance >>> 8));
|
||||
output[outOffset + op++] = (byte) (MAX_LEN - 2 - 7 - 2);
|
||||
output[outOffset + op++] = (byte) (distance & 255);
|
||||
len -= MAX_LEN - 2;
|
||||
}
|
||||
}
|
||||
|
||||
if (len < 7) {
|
||||
output[outOffset + op++] = (byte) ((len << 5) + (distance >>> 8));
|
||||
output[outOffset + op++] = (byte) (distance & 255);
|
||||
} else {
|
||||
output[outOffset + op++] = (byte) ((7 << 5) + (distance >>> 8));
|
||||
output[outOffset + op++] = (byte) (len - 7);
|
||||
output[outOffset + op++] = (byte) (distance & 255);
|
||||
}
|
||||
}
|
||||
|
||||
/* update the hash at match boundary */
|
||||
//HASH_FUNCTION(hval,ip);
|
||||
hval = hashFunction(input, inOffset + ip);
|
||||
htab[hval] = ip++;
|
||||
|
||||
//HASH_FUNCTION(hval,ip);
|
||||
hval = hashFunction(input, inOffset + ip);
|
||||
htab[hval] = ip++;
|
||||
|
||||
/* assuming literal copy */
|
||||
output[outOffset + op++] = MAX_COPY - 1;
|
||||
|
||||
continue;
|
||||
|
||||
// Moved to be inline, with a 'continue'
|
||||
/*
|
||||
* literal:
|
||||
*
|
||||
output[outOffset + op++] = input[inOffset + anchor++];
|
||||
ip = anchor;
|
||||
copy++;
|
||||
if(copy == MAX_COPY){
|
||||
copy = 0;
|
||||
output[outOffset + op++] = MAX_COPY-1;
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
/* left-over as literal copy */
|
||||
ipBound++;
|
||||
while (ip <= ipBound) {
|
||||
output[outOffset + op++] = input[inOffset + ip++];
|
||||
copy++;
|
||||
if (copy == MAX_COPY) {
|
||||
copy = 0;
|
||||
output[outOffset + op++] = MAX_COPY - 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* if we have copied something, adjust the copy length */
|
||||
if (copy != 0) {
|
||||
//*(op-copy-1) = copy-1;
|
||||
output[outOffset + op - copy - 1] = (byte) (copy - 1);
|
||||
} else {
|
||||
op--;
|
||||
}
|
||||
|
||||
if (level == LEVEL_2) {
|
||||
/* marker for fastlz2 */
|
||||
output[outOffset] |= 1 << 5;
|
||||
}
|
||||
|
||||
return op;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decompress a block of compressed data and returns the size of the decompressed block.
|
||||
* If error occurs, e.g. the compressed data is corrupted or the output buffer is not large
|
||||
* enough, then 0 (zero) will be returned instead.
|
||||
*
|
||||
* Decompression is memory safe and guaranteed not to write the output buffer
|
||||
* more than what is specified in outLength.
|
||||
*/
|
||||
static int decompress(final byte[] input, final int inOffset, final int inLength,
|
||||
final byte[] output, final int outOffset, final int outLength) {
|
||||
//int level = ((*(const flzuint8*)input) >> 5) + 1;
|
||||
final int level = (input[inOffset] >> 5) + 1;
|
||||
if (level != LEVEL_1 && level != LEVEL_2) {
|
||||
throw new DecompressionException(String.format(
|
||||
"invalid level: %d (expected: %d or %d)", level, LEVEL_1, LEVEL_2
|
||||
));
|
||||
}
|
||||
|
||||
// const flzuint8* ip = (const flzuint8*) input;
|
||||
int ip = 0;
|
||||
// flzuint8* op = (flzuint8*) output;
|
||||
int op = 0;
|
||||
// flzuint32 ctrl = (*ip++) & 31;
|
||||
long ctrl = input[inOffset + ip++] & 31;
|
||||
|
||||
int loop = 1;
|
||||
do {
|
||||
// const flzuint8* ref = op;
|
||||
int ref = op;
|
||||
// flzuint32 len = ctrl >> 5;
|
||||
long len = ctrl >> 5;
|
||||
// flzuint32 ofs = (ctrl & 31) << 8;
|
||||
long ofs = (ctrl & 31) << 8;
|
||||
|
||||
if (ctrl >= 32) {
|
||||
len--;
|
||||
// ref -= ofs;
|
||||
ref -= ofs;
|
||||
|
||||
int code;
|
||||
if (len == 6) {
|
||||
if (level == LEVEL_1) {
|
||||
// len += *ip++;
|
||||
len += input[inOffset + ip++] & 0xFF;
|
||||
} else {
|
||||
do {
|
||||
code = input[inOffset + ip++] & 0xFF;
|
||||
len += code;
|
||||
} while (code == 255);
|
||||
}
|
||||
}
|
||||
if (level == LEVEL_1) {
|
||||
// ref -= *ip++;
|
||||
ref -= input[inOffset + ip++] & 0xFF;
|
||||
} else {
|
||||
code = input[inOffset + ip++] & 0xFF;
|
||||
ref -= code;
|
||||
|
||||
/* match from 16-bit distance */
|
||||
// if(FASTLZ_UNEXPECT_CONDITIONAL(code==255))
|
||||
// if(FASTLZ_EXPECT_CONDITIONAL(ofs==(31 << 8)))
|
||||
if (code == 255 && ofs == 31 << 8) {
|
||||
ofs = (input[inOffset + ip++] & 0xFF) << 8;
|
||||
ofs += input[inOffset + ip++] & 0xFF;
|
||||
|
||||
ref = (int) (op - ofs - MAX_DISTANCE);
|
||||
}
|
||||
}
|
||||
|
||||
// if the output index + length of block(?) + 3(?) is over the output limit?
|
||||
if (op + len + 3 > outLength) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// if (FASTLZ_UNEXPECT_CONDITIONAL(ref-1 < (flzuint8 *)output))
|
||||
// if the address space of ref-1 is < the address of output?
|
||||
// if we are still at the beginning of the output address?
|
||||
if (ref - 1 < 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (ip < inLength) {
|
||||
ctrl = input[inOffset + ip++] & 0xFF;
|
||||
} else {
|
||||
loop = 0;
|
||||
}
|
||||
|
||||
if (ref == op) {
|
||||
/* optimize copy for a run */
|
||||
// flzuint8 b = ref[-1];
|
||||
byte b = output[outOffset + ref - 1];
|
||||
output[outOffset + op++] = b;
|
||||
output[outOffset + op++] = b;
|
||||
output[outOffset + op++] = b;
|
||||
while (len != 0) {
|
||||
output[outOffset + op++] = b;
|
||||
--len;
|
||||
}
|
||||
} else {
|
||||
/* copy from reference */
|
||||
ref--;
|
||||
|
||||
// *op++ = *ref++;
|
||||
output[outOffset + op++] = output[outOffset + ref++];
|
||||
output[outOffset + op++] = output[outOffset + ref++];
|
||||
output[outOffset + op++] = output[outOffset + ref++];
|
||||
|
||||
while (len != 0) {
|
||||
output[outOffset + op++] = output[outOffset + ref++];
|
||||
--len;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ctrl++;
|
||||
|
||||
if (op + ctrl > outLength) {
|
||||
return 0;
|
||||
}
|
||||
if (ip + ctrl > inLength) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
//*op++ = *ip++;
|
||||
output[outOffset + op++] = input[inOffset + ip++];
|
||||
|
||||
for (--ctrl; ctrl != 0; ctrl--) {
|
||||
// *op++ = *ip++;
|
||||
output[outOffset + op++] = input[inOffset + ip++];
|
||||
}
|
||||
|
||||
loop = ip < inLength ? 1 : 0;
|
||||
if (loop != 0) {
|
||||
// ctrl = *ip++;
|
||||
ctrl = input[inOffset + ip++] & 0xFF;
|
||||
}
|
||||
}
|
||||
|
||||
// while(FASTLZ_EXPECT_CONDITIONAL(loop));
|
||||
} while (loop != 0);
|
||||
|
||||
// return op - (flzuint8*)output;
|
||||
return op;
|
||||
}
|
||||
|
||||
private static int hashFunction(byte[] p, int offset) {
|
||||
int v = readU16(p, offset);
|
||||
v ^= readU16(p, offset + 1) ^ v >> 16 - HASH_LOG;
|
||||
v &= HASH_MASK;
|
||||
return v;
|
||||
}
|
||||
|
||||
private static int readU16(byte[] data, int offset) {
|
||||
if (offset + 1 >= data.length) {
|
||||
return data[offset] & 0xff;
|
||||
}
|
||||
return (data[offset + 1] & 0xff) << 8 | data[offset] & 0xff;
|
||||
}
|
||||
|
||||
private FastLz() { }
|
||||
}
|
||||
|
|
@ -0,0 +1,211 @@
|
|||
/*
|
||||
* Copyright 2014 The Netty Project
|
||||
*
|
||||
* The Netty Project licenses this file to you under the Apache License,
|
||||
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at:
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package io.netty.handler.codec.compression;
|
||||
|
||||
import io.netty.buffer.ByteBuf;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.handler.codec.ByteToMessageDecoder;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.zip.Adler32;
|
||||
import java.util.zip.Checksum;
|
||||
|
||||
import static io.netty.handler.codec.compression.FastLz.*;
|
||||
|
||||
/**
|
||||
* Uncompresses a {@link ByteBuf} encoded with the Bzip2 format.
|
||||
*
|
||||
* See <a href="https://github.com/netty/netty/issues/2750">FastLZ format</a>.
|
||||
*/
|
||||
public class FastLzFramedDecoder extends ByteToMessageDecoder {
|
||||
/**
|
||||
* Current state of decompression.
|
||||
*/
|
||||
private enum State {
|
||||
INIT_BLOCK,
|
||||
INIT_BLOCK_PARAMS,
|
||||
DECOMPRESS_DATA,
|
||||
CORRUPTED
|
||||
}
|
||||
|
||||
private State currentState = State.INIT_BLOCK;
|
||||
|
||||
/**
|
||||
* Underlying checksum calculator in use.
|
||||
*/
|
||||
private final Checksum checksum;
|
||||
|
||||
/**
|
||||
* Length of current received chunk of data.
|
||||
*/
|
||||
private int chunkLength;
|
||||
|
||||
/**
|
||||
* Original of current received chunk of data.
|
||||
* It is equal to {@link #chunkLength} for non compressed chunks.
|
||||
*/
|
||||
private int originalLength;
|
||||
|
||||
/**
|
||||
* Indicates is this chunk compressed or not.
|
||||
*/
|
||||
private boolean isCompressed;
|
||||
|
||||
/**
|
||||
* Indicates is this chunk has checksum or not.
|
||||
*/
|
||||
private boolean hasChecksum;
|
||||
|
||||
/**
|
||||
* Chechsum value of current received chunk of data which has checksum.
|
||||
*/
|
||||
private int currentChecksum;
|
||||
|
||||
/**
|
||||
* Creates the fastest FastLZ decoder without checksum calculation.
|
||||
*/
|
||||
public FastLzFramedDecoder() {
|
||||
this(false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a FastLZ decoder with calculation of checksums as specified.
|
||||
*
|
||||
* @param validateChecksums
|
||||
* If true, the checksum field will be validated against the actual
|
||||
* uncompressed data, and if the checksums do not match, a suitable
|
||||
* {@link DecompressionException} will be thrown.
|
||||
* Note, that in this case decoder will use {@link java.util.zip.Adler32}
|
||||
* as a default checksum calculator.
|
||||
*/
|
||||
public FastLzFramedDecoder(boolean validateChecksums) {
|
||||
this(validateChecksums ? new Adler32() : null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a FastLZ decoder with specified checksum calculator.
|
||||
*
|
||||
* @param checksum
|
||||
* the {@link Checksum} instance to use to check data for integrity.
|
||||
* You may set {@code null} if you do not want to validate checksum of each block.
|
||||
*/
|
||||
public FastLzFramedDecoder(Checksum checksum) {
|
||||
this.checksum = checksum;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
|
||||
for (;;) {
|
||||
try {
|
||||
switch (currentState) {
|
||||
case INIT_BLOCK:
|
||||
if (in.readableBytes() < 4) {
|
||||
return;
|
||||
}
|
||||
|
||||
final int magic = in.readUnsignedMedium();
|
||||
if (magic != MAGIC_NUMBER) {
|
||||
throw new DecompressionException("unexpected block identifier");
|
||||
}
|
||||
|
||||
final byte options = in.readByte();
|
||||
isCompressed = (options & 0x01) == BLOCK_TYPE_COMPRESSED;
|
||||
hasChecksum = (options & 0x10) == BLOCK_WITH_CHECKSUM;
|
||||
|
||||
currentState = State.INIT_BLOCK_PARAMS;
|
||||
case INIT_BLOCK_PARAMS:
|
||||
if (in.readableBytes() < 2 + (isCompressed ? 2 : 0) + (hasChecksum ? 4 : 0)) {
|
||||
return;
|
||||
}
|
||||
currentChecksum = hasChecksum ? in.readInt() : 0;
|
||||
chunkLength = in.readUnsignedShort();
|
||||
originalLength = isCompressed ? in.readUnsignedShort() : chunkLength;
|
||||
|
||||
currentState = State.DECOMPRESS_DATA;
|
||||
case DECOMPRESS_DATA:
|
||||
final int chunkLength = this.chunkLength;
|
||||
if (in.readableBytes() < chunkLength) {
|
||||
return;
|
||||
}
|
||||
|
||||
final int idx = in.readerIndex();
|
||||
final int originalLength = this.originalLength;
|
||||
|
||||
ByteBuf uncompressed = ctx.alloc().heapBuffer(originalLength, originalLength);
|
||||
final byte[] output = uncompressed.array();
|
||||
final int outputPtr = uncompressed.arrayOffset() + uncompressed.writerIndex();
|
||||
|
||||
boolean success = false;
|
||||
try {
|
||||
if (isCompressed) {
|
||||
final byte[] input;
|
||||
final int inputPtr;
|
||||
if (in.hasArray()) {
|
||||
input = in.array();
|
||||
inputPtr = in.arrayOffset() + idx;
|
||||
} else {
|
||||
input = new byte[chunkLength];
|
||||
in.getBytes(idx, input);
|
||||
inputPtr = 0;
|
||||
}
|
||||
|
||||
final int decompressedBytes = decompress(input, inputPtr, chunkLength,
|
||||
output, outputPtr, originalLength);
|
||||
if (originalLength != decompressedBytes) {
|
||||
throw new DecompressionException(String.format(
|
||||
"stream corrupted: originalLength(%d) and actual length(%d) mismatch",
|
||||
originalLength, decompressedBytes));
|
||||
}
|
||||
} else {
|
||||
in.getBytes(idx, output, outputPtr, chunkLength);
|
||||
}
|
||||
|
||||
final Checksum checksum = this.checksum;
|
||||
if (hasChecksum && checksum != null) {
|
||||
checksum.reset();
|
||||
checksum.update(output, outputPtr, originalLength);
|
||||
final int checksumResult = (int) checksum.getValue();
|
||||
if (checksumResult != currentChecksum) {
|
||||
throw new DecompressionException(String.format(
|
||||
"stream corrupted: mismatching checksum: %d (expected: %d)",
|
||||
checksumResult, currentChecksum));
|
||||
}
|
||||
}
|
||||
uncompressed.writerIndex(uncompressed.writerIndex() + originalLength);
|
||||
out.add(uncompressed);
|
||||
in.skipBytes(chunkLength);
|
||||
|
||||
currentState = State.INIT_BLOCK;
|
||||
success = true;
|
||||
} finally {
|
||||
if (!success) {
|
||||
uncompressed.release();
|
||||
}
|
||||
}
|
||||
break;
|
||||
case CORRUPTED:
|
||||
in.skipBytes(in.readableBytes());
|
||||
return;
|
||||
default:
|
||||
throw new IllegalStateException();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
currentState = State.CORRUPTED;
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,186 @@
|
|||
/*
|
||||
* Copyright 2014 The Netty Project
|
||||
*
|
||||
* The Netty Project licenses this file to you under the Apache License,
|
||||
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at:
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package io.netty.handler.codec.compression;
|
||||
|
||||
import io.netty.buffer.ByteBuf;
|
||||
import io.netty.channel.ChannelHandlerContext;
|
||||
import io.netty.handler.codec.MessageToByteEncoder;
|
||||
|
||||
import java.util.zip.Adler32;
|
||||
import java.util.zip.Checksum;
|
||||
|
||||
import static io.netty.handler.codec.compression.FastLz.*;
|
||||
|
||||
/**
|
||||
* Compresses a {@link ByteBuf} using the FastLZ algorithm.
|
||||
*
|
||||
* See <a href="https://github.com/netty/netty/issues/2750">FastLZ format</a>.
|
||||
*/
|
||||
public class FastLzFramedEncoder extends MessageToByteEncoder<ByteBuf> {
|
||||
/**
|
||||
* Compression level.
|
||||
*/
|
||||
private final int level;
|
||||
|
||||
/**
|
||||
* Underlying checksum calculator in use.
|
||||
*/
|
||||
private final Checksum checksum;
|
||||
|
||||
/**
|
||||
* Creates a FastLZ encoder without checksum calculator and with auto detection of compression level.
|
||||
*/
|
||||
public FastLzFramedEncoder() {
|
||||
this(LEVEL_AUTO, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a FastLZ encoder with specified compression level and without checksum calculator.
|
||||
*
|
||||
* @param level supports only these values:
|
||||
* 0 - Encoder will choose level automatically depending on the length of the input buffer.
|
||||
* 1 - Level 1 is the fastest compression and generally useful for short data.
|
||||
* 2 - Level 2 is slightly slower but it gives better compression ratio.
|
||||
*/
|
||||
public FastLzFramedEncoder(int level) {
|
||||
this(level, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a FastLZ encoder with auto detection of compression
|
||||
* level and calculation of checksums as specified.
|
||||
*
|
||||
* @param validateChecksums
|
||||
* If true, the checksum of each block will be calculated and this value
|
||||
* will be added to the header of block.
|
||||
* By default {@link FastLzFramedEncoder} uses {@link java.util.zip.Adler32}
|
||||
* for checksum calculation.
|
||||
*/
|
||||
public FastLzFramedEncoder(boolean validateChecksums) {
|
||||
this(LEVEL_AUTO, validateChecksums ? new Adler32() : null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a FastLZ encoder with specified compression level and checksum calculator.
|
||||
*
|
||||
* @param level supports only these values:
|
||||
* 0 - Encoder will choose level automatically depending on the length of the input buffer.
|
||||
* 1 - Level 1 is the fastest compression and generally useful for short data.
|
||||
* 2 - Level 2 is slightly slower but it gives better compression ratio.
|
||||
* @param checksum
|
||||
* the {@link Checksum} instance to use to check data for integrity.
|
||||
* You may set {@code null} if you don't want to validate checksum of each block.
|
||||
*/
|
||||
public FastLzFramedEncoder(int level, Checksum checksum) {
|
||||
super(false);
|
||||
if (level != LEVEL_AUTO && level != LEVEL_1 && level != LEVEL_2) {
|
||||
throw new IllegalArgumentException(String.format(
|
||||
"level: %d (expected: %d or %d or %d)", level, LEVEL_AUTO, LEVEL_1, LEVEL_2));
|
||||
}
|
||||
this.level = level;
|
||||
this.checksum = checksum;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void encode(ChannelHandlerContext ctx, ByteBuf in, ByteBuf out) throws Exception {
|
||||
final Checksum checksum = this.checksum;
|
||||
|
||||
for (;;) {
|
||||
if (!in.isReadable()) {
|
||||
return;
|
||||
}
|
||||
final int idx = in.readerIndex();
|
||||
final int length = Math.min(in.readableBytes(), MAX_CHUNK_LENGTH);
|
||||
|
||||
final int outputIdx = out.writerIndex();
|
||||
out.setMedium(outputIdx, MAGIC_NUMBER);
|
||||
int outputOffset = outputIdx + CHECKSUM_OFFSET + (checksum != null ? 4 : 0);
|
||||
|
||||
final byte blockType;
|
||||
final int chunkLength;
|
||||
if (length < MIN_LENGTH_TO_COMPRESSION) {
|
||||
blockType = BLOCK_TYPE_NON_COMPRESSED;
|
||||
|
||||
out.ensureWritable(outputOffset + 2 + length);
|
||||
final byte[] output = out.array();
|
||||
final int outputPtr = out.arrayOffset() + outputOffset + 2;
|
||||
|
||||
if (checksum != null) {
|
||||
final byte[] input;
|
||||
final int inputPtr;
|
||||
if (in.hasArray()) {
|
||||
input = in.array();
|
||||
inputPtr = in.arrayOffset() + idx;
|
||||
} else {
|
||||
input = new byte[length];
|
||||
in.getBytes(idx, input);
|
||||
inputPtr = 0;
|
||||
}
|
||||
|
||||
checksum.reset();
|
||||
checksum.update(input, inputPtr, length);
|
||||
out.setInt(outputIdx + CHECKSUM_OFFSET, (int) checksum.getValue());
|
||||
|
||||
System.arraycopy(input, inputPtr, output, outputPtr, length);
|
||||
} else {
|
||||
in.getBytes(idx, output, outputPtr, length);
|
||||
}
|
||||
chunkLength = length;
|
||||
} else {
|
||||
// try to compress
|
||||
final byte[] input;
|
||||
final int inputPtr;
|
||||
if (in.hasArray()) {
|
||||
input = in.array();
|
||||
inputPtr = in.arrayOffset() + idx;
|
||||
} else {
|
||||
input = new byte[length];
|
||||
in.getBytes(idx, input);
|
||||
inputPtr = 0;
|
||||
}
|
||||
|
||||
if (checksum != null) {
|
||||
checksum.reset();
|
||||
checksum.update(input, inputPtr, length);
|
||||
out.setInt(outputIdx + CHECKSUM_OFFSET, (int) checksum.getValue());
|
||||
}
|
||||
|
||||
final int maxOutputLength = calculateOutputBufferLength(length);
|
||||
out.ensureWritable(outputOffset + 4 + maxOutputLength);
|
||||
final byte[] output = out.array();
|
||||
final int outputPtr = out.arrayOffset() + outputOffset + 4;
|
||||
final int compressedLength = compress(input, inputPtr, length, output, outputPtr, level);
|
||||
if (compressedLength < length) {
|
||||
blockType = BLOCK_TYPE_COMPRESSED;
|
||||
chunkLength = compressedLength;
|
||||
|
||||
out.setShort(outputOffset, chunkLength);
|
||||
outputOffset += 2;
|
||||
} else {
|
||||
blockType = BLOCK_TYPE_NON_COMPRESSED;
|
||||
System.arraycopy(input, inputPtr, output, outputPtr - 2, length);
|
||||
chunkLength = length;
|
||||
}
|
||||
}
|
||||
out.setShort(outputOffset, length);
|
||||
|
||||
out.setByte(outputIdx + OPTIONS_OFFSET,
|
||||
blockType | (checksum != null ? BLOCK_WITH_CHECKSUM : BLOCK_WITHOUT_CHECKSUM));
|
||||
out.writerIndex(outputOffset + 2 + chunkLength);
|
||||
in.skipBytes(length);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,136 @@
|
|||
/*
|
||||
* Copyright 2014 The Netty Project
|
||||
*
|
||||
* The Netty Project licenses this file to you under the Apache License,
|
||||
* version 2.0 (the "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at:
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package io.netty.handler.codec.compression;
|
||||
|
||||
import io.netty.buffer.ByteBuf;
|
||||
import io.netty.buffer.CompositeByteBuf;
|
||||
import io.netty.buffer.Unpooled;
|
||||
import io.netty.channel.embedded.EmbeddedChannel;
|
||||
import io.netty.util.ReferenceCountUtil;
|
||||
|
||||
import static org.hamcrest.Matchers.*;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
public class FastLzIntegrationTest extends IntegrationTest {
|
||||
|
||||
public static class TestWithChecksum extends IntegrationTest {
|
||||
|
||||
@Override
|
||||
protected EmbeddedChannel createEncoderEmbeddedChannel() {
|
||||
return new EmbeddedChannel(new FastLzFramedEncoder(true));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected EmbeddedChannel createDecoderEmbeddedChannel() {
|
||||
return new EmbeddedChannel(new FastLzFramedDecoder(true));
|
||||
}
|
||||
}
|
||||
|
||||
public static class TestRandomChecksum extends IntegrationTest {
|
||||
|
||||
@Override
|
||||
protected EmbeddedChannel createEncoderEmbeddedChannel() {
|
||||
return new EmbeddedChannel(new FastLzFramedEncoder(rand.nextBoolean()));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected EmbeddedChannel createDecoderEmbeddedChannel() {
|
||||
return new EmbeddedChannel(new FastLzFramedDecoder(rand.nextBoolean()));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected EmbeddedChannel createEncoderEmbeddedChannel() {
|
||||
return new EmbeddedChannel(new FastLzFramedEncoder(rand.nextBoolean()));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected EmbeddedChannel createDecoderEmbeddedChannel() {
|
||||
return new EmbeddedChannel(new FastLzFramedDecoder(rand.nextBoolean()));
|
||||
}
|
||||
|
||||
@Override // test batched flow of data
|
||||
protected void testIdentity(final byte[] data) {
|
||||
final ByteBuf original = Unpooled.wrappedBuffer(data);
|
||||
final EmbeddedChannel encoder = createEncoderEmbeddedChannel();
|
||||
final EmbeddedChannel decoder = createDecoderEmbeddedChannel();
|
||||
|
||||
try {
|
||||
int written = 0, length = rand.nextInt(100);
|
||||
while (written + length < data.length) {
|
||||
ByteBuf in = Unpooled.wrappedBuffer(data, written, length);
|
||||
encoder.writeOutbound(in);
|
||||
written += length;
|
||||
length = rand.nextInt(100);
|
||||
}
|
||||
ByteBuf in = Unpooled.wrappedBuffer(data, written, data.length - written);
|
||||
encoder.writeOutbound(in);
|
||||
encoder.finish();
|
||||
|
||||
ByteBuf msg;
|
||||
final CompositeByteBuf compressed = Unpooled.compositeBuffer();
|
||||
while ((msg = encoder.readOutbound()) != null) {
|
||||
compressed.addComponent(msg);
|
||||
compressed.writerIndex(compressed.writerIndex() + msg.readableBytes());
|
||||
}
|
||||
assertThat(compressed, is(notNullValue()));
|
||||
|
||||
final byte[] compressedArray = new byte[compressed.readableBytes()];
|
||||
compressed.readBytes(compressedArray);
|
||||
written = 0;
|
||||
length = rand.nextInt(100);
|
||||
while (written + length < compressedArray.length) {
|
||||
in = Unpooled.wrappedBuffer(compressedArray, written, length);
|
||||
decoder.writeInbound(in);
|
||||
written += length;
|
||||
length = rand.nextInt(100);
|
||||
}
|
||||
in = Unpooled.wrappedBuffer(compressedArray, written, compressedArray.length - written);
|
||||
decoder.writeInbound(in);
|
||||
|
||||
assertFalse(compressed.isReadable());
|
||||
final CompositeByteBuf decompressed = Unpooled.compositeBuffer();
|
||||
while ((msg = decoder.readInbound()) != null) {
|
||||
decompressed.addComponent(msg);
|
||||
decompressed.writerIndex(decompressed.writerIndex() + msg.readableBytes());
|
||||
}
|
||||
assertEquals(original, decompressed);
|
||||
|
||||
compressed.release();
|
||||
decompressed.release();
|
||||
original.release();
|
||||
} finally {
|
||||
encoder.close();
|
||||
decoder.close();
|
||||
|
||||
for (;;) {
|
||||
Object msg = encoder.readOutbound();
|
||||
if (msg == null) {
|
||||
break;
|
||||
}
|
||||
ReferenceCountUtil.release(msg);
|
||||
}
|
||||
|
||||
for (;;) {
|
||||
Object msg = decoder.readInbound();
|
||||
if (msg == null) {
|
||||
break;
|
||||
}
|
||||
ReferenceCountUtil.release(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
The MIT License
|
||||
|
||||
Copyright (c) 2009 William Kinney
|
||||
|
||||
Permission is hereby granted, free of charge, to any person
|
||||
obtaining a copy of this software and associated documentation
|
||||
files (the "Software"), to deal in the Software without
|
||||
restriction, including without limitation the rights to use,
|
||||
copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the
|
||||
Software is furnished to do so, subject to the following
|
||||
conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
OTHER DEALINGS IN THE SOFTWARE.
|
||||
Loading…
Reference in New Issue