big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17877. BuiltInGzipCompressor header and trailer should not be static variables (#3350)

Browse Source
This commit is contained in:
Liang-Chi Hsieh 2021-08-29 08:21:55 -07:00 committed by GitHub
parent 16e6030e25
commit 73a0c31370
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 20 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/BuiltInGzipCompressor.java
View File

@ -39,15 +39,15 @@ public class BuiltInGzipCompressor implements Compressor {
* Fixed ten-byte gzip header. See {@link GZIPOutputStream}'s source for
* details.
*/
private static final byte[] GZIP_HEADER = new byte[]{
private final byte[] gzipHeader = new byte[]{
0x1f, (byte) 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
// The trailer will be overwritten based on crc and output size.
private static final byte[] GZIP_TRAILER = new byte[]{
private final byte[] gzipTrailer = new byte[]{
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
private static final int GZIP_HEADER_LEN = GZIP_HEADER.length;
private static final int GZIP_TRAILER_LEN = GZIP_TRAILER.length;
private final int gzipHeaderLen = gzipHeader.length;
private final int gzipTrailerLen = gzipTrailer.length;
private Deflater deflater;
@ -210,12 +210,12 @@ private int writeHeader(byte[] b, int off, int len) {
return 0;
}
int n = Math.min(len, GZIP_HEADER_LEN - headerOff);
System.arraycopy(GZIP_HEADER, headerOff, b, off, n);
int n = Math.min(len, gzipHeaderLen - headerOff);
System.arraycopy(gzipHeader, headerOff, b, off, n);
headerOff += n;
// Completes header output.
if (headerOff == GZIP_HEADER_LEN) {
if (headerOff == gzipHeaderLen) {
state = BuiltInGzipDecompressor.GzipStateLabel.INFLATE_STREAM;
}
@ -225,15 +225,15 @@ private int writeHeader(byte[] b, int off, int len) {
private void fillTrailer() {
if (state == BuiltInGzipDecompressor.GzipStateLabel.TRAILER_CRC) {
int streamCrc = (int) crc.getValue();
GZIP_TRAILER[0] = (byte) (streamCrc & 0x000000ff);
GZIP_TRAILER[1] = (byte) ((streamCrc & 0x0000ff00) >> 8);
GZIP_TRAILER[2] = (byte) ((streamCrc & 0x00ff0000) >> 16);
GZIP_TRAILER[3] = (byte) ((streamCrc & 0xff000000) >> 24);
gzipTrailer[0] = (byte) (streamCrc & 0x000000ff);
gzipTrailer[1] = (byte) ((streamCrc & 0x0000ff00) >> 8);
gzipTrailer[2] = (byte) ((streamCrc & 0x00ff0000) >> 16);
gzipTrailer[3] = (byte) ((streamCrc & 0xff000000) >> 24);
GZIP_TRAILER[4] = (byte) (accuBufLen & 0x000000ff);
GZIP_TRAILER[5] = (byte) ((accuBufLen & 0x0000ff00) >> 8);
GZIP_TRAILER[6] = (byte) ((accuBufLen & 0x00ff0000) >> 16);
GZIP_TRAILER[7] = (byte) ((accuBufLen & 0xff000000) >> 24);
gzipTrailer[4] = (byte) (accuBufLen & 0x000000ff);
gzipTrailer[5] = (byte) ((accuBufLen & 0x0000ff00) >> 8);
gzipTrailer[6] = (byte) ((accuBufLen & 0x00ff0000) >> 16);
gzipTrailer[7] = (byte) ((accuBufLen & 0xff000000) >> 24);
crc.reset();
accuBufLen = 0;
@ -245,11 +245,11 @@ private int writeTrailer(byte[] b, int off, int len) {
return 0;
}
int n = Math.min(len, GZIP_TRAILER_LEN - trailerOff);
System.arraycopy(GZIP_TRAILER, trailerOff, b, off, n);
int n = Math.min(len, gzipTrailerLen - trailerOff);
System.arraycopy(gzipTrailer, trailerOff, b, off, n);
trailerOff += n;
if (trailerOff == GZIP_TRAILER_LEN) {
if (trailerOff == gzipTrailerLen) {
state = BuiltInGzipDecompressor.GzipStateLabel.FINISHED;
currentBufLen = 0;
headerOff = 0;

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java
View File

@ -109,7 +109,7 @@ public void testGzipCodec() throws IOException {
}
// without hadoop-native installed.
ZlibFactory.setNativeZlibLoaded(false);
assumeTrue(ZlibFactory.isNativeZlibLoaded(conf));
assertFalse(ZlibFactory.isNativeZlibLoaded(conf));
codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.GzipCodec");
codecTest(conf, seed, count, "org.apache.hadoop.io.compress.GzipCodec");
}
@ -570,7 +570,7 @@ public void testSequenceFileGzipCodec() throws IOException, ClassNotFoundExcepti
}
// without hadoop-native installed.
ZlibFactory.setNativeZlibLoaded(false);
assumeTrue(ZlibFactory.isNativeZlibLoaded(conf));
assertFalse(ZlibFactory.isNativeZlibLoaded(conf));
sequenceFileCodecTest(conf, 100, "org.apache.hadoop.io.compress.GzipCodec", 5);
sequenceFileCodecTest(conf, 100, "org.apache.hadoop.io.compress.GzipCodec", 100);
sequenceFileCodecTest(conf, 200000, "org.apache.hadoop.io.compress.GzipCodec", 1000000);