big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-8926. hadoop.util.PureJavaCrc32 cache hit-ratio is low for static data (Gopal V via bobby)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1399005 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans 2012-10-16 21:52:10 +00:00
parent be2b9d0236
commit 224de4f92c
4 changed files with 127 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -330,6 +330,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
via atm)
HADOOP-8926. hadoop.util.PureJavaCrc32 cache hit-ratio is low for static
data (Gopal V via bobby)
BUG FIXES
HADOOP-8795. BASH tab completion doesn't look in PATH, assumes path to

87
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32.java
View File

@ -59,22 +59,38 @@ public void reset() {
@Override
public void update(byte[] b, int off, int len) {
int localCrc = crc;
while(len > 7) {
int c0 = b[off++] ^ localCrc;
int c1 = b[off++] ^ (localCrc >>>= 8);
int c2 = b[off++] ^ (localCrc >>>= 8);
int c3 = b[off++] ^ (localCrc >>>= 8);
localCrc = (T8_7[c0 & 0xff] ^ T8_6[c1 & 0xff])
^ (T8_5[c2 & 0xff] ^ T8_4[c3 & 0xff]);
final int c0 =(b[off+0] ^ localCrc) & 0xff;
final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
localCrc ^= (T8_3[b[off++] & 0xff] ^ T8_2[b[off++] & 0xff])
^ (T8_1[b[off++] & 0xff] ^ T8_0[b[off++] & 0xff]);
final int c4 = b[off+4] & 0xff;
final int c5 = b[off+5] & 0xff;
final int c6 = b[off+6] & 0xff;
final int c7 = b[off+7] & 0xff;
localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
off += 8;
len -= 8;
}
while(len > 0) {
localCrc = (localCrc >>> 8) ^ T8_0[(localCrc ^ b[off++]) & 0xff];
len--;
/* loop unroll - duff's device style */
switch(len) {
case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
default:
/* nothing */
}
// Publish crc out to object
@ -83,14 +99,24 @@ public void update(byte[] b, int off, int len) {
@Override
final public void update(int b) {
crc = (crc >>> 8) ^ T8_0[(crc ^ b) & 0xff];
crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
}
/*
* CRC-32 lookup tables generated by the polynomial 0xEDB88320.
* See also TestPureJavaCrc32.Table.
*/
private static final int[] T8_0 = new int[] {
private static final int T8_0_start = 0*256;
private static final int T8_1_start = 1*256;
private static final int T8_2_start = 2*256;
private static final int T8_3_start = 3*256;
private static final int T8_4_start = 4*256;
private static final int T8_5_start = 5*256;
private static final int T8_6_start = 6*256;
private static final int T8_7_start = 7*256;
private static final int[] T = new int[] {
/* T8_0 */
0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA,
0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3,
0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988,
@ -154,9 +180,8 @@ final public void update(int b) {
0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6,
0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF,
0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94,
0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D
};
private static final int[] T8_1 = new int[] {
0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D,
/* T8_1 */
0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3,
0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7,
0xC8D98A08, 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB,
@ -220,9 +245,8 @@ final public void update(int b) {
0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, 0x3F91B27E,
0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A,
0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876,
0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72
};
private static final int[] T8_2 = new int[] {
0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72,
/* T8_2 */
0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59,
0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685,
0x0E1351B8, 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1,
@ -286,9 +310,8 @@ final public void update(int b) {
0xB5C473D0, 0xB40619E7, 0xB640A7BE, 0xB782CD89,
0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555,
0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31,
0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED
};
private static final int[] T8_3 = new int[] {
0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED,
/* T8_3 */
0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE,
0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9,
0xC5B428EF, 0x7D084F8A, 0x6FBDE064, 0xD7018701,
@ -352,9 +375,8 @@ final public void update(int b) {
0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, 0x94D3B949,
0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E,
0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6,
0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1
};
private static final int[] T8_4 = new int[] {
0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1,
/* T8_4 */
0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0,
0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10,
0x30704BC1, 0x0D106271, 0x4AB018A1, 0x77D03111,
@ -418,9 +440,8 @@ final public void update(int b) {
0x4834505D, 0x755479ED, 0x32F4033D, 0x0F942A8D,
0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D,
0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C,
0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C
};
private static final int[] T8_5 = new int[] {
0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C,
/* T8_5 */
0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE,
0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8,
0xEC53826D, 0x270F51C8, 0xA19B2366, 0x6AC7F0C3,
@ -484,9 +505,8 @@ final public void update(int b) {
0x15921919, 0xDECECABC, 0x585AB812, 0x93066BB7,
0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1,
0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA,
0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC
};
private static final int[] T8_6 = new int[] {
0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC,
/* T8_6 */
0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D,
0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E,
0x33EF4E67, 0x959845D3, 0xA4705F4E, 0x020754FA,
@ -550,9 +570,8 @@ final public void update(int b) {
0x647E3AD9, 0xC209316D, 0xF3E12BF0, 0x55962044,
0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457,
0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23,
0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30
};
private static final int[] T8_7 = new int[] {
0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30,
/* T8_7 */
0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3,
0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919,
0xD3E51BB5, 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56,

89
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PureJavaCrc32C.java
View File

@ -56,22 +56,38 @@ public void reset() {
@Override
public void update(byte[] b, int off, int len) {
int localCrc = crc;
while(len > 7) {
int c0 = b[off++] ^ localCrc;
int c1 = b[off++] ^ (localCrc >>>= 8);
int c2 = b[off++] ^ (localCrc >>>= 8);
int c3 = b[off++] ^ (localCrc >>>= 8);
localCrc = (T8_7[c0 & 0xff] ^ T8_6[c1 & 0xff])
^ (T8_5[c2 & 0xff] ^ T8_4[c3 & 0xff]);
final int c0 =(b[off+0] ^ localCrc) & 0xff;
final int c1 =(b[off+1] ^ (localCrc >>>= 8)) & 0xff;
final int c2 =(b[off+2] ^ (localCrc >>>= 8)) & 0xff;
final int c3 =(b[off+3] ^ (localCrc >>>= 8)) & 0xff;
localCrc = (T[T8_7_start + c0] ^ T[T8_6_start + c1])
^ (T[T8_5_start + c2] ^ T[T8_4_start + c3]);
localCrc ^= (T8_3[b[off++] & 0xff] ^ T8_2[b[off++] & 0xff])
^ (T8_1[b[off++] & 0xff] ^ T8_0[b[off++] & 0xff]);
final int c4 = b[off+4] & 0xff;
final int c5 = b[off+5] & 0xff;
final int c6 = b[off+6] & 0xff;
final int c7 = b[off+7] & 0xff;
localCrc ^= (T[T8_3_start + c4] ^ T[T8_2_start + c5])
^ (T[T8_1_start + c6] ^ T[T8_0_start + c7]);
off += 8;
len -= 8;
}
while(len > 0) {
localCrc = (localCrc >>> 8) ^ T8_0[(localCrc ^ b[off++]) & 0xff];
len--;
/* loop unroll - duff's device style */
switch(len) {
case 7: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
case 6: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
case 5: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
case 4: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
case 3: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
case 2: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
case 1: localCrc = (localCrc >>> 8) ^ T[T8_0_start + ((localCrc ^ b[off++]) & 0xff)];
default:
/* nothing */
}
// Publish crc out to object
@ -80,14 +96,24 @@ public void update(byte[] b, int off, int len) {
@Override
final public void update(int b) {
crc = (crc >>> 8) ^ T8_0[(crc ^ b) & 0xff];
crc = (crc >>> 8) ^ T[T8_0_start + ((crc ^ b) & 0xff)];
}
// CRC polynomial tables generated by:
// java -cp build/test/classes/:build/classes/ \
// org.apache.hadoop.util.TestPureJavaCrc32\$Table 82F63B78
static final int[] T8_0 = new int[] {
private static final int T8_0_start = 0*256;
private static final int T8_1_start = 1*256;
private static final int T8_2_start = 2*256;
private static final int T8_3_start = 3*256;
private static final int T8_4_start = 4*256;
private static final int T8_5_start = 5*256;
private static final int T8_6_start = 6*256;
private static final int T8_7_start = 7*256;
private static final int[] T = new int[] {
/* T8_0 */
0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4,
0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB,
0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B,
@ -151,9 +177,8 @@ final public void update(int b) {
0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81,
0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E,
0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E,
0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351
};
static final int[] T8_1 = new int[] {
0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351,
/* T8_1 */
0x00000000, 0x13A29877, 0x274530EE, 0x34E7A899,
0x4E8A61DC, 0x5D28F9AB, 0x69CF5132, 0x7A6DC945,
0x9D14C3B8, 0x8EB65BCF, 0xBA51F356, 0xA9F36B21,
@ -217,9 +242,8 @@ final public void update(int b) {
0x449A2E7E, 0x5738B609, 0x63DF1E90, 0x707D86E7,
0x0A104FA2, 0x19B2D7D5, 0x2D557F4C, 0x3EF7E73B,
0xD98EEDC6, 0xCA2C75B1, 0xFECBDD28, 0xED69455F,
0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483
};
static final int[] T8_2 = new int[] {
0x97048C1A, 0x84A6146D, 0xB041BCF4, 0xA3E32483,
/* T8_2 */
0x00000000, 0xA541927E, 0x4F6F520D, 0xEA2EC073,
0x9EDEA41A, 0x3B9F3664, 0xD1B1F617, 0x74F06469,
0x38513EC5, 0x9D10ACBB, 0x773E6CC8, 0xD27FFEB6,
@ -283,9 +307,8 @@ final public void update(int b) {
0xDDA47104, 0x78E5E37A, 0x92CB2309, 0x378AB177,
0x437AD51E, 0xE63B4760, 0x0C158713, 0xA954156D,
0xE5F54FC1, 0x40B4DDBF, 0xAA9A1DCC, 0x0FDB8FB2,
0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8
};
static final int[] T8_3 = new int[] {
0x7B2BEBDB, 0xDE6A79A5, 0x3444B9D6, 0x91052BA8,
/* T8_3 */
0x00000000, 0xDD45AAB8, 0xBF672381, 0x62228939,
0x7B2231F3, 0xA6679B4B, 0xC4451272, 0x1900B8CA,
0xF64463E6, 0x2B01C95E, 0x49234067, 0x9466EADF,
@ -349,9 +372,8 @@ final public void update(int b) {
0xC747336E, 0x1A0299D6, 0x782010EF, 0xA565BA57,
0xBC65029D, 0x6120A825, 0x0302211C, 0xDE478BA4,
0x31035088, 0xEC46FA30, 0x8E647309, 0x5321D9B1,
0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842
};
static final int[] T8_4 = new int[] {
0x4A21617B, 0x9764CBC3, 0xF54642FA, 0x2803E842,
/* T8_4 */
0x00000000, 0x38116FAC, 0x7022DF58, 0x4833B0F4,
0xE045BEB0, 0xD854D11C, 0x906761E8, 0xA8760E44,
0xC5670B91, 0xFD76643D, 0xB545D4C9, 0x8D54BB65,
@ -415,9 +437,8 @@ final public void update(int b) {
0xCD796B76, 0xF56804DA, 0xBD5BB42E, 0x854ADB82,
0x2D3CD5C6, 0x152DBA6A, 0x5D1E0A9E, 0x650F6532,
0x081E60E7, 0x300F0F4B, 0x783CBFBF, 0x402DD013,
0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3
};
static final int[] T8_5 = new int[] {
0xE85BDE57, 0xD04AB1FB, 0x9879010F, 0xA0686EA3,
/* T8_5 */
0x00000000, 0xEF306B19, 0xDB8CA0C3, 0x34BCCBDA,
0xB2F53777, 0x5DC55C6E, 0x697997B4, 0x8649FCAD,
0x6006181F, 0x8F367306, 0xBB8AB8DC, 0x54BAD3C5,
@ -481,9 +502,8 @@ final public void update(int b) {
0x57F4CA8E, 0xB8C4A197, 0x8C786A4D, 0x63480154,
0xE501FDF9, 0x0A3196E0, 0x3E8D5D3A, 0xD1BD3623,
0x37F2D291, 0xD8C2B988, 0xEC7E7252, 0x034E194B,
0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C
};
static final int[] T8_6 = new int[] {
0x8507E5E6, 0x6A378EFF, 0x5E8B4525, 0xB1BB2E3C,
/* T8_6 */
0x00000000, 0x68032CC8, 0xD0065990, 0xB8057558,
0xA5E0C5D1, 0xCDE3E919, 0x75E69C41, 0x1DE5B089,
0x4E2DFD53, 0x262ED19B, 0x9E2BA4C3, 0xF628880B,
@ -547,9 +567,8 @@ final public void update(int b) {
0x2ED97095, 0x46DA5C5D, 0xFEDF2905, 0x96DC05CD,
0x8B39B544, 0xE33A998C, 0x5B3FECD4, 0x333CC01C,
0x60F48DC6, 0x08F7A10E, 0xB0F2D456, 0xD8F1F89E,
0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F
};
static final int[] T8_7 = new int[] {
0xC5144817, 0xAD1764DF, 0x15121187, 0x7D113D4F,
/* T8_7 */
0x00000000, 0x493C7D27, 0x9278FA4E, 0xDB448769,
0x211D826D, 0x6821FF4A, 0xB3657823, 0xFA590504,
0x423B04DA, 0x0B0779FD, 0xD043FE94, 0x997F83B3,
@ -613,6 +632,6 @@ final public void update(int b) {
0xA777317B, 0xEE4B4C5C, 0x350FCB35, 0x7C33B612,
0x866AB316, 0xCF56CE31, 0x14124958, 0x5D2E347F,
0xE54C35A1, 0xAC704886, 0x7734CFEF, 0x3E08B2C8,
0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5
0xC451B7CC, 0x8D6DCAEB, 0x56294D82, 0x1F1530A5
};
}

23
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestPureJavaCrc32.java
View File

@ -142,16 +142,14 @@ String[] toStrings(String nameformat) {
for (int j = 0; j < tables.length; j++) {
final int[] t = tables[j];
final StringBuilder b = new StringBuilder();
b.append(String.format(" static final int[] " + nameformat
+ " = new int[] {", j));
b.append(String.format(" /* "+ nameformat +" */", j));
for (int i = 0; i < t.length;) {
b.append("\n ");
for(int k = 0; k < 4; k++) {
b.append(String.format("0x%08X, ", t[i++]));
}
}
b.setCharAt(b.length() - 2, '\n');
s[j] = b.toString() + " };\n";
s[j] = b.toString();
}
return s;
}
@ -159,10 +157,23 @@ String[] toStrings(String nameformat) {
@Override
public String toString() {
final StringBuilder b = new StringBuilder();
for(String s : toStrings(String.format("T%d_",
Integer.numberOfTrailingZeros(tables[0].length)) + "%d")) {
final String tableFormat = String.format("T%d_",
Integer.numberOfTrailingZeros(tables[0].length)) + "%d";
final String startFormat = " private static final int "+tableFormat+"_start = %d*256;";
for (int j = 0; j < tables.length; j++) {
b.append(String.format(startFormat, j, j));
b.append("\n");
}
b.append(" private static final int[] T = new int[] {");
for(String s : toStrings(tableFormat)) {
b.append("\n");
b.append(s);
}
b.setCharAt(b.length() - 2, '\n');
b.append(" };\n");
return b.toString();
}