diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java index d8963df986..34dfc3a6db 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/Crc32PerformanceTest.java @@ -33,7 +33,7 @@ import org.apache.log4j.Level; /** - * Performance tests to compare performance of Crc32 implementations + * Performance tests to compare performance of Crc32|Crc32C implementations * This can be run from the command line with: * * java -cp path/to/test/classes:path/to/common/classes \ @@ -43,17 +43,21 @@ * * hadoop org.apache.hadoop.util.Crc32PerformanceTest * + * If any argument is provided, this test will run with non-directly buffer. + * * The output is in JIRA table format. */ public class Crc32PerformanceTest { static final int MB = 1024 * 1024; - static interface Crc32 { + interface Crc32 { - public void verifyChunked(ByteBuffer data, int bytesPerCrc, ByteBuffer crcs, + void verifyChunked(ByteBuffer data, int bytesPerCrc, ByteBuffer crcs, String filename, long basePos) throws ChecksumException; - static final class Native implements Crc32 { + DataChecksum.Type crcType(); + + final class Native implements Crc32 { @Override public void verifyChunked(ByteBuffer data, int bytesPerSum, ByteBuffer sums, String fileName, long basePos) @@ -61,40 +65,91 @@ public void verifyChunked(ByteBuffer data, int bytesPerSum, NativeCrc32.verifyChunkedSums(bytesPerSum, DataChecksum.Type.CRC32.id, sums, data, fileName, basePos); } + + @Override + public DataChecksum.Type crcType() { + return DataChecksum.Type.CRC32; + } } + final class NativeC implements Crc32 { + @Override + public void verifyChunked(ByteBuffer data, int bytesPerSum, + ByteBuffer sums, String fileName, long basePos) + throws ChecksumException { - static abstract class AbstractCrc32 implements Crc32 { + if (data.isDirect()) { + NativeCrc32.verifyChunkedSums(bytesPerSum, + DataChecksum.Type.CRC32C.id, sums, data, fileName, basePos); + } else { + final int dataOffset = data.arrayOffset() + data.position(); + final int crcsOffset = sums.arrayOffset() + sums.position(); + NativeCrc32.verifyChunkedSumsByteArray(bytesPerSum, + DataChecksum.Type.CRC32C.id, sums.array(), crcsOffset, + data.array(), dataOffset, data.remaining(), fileName, basePos); + } + } + + @Override + public DataChecksum.Type crcType() { + return DataChecksum.Type.CRC32C; + } + } + + abstract class AbstractCrc32 implements Crc32 { abstract T newAlgorithm(); @Override public void verifyChunked(ByteBuffer data, int bytesPerCrc, - ByteBuffer crcs, String filename, long basePos) + ByteBuffer sums, String filename, long basePos) throws ChecksumException { final Checksum algorithm = newAlgorithm(); - if (data.hasArray() && crcs.hasArray()) { - DataChecksum.verifyChunked(DataChecksum.Type.CRC32, algorithm, - data.array(), data.position(), data.remaining(), bytesPerCrc, - crcs.array(), crcs.position(), filename, basePos); + final DataChecksum.Type type = crcType(); + if (data.hasArray() && sums.hasArray()) { + DataChecksum.verifyChunked(type, algorithm, data.array(), + data.position(), data.remaining(), bytesPerCrc, sums.array(), + sums.position(), filename, basePos); } else { - DataChecksum.verifyChunked(DataChecksum.Type.CRC32, algorithm, - data, bytesPerCrc, crcs, filename, basePos); + DataChecksum.verifyChunked(type, algorithm, data, bytesPerCrc, + sums, filename, basePos); } } } - static final class Zip extends AbstractCrc32 { + final class Zip extends AbstractCrc32 { @Override public CRC32 newAlgorithm() { return new CRC32(); } + + @Override + public DataChecksum.Type crcType() { + return DataChecksum.Type.CRC32; + } } - static final class PureJava extends AbstractCrc32 { + final class PureJava extends AbstractCrc32 { @Override public PureJavaCrc32 newAlgorithm() { return new PureJavaCrc32(); } + + @Override + public DataChecksum.Type crcType() { + return DataChecksum.Type.CRC32; + } + } + + final class PureJavaC extends AbstractCrc32 { + @Override + public PureJavaCrc32C newAlgorithm() { + return new PureJavaCrc32C(); + } + + @Override + public DataChecksum.Type crcType() { + return DataChecksum.Type.CRC32C; + } } } @@ -114,9 +169,13 @@ public PureJavaCrc32 newAlgorithm() { crcs.add(Crc32.Zip.class); crcs.add(Crc32.PureJava.class); + crcs.add(Crc32.PureJavaC.class); - if (direct && NativeCrc32.isAvailable()) { - crcs.add(Crc32.Native.class); + if (NativeCrc32.isAvailable()) { + if (direct) { + crcs.add(Crc32.Native.class); + } + crcs.add(Crc32.NativeC.class); ((Log4JLogger)LogFactory.getLog(NativeCodeLoader.class)) .getLogger().setLevel(Level.ALL); } @@ -131,13 +190,18 @@ void run() throws Exception { out.printf("Elapsed %.1fs\n", secondsElapsed(startTime)); } - public static void main(String args[]) throws Exception { - new Crc32PerformanceTest(64, 5, true).run(); + public static void main(String[] args) throws Exception { + boolean isdirect = true; + + if (args.length > 0) { + isdirect = false; + } + new Crc32PerformanceTest(64, 5, isdirect).run(); } - private static void printCell(String s, int width, PrintStream out) { + private static void printCell(String s, int width, PrintStream outCrc) { final int w = s.length() > width? s.length(): width; - out.printf(" %" + w + "s |", s); + outCrc.printf(" %" + w + "s |", s); } private ByteBuffer allocateByteBuffer(int length) { @@ -155,16 +219,29 @@ private ByteBuffer newData() { return dataBufs; } - private ByteBuffer computeCrc(ByteBuffer dataBufs, int bytePerCrc) { + private ByteBuffer computeCrc(ByteBuffer dataBufs, int bytePerCrc, + DataChecksum.Type type) { final int size = 4 * (dataBufs.remaining() - 1) / bytePerCrc + 1; final ByteBuffer crcBufs = allocateByteBuffer(size); final DataChecksum checksum = DataChecksum.newDataChecksum( - DataChecksum.Type.CRC32, bytePerCrc); + type, bytePerCrc); checksum.calculateChunkedSums(dataBufs, crcBufs); return crcBufs; } - private void doBench(final List> crcs) + private ByteBuffer computeCrc(Class clazz, + ByteBuffer dataBufs, int bytePerCrc) throws Exception { + final Constructor ctor = clazz.getConstructor(); + final Crc32 crc = ctor.newInstance(); + final int size = 4 * (dataBufs.remaining() - 1) / bytePerCrc + 1; + final ByteBuffer crcBufs = allocateByteBuffer(size); + final DataChecksum checksum = DataChecksum.newDataChecksum( + crc.crcType(), bytePerCrc); + checksum.calculateChunkedSums(dataBufs, crcBufs); + return crcBufs; + } + + private void doBench(final List> crcTargets) throws Exception { final ByteBuffer[] dataBufs = new ByteBuffer[16]; for(int i = 0; i < dataBufs.length; i++) { @@ -176,9 +253,9 @@ private void doBench(final List> crcs) out.printf(" (bpc: byte-per-crc in MB/sec; #T: #Theads)\n"); // Warm up implementations to get jit going. - final ByteBuffer[] crc32 = {computeCrc(dataBufs[0], 32)}; - final ByteBuffer[] crc512 = {computeCrc(dataBufs[0], 512)}; - for (Class c : crcs) { + for (Class c : crcTargets) { + final ByteBuffer[] crc32 = {computeCrc(c, dataBufs[0], 32)}; + final ByteBuffer[] crc512 = {computeCrc(c, dataBufs[0], 512)}; doBench(c, 1, dataBufs, crc32, 32); doBench(c, 1, dataBufs, crc512, 512); } @@ -189,57 +266,69 @@ private void doBench(final List> crcs) } } - private void doBench(final List> crcs, - final ByteBuffer[] dataBufs, final int bytePerCrc, final PrintStream out) + private void doBench(final List> crcTargets, + final ByteBuffer[] dataBufs, final int bytePerCrc, + final PrintStream outCrc) throws Exception { final ByteBuffer[] crcBufs = new ByteBuffer[dataBufs.length]; - for(int i = 0; i < crcBufs.length; i++) { - crcBufs[i] = computeCrc(dataBufs[i], bytePerCrc); + final ByteBuffer[] crcBufsC = new ByteBuffer[dataBufs.length]; + for(int i = 0; i < dataBufs.length; i++) { + crcBufs[i] = computeCrc(dataBufs[i], bytePerCrc, + DataChecksum.Type.CRC32); + crcBufsC[i] = computeCrc(dataBufs[i], bytePerCrc, + DataChecksum.Type.CRC32C); } final String numBytesStr = " bpc "; final String numThreadsStr = "#T"; final String diffStr = "% diff"; - out.print('|'); - printCell(numBytesStr, 0, out); - printCell(numThreadsStr, 0, out); - for (int i = 0; i < crcs.size(); i++) { - final Class c = crcs.get(i); - out.print('|'); - printCell(c.getSimpleName(), 8, out); - for(int j = 0; j < i; j++) { - printCell(diffStr, diffStr.length(), out); + outCrc.print('|'); + printCell(numBytesStr, 0, outCrc); + printCell(numThreadsStr, 0, outCrc); + for (int i = 0; i < crcTargets.size(); i++) { + final Class c = crcTargets.get(i); + outCrc.print('|'); + printCell(c.getSimpleName(), 8, outCrc); + if (i > 0) { + printCell(diffStr, diffStr.length(), outCrc); } } - out.printf("\n"); + outCrc.printf("\n"); for(int numThreads = 1; numThreads <= dataBufs.length; numThreads <<= 1) { - out.printf("|"); - printCell(String.valueOf(bytePerCrc), numBytesStr.length(), out); - printCell(String.valueOf(numThreads), numThreadsStr.length(), out); + outCrc.printf("|"); + printCell(String.valueOf(bytePerCrc), numBytesStr.length(), outCrc); + printCell(String.valueOf(numThreads), numThreadsStr.length(), outCrc); final List previous = new ArrayList(); - for(Class c : crcs) { + for(Class c : crcTargets) { System.gc(); - final BenchResult result = doBench(c, numThreads, dataBufs, crcBufs, - bytePerCrc); + final BenchResult result; + final Constructor ctor = c.getConstructor(); + final Crc32 crc = ctor.newInstance(); + if (crc.crcType() == DataChecksum.Type.CRC32) { + result = doBench(c, numThreads, dataBufs, crcBufs, bytePerCrc); + } else { + result = doBench(c, numThreads, dataBufs, crcBufsC, bytePerCrc); + } printCell(String.format("%9.1f", result.mbps), - c.getSimpleName().length() + 1, out); + c.getSimpleName().length() + 1, outCrc); - //compare result with previous - for(BenchResult p : previous) { + //compare result with the last previous. + final int size = previous.size(); + if (size > 0) { + BenchResult p = previous.get(size - 1); final double diff = (result.mbps - p.mbps) / p.mbps * 100; - printCell(String.format("%5.1f%%", diff), diffStr.length(), out); + printCell(String.format("%5.1f%%", diff), diffStr.length(), outCrc); } previous.add(result); } - out.printf("\n"); + outCrc.printf("\n"); } } - private BenchResult doBench(Class clazz, final int numThreads, final ByteBuffer[] dataBufs, final ByteBuffer[] crcBufs, final int bytePerCrc) @@ -248,36 +337,34 @@ private BenchResult doBench(Class clazz, final Thread[] threads = new Thread[numThreads]; final BenchResult[] results = new BenchResult[threads.length]; - { - final Constructor ctor = clazz.getConstructor(); + final Constructor ctor = clazz.getConstructor(); - for(int i = 0; i < threads.length; i++) { - final Crc32 crc = ctor.newInstance(); - final long byteProcessed = dataBufs[i].remaining() * trials; - final int index = i; - threads[i] = new Thread() { - @Override - public void run() { - final long startTime = System.nanoTime(); - for (int i = 0; i < trials; i++) { - dataBufs[index].mark(); - crcBufs[index].mark(); - try { - crc.verifyChunked(dataBufs[index], bytePerCrc, crcBufs[index], - crc.getClass().getSimpleName(), dataBufs[index].position()); - } catch (Throwable t) { - results[index] = new BenchResult(t); - return; - } finally { - dataBufs[index].reset(); - crcBufs[index].reset(); - } + for(int i = 0; i < threads.length; i++) { + final Crc32 crc = ctor.newInstance(); + final long byteProcessed = dataBufs[i].remaining() * trials; + final int index = i; + threads[i] = new Thread() { + @Override + public void run() { + final long startTime = System.nanoTime(); + for (int i = 0; i < trials; i++) { + dataBufs[index].mark(); + crcBufs[index].mark(); + try { + crc.verifyChunked(dataBufs[index], bytePerCrc, crcBufs[index], + crc.getClass().getSimpleName(), dataBufs[index].position()); + } catch (Throwable t) { + results[index] = new BenchResult(t); + return; + } finally { + dataBufs[index].reset(); + crcBufs[index].reset(); } - final double secsElapsed = secondsElapsed(startTime); - results[index] = new BenchResult(byteProcessed/secsElapsed/MB); } - }; - } + final double secsElapsed = secondsElapsed(startTime); + results[index] = new BenchResult(byteProcessed/secsElapsed/MB); + } + }; } for(Thread t : threads) { @@ -295,7 +382,7 @@ public void run() { } private static class BenchResult { - /** Speed (MB per second) */ + /** Speed (MB per second). */ final double mbps; final Throwable thrown; @@ -321,7 +408,7 @@ static double secondsElapsed(final long startTime) { return (System.nanoTime() - startTime) / 1000000000.0d; } - static void printSystemProperties(PrintStream out) { + static void printSystemProperties(PrintStream outCrc) { final String[] names = { "java.version", "java.runtime.name", @@ -344,7 +431,7 @@ static void printSystemProperties(PrintStream out) { final Properties p = System.getProperties(); for(String n : names) { - out.printf("%" + max + "s = %s\n", n, p.getProperty(n)); + outCrc.printf("%" + max + "s = %s\n", n, p.getProperty(n)); } } } \ No newline at end of file