diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1d8c25975c..ce3643fb34 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -397,6 +397,8 @@ Branch-2 ( Unreleased changes ) HDFS-3802. StartupOption.name in HdfsServerConstants should be final. (Jing Zhao via szetszwo) + HDFS-3796. Speed up edit log tests by avoiding fsync() (todd) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java index bb181d743b..f7a8b337a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java @@ -49,6 +49,8 @@ public class EditLogFileOutputStream extends EditLogOutputStream { private EditsDoubleBuffer doubleBuf; static ByteBuffer fill = ByteBuffer.allocateDirect(MIN_PREALLOCATION_LENGTH); + private static boolean shouldSkipFsyncForTests = false; + static { fill.position(0); for (int i = 0; i < fill.capacity(); i++) { @@ -184,7 +186,9 @@ public void flushAndSync() throws IOException { } preallocate(); // preallocate file if necessay doubleBuf.flushTo(fp); - fc.force(false); // metadata updates not needed + if (!shouldSkipFsyncForTests) { + fc.force(false); // metadata updates not needed + } } /** @@ -247,4 +251,15 @@ public void setFileChannelForTesting(FileChannel fc) { public FileChannel getFileChannelForTesting() { return fc; } + + /** + * For the purposes of unit tests, we don't need to actually + * write durably to disk. So, we can skip the fsync() calls + * for a speed improvement. + * @param skip true if fsync should not be called + */ + @VisibleForTesting + public static void setShouldSkipFsyncForTesting(boolean skip) { + shouldSkipFsyncForTests = skip; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java index e7abc1132d..d9ac54ed0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java @@ -119,6 +119,11 @@ public class TestEditLog { "a4ff 0000 0000 0000 0000 0000 0000 0000" ).replace(" ","")); + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } static final byte TRAILER_BYTE = FSEditLogOpCodes.OP_INVALID.getOpCode(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java index ead94968ee..22ab02d2a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java @@ -40,6 +40,12 @@ public class TestEditLogFileOutputStream { final static int MIN_PREALLOCATION_LENGTH = EditLogFileOutputStream.MIN_PREALLOCATION_LENGTH; + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } + @Before @After public void deleteEditsFile() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java index 1a968c7d1d..722e4e24bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java @@ -51,6 +51,12 @@ public class TestFileJournalManager { static final Log LOG = LogFactory.getLog(TestFileJournalManager.class); + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } + /** * Find out how many transactions we can read from a * FileJournalManager, starting at a given transaction ID. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java index 1717bb0412..23fd3b51a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRecovery.java @@ -57,6 +57,7 @@ public class TestNameNodeRecovery { static { recoverStartOpt.setForce(MetaRecoveryContext.FORCE_ALL); + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); } static void runEditLogTest(EditLogTestSetup elts) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java index e3056e9b0b..dd679d1a9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java @@ -49,6 +49,12 @@ public class TestSecurityTokenEditLog { static final int NUM_THREADS = 100; static final int opsPerTrans = 3; + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } + // // an object that does a bunch of transactions // diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java index dd5c1bab75..5e18381e7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogsDuringFailover.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; import org.apache.hadoop.hdfs.server.namenode.NNStorage; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; @@ -52,6 +53,12 @@ public class TestEditLogsDuringFailover { private static final Log LOG = LogFactory.getLog(TestEditLogsDuringFailover.class); private static final int NUM_DIRS_IN_LOG = 5; + + static { + // No need to fsync for the purposes of tests. This makes + // the tests run much faster. + EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); + } @Test public void testStartup() throws Exception {