diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 99d5c01cb4..77832433c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -518,6 +518,9 @@ Release 2.6.0 - UNRELEASED HDFS-6690. Deduplicate xattr names in memory. (wang) + HDFS-6773. MiniDFSCluster should skip edit log fsync by default (Stephen + Chu via Colin Patrick McCabe) + BUG FIXES HDFS-6823. dfs.web.authentication.kerberos.principal shows up in logs for diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 98ca316004..0e49cfec05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -93,6 +93,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetUtil; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; +import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; @@ -172,6 +173,7 @@ public static class Builder { private boolean checkDataNodeAddrConfig = false; private boolean checkDataNodeHostConfig = false; private Configuration[] dnConfOverlays; + private boolean skipFsyncForTesting = true; public Builder(Configuration conf) { this.conf = conf; @@ -405,6 +407,15 @@ public Builder dataNodeConfOverlays(Configuration[] dnConfOverlays) { this.dnConfOverlays = dnConfOverlays; return this; } + + /** + * Default: true + * When true, we skip fsync() calls for speed improvements. + */ + public Builder skipFsyncForTesting(boolean val) { + this.skipFsyncForTesting = val; + return this; + } /** * Construct the actual MiniDFSCluster @@ -472,7 +483,8 @@ protected MiniDFSCluster(Builder builder) throws IOException { builder.checkExitOnShutdown, builder.checkDataNodeAddrConfig, builder.checkDataNodeHostConfig, - builder.dnConfOverlays); + builder.dnConfOverlays, + builder.skipFsyncForTesting); } public class DataNodeProperties { @@ -727,7 +739,8 @@ public MiniDFSCluster(int nameNodePort, manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs, operation, null, racks, hosts, null, simulatedCapacities, null, true, false, - MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false, false, null); + MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), + true, false, false, null, true); } private void initMiniDFSCluster( @@ -742,7 +755,8 @@ private void initMiniDFSCluster( MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown, boolean checkDataNodeAddrConfig, boolean checkDataNodeHostConfig, - Configuration[] dnConfOverlays) + Configuration[] dnConfOverlays, + boolean skipFsyncForTesting) throws IOException { boolean success = false; try { @@ -782,6 +796,8 @@ private void initMiniDFSCluster( + "Standby node since no IPC ports have been specified."); conf.setInt(DFS_HA_LOGROLL_PERIOD_KEY, -1); } + + EditLogFileOutputStream.setShouldSkipFsyncForTesting(skipFsyncForTesting); federation = nnTopology.isFederated(); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java index 5ac13eec27..d6e70d8003 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java @@ -108,7 +108,6 @@ public class TestFsDatasetCache { private static CacheManipulator prevCacheManipulator; static { - EditLogFileOutputStream.setShouldSkipFsyncForTesting(false); LogManager.getLogger(FsDatasetCache.class).setLevel(Level.DEBUG); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index d54b90e663..93076928f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -110,7 +110,6 @@ public class TestCacheDirectives { static { NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator()); - EditLogFileOutputStream.setShouldSkipFsyncForTesting(false); } private static final long BLOCK_SIZE = 4096;