big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-6773. MiniDFSCluster should skip edit log fsync by default. Contributed by Stephen Chu.

Browse Source
This commit is contained in:
Colin Patrick Mccabe 2014-08-27 13:33:02 -07:00
parent fdd3bc5f45
commit d805cc27a9
4 changed files with 22 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -518,6 +518,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6690. Deduplicate xattr names in memory. (wang) HDFS-6690. Deduplicate xattr names in memory. (wang)
HDFS-6773. MiniDFSCluster should skip edit log fsync by default (Stephen
Chu via Colin Patrick McCabe)
BUG FIXES BUG FIXES
HDFS-6823. dfs.web.authentication.kerberos.principal shows up in logs for HDFS-6823. dfs.web.authentication.kerberos.principal shows up in logs for

22
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
View File

@ -93,6 +93,7 @@
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetUtil; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetUtil;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
@ -172,6 +173,7 @@ public static class Builder {
private boolean checkDataNodeAddrConfig = false; private boolean checkDataNodeAddrConfig = false;
private boolean checkDataNodeHostConfig = false; private boolean checkDataNodeHostConfig = false;
private Configuration[] dnConfOverlays; private Configuration[] dnConfOverlays;
private boolean skipFsyncForTesting = true;
public Builder(Configuration conf) { public Builder(Configuration conf) {
this.conf = conf; this.conf = conf;
@ -406,6 +408,15 @@ public Builder dataNodeConfOverlays(Configuration[] dnConfOverlays) {
return this; return this;
} }
/**
* Default: true
* When true, we skip fsync() calls for speed improvements.
*/
public Builder skipFsyncForTesting(boolean val) {
this.skipFsyncForTesting = val;
return this;
}
/** /**
* Construct the actual MiniDFSCluster * Construct the actual MiniDFSCluster
*/ */
@ -472,7 +483,8 @@ protected MiniDFSCluster(Builder builder) throws IOException {
builder.checkExitOnShutdown, builder.checkExitOnShutdown,
builder.checkDataNodeAddrConfig, builder.checkDataNodeAddrConfig,
builder.checkDataNodeHostConfig, builder.checkDataNodeHostConfig,
builder.dnConfOverlays); builder.dnConfOverlays,
builder.skipFsyncForTesting);
} }
public class DataNodeProperties { public class DataNodeProperties {
@ -727,7 +739,8 @@ public MiniDFSCluster(int nameNodePort,
manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs, manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs,
operation, null, racks, hosts, operation, null, racks, hosts,
null, simulatedCapacities, null, true, false, null, simulatedCapacities, null, true, false,
MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0), true, false, false, null); MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0),
true, false, false, null, true);
} }
private void initMiniDFSCluster( private void initMiniDFSCluster(
@ -742,7 +755,8 @@ private void initMiniDFSCluster(
MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown, MiniDFSNNTopology nnTopology, boolean checkExitOnShutdown,
boolean checkDataNodeAddrConfig, boolean checkDataNodeAddrConfig,
boolean checkDataNodeHostConfig, boolean checkDataNodeHostConfig,
Configuration[] dnConfOverlays) Configuration[] dnConfOverlays,
boolean skipFsyncForTesting)
throws IOException { throws IOException {
boolean success = false; boolean success = false;
try { try {
@ -783,6 +797,8 @@ private void initMiniDFSCluster(
conf.setInt(DFS_HA_LOGROLL_PERIOD_KEY, -1); conf.setInt(DFS_HA_LOGROLL_PERIOD_KEY, -1);
} }
EditLogFileOutputStream.setShouldSkipFsyncForTesting(skipFsyncForTesting);
federation = nnTopology.isFederated(); federation = nnTopology.isFederated();
try { try {
createNameNodesAndSetConf( createNameNodesAndSetConf(

1
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
View File

@ -108,7 +108,6 @@ public class TestFsDatasetCache {
private static CacheManipulator prevCacheManipulator; private static CacheManipulator prevCacheManipulator;
static { static {
EditLogFileOutputStream.setShouldSkipFsyncForTesting(false);
LogManager.getLogger(FsDatasetCache.class).setLevel(Level.DEBUG); LogManager.getLogger(FsDatasetCache.class).setLevel(Level.DEBUG);
} }

1
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java
View File

@ -110,7 +110,6 @@ public class TestCacheDirectives {
static { static {
NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator()); NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator());
EditLogFileOutputStream.setShouldSkipFsyncForTesting(false);
} }
private static final long BLOCK_SIZE = 4096; private static final long BLOCK_SIZE = 4096;