big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12603. Enable async edit logging by default.

Browse Source
This commit is contained in:
Andrew Wang 2017-10-09 11:20:00 -07:00
parent 09ad848b64
commit afb42aeabf
5 changed files with 28 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -326,7 +326,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_EDITS_ASYNC_LOGGING =
"dfs.namenode.edits.asynclogging";
public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = false;
public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = true;
public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
public static final int DFS_LIST_LIMIT_DEFAULT = 1000;

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
View File

@ -130,7 +130,7 @@ public class FSEditLog implements LogsPurgeable {
*
* In a non-HA setup:
*
* The log starts in UNITIALIZED state upon construction. Once it's
* The log starts in UNINITIALIZED state upon construction. Once it's
* initialized, it is usually in IN_SEGMENT state, indicating that edits may
* be written. In the middle of a roll, or while saving the namespace, it
* briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the previous
@ -1837,4 +1837,9 @@ public long getTotalSyncCount() {
}
return count;
}
@Override
public String toString() {
return super.toString();
}
}

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
View File

@ -319,4 +319,9 @@ public String toString() {
return "["+getClass().getSimpleName()+" op:"+op+" call:"+call+"]";
}
}
@Override
public String toString() {
return super.toString();
}
}

2
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -4105,7 +4105,7 @@
<property>
<name>dfs.namenode.edits.asynclogging</name>
<value>false</value>
<value>true</value>
<description>
If set to true, enables asynchronous edit logs in the Namenode. If set
to false, the Namenode uses the traditional synchronous edit logs.

22
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
View File

@ -75,6 +75,7 @@ public class TestFailureToReadEdits {
private static final Random RANDOM = new Random();
private final TestType clusterType;
private final boolean useAsyncEditLogging;
private Configuration conf;
private MiniDFSCluster cluster;
private MiniQJMHACluster miniQjmHaCluster; // for QJM case only
@ -88,18 +89,23 @@ private enum TestType {
};
/**
* Run this suite of tests both for QJM-based HA and for file-based
* HA.
* Run this suite of tests for {QJM-based, file-based HA} x {async
* edit logging enabled, disabled}.
*/
@Parameters
public static Iterable<Object[]> data() {
return Arrays.asList(new Object[][] {
{ TestType.SHARED_DIR_HA },
{ TestType.QJM_HA } });
return Arrays.asList(new Object[][]{
{TestType.SHARED_DIR_HA, Boolean.FALSE},
{TestType.SHARED_DIR_HA, Boolean.TRUE},
{TestType.QJM_HA, Boolean.FALSE},
{TestType.QJM_HA, Boolean.TRUE},
});
}
public TestFailureToReadEdits(TestType clusterType) {
public TestFailureToReadEdits(TestType clusterType, Boolean
useAsyncEditLogging) {
this.clusterType = clusterType;
this.useAsyncEditLogging = useAsyncEditLogging;
}
@Before
@ -109,6 +115,8 @@ public void setUpCluster() throws Exception {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
useAsyncEditLogging);
HAUtil.setAllowStandbyReads(conf, true);
if (clusterType == TestType.SHARED_DIR_HA) {