big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12603. Enable async edit logging by default.

Browse Source
This commit is contained in:
Andrew Wang 2017-10-09 11:20:00 -07:00
parent 09ad848b64
commit afb42aeabf
5 changed files with 28 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -326,7 +326,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_EDITS_ASYNC_LOGGING = public static final String DFS_NAMENODE_EDITS_ASYNC_LOGGING =
"dfs.namenode.edits.asynclogging"; "dfs.namenode.edits.asynclogging";
public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = false; public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = true;
public static final String DFS_LIST_LIMIT = "dfs.ls.limit"; public static final String DFS_LIST_LIMIT = "dfs.ls.limit";
public static final int DFS_LIST_LIMIT_DEFAULT = 1000; public static final int DFS_LIST_LIMIT_DEFAULT = 1000;

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
View File

@ -130,7 +130,7 @@ public class FSEditLog implements LogsPurgeable {
* *
* In a non-HA setup: * In a non-HA setup:
* *
* The log starts in UNITIALIZED state upon construction. Once it's * The log starts in UNINITIALIZED state upon construction. Once it's
* initialized, it is usually in IN_SEGMENT state, indicating that edits may * initialized, it is usually in IN_SEGMENT state, indicating that edits may
* be written. In the middle of a roll, or while saving the namespace, it * be written. In the middle of a roll, or while saving the namespace, it
* briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the previous * briefly enters the BETWEEN_LOG_SEGMENTS state, indicating that the previous
@ -1837,4 +1837,9 @@ public long getTotalSyncCount() {
} }
return count; return count;
} }
@Override
public String toString() {
return super.toString();
}
} }

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
View File

@ -319,4 +319,9 @@ public String toString() {
return "["+getClass().getSimpleName()+" op:"+op+" call:"+call+"]"; return "["+getClass().getSimpleName()+" op:"+op+" call:"+call+"]";
} }
} }
@Override
public String toString() {
return super.toString();
}
} }

2
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -4105,7 +4105,7 @@
<property> <property>
<name>dfs.namenode.edits.asynclogging</name> <name>dfs.namenode.edits.asynclogging</name>
<value>false</value> <value>true</value>
<description> <description>
If set to true, enables asynchronous edit logs in the Namenode. If set If set to true, enables asynchronous edit logs in the Namenode. If set
to false, the Namenode uses the traditional synchronous edit logs. to false, the Namenode uses the traditional synchronous edit logs.

20
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java
View File

@ -75,6 +75,7 @@ public class TestFailureToReadEdits {
private static final Random RANDOM = new Random(); private static final Random RANDOM = new Random();
private final TestType clusterType; private final TestType clusterType;
private final boolean useAsyncEditLogging;
private Configuration conf; private Configuration conf;
private MiniDFSCluster cluster; private MiniDFSCluster cluster;
private MiniQJMHACluster miniQjmHaCluster; // for QJM case only private MiniQJMHACluster miniQjmHaCluster; // for QJM case only
@ -88,18 +89,23 @@ private enum TestType {
}; };
/** /**
* Run this suite of tests both for QJM-based HA and for file-based * Run this suite of tests for {QJM-based, file-based HA} x {async
* HA. * edit logging enabled, disabled}.
*/ */
@Parameters @Parameters
public static Iterable<Object[]> data() { public static Iterable<Object[]> data() {
return Arrays.asList(new Object[][] { return Arrays.asList(new Object[][]{
{ TestType.SHARED_DIR_HA }, {TestType.SHARED_DIR_HA, Boolean.FALSE},
{ TestType.QJM_HA } }); {TestType.SHARED_DIR_HA, Boolean.TRUE},
{TestType.QJM_HA, Boolean.FALSE},
{TestType.QJM_HA, Boolean.TRUE},
});
} }
public TestFailureToReadEdits(TestType clusterType) { public TestFailureToReadEdits(TestType clusterType, Boolean
useAsyncEditLogging) {
this.clusterType = clusterType; this.clusterType = clusterType;
this.useAsyncEditLogging = useAsyncEditLogging;
} }
@Before @Before
@ -109,6 +115,8 @@ public void setUpCluster() throws Exception {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1); conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10); conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10);
conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING,
useAsyncEditLogging);
HAUtil.setAllowStandbyReads(conf, true); HAUtil.setAllowStandbyReads(conf, true);
if (clusterType == TestType.SHARED_DIR_HA) { if (clusterType == TestType.SHARED_DIR_HA) {