big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14806. Bootstrap standby may fail if with in-progress tailing. Contributed by Chen Liang and Erik Krogen.

Browse Source
This commit is contained in:
Chen Liang 2019-11-06 09:30:04 -08:00
parent 9e287054a8
commit 9d0d580031
3 changed files with 56 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
View File

@ -108,6 +108,9 @@ public class BootstrapStandby implements Tool, Configurable {
@Override
public int run(String[] args) throws Exception {
parseArgs(args);
// Disable using the RPC tailing mechanism for bootstrapping the standby
// since it is less efficient in this case; see HDFS-14806
conf.setBoolean(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, false);
parseConfAndFindOtherNN();
NameNode.checkAllowFormat(conf);

46
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithInProgressTailing.java Normal file
View File

@ -0,0 +1,46 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode.ha;
import org.apache.hadoop.conf.Configuration;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY;
/**
* Test in progress tailing with small txn id per call.
*
* The number of edits that needs to be tailed during
* bootstrapStandby can be large, but the number of edits
* that can be tailed using RPC call can be limited
* (configured by dfs.ha.tail-edits.qjm.rpc.max-txns).
* This is to test that even with small number of configured
* txnid, bootstrapStandby can still work. See HDFS-14806.
*/
public class TestBootstrapStandbyWithInProgressTailing
extends TestBootstrapStandbyWithQJM {
@Override
public Configuration createConfig() {
Configuration conf = super.createConfig();
conf.setBoolean(DFS_HA_TAILEDITS_INPROGRESS_KEY, true);
conf.setInt("dfs.ha.tail-edits.qjm.rpc.max-txns", 1);
return conf;
}
}

9
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandbyWithQJM.java
View File

@ -54,14 +54,19 @@ enum UpgradeState {
private MiniJournalCluster jCluster;
private int nnCount = 3;
@Before
public void setup() throws Exception {
protected Configuration createConfig() {
Configuration conf = new Configuration();
// Turn off IPC client caching, so that the suite can handle
// the restart of the daemons between test cases.
conf.setInt(
CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
0);
return conf;
}
@Before
public void setup() throws Exception {
Configuration conf = createConfig();
MiniQJMHACluster miniQjmHaCluster =
new MiniQJMHACluster.Builder(conf).setNumNameNodes(nnCount).build();