big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15551. Tiny Improve for DeadNode detector (#2265)

Browse Source 
Contributed by imbajin.

Reviewed-by: leosunli <lisheng.sun08@gmail.com>
Signed-off-by: He Xiaoqiao <hexiaoqiao@apache.org>
(cherry picked from commit 89428f142f)
This commit is contained in:
imbajin 2020-09-11 15:44:03 +08:00 committed by Wei-Chiu Chuang
parent 3ecd02ac78
commit 4f79df4da1
2 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
View File

@ -181,10 +181,13 @@ private boolean isPeriodicRefreshEnabled() {
private byte[] oneByteBuf; // used for 'int read()' private byte[] oneByteBuf; // used for 'int read()'
protected void addToLocalDeadNodes(DatanodeInfo dnInfo) { protected void addToLocalDeadNodes(DatanodeInfo dnInfo) {
DFSClient.LOG.debug("Add {} to local dead nodes, previously was {}.",
dnInfo, deadNodes);
deadNodes.put(dnInfo, dnInfo); deadNodes.put(dnInfo, dnInfo);
} }
protected void removeFromLocalDeadNodes(DatanodeInfo dnInfo) { protected void removeFromLocalDeadNodes(DatanodeInfo dnInfo) {
DFSClient.LOG.debug("Remove {} from local dead nodes.", dnInfo);
deadNodes.remove(dnInfo); deadNodes.remove(dnInfo);
} }

11
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java
View File

@ -294,7 +294,7 @@ void startProbeScheduler() {
} }
/** /**
* Prode datanode by probe byte. * Prode datanode by probe type.
*/ */
private void scheduleProbe(ProbeType type) { private void scheduleProbe(ProbeType type) {
LOG.debug("Schedule probe datanode for probe type: {}.", type); LOG.debug("Schedule probe datanode for probe type: {}.", type);
@ -376,11 +376,10 @@ public DatanodeLocalInfo call() throws Exception {
} catch (Exception e) { } catch (Exception e) {
LOG.error("Probe failed, datanode: {}, type: {}.", datanodeInfo, type, LOG.error("Probe failed, datanode: {}, type: {}.", datanodeInfo, type,
e); e);
}
deadNodeDetector.probeCallBack(this, false); deadNodeDetector.probeCallBack(this, false);
} }
} }
}
/** /**
* Handle data node, according to probe result. When ProbeType is CHECK_DEAD, * Handle data node, according to probe result. When ProbeType is CHECK_DEAD,
@ -402,7 +401,7 @@ private void probeCallBack(Probe probe, boolean success) {
} }
} else { } else {
if (probe.getType() == ProbeType.CHECK_SUSPECT) { if (probe.getType() == ProbeType.CHECK_SUSPECT) {
LOG.info("Add the node to dead node list: {}.", LOG.warn("Probe failed, add suspect node to dead node list: {}.",
probe.getDatanodeInfo()); probe.getDatanodeInfo());
addToDead(probe.getDatanodeInfo()); addToDead(probe.getDatanodeInfo());
} }
@ -415,11 +414,12 @@ private void probeCallBack(Probe probe, boolean success) {
private void checkDeadNodes() { private void checkDeadNodes() {
Set<DatanodeInfo> datanodeInfos = clearAndGetDetectedDeadNodes(); Set<DatanodeInfo> datanodeInfos = clearAndGetDetectedDeadNodes();
for (DatanodeInfo datanodeInfo : datanodeInfos) { for (DatanodeInfo datanodeInfo : datanodeInfos) {
LOG.debug("Add dead node to check: {}.", datanodeInfo);
if (!deadNodesProbeQueue.offer(datanodeInfo)) { if (!deadNodesProbeQueue.offer(datanodeInfo)) {
LOG.debug("Skip to add dead node {} to check " + LOG.debug("Skip to add dead node {} to check " +
"since the probe queue is full.", datanodeInfo); "since the probe queue is full.", datanodeInfo);
break; break;
} else {
LOG.debug("Add dead node to check: {}.", datanodeInfo);
} }
} }
state = State.IDLE; state = State.IDLE;
@ -475,6 +475,7 @@ public synchronized void addNodeToDetect(DFSInputStream dfsInputStream,
datanodeInfos.add(datanodeInfo); datanodeInfos.add(datanodeInfo);
} }
LOG.debug("Add datanode {} to suspectAndDeadNodes.", datanodeInfo);
addSuspectNodeToDetect(datanodeInfo); addSuspectNodeToDetect(datanodeInfo);
} }