big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15806. DeadNodeDetector should close all the threads when it is closed. Contributed by Jinglun.

Browse Source 
(cherry picked from commit ff84a57483)
This commit is contained in:
Ayush Saxena 2021-02-20 19:48:00 +05:30 committed by Wei-Chiu Chuang
parent 4f79df4da1
commit 94766fdb13
3 changed files with 44 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
View File

@ -321,13 +321,7 @@ synchronized void unreference() {
Preconditions.checkState(counter > 0);
counter--;
if (counter == 0 && deadNodeDetectionEnabled && deadNodeDetector != null) {
deadNodeDetector.interrupt();
try {
deadNodeDetector.join();
} catch (InterruptedException e) {
LOG.warn("Encountered exception while waiting to join on dead " +
"node detector thread.", e);
}
deadNodeDetector.shutdown();
deadNodeDetector = null;
}
}

31
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DeadNodeDetector.java
View File

@ -271,6 +271,37 @@ public void run() {
}
}
/**
* Shutdown all the threads.
*/
public void shutdown() {
threadShutDown(this);
threadShutDown(probeDeadNodesSchedulerThr);
threadShutDown(probeSuspectNodesSchedulerThr);
probeDeadNodesThreadPool.shutdown();
probeSuspectNodesThreadPool.shutdown();
rpcThreadPool.shutdown();
}
private static void threadShutDown(Thread thread) {
if (thread != null && thread.isAlive()) {
thread.interrupt();
try {
thread.join();
} catch (InterruptedException e) {
}
}
}
@VisibleForTesting
boolean isThreadsShutdown() {
return !this.isAlive() && !probeDeadNodesSchedulerThr.isAlive()
&& !probeSuspectNodesSchedulerThr.isAlive()
&& probeDeadNodesThreadPool.isShutdown()
&& probeSuspectNodesThreadPool.isShutdown()
&& rpcThreadPool.isShutdown();
}
@VisibleForTesting
static void setDisabledProbeThreadForTest(
boolean disabledProbeThreadForTest) {

12
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDeadNodeDetection.java
View File

@ -357,6 +357,18 @@ public void testCloseDeadNodeDetector() throws Exception {
dfs1.close();
}
@Test
public void testDeadNodeDetectorThreadsShutdown() throws Exception {
DistributedFileSystem dfs = (DistributedFileSystem) FileSystem
.newInstance(new URI("hdfs://127.0.0.1:2001/"), conf);
DeadNodeDetector detector = dfs.getClient().getDeadNodeDetector();
assertNotNull(detector);
dfs.close();
assertTrue(detector.isThreadsShutdown());
detector = dfs.getClient().getDeadNodeDetector();
assertNull(detector);
}
private void createFile(FileSystem fs, Path filePath) throws IOException {
FSDataOutputStream out = null;
try {