From c6b48391680c1b81a86aabc3ad4c725bfade6d2e Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Tue, 3 May 2016 17:27:28 +0000 Subject: [PATCH] YARN-4834. ProcfsBasedProcessTree doesn't track daemonized processes. Contributed by Nathan Roberts --- .../yarn/util/ProcfsBasedProcessTree.java | 19 ++++++++- .../yarn/util/TestProcfsBasedProcessTree.java | 39 +++++++++++-------- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java index 7bd5390f3d..4fb4be3053 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java @@ -216,7 +216,16 @@ public void updateProcessTree() { String pID = entry.getKey(); if (!pID.equals("1")) { ProcessInfo pInfo = entry.getValue(); - ProcessInfo parentPInfo = allProcessInfo.get(pInfo.getPpid()); + String ppid = pInfo.getPpid(); + // If parent is init and process is not session leader, + // attach to sessionID + if (ppid.equals("1")) { + String sid = pInfo.getSessionId().toString(); + if (!pID.equals(sid)) { + ppid = sid; + } + } + ProcessInfo parentPInfo = allProcessInfo.get(ppid); if (parentPInfo != null) { parentPInfo.addChild(pInfo); } @@ -571,6 +580,14 @@ public String toString() { return pTree.substring(0, pTree.length()) + "]"; } +/** + * Returns boolean indicating whether pid + * is in process tree. + */ + public boolean contains(String pid) { + return processTree.containsKey(pid); + } + /** * * Class containing information of a process. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java index da7849d4fa..6fbbfbebdf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java @@ -36,6 +36,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -65,7 +66,7 @@ public class TestProcfsBasedProcessTree { TestProcfsBasedProcessTree.class.getName() + "-localDir"); private ShellCommandExecutor shexec = null; - private String pidFile, lowestDescendant; + private String pidFile, lowestDescendant, lostDescendant; private String shellScript; private static final int N = 6; // Controls the RogueTask @@ -144,19 +145,17 @@ public void testProcessTree() throws Exception { lowestDescendant = TEST_ROOT_DIR + File.separator + "lowestDescendantPidFile"; + lostDescendant = + TEST_ROOT_DIR + File.separator + "lostDescendantPidFile"; // write to shell-script - try { - FileWriter fWriter = new FileWriter(shellScript); - fWriter.write("# rogue task\n" + "sleep 1\n" + "echo hello\n" - + "if [ $1 -ne 0 ]\n" + "then\n" + " sh " + shellScript - + " $(($1-1))\n" + "else\n" + " echo $$ > " + lowestDescendant + "\n" - + " while true\n do\n" + " sleep 5\n" + " done\n" + "fi"); - fWriter.close(); - } catch (IOException ioe) { - LOG.info("Error: " + ioe); - return; - } + File file = new File(shellScript); + FileUtils.writeStringToFile(file, "# rogue task\n" + "sleep 1\n" + "echo hello\n" + + "if [ $1 -ne 0 ]\n" + "then\n" + " sh " + shellScript + + " $(($1-1))\n" + "else\n" + " echo $$ > " + lowestDescendant + "\n" + + "(sleep 300&\n" + + "echo $! > " + lostDescendant + ")\n" + + " while true\n do\n" + " sleep 5\n" + " done\n" + "fi"); Thread t = new RogueTaskThread(); t.start(); @@ -179,6 +178,12 @@ public void testProcessTree() throws Exception { p.updateProcessTree(); // reconstruct LOG.info("ProcessTree: " + p.toString()); + // Verify the orphaned pid is In process tree + String lostpid = getPidFromPidFile(lostDescendant); + LOG.info("Orphaned pid: " + lostpid); + Assert.assertTrue("Child process owned by init escaped process tree.", + p.contains(lostpid)); + // Get the process-tree dump String processTreeDump = p.getProcessTreeDump(); @@ -229,10 +234,12 @@ public void testProcessTree() throws Exception { Assert.assertFalse("ProcessTree must have been gone", isAlive(pid)); Assert.assertTrue( "vmem for the gone-process is " + p.getVirtualMemorySize() - + " . It should be zero.", p.getVirtualMemorySize() == 0); + + " . It should be UNAVAILABLE(-1).", + p.getVirtualMemorySize() == UNAVAILABLE); Assert.assertTrue( "vmem (old API) for the gone-process is " + p.getCumulativeVmem() - + " . It should be zero.", p.getCumulativeVmem() == 0); + + " . It should be UNAVAILABLE(-1).", + p.getCumulativeVmem() == UNAVAILABLE); Assert.assertTrue(p.toString().equals("[ ]")); } @@ -246,7 +253,7 @@ protected ProcfsBasedProcessTree createProcessTree(String pid, } protected void destroyProcessTree(String pid) throws IOException { - sendSignal(pid, 9); + sendSignal("-"+pid, 9); } /** @@ -910,7 +917,7 @@ private static boolean isAlive(String pid) { private static void sendSignal(String pid, int signal) throws IOException { ShellCommandExecutor shexec = null; - String[] arg = { "kill", "-" + signal, pid }; + String[] arg = { "kill", "-" + signal, "--", pid }; shexec = new ShellCommandExecutor(arg); shexec.execute(); }