YARN-6500. Do not mount inaccessible cgroups directories in CgroupsLCEResourcesHandler. (Miklos Szegedi via Haibo Chen)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java

@@ -232,7 +232,8 @@ private static Map<String, List<String>> parseMtab(String mtab)
    * @param entries map of paths to mount options
    * @return the first mount path that has the requested subsystem
    */
-  private static String findControllerInMtab(String controller,
+  @VisibleForTesting
+  static String findControllerInMtab(String controller,
       Map<String, List<String>> entries) {
     for (Map.Entry<String, List<String>> e : entries.entrySet()) {
       if (e.getValue().contains(controller)) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java

@@ -428,11 +428,18 @@ private Map<String, List<String>> parseMtab() throws IOException {
     return ret;
   }
 
-  private String findControllerInMtab(String controller,
+  @VisibleForTesting
+  String findControllerInMtab(String controller,
                                       Map<String, List<String>> entries) {
     for (Entry<String, List<String>> e : entries.entrySet()) {
-      if (e.getValue().contains(controller))
+      if (e.getValue().contains(controller)) {
-        return e.getKey();
+        if (new File(e.getKey()).canRead()) {
+          return e.getKey();
+        } else {
+          LOG.warn(String.format(
+              "Skipping inaccessible cgroup mount point %s", e.getKey()));
+        }
+      }
     }
 
     return null;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java

@@ -40,6 +40,9 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.security.Permission;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 
@@ -432,6 +435,35 @@ private void testPreMountedControllerInitialization(String myHierarchy)
     }
   }
 
+  @Test
+  public void testSelectCgroup() throws Exception {
+    File cpu = new File(tmpPath, "cpu");
+    File cpuNoExist = new File(tmpPath, "cpuNoExist");
+    File memory = new File(tmpPath, "memory");
+    try {
+      CGroupsHandlerImpl handler = new CGroupsHandlerImpl(
+          conf,
+          privilegedOperationExecutorMock);
+      Map<String, List<String>> cgroups = new LinkedHashMap<>();
+
+      Assert.assertTrue("temp dir should be created", cpu.mkdirs());
+      Assert.assertTrue("temp dir should be created", memory.mkdirs());
+      Assert.assertFalse("temp dir should not be created", cpuNoExist.exists());
+
+      cgroups.put(
+          memory.getAbsolutePath(), Collections.singletonList("memory"));
+      cgroups.put(
+          cpuNoExist.getAbsolutePath(), Collections.singletonList("cpu"));
+      cgroups.put(cpu.getAbsolutePath(), Collections.singletonList("cpu"));
+      String selectedCPU = handler.findControllerInMtab("cpu", cgroups);
+      Assert.assertEquals("Wrong CPU mount point selected",
+          cpu.getAbsolutePath(), selectedCPU);
+    } finally {
+      FileUtils.deleteQuietly(cpu);
+      FileUtils.deleteQuietly(memory);
+    }
+  }
+
   @After
   public void teardown() {
     FileUtil.fullyDelete(new File(tmpPath));

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java

@@ -33,7 +33,10 @@
 import org.mockito.Mockito;
 
 import java.io.*;
+import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Scanner;
 import java.util.concurrent.CountDownLatch;
 
@@ -342,4 +345,30 @@ public void testContainerLimits() throws IOException {
     FileUtils.deleteQuietly(cgroupDir);
   }
 
+  @Test
+  public void testSelectCgroup() {
+    File cpu = new File(cgroupDir, "cpu");
+    File cpuNoExist = new File(cgroupDir, "cpuNoExist");
+    File memory = new File(cgroupDir, "memory");
+    try {
+      CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler();
+      Map<String, List<String>> cgroups = new LinkedHashMap<>();
+
+      Assert.assertTrue("temp dir should be created", cpu.mkdirs());
+      Assert.assertTrue("temp dir should be created", memory.mkdirs());
+      Assert.assertFalse("temp dir should not be created", cpuNoExist.exists());
+
+      cgroups.put(
+          memory.getAbsolutePath(), Collections.singletonList("memory"));
+      cgroups.put(
+          cpuNoExist.getAbsolutePath(), Collections.singletonList("cpu"));
+      cgroups.put(cpu.getAbsolutePath(), Collections.singletonList("cpu"));
+      String selectedCPU = handler.findControllerInMtab("cpu", cgroups);
+      Assert.assertEquals("Wrong CPU mount point selected",
+          cpu.getAbsolutePath(), selectedCPU);
+    } finally {
+      FileUtils.deleteQuietly(cpu);
+      FileUtils.deleteQuietly(memory);
+    }
+  }
 }