YARN-7328. ResourceUtils allows yarn.nodemanager.resource-types.memory-mb and .vcores to override yarn.nodemanager.resource.memory-mb and .cpu-vcores. (lovekesh bansal via wangda)

Change-Id: Ibb1faf0beefec079dae2208986976b2f64650672

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java

@@ -84,20 +84,22 @@ private static void checkMandatoryResources(
       Map<String, ResourceInformation> resourceInformationMap)
       throws YarnRuntimeException {
     /*
-     * Supporting 'memory' also as invalid resource name, in addition to
+     * Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource names, in addition to
      * 'MEMORY' for historical reasons
      */
-    String key = "memory";
+    String keys[] = { "memory", ResourceInformation.MEMORY_URI,
-    if (resourceInformationMap.containsKey(key)) {
+        ResourceInformation.VCORES_URI };
-      LOG.warn(
+    for(String key : keys) {
-          "Attempt to define resource '" + key + "', but it is not allowed.");
+      if (resourceInformationMap.containsKey(key)) {
-      throw new YarnRuntimeException(
+        LOG.warn("Attempt to define resource '" + key + "', but it is not allowed.");
-          "Attempt to re-define mandatory resource '" + key + "'.");
+        throw new YarnRuntimeException(
+            "Attempt to re-define mandatory resource '" + key + "'.");
+      }
     }
 
     for (Map.Entry<String, ResourceInformation> mandatoryResourceEntry :
         ResourceInformation.MANDATORY_RESOURCES.entrySet()) {
-      key = mandatoryResourceEntry.getKey();
+      String key = mandatoryResourceEntry.getKey();
       ResourceInformation mandatoryRI = mandatoryResourceEntry.getValue();
 
       ResourceInformation newDefinedRI = resourceInformationMap.get(key);
@@ -485,8 +487,8 @@ public static Map<String, ResourceInformation> getNodeResourceInformation(
         if (!initializedNodeResources) {
           Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation(
               conf);
-          addMandatoryResources(nodeResources);
           checkMandatoryResources(nodeResources);
+          addMandatoryResources(nodeResources);
           setAllocationForMandatoryResources(nodeResources, conf);
           readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
           initializedNodeResources = true;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java

@@ -287,7 +287,7 @@ public void testGetResourceInformation() throws Exception {
     Map<String, Resource> testRun = new HashMap<>();
     setupResourceTypes(conf, "resource-types-4.xml");
     // testRun.put("node-resources-1.xml", Resource.newInstance(1024, 1));
-    Resource test3Resources = Resource.newInstance(1024, 1);
+    Resource test3Resources = Resource.newInstance(0, 0);
     test3Resources.setResourceInformation("resource1",
         ResourceInformation.newInstance("resource1", "Gi", 5L));
     test3Resources.setResourceInformation("resource2",
@@ -315,6 +315,32 @@ public void testGetResourceInformation() throws Exception {
     }
   }
 
+  @Test
+  public void testGetNodeResourcesConfigErrors() throws Exception {
+    Configuration conf = new YarnConfiguration();
+    Map<String, Resource> testRun = new HashMap<>();
+    setupResourceTypes(conf, "resource-types-4.xml");
+    String invalidNodeResFiles[] = { "node-resources-error-1.xml"};
+
+    for (String resourceFile : invalidNodeResFiles) {
+      ResourceUtils.resetNodeResources();
+      File dest = null;
+      try {
+        File source = new File(conf.getClassLoader().getResource(resourceFile).getFile());
+        dest = new File(source.getParent(), "node-resources.xml");
+        FileUtils.copyFile(source, dest);
+        Map<String, ResourceInformation> actual = ResourceUtils.getNodeResourceInformation(conf);
+        Assert.fail("Expected error with file " + resourceFile);
+      } catch (NullPointerException ne) {
+        throw ne;
+      } catch (Exception e) {
+        if (dest != null) {
+          dest.delete();
+        }
+      }
+    }
+  }
+
   @Test
   public void testResourceNameFormatValidation() throws Exception {
     String[] validNames = new String[] {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml

@@ -15,17 +15,6 @@ limitations under the License. See accompanying LICENSE file.
 -->
 
 <configuration>
-
- <property>
-   <name>yarn.nodemanager.resource-type.memory-mb</name>
-   <value>1024Mi</value>
- </property>
-
- <property>
-   <name>yarn.nodemanager.resource-type.vcores</name>
-   <value>1</value>
- </property>
-
  <property>
    <name>yarn.nodemanager.resource-type.resource1</name>
    <value>5Gi</value>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-error-1.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+    <property>
+        <name>yarn.nodemanager.resource-type.vcores</name>
+        <value>1024</value>
+    </property>
+
+    <property>
+        <name>yarn.nodemanager.resource.resource1</name>
+        <value>1</value>
+    </property>
+
+</configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml

@@ -18,7 +18,7 @@ limitations under the License. See accompanying LICENSE file.
 
  <property>
    <name>yarn.resource-types</name>
-   <value>vcores,resource1</value>
+   <value>resource1,resource2</value>
  </property>
 
  <property>
@@ -27,7 +27,7 @@ limitations under the License. See accompanying LICENSE file.
  </property>
 
  <property>
-   <name>yarn.resource-types.vcores.units</name>
+   <name>yarn.resource-types.resource2.units</name>
    <value>Az</value>
  </property>
 </configuration>