diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 5a660cea8a..f211f49ff0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -84,20 +84,22 @@ private static void checkMandatoryResources( Map resourceInformationMap) throws YarnRuntimeException { /* - * Supporting 'memory' also as invalid resource name, in addition to + * Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource names, in addition to * 'MEMORY' for historical reasons */ - String key = "memory"; - if (resourceInformationMap.containsKey(key)) { - LOG.warn( - "Attempt to define resource '" + key + "', but it is not allowed."); - throw new YarnRuntimeException( - "Attempt to re-define mandatory resource '" + key + "'."); + String keys[] = { "memory", ResourceInformation.MEMORY_URI, + ResourceInformation.VCORES_URI }; + for(String key : keys) { + if (resourceInformationMap.containsKey(key)) { + LOG.warn("Attempt to define resource '" + key + "', but it is not allowed."); + throw new YarnRuntimeException( + "Attempt to re-define mandatory resource '" + key + "'."); + } } for (Map.Entry mandatoryResourceEntry : ResourceInformation.MANDATORY_RESOURCES.entrySet()) { - key = mandatoryResourceEntry.getKey(); + String key = mandatoryResourceEntry.getKey(); ResourceInformation mandatoryRI = mandatoryResourceEntry.getValue(); ResourceInformation newDefinedRI = resourceInformationMap.get(key); @@ -485,8 +487,8 @@ public static Map getNodeResourceInformation( if (!initializedNodeResources) { Map nodeResources = initializeNodeResourceInformation( conf); - addMandatoryResources(nodeResources); checkMandatoryResources(nodeResources); + addMandatoryResources(nodeResources); setAllocationForMandatoryResources(nodeResources, conf); readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); initializedNodeResources = true; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java index 1111ca6d84..2671de8505 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java @@ -287,7 +287,7 @@ public void testGetResourceInformation() throws Exception { Map testRun = new HashMap<>(); setupResourceTypes(conf, "resource-types-4.xml"); // testRun.put("node-resources-1.xml", Resource.newInstance(1024, 1)); - Resource test3Resources = Resource.newInstance(1024, 1); + Resource test3Resources = Resource.newInstance(0, 0); test3Resources.setResourceInformation("resource1", ResourceInformation.newInstance("resource1", "Gi", 5L)); test3Resources.setResourceInformation("resource2", @@ -315,6 +315,32 @@ public void testGetResourceInformation() throws Exception { } } + @Test + public void testGetNodeResourcesConfigErrors() throws Exception { + Configuration conf = new YarnConfiguration(); + Map testRun = new HashMap<>(); + setupResourceTypes(conf, "resource-types-4.xml"); + String invalidNodeResFiles[] = { "node-resources-error-1.xml"}; + + for (String resourceFile : invalidNodeResFiles) { + ResourceUtils.resetNodeResources(); + File dest = null; + try { + File source = new File(conf.getClassLoader().getResource(resourceFile).getFile()); + dest = new File(source.getParent(), "node-resources.xml"); + FileUtils.copyFile(source, dest); + Map actual = ResourceUtils.getNodeResourceInformation(conf); + Assert.fail("Expected error with file " + resourceFile); + } catch (NullPointerException ne) { + throw ne; + } catch (Exception e) { + if (dest != null) { + dest.delete(); + } + } + } + } + @Test public void testResourceNameFormatValidation() throws Exception { String[] validNames = new String[] { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml index 382d5dd2ca..3323294252 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml @@ -15,17 +15,6 @@ limitations under the License. See accompanying LICENSE file. --> - - - yarn.nodemanager.resource-type.memory-mb - 1024Mi - - - - yarn.nodemanager.resource-type.vcores - 1 - - yarn.nodemanager.resource-type.resource1 5Gi diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-error-1.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-error-1.xml new file mode 100644 index 0000000000..4ea3c7445a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-error-1.xml @@ -0,0 +1,29 @@ + + + + + + + + yarn.nodemanager.resource-type.vcores + 1024 + + + + yarn.nodemanager.resource.resource1 + 1 + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml index fa43b6c14e..a34db38545 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml @@ -18,7 +18,7 @@ limitations under the License. See accompanying LICENSE file. yarn.resource-types - vcores,resource1 + resource1,resource2 @@ -27,7 +27,7 @@ limitations under the License. See accompanying LICENSE file. - yarn.resource-types.vcores.units + yarn.resource-types.resource2.units Az