big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9124. Resolve contradiction in ResourceUtils: addMandatoryResources / checkMandatoryResources work differently (#1121)

Browse Source
This commit is contained in:
Adam Antal 2019-08-02 13:04:34 +02:00 committed by Szilard Nemeth
parent 95fc38f2e9
commit cbcada804d
2 changed files with 33 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
View File

@ -59,7 +59,11 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
public static final ResourceInformation FPGAS = public static final ResourceInformation FPGAS =
ResourceInformation.newInstance(FPGA_URI); ResourceInformation.newInstance(FPGA_URI);
public static final Map<String, ResourceInformation> MANDATORY_RESOURCES = /**
* Special resources that should be treated separately
* from arbitrary resource types.
*/
public static final Map<String, ResourceInformation> SPECIAL_RESOURCES =
ImmutableMap.of(MEMORY_URI, MEMORY_MB, VCORES_URI, VCORES, GPU_URI, GPUS, FPGA_URI, FPGAS); ImmutableMap.of(MEMORY_URI, MEMORY_MB, VCORES_URI, VCORES, GPU_URI, GPUS, FPGA_URI, FPGAS);
/** /**

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
View File

@ -93,14 +93,28 @@ public class ResourceUtils {
private ResourceUtils() { private ResourceUtils() {
} }
private static void checkMandatoryResources( /**
* Ensures that historical resource types (like {@link
* ResourceInformation#MEMORY_URI}, {@link ResourceInformation#VCORES_URI})
* are not getting overridden in the resourceInformationMap.
*
* Also checks whether {@link ResourceInformation#SPECIAL_RESOURCES} are not
* configured poorly: having their proper units and types.
*
* @param resourceInformationMap Map object having keys as resources names
* and {@link ResourceInformation} objects as
* values
* @throws YarnRuntimeException if either of the two above
* conditions do not hold
*/
private static void checkSpecialResources(
Map<String, ResourceInformation> resourceInformationMap) Map<String, ResourceInformation> resourceInformationMap)
throws YarnRuntimeException { throws YarnRuntimeException {
/* /*
* Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource names, in addition to * Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource
* 'MEMORY' for historical reasons * names, in addition to 'MEMORY' for historical reasons
*/ */
String keys[] = { "memory", ResourceInformation.MEMORY_URI, String[] keys = { "memory", ResourceInformation.MEMORY_URI,
ResourceInformation.VCORES_URI }; ResourceInformation.VCORES_URI };
for(String key : keys) { for(String key : keys) {
if (resourceInformationMap.containsKey(key)) { if (resourceInformationMap.containsKey(key)) {
@ -111,7 +125,7 @@ private static void checkMandatoryResources(
} }
for (Map.Entry<String, ResourceInformation> mandatoryResourceEntry : for (Map.Entry<String, ResourceInformation> mandatoryResourceEntry :
ResourceInformation.MANDATORY_RESOURCES.entrySet()) { ResourceInformation.SPECIAL_RESOURCES.entrySet()) {
String key = mandatoryResourceEntry.getKey(); String key = mandatoryResourceEntry.getKey();
ResourceInformation mandatoryRI = mandatoryResourceEntry.getValue(); ResourceInformation mandatoryRI = mandatoryResourceEntry.getValue();
@ -134,6 +148,13 @@ private static void checkMandatoryResources(
} }
} }
/**
* Ensures that {@link ResourceUtils#MEMORY} and {@link ResourceUtils#VCORES}
* resources are contained in the map received as parameter.
*
* @param res Map object having keys as resources names
* and {@link ResourceInformation} objects as values
*/
private static void addMandatoryResources( private static void addMandatoryResources(
Map<String, ResourceInformation> res) { Map<String, ResourceInformation> res) {
ResourceInformation ri; ResourceInformation ri;
@ -272,7 +293,7 @@ private static Map<String, ResourceInformation> getResourceInformationMapFromCon
validateNameOfResourceNameAndThrowException(name); validateNameOfResourceNameAndThrowException(name);
} }
checkMandatoryResources(resourceInformationMap); checkSpecialResources(resourceInformationMap);
addMandatoryResources(resourceInformationMap); addMandatoryResources(resourceInformationMap);
setAllocationForMandatoryResources(resourceInformationMap, conf); setAllocationForMandatoryResources(resourceInformationMap, conf);
@ -522,7 +543,7 @@ public static Map<String, ResourceInformation> getNodeResourceInformation(
if (!initializedNodeResources) { if (!initializedNodeResources) {
Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation( Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation(
conf); conf);
checkMandatoryResources(nodeResources); checkSpecialResources(nodeResources);
addMandatoryResources(nodeResources); addMandatoryResources(nodeResources);
setAllocationForMandatoryResources(nodeResources, conf); setAllocationForMandatoryResources(nodeResources, conf);
readOnlyNodeResources = Collections.unmodifiableMap(nodeResources); readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);