big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9124. Resolve contradiction in ResourceUtils: addMandatoryResources / checkMandatoryResources work differently (#1121)

Browse Source
This commit is contained in:
Adam Antal 2019-08-02 13:04:34 +02:00 committed by Szilard Nemeth
parent 95fc38f2e9
commit cbcada804d
2 changed files with 33 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
View File

@ -59,7 +59,11 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
public static final ResourceInformation FPGAS =
ResourceInformation.newInstance(FPGA_URI);
public static final Map<String, ResourceInformation> MANDATORY_RESOURCES =
/**
* Special resources that should be treated separately
* from arbitrary resource types.
*/
public static final Map<String, ResourceInformation> SPECIAL_RESOURCES =
ImmutableMap.of(MEMORY_URI, MEMORY_MB, VCORES_URI, VCORES, GPU_URI, GPUS, FPGA_URI, FPGAS);
/**

35
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
View File

@ -93,14 +93,28 @@ public class ResourceUtils {
private ResourceUtils() {
}
private static void checkMandatoryResources(
/**
* Ensures that historical resource types (like {@link
* ResourceInformation#MEMORY_URI}, {@link ResourceInformation#VCORES_URI})
* are not getting overridden in the resourceInformationMap.
*
* Also checks whether {@link ResourceInformation#SPECIAL_RESOURCES} are not
* configured poorly: having their proper units and types.
*
* @param resourceInformationMap Map object having keys as resources names
* and {@link ResourceInformation} objects as
* values
* @throws YarnRuntimeException if either of the two above
* conditions do not hold
*/
private static void checkSpecialResources(
Map<String, ResourceInformation> resourceInformationMap)
throws YarnRuntimeException {
/*
* Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource names, in addition to
* 'MEMORY' for historical reasons
* Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource
* names, in addition to 'MEMORY' for historical reasons
*/
String keys[] = { "memory", ResourceInformation.MEMORY_URI,
String[] keys = { "memory", ResourceInformation.MEMORY_URI,
ResourceInformation.VCORES_URI };
for(String key : keys) {
if (resourceInformationMap.containsKey(key)) {
@ -111,7 +125,7 @@ private static void checkMandatoryResources(
}
for (Map.Entry<String, ResourceInformation> mandatoryResourceEntry :
ResourceInformation.MANDATORY_RESOURCES.entrySet()) {
ResourceInformation.SPECIAL_RESOURCES.entrySet()) {
String key = mandatoryResourceEntry.getKey();
ResourceInformation mandatoryRI = mandatoryResourceEntry.getValue();
@ -134,6 +148,13 @@ private static void checkMandatoryResources(
}
}
/**
* Ensures that {@link ResourceUtils#MEMORY} and {@link ResourceUtils#VCORES}
* resources are contained in the map received as parameter.
*
* @param res Map object having keys as resources names
* and {@link ResourceInformation} objects as values
*/
private static void addMandatoryResources(
Map<String, ResourceInformation> res) {
ResourceInformation ri;
@ -272,7 +293,7 @@ private static Map<String, ResourceInformation> getResourceInformationMapFromCon
validateNameOfResourceNameAndThrowException(name);
}
checkMandatoryResources(resourceInformationMap);
checkSpecialResources(resourceInformationMap);
addMandatoryResources(resourceInformationMap);
setAllocationForMandatoryResources(resourceInformationMap, conf);
@ -522,7 +543,7 @@ public static Map<String, ResourceInformation> getNodeResourceInformation(
if (!initializedNodeResources) {
Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation(
conf);
checkMandatoryResources(nodeResources);
checkSpecialResources(nodeResources);
addMandatoryResources(nodeResources);
setAllocationForMandatoryResources(nodeResources, conf);
readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);