big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-5242. Update DominantResourceCalculator to consider all resource types in calculations. Contributed by Varun Vasudev.

Browse Source
This commit is contained in:
Rohith Sharma K S 2016-07-26 14:13:03 +05:30 committed by Wangda Tan
parent 759114b006
commit 9e4ba6aff5
5 changed files with 31 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
View File

@ -327,6 +327,8 @@ public int compareTo(Resource other) {
otherResources = other.getResources();
long diff = thisResources.size() - otherResources.size();
if (diff == 0) {
// compare memory and vcores first(in that order) to preserve
// existing behaviour
if (thisResources.keySet().equals(otherResources.keySet())) {
diff = this.getMemorySize() - other.getMemorySize();
if (diff == 0) {
@ -335,6 +337,11 @@ public int compareTo(Resource other) {
if (diff == 0) {
for (Map.Entry<String, ResourceInformation> entry : thisResources
.entrySet()) {
if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
|| entry.getKey()
.equals(ResourceInformation.VCORES.getName())) {
continue;
}
diff =
entry.getValue().compareTo(otherResources.get(entry.getKey()));
if (diff != 0) {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
View File

@ -242,7 +242,7 @@ synchronized private void mergeLocalToBuilder() {
builder.addResourceValueMap(e);
}
}
builder.setMemory(this.getMemory());
builder.setMemory(this.getMemorySize());
builder.setVirtualCores(this.getVirtualCores());
}

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
View File

@ -397,10 +397,25 @@ private Resource multiplyAndNormalize(Resource r, double by,
}
@Override
public boolean fitsIn(Resource cluster,
Resource smaller, Resource bigger) {
return smaller.getMemorySize() <= bigger.getMemorySize()
&& smaller.getVirtualCores() <= bigger.getVirtualCores();
public boolean fitsIn(Resource cluster, Resource smaller, Resource bigger) {
for (String resource : resourceNames) {
try {
ResourceInformation sResourceInformation =
smaller.getResourceInformation(resource);
ResourceInformation bResourceInformation =
bigger.getResourceInformation(resource);
Long sResourceValue = UnitsConversionUtil
.convert(sResourceInformation.getUnits(),
bResourceInformation.getUnits(),
sResourceInformation.getValue());
if(sResourceValue > bResourceInformation.getValue()) {
return false;
}
} catch (YarnException ye) {
return false;
}
}
return true;
}
@Override

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
View File

@ -178,7 +178,6 @@ private static Map<String, ResourceInformation> getResourceTypes(
synchronized (ResourceUtils.class) {
if (lock == null) {
synchronized (ResourceUtils.class) {
lock = new Object();
Map<String, ResourceInformation> resources = new HashMap<>();
if (conf == null) {
conf = new YarnConfiguration();
@ -187,10 +186,12 @@ private static Map<String, ResourceInformation> getResourceTypes(
addResourcesFileToConf(resourceFile, conf);
LOG.debug("Found " + resourceFile + ", adding to configuration");
initializeResourcesMap(conf, resources);
lock = new Object();
} catch (FileNotFoundException fe) {
LOG.info("Unable to find '" + resourceFile
+ "'. Falling back to memory and vcores as resources", fe);
initializeResourcesMap(conf, resources);
lock = new Object();
}
}
}
@ -268,12 +269,12 @@ public static Map<String, ResourceInformation> getNodeResourceInformation(
synchronized (ResourceUtils.class) {
if (nodeLock == null) {
synchronized (ResourceUtils.class) {
nodeLock = new Object();
Map<String, ResourceInformation> nodeResources =
initializeNodeResourceInformation(conf);
addManadtoryResources(nodeResources);
checkMandatatoryResources(nodeResources);
readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
nodeLock = new Object();
}
}
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
View File

@ -134,16 +134,12 @@ public void setResourceValue(String resource, Long value)
private Map<String, ResourceInformation> initResourceMap() {
Map<String, ResourceInformation> tmp = new HashMap<>();
// Due to backwards compat, the max value for memory and vcores
// needs to be Integer.MAX_VALUE
int max = resourceValue > Integer.MAX_VALUE ? Integer.MAX_VALUE :
resourceValue.intValue();
Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
if (types != null) {
for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
tmp.put(entry.getKey(),
ResourceInformation.newInstance(entry.getValue()));
tmp.get(entry.getKey()).setValue((long) max);
tmp.get(entry.getKey()).setValue(resourceValue);
}
}
return tmp;