YARN-7159. Normalize unit of resource objects in RM to avoid unit conversion in critical path. Contributed by Manikandan R.
This commit is contained in:
Sunil G
parent
7526815e32
commit
12a095a496
@ -202,7 +202,7 @@ public static ResourceInformation newInstance(String name, String units,
|
||||
ResourceInformation ret = new ResourceInformation();
|
||||
ret.setName(name);
|
||||
ret.setResourceType(type);
|
||||
ret.setUnits(units);
|
||||
ret.setUnitsWithoutValidation(units);
|
||||
ret.setValue(value);
|
||||
ret.setMinimumAllocation(minimumAllocation);
|
||||
ret.setMaximumAllocation(maximumAllocation);
|
||||
|
||||
@ -30,6 +30,7 @@
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
@ -527,6 +528,12 @@ private static void addResourceTypeInformation(String prop, String value,
|
||||
String units = getUnits(value);
|
||||
Long resourceValue =
|
||||
Long.valueOf(value.substring(0, value.length() - units.length()));
|
||||
String destUnit = getDefaultUnit(resourceType);
|
||||
if(!units.equals(destUnit)) {
|
||||
resourceValue = UnitsConversionUtil.convert(
|
||||
units, destUnit, resourceValue);
|
||||
units = destUnit;
|
||||
}
|
||||
nodeResources.get(resourceType).setValue(resourceValue);
|
||||
nodeResources.get(resourceType).setUnits(units);
|
||||
if (LOG.isDebugEnabled()) {
|
||||
|
||||
@ -43,7 +43,7 @@ public void testUnits() {
|
||||
Assert.assertEquals("Resource units incorrect", units, ri.getUnits());
|
||||
units = "z";
|
||||
try {
|
||||
ResourceInformation.newInstance(name, units);
|
||||
ResourceInformation.newInstance(name, units).setUnits(units);
|
||||
Assert.fail(units + "is not a valid unit");
|
||||
} catch (IllegalArgumentException ie) {
|
||||
// do nothing
|
||||
|
||||
@ -173,9 +173,17 @@ private static ResourceInformation newDefaultInformation(
|
||||
ri.setResourceType(entry.hasType()
|
||||
? ProtoUtils.convertFromProtoFormat(entry.getType())
|
||||
: ResourceTypes.COUNTABLE);
|
||||
ri.setUnits(
|
||||
entry.hasUnits() ? entry.getUnits() : resourceInformation.getUnits());
|
||||
ri.setValue(entry.hasValue() ? entry.getValue() : 0L);
|
||||
String units = entry.hasUnits() ? entry.getUnits() :
|
||||
ResourceUtils.getDefaultUnit(entry.getKey());
|
||||
long value = entry.hasValue() ? entry.getValue() : 0L;
|
||||
String destUnit = ResourceUtils.getDefaultUnit(entry.getKey());
|
||||
if(!units.equals(destUnit)) {
|
||||
ri.setValue(UnitsConversionUtil.convert(units, destUnit, value));
|
||||
ri.setUnits(destUnit);
|
||||
} else {
|
||||
ri.setUnits(units);
|
||||
ri.setValue(value);
|
||||
}
|
||||
return ri;
|
||||
}
|
||||
|
||||
|
||||
@ -24,7 +24,6 @@
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
|
||||
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
@ -298,11 +297,7 @@ private void calculateShares(ResourceInformation[] clusterRes, Resource first,
|
||||
*/
|
||||
private double calculateShare(ResourceInformation clusterRes,
|
||||
ResourceInformation res) {
|
||||
// Convert the resources' units into the cluster resource's units
|
||||
long value = UnitsConversionUtil.convert(res.getUnits(),
|
||||
clusterRes.getUnits(), res.getValue());
|
||||
|
||||
return (double) value / clusterRes.getValue();
|
||||
return (double) res.getValue() / clusterRes.getValue();
|
||||
}
|
||||
|
||||
/**
|
||||
@ -340,11 +335,8 @@ public long computeAvailableContainers(Resource available,
|
||||
ResourceInformation availableResource = available
|
||||
.getResourceInformation(i);
|
||||
ResourceInformation requiredResource = required.getResourceInformation(i);
|
||||
long requiredResourceValue = UnitsConversionUtil.convert(
|
||||
requiredResource.getUnits(), availableResource.getUnits(),
|
||||
requiredResource.getValue());
|
||||
if (requiredResourceValue != 0) {
|
||||
long tmp = availableResource.getValue() / requiredResourceValue;
|
||||
if (requiredResource.getValue() != 0) {
|
||||
long tmp = availableResource.getValue() / requiredResource.getValue();
|
||||
min = min < tmp ? min : tmp;
|
||||
}
|
||||
}
|
||||
@ -387,11 +379,8 @@ public float ratio(Resource a, Resource b) {
|
||||
for (int i = 0; i < maxLength; i++) {
|
||||
ResourceInformation aResourceInformation = a.getResourceInformation(i);
|
||||
ResourceInformation bResourceInformation = b.getResourceInformation(i);
|
||||
long bResourceValue = UnitsConversionUtil.convert(
|
||||
bResourceInformation.getUnits(), aResourceInformation.getUnits(),
|
||||
bResourceInformation.getValue());
|
||||
float tmp = (float) aResourceInformation.getValue()
|
||||
/ (float) bResourceValue;
|
||||
/ (float) bResourceInformation.getValue();
|
||||
ratio = ratio > tmp ? ratio : tmp;
|
||||
}
|
||||
return ratio;
|
||||
@ -437,23 +426,11 @@ public Resource normalize(Resource r, Resource minimumResource,
|
||||
ResourceInformation tmp = ret.getResourceInformation(i);
|
||||
|
||||
long rValue = rResourceInformation.getValue();
|
||||
long minimumValue = UnitsConversionUtil.convert(
|
||||
minimumResourceInformation.getUnits(),
|
||||
rResourceInformation.getUnits(),
|
||||
minimumResourceInformation.getValue());
|
||||
long maximumValue = UnitsConversionUtil.convert(
|
||||
maximumResourceInformation.getUnits(),
|
||||
rResourceInformation.getUnits(),
|
||||
maximumResourceInformation.getValue());
|
||||
long stepFactorValue = UnitsConversionUtil.convert(
|
||||
stepFactorResourceInformation.getUnits(),
|
||||
rResourceInformation.getUnits(),
|
||||
stepFactorResourceInformation.getValue());
|
||||
long value = Math.max(rValue, minimumValue);
|
||||
if (stepFactorValue != 0) {
|
||||
value = roundUp(value, stepFactorValue);
|
||||
long value = Math.max(rValue, minimumResourceInformation.getValue());
|
||||
if (stepFactorResourceInformation.getValue() != 0) {
|
||||
value = roundUp(value, stepFactorResourceInformation.getValue());
|
||||
}
|
||||
tmp.setValue(Math.min(value, maximumValue));
|
||||
tmp.setValue(Math.min(value, maximumResourceInformation.getValue()));
|
||||
ret.setResourceInformation(i, tmp);
|
||||
}
|
||||
return ret;
|
||||
@ -478,10 +455,7 @@ private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) {
|
||||
.getResourceInformation(i);
|
||||
|
||||
long rValue = rResourceInformation.getValue();
|
||||
long stepFactorValue = UnitsConversionUtil.convert(
|
||||
stepFactorResourceInformation.getUnits(),
|
||||
rResourceInformation.getUnits(),
|
||||
stepFactorResourceInformation.getValue());
|
||||
long stepFactorValue = stepFactorResourceInformation.getValue();
|
||||
long value = rValue;
|
||||
if (stepFactorValue != 0) {
|
||||
value = roundUp
|
||||
@ -506,10 +480,7 @@ public Resource multiplyAndNormalizeUp(Resource r, double[] by,
|
||||
.getResourceInformation(i);
|
||||
|
||||
long rValue = rResourceInformation.getValue();
|
||||
long stepFactorValue = UnitsConversionUtil.convert(
|
||||
stepFactorResourceInformation.getUnits(),
|
||||
rResourceInformation.getUnits(),
|
||||
stepFactorResourceInformation.getValue());
|
||||
long stepFactorValue = stepFactorResourceInformation.getValue();
|
||||
ret.setResourceValue(i, ResourceCalculator
|
||||
.roundUp((long) Math.ceil(rValue * by[i]), stepFactorValue));
|
||||
}
|
||||
@ -539,10 +510,7 @@ private Resource multiplyAndNormalize(Resource r, double by,
|
||||
ResourceInformation tmp = ret.getResourceInformation(i);
|
||||
|
||||
long rValue = rResourceInformation.getValue();
|
||||
long stepFactorValue = UnitsConversionUtil.convert(
|
||||
stepFactorResourceInformation.getUnits(),
|
||||
rResourceInformation.getUnits(),
|
||||
stepFactorResourceInformation.getValue());
|
||||
long stepFactorValue = stepFactorResourceInformation.getValue();
|
||||
long value;
|
||||
if (stepFactorValue != 0) {
|
||||
value = roundUp
|
||||
@ -566,10 +534,7 @@ public boolean fitsIn(Resource smaller, Resource bigger) {
|
||||
.getResourceInformation(i);
|
||||
ResourceInformation bResourceInformation = bigger
|
||||
.getResourceInformation(i);
|
||||
long sResourceValue = UnitsConversionUtil.convert(
|
||||
sResourceInformation.getUnits(), bResourceInformation.getUnits(),
|
||||
sResourceInformation.getValue());
|
||||
if (sResourceValue > bResourceInformation.getValue()) {
|
||||
if (sResourceInformation.getValue() > bResourceInformation.getValue()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -587,11 +552,7 @@ public Resource normalizeDown(Resource r, Resource stepFactor) {
|
||||
ResourceInformation tmp = ret.getResourceInformation(i);
|
||||
|
||||
long rValue = rResourceInformation.getValue();
|
||||
long stepFactorValue = UnitsConversionUtil.convert(
|
||||
stepFactorResourceInformation.getUnits(),
|
||||
rResourceInformation.getUnits(),
|
||||
stepFactorResourceInformation.getValue());
|
||||
|
||||
long stepFactorValue = stepFactorResourceInformation.getValue();
|
||||
long value = rValue;
|
||||
if (stepFactorValue != 0) {
|
||||
value = roundDown(rValue, stepFactorValue);
|
||||
|
||||
@ -27,7 +27,6 @@
|
||||
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
||||
import org.apache.hadoop.yarn.api.records.impl.LightWeightResource;
|
||||
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
|
||||
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
|
||||
|
||||
/**
|
||||
* Resources is a computation class which provides a set of apis to do
|
||||
@ -257,12 +256,7 @@ public static Resource addTo(Resource lhs, Resource rhs) {
|
||||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(i);
|
||||
ResourceInformation lhsValue = lhs.getResourceInformation(i);
|
||||
|
||||
long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
|
||||
? rhsValue.getValue()
|
||||
: UnitsConversionUtil.convert(rhsValue.getUnits(),
|
||||
lhsValue.getUnits(), rhsValue.getValue());
|
||||
lhs.setResourceValue(i, lhsValue.getValue() + convertedRhs);
|
||||
lhs.setResourceValue(i, lhsValue.getValue() + rhsValue.getValue());
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
LOG.warn("Resource is missing:" + ye.getMessage());
|
||||
continue;
|
||||
@ -281,12 +275,7 @@ public static Resource subtractFrom(Resource lhs, Resource rhs) {
|
||||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(i);
|
||||
ResourceInformation lhsValue = lhs.getResourceInformation(i);
|
||||
|
||||
long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
|
||||
? rhsValue.getValue()
|
||||
: UnitsConversionUtil.convert(rhsValue.getUnits(),
|
||||
lhsValue.getUnits(), rhsValue.getValue());
|
||||
lhs.setResourceValue(i, lhsValue.getValue() - convertedRhs);
|
||||
lhs.setResourceValue(i, lhsValue.getValue() - rhsValue.getValue());
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
LOG.warn("Resource is missing:" + ye.getMessage());
|
||||
continue;
|
||||
@ -365,12 +354,7 @@ public static Resource multiplyAndAddTo(
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(i);
|
||||
ResourceInformation lhsValue = lhs.getResourceInformation(i);
|
||||
|
||||
long convertedRhs = (long) (((rhsValue.getUnits()
|
||||
.equals(lhsValue.getUnits()))
|
||||
? rhsValue.getValue()
|
||||
: UnitsConversionUtil.convert(rhsValue.getUnits(),
|
||||
lhsValue.getUnits(), rhsValue.getValue()))
|
||||
* by);
|
||||
long convertedRhs = (long) (rhsValue.getValue() * by);
|
||||
lhs.setResourceValue(i, lhsValue.getValue() + convertedRhs);
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
LOG.warn("Resource is missing:" + ye.getMessage());
|
||||
@ -511,12 +495,7 @@ public static boolean fitsIn(Resource smaller, Resource bigger) {
|
||||
try {
|
||||
ResourceInformation rhsValue = bigger.getResourceInformation(i);
|
||||
ResourceInformation lhsValue = smaller.getResourceInformation(i);
|
||||
|
||||
long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
|
||||
? rhsValue.getValue()
|
||||
: UnitsConversionUtil.convert(rhsValue.getUnits(),
|
||||
lhsValue.getUnits(), rhsValue.getValue());
|
||||
if (lhsValue.getValue() > convertedRhs) {
|
||||
if (lhsValue.getValue() > rhsValue.getValue()) {
|
||||
return false;
|
||||
}
|
||||
} catch (ResourceNotFoundException ye) {
|
||||
@ -539,12 +518,7 @@ public static Resource componentwiseMin(Resource lhs, Resource rhs) {
|
||||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(i);
|
||||
ResourceInformation lhsValue = lhs.getResourceInformation(i);
|
||||
|
||||
long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
|
||||
? rhsValue.getValue()
|
||||
: UnitsConversionUtil.convert(rhsValue.getUnits(),
|
||||
lhsValue.getUnits(), rhsValue.getValue());
|
||||
ResourceInformation outInfo = lhsValue.getValue() < convertedRhs
|
||||
ResourceInformation outInfo = lhsValue.getValue() < rhsValue.getValue()
|
||||
? lhsValue
|
||||
: rhsValue;
|
||||
ret.setResourceInformation(i, outInfo);
|
||||
@ -563,12 +537,7 @@ public static Resource componentwiseMax(Resource lhs, Resource rhs) {
|
||||
try {
|
||||
ResourceInformation rhsValue = rhs.getResourceInformation(i);
|
||||
ResourceInformation lhsValue = lhs.getResourceInformation(i);
|
||||
|
||||
long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
|
||||
? rhsValue.getValue()
|
||||
: UnitsConversionUtil.convert(rhsValue.getUnits(),
|
||||
lhsValue.getUnits(), rhsValue.getValue());
|
||||
ResourceInformation outInfo = lhsValue.getValue() > convertedRhs
|
||||
ResourceInformation outInfo = lhsValue.getValue() > rhsValue.getValue()
|
||||
? lhsValue
|
||||
: rhsValue;
|
||||
ret.setResourceInformation(i, outInfo);
|
||||
|
||||
@ -18,11 +18,19 @@
|
||||
|
||||
package org.apache.hadoop.yarn.api;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
||||
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos;
|
||||
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
||||
import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
@ -31,6 +39,27 @@
|
||||
* Test class to handle various proto related tests for resources.
|
||||
*/
|
||||
public class TestResourcePBImpl {
|
||||
|
||||
@Before
|
||||
public void setup() throws Exception {
|
||||
ResourceUtils.resetResourceTypes();
|
||||
|
||||
String resourceTypesFile = "resource-types-4.xml";
|
||||
Configuration conf = new YarnConfiguration();
|
||||
TestResourceUtils.setupResourceTypes(conf, resourceTypesFile);
|
||||
}
|
||||
|
||||
@After
|
||||
public void teardown() {
|
||||
Configuration conf = new YarnConfiguration();
|
||||
File source = new File(
|
||||
conf.getClassLoader().getResource("resource-types-4.xml").getFile());
|
||||
File dest = new File(source.getParent(), "resource-types.xml");
|
||||
if (dest.exists()) {
|
||||
dest.delete();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEmptyResourcePBInit() throws Exception {
|
||||
Resource res = new ResourcePBImpl();
|
||||
@ -85,4 +114,65 @@ public void testGetVirtualCores() {
|
||||
assertEquals("Cast to Integer.MAX_VALUE if the long is greater than "
|
||||
+ "Integer.MAX_VALUE", Integer.MAX_VALUE, res.getVirtualCores());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testResourcePBWithExtraResources() throws Exception {
|
||||
|
||||
//Resource 'resource1' has been passed as 4T
|
||||
//4T should be converted to 4000G
|
||||
YarnProtos.ResourceInformationProto riProto =
|
||||
YarnProtos.ResourceInformationProto.newBuilder().setType(
|
||||
YarnProtos.ResourceTypeInfoProto.newBuilder().
|
||||
setName("resource1").setType(
|
||||
YarnProtos.ResourceTypesProto.COUNTABLE).getType()).
|
||||
setValue(4).setUnits("T").setKey("resource1").build();
|
||||
|
||||
YarnProtos.ResourceProto proto =
|
||||
YarnProtos.ResourceProto.newBuilder().setMemory(1024).
|
||||
setVirtualCores(3).addResourceValueMap(riProto).build();
|
||||
Resource res = new ResourcePBImpl(proto);
|
||||
|
||||
Assert.assertEquals(4000,
|
||||
res.getResourceInformation("resource1").getValue());
|
||||
Assert.assertEquals("G",
|
||||
res.getResourceInformation("resource1").getUnits());
|
||||
|
||||
//Resource 'resource2' has been passed as 4M
|
||||
//4M should be converted to 4000000000m
|
||||
YarnProtos.ResourceInformationProto riProto1 =
|
||||
YarnProtos.ResourceInformationProto.newBuilder().setType(
|
||||
YarnProtos.ResourceTypeInfoProto.newBuilder().
|
||||
setName("resource2").setType(
|
||||
YarnProtos.ResourceTypesProto.COUNTABLE).getType()).
|
||||
setValue(4).setUnits("M").setKey("resource2").build();
|
||||
|
||||
YarnProtos.ResourceProto proto1 =
|
||||
YarnProtos.ResourceProto.newBuilder().setMemory(1024).
|
||||
setVirtualCores(3).addResourceValueMap(riProto1).build();
|
||||
Resource res1 = new ResourcePBImpl(proto1);
|
||||
|
||||
Assert.assertEquals(4000000000L,
|
||||
res1.getResourceInformation("resource2").getValue());
|
||||
Assert.assertEquals("m",
|
||||
res1.getResourceInformation("resource2").getUnits());
|
||||
|
||||
//Resource 'resource1' has been passed as 3M
|
||||
//3M should be converted to 0G
|
||||
YarnProtos.ResourceInformationProto riProto2 =
|
||||
YarnProtos.ResourceInformationProto.newBuilder().setType(
|
||||
YarnProtos.ResourceTypeInfoProto.newBuilder().
|
||||
setName("resource1").setType(
|
||||
YarnProtos.ResourceTypesProto.COUNTABLE).getType()).
|
||||
setValue(3).setUnits("M").setKey("resource1").build();
|
||||
|
||||
YarnProtos.ResourceProto proto2 =
|
||||
YarnProtos.ResourceProto.newBuilder().setMemory(1024).
|
||||
setVirtualCores(3).addResourceValueMap(riProto2).build();
|
||||
Resource res2 = new ResourcePBImpl(proto2);
|
||||
|
||||
Assert.assertEquals(0,
|
||||
res2.getResourceInformation("resource1").getValue());
|
||||
Assert.assertEquals("G",
|
||||
res2.getResourceInformation("resource1").getUnits());
|
||||
}
|
||||
}
|
||||
|
||||
@ -377,6 +377,46 @@ public void testResourceNameFormatValidation() throws Exception {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetResourceInformationWithDiffUnits() throws Exception {
|
||||
|
||||
Configuration conf = new YarnConfiguration();
|
||||
Map<String, Resource> testRun = new HashMap<>();
|
||||
setupResourceTypes(conf, "resource-types-4.xml");
|
||||
Resource test3Resources = Resource.newInstance(0, 0);
|
||||
|
||||
//Resource 'resource1' has been passed as 5T
|
||||
//5T should be converted to 5000G
|
||||
test3Resources.setResourceInformation("resource1",
|
||||
ResourceInformation.newInstance("resource1", "T", 5L));
|
||||
|
||||
//Resource 'resource2' has been passed as 2M
|
||||
//2M should be converted to 2000000000m
|
||||
test3Resources.setResourceInformation("resource2",
|
||||
ResourceInformation.newInstance("resource2", "M", 2L));
|
||||
test3Resources.setResourceInformation("yarn.io/gpu",
|
||||
ResourceInformation.newInstance("yarn.io/gpu", "", 1));
|
||||
testRun.put("node-resources-3.xml", test3Resources);
|
||||
|
||||
for (Map.Entry<String, Resource> entry : testRun.entrySet()) {
|
||||
String resourceFile = entry.getKey();
|
||||
ResourceUtils.resetNodeResources();
|
||||
File dest;
|
||||
File source = new File(
|
||||
conf.getClassLoader().getResource(resourceFile).getFile());
|
||||
dest = new File(source.getParent(), "node-resources.xml");
|
||||
FileUtils.copyFile(source, dest);
|
||||
Map<String, ResourceInformation> actual = ResourceUtils
|
||||
.getNodeResourceInformation(conf);
|
||||
Assert.assertEquals(actual.size(),
|
||||
entry.getValue().getResources().length);
|
||||
for (ResourceInformation resInfo : entry.getValue().getResources()) {
|
||||
Assert.assertEquals(resInfo, actual.get(resInfo.getName()));
|
||||
}
|
||||
dest.delete();
|
||||
}
|
||||
}
|
||||
|
||||
public static String setupResourceTypes(Configuration conf, String filename)
|
||||
throws Exception {
|
||||
File source = new File(
|
||||
|
||||
@ -0,0 +1,33 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. See accompanying LICENSE file.
|
||||
-->
|
||||
|
||||
<configuration>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.resource-type.resource1</name>
|
||||
<value>5T</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.resource-type.resource2</name>
|
||||
<value>2M</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>yarn.nodemanager.resource-type.yarn.io/gpu</name>
|
||||
<value>1</value>
|
||||
</property>
|
||||
</configuration>
|
||||
@ -33,6 +33,7 @@
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.security.AccessControlException;
|
||||
import java.security.PrivilegedExceptionAction;
|
||||
@ -49,6 +50,7 @@
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.CyclicBarrier;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
@ -157,7 +159,9 @@
|
||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||
import org.apache.hadoop.yarn.util.Clock;
|
||||
import org.apache.hadoop.yarn.util.UTCClock;
|
||||
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
|
||||
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
|
||||
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Assume;
|
||||
@ -2243,4 +2247,88 @@ public void handle(Event event) {
|
||||
rmService.getApplications(request).getApplicationList().size());
|
||||
rmService.setDisplayPerUserApps(false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRegisterNMWithDiffUnits() throws Exception {
|
||||
ResourceUtils.resetResourceTypes();
|
||||
Configuration yarnConf = new YarnConfiguration();
|
||||
String resourceTypesFile = "resource-types-4.xml";
|
||||
InputStream source =
|
||||
yarnConf.getClassLoader().getResourceAsStream(resourceTypesFile);
|
||||
File dest = new File(yarnConf.getClassLoader().
|
||||
getResource(".").getPath(), "resource-types.xml");
|
||||
FileUtils.copyInputStreamToFile(source, dest);
|
||||
ResourceUtils.getResourceTypes();
|
||||
|
||||
yarnConf.setClass(
|
||||
CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
|
||||
DominantResourceCalculator.class, ResourceCalculator.class);
|
||||
|
||||
MockRM rm = new MockRM(yarnConf) {
|
||||
protected ClientRMService createClientRMService() {
|
||||
return new ClientRMService(this.rmContext, scheduler,
|
||||
this.rmAppManager, this.applicationACLsManager, this.queueACLsManager,
|
||||
this.getRMContext().getRMDelegationTokenSecretManager());
|
||||
};
|
||||
};
|
||||
rm.start();
|
||||
|
||||
Resource resource = BuilderUtils.newResource(1024, 1);
|
||||
resource.setResourceInformation("memory-mb",
|
||||
ResourceInformation.newInstance("memory-mb", "G", 1024));
|
||||
resource.setResourceInformation("resource1",
|
||||
ResourceInformation.newInstance("resource1", "T", 1));
|
||||
resource.setResourceInformation("resource2",
|
||||
ResourceInformation.newInstance("resource2", "M", 1));
|
||||
|
||||
MockNM node = rm.registerNode("host1:1234", resource);
|
||||
node.nodeHeartbeat(true);
|
||||
|
||||
// Create a client.
|
||||
Configuration conf = new Configuration();
|
||||
YarnRPC rpc = YarnRPC.create(conf);
|
||||
InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
|
||||
LOG.info("Connecting to ResourceManager at " + rmAddress);
|
||||
ApplicationClientProtocol client =
|
||||
(ApplicationClientProtocol) rpc
|
||||
.getProxy(ApplicationClientProtocol.class, rmAddress, conf);
|
||||
|
||||
// Make call
|
||||
GetClusterNodesRequest request =
|
||||
GetClusterNodesRequest.newInstance(EnumSet.of(NodeState.RUNNING));
|
||||
List<NodeReport> nodeReports =
|
||||
client.getClusterNodes(request).getNodeReports();
|
||||
Assert.assertEquals(1, nodeReports.size());
|
||||
Assert.assertNotSame("Node is expected to be healthy!", NodeState.UNHEALTHY,
|
||||
nodeReports.get(0).getNodeState());
|
||||
Assert.assertEquals(1, nodeReports.size());
|
||||
|
||||
//Resource 'resource1' has been passed as 1T while registering NM.
|
||||
//1T should be converted to 1000G
|
||||
Assert.assertEquals("G", nodeReports.get(0).getCapability().
|
||||
getResourceInformation("resource1").getUnits());
|
||||
Assert.assertEquals(1000, nodeReports.get(0).getCapability().
|
||||
getResourceInformation("resource1").getValue());
|
||||
|
||||
//Resource 'resource2' has been passed as 1M while registering NM
|
||||
//1M should be converted to 1000000000M
|
||||
Assert.assertEquals("m", nodeReports.get(0).getCapability().
|
||||
getResourceInformation("resource2").getUnits());
|
||||
Assert.assertEquals(1000000000, nodeReports.get(0).getCapability().
|
||||
getResourceInformation("resource2").getValue());
|
||||
|
||||
//Resource 'memory-mb' has been passed as 1024G while registering NM
|
||||
//1024G should be converted to 976562Mi
|
||||
Assert.assertEquals("Mi", nodeReports.get(0).getCapability().
|
||||
getResourceInformation("memory-mb").getUnits());
|
||||
Assert.assertEquals(976562, nodeReports.get(0).getCapability().
|
||||
getResourceInformation("memory-mb").getValue());
|
||||
|
||||
rpc.stopProxy(client, conf);
|
||||
rm.close();
|
||||
|
||||
if (dest.exists()) {
|
||||
dest.delete();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -414,7 +414,7 @@ public void testAllocationIncrementCustomResource() throws Exception {
|
||||
calculator.normalize(customResource(10000L, ""), min, max, increment)
|
||||
.getResourceInformation(A_CUSTOM_RESOURCE));
|
||||
assertEquals(customResourceInformation(20000L, ""),
|
||||
calculator.normalize(customResource(10001L, ""), min, max, increment)
|
||||
calculator.normalize(customResource(19999L, ""), min, max, increment)
|
||||
.getResourceInformation(A_CUSTOM_RESOURCE));
|
||||
assertEquals(customResourceInformation(10L, "k"),
|
||||
calculator.normalize(customResource(9L, "k"), min, max, increment)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user