YARN-6264. AM not launched when a single vcore is available on the cluster. (Yufei Gu via kasha)
This commit is contained in:
parent
c5ee7fded4
commit
a96afae125
@ -242,6 +242,13 @@ public static Resource multiplyAndRoundDown(Resource lhs, double by) {
|
||||
out.setVirtualCores((int)(lhs.getVirtualCores() * by));
|
||||
return out;
|
||||
}
|
||||
|
||||
public static Resource multiplyAndRoundUp(Resource lhs, double by) {
|
||||
Resource out = clone(lhs);
|
||||
out.setMemorySize((long)Math.ceil(lhs.getMemorySize() * by));
|
||||
out.setVirtualCores((int)Math.ceil(lhs.getVirtualCores() * by));
|
||||
return out;
|
||||
}
|
||||
|
||||
public static Resource normalize(
|
||||
ResourceCalculator calculator, Resource lhs, Resource min,
|
||||
|
@ -20,6 +20,8 @@
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
public class TestResources {
|
||||
@ -46,5 +48,25 @@ public void testCompareToWithNoneResource() {
|
||||
assertTrue(Resources.none().compareTo(
|
||||
createResource(0, 1)) < 0);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testMultipleRoundUp() {
|
||||
final double by = 0.5;
|
||||
final String memoryErrorMsg = "Invalid memory size.";
|
||||
final String vcoreErrorMsg = "Invalid virtual core number.";
|
||||
Resource resource = Resources.createResource(1, 1);
|
||||
Resource result = Resources.multiplyAndRoundUp(resource, by);
|
||||
assertEquals(memoryErrorMsg, result.getMemorySize(), 1);
|
||||
assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1);
|
||||
|
||||
resource = Resources.createResource(2, 2);
|
||||
result = Resources.multiplyAndRoundUp(resource, by);
|
||||
assertEquals(memoryErrorMsg, result.getMemorySize(), 1);
|
||||
assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1);
|
||||
|
||||
resource = Resources.createResource(0, 0);
|
||||
result = Resources.multiplyAndRoundUp(resource, by);
|
||||
assertEquals(memoryErrorMsg, result.getMemorySize(), 0);
|
||||
assertEquals(vcoreErrorMsg, result.getVirtualCores(), 0);
|
||||
}
|
||||
}
|
||||
|
@ -515,7 +515,8 @@ private Resource computeMaxAMResource() {
|
||||
getMaxShare().getVirtualCores()));
|
||||
}
|
||||
|
||||
return Resources.multiply(maxResource, maxAMShare);
|
||||
// Round up to allow AM to run when there is only one vcore on the cluster
|
||||
return Resources.multiplyAndRoundUp(maxResource, maxAMShare);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -659,15 +659,13 @@ public void testComputeMaxAMResource() throws IOException {
|
||||
// case, we use maxShare, since it is smaller than available resource.
|
||||
assertEquals("QueueFSZeroWithMax's fair share should be zero",
|
||||
0, queueFSZeroWithMax.getFairShare().getMemorySize());
|
||||
Resource expectedAMResource = Resources.multiplyAndRoundUp(
|
||||
queueFSZeroWithMax.getMaxShare(), queueFSZeroWithMax.getMaxAMShare());
|
||||
assertEquals("QueueFSZeroWithMax's maximum AM resource should be "
|
||||
+ "maxShare * maxAMShare",
|
||||
(long)(queueFSZeroWithMax.getMaxShare().getMemorySize() *
|
||||
queueFSZeroWithMax.getMaxAMShare()),
|
||||
+ "maxShare * maxAMShare", expectedAMResource.getMemorySize(),
|
||||
queueFSZeroWithMax.getMetrics().getMaxAMShareMB());
|
||||
assertEquals("QueueFSZeroWithMax's maximum AM resource should be "
|
||||
+ "maxShare * maxAMShare",
|
||||
(long)(queueFSZeroWithMax.getMaxShare().getVirtualCores() *
|
||||
queueFSZeroWithMax.getMaxAMShare()),
|
||||
+ "maxShare * maxAMShare", expectedAMResource.getVirtualCores(),
|
||||
queueFSZeroWithMax.getMetrics().getMaxAMShareVCores());
|
||||
assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to "
|
||||
+ "AM resource request",
|
||||
@ -689,17 +687,19 @@ public void testComputeMaxAMResource() throws IOException {
|
||||
// the min(maxShare, available resource) to compute maxAMShare, in this
|
||||
// case, we use available resource since it is smaller than the
|
||||
// default maxShare.
|
||||
expectedAMResource = Resources.multiplyAndRoundUp(
|
||||
Resources.createResource(memCapacity - amResource.getMemorySize(),
|
||||
cpuCapacity - amResource.getVirtualCores()),
|
||||
queueFSZeroWithAVL.getMaxAMShare());
|
||||
assertEquals("QueueFSZeroWithAVL's fair share should be zero",
|
||||
0, queueFSZeroWithAVL.getFairShare().getMemorySize());
|
||||
assertEquals("QueueFSZeroWithAVL's maximum AM resource should be "
|
||||
+ " available resource * maxAMShare",
|
||||
(long) ((memCapacity - amResource.getMemorySize()) *
|
||||
queueFSZeroWithAVL.getMaxAMShare()),
|
||||
expectedAMResource.getMemorySize(),
|
||||
queueFSZeroWithAVL.getMetrics().getMaxAMShareMB());
|
||||
assertEquals("QueueFSZeroWithAVL's maximum AM resource should be "
|
||||
+ " available resource * maxAMShare",
|
||||
(long) ((cpuCapacity - amResource.getVirtualCores()) *
|
||||
queueFSZeroWithAVL.getMaxAMShare()),
|
||||
expectedAMResource.getVirtualCores(),
|
||||
queueFSZeroWithAVL.getMetrics().getMaxAMShareVCores());
|
||||
assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to "
|
||||
+ "AM resource request",
|
||||
@ -721,13 +721,13 @@ public void testComputeMaxAMResource() throws IOException {
|
||||
// fair share to compute maxAMShare
|
||||
assertNotEquals("QueueFSNonZero's fair share shouldn't be zero",
|
||||
0, queueFSNonZero.getFairShare().getMemorySize());
|
||||
expectedAMResource = Resources.multiplyAndRoundUp(
|
||||
queueFSNonZero.getFairShare(), queueFSNonZero.getMaxAMShare());
|
||||
assertEquals("QueueFSNonZero's maximum AM resource should be "
|
||||
+ " fair share * maxAMShare",
|
||||
(long)(memCapacity * queueFSNonZero.getMaxAMShare()),
|
||||
+ " fair share * maxAMShare", expectedAMResource.getMemorySize(),
|
||||
queueFSNonZero.getMetrics().getMaxAMShareMB());
|
||||
assertEquals("QueueFSNonZero's maximum AM resource should be "
|
||||
+ " fair share * maxAMShare",
|
||||
(long)(cpuCapacity * queueFSNonZero.getMaxAMShare()),
|
||||
+ " fair share * maxAMShare", expectedAMResource.getVirtualCores(),
|
||||
queueFSNonZero.getMetrics().getMaxAMShareVCores());
|
||||
assertEquals("QueueFSNonZero's AM resource usage should be the same to "
|
||||
+ "AM resource request",
|
||||
|
Loading…
Reference in New Issue
Block a user