YARN-6264. AM not launched when a single vcore is available on the cluster. (Yufei Gu via kasha)
This commit is contained in:
parent
c5ee7fded4
commit
a96afae125
@ -242,6 +242,13 @@ public static Resource multiplyAndRoundDown(Resource lhs, double by) {
|
|||||||
out.setVirtualCores((int)(lhs.getVirtualCores() * by));
|
out.setVirtualCores((int)(lhs.getVirtualCores() * by));
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static Resource multiplyAndRoundUp(Resource lhs, double by) {
|
||||||
|
Resource out = clone(lhs);
|
||||||
|
out.setMemorySize((long)Math.ceil(lhs.getMemorySize() * by));
|
||||||
|
out.setVirtualCores((int)Math.ceil(lhs.getVirtualCores() * by));
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
public static Resource normalize(
|
public static Resource normalize(
|
||||||
ResourceCalculator calculator, Resource lhs, Resource min,
|
ResourceCalculator calculator, Resource lhs, Resource min,
|
||||||
|
@ -20,6 +20,8 @@
|
|||||||
|
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
public class TestResources {
|
public class TestResources {
|
||||||
@ -46,5 +48,25 @@ public void testCompareToWithNoneResource() {
|
|||||||
assertTrue(Resources.none().compareTo(
|
assertTrue(Resources.none().compareTo(
|
||||||
createResource(0, 1)) < 0);
|
createResource(0, 1)) < 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testMultipleRoundUp() {
|
||||||
|
final double by = 0.5;
|
||||||
|
final String memoryErrorMsg = "Invalid memory size.";
|
||||||
|
final String vcoreErrorMsg = "Invalid virtual core number.";
|
||||||
|
Resource resource = Resources.createResource(1, 1);
|
||||||
|
Resource result = Resources.multiplyAndRoundUp(resource, by);
|
||||||
|
assertEquals(memoryErrorMsg, result.getMemorySize(), 1);
|
||||||
|
assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1);
|
||||||
|
|
||||||
|
resource = Resources.createResource(2, 2);
|
||||||
|
result = Resources.multiplyAndRoundUp(resource, by);
|
||||||
|
assertEquals(memoryErrorMsg, result.getMemorySize(), 1);
|
||||||
|
assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1);
|
||||||
|
|
||||||
|
resource = Resources.createResource(0, 0);
|
||||||
|
result = Resources.multiplyAndRoundUp(resource, by);
|
||||||
|
assertEquals(memoryErrorMsg, result.getMemorySize(), 0);
|
||||||
|
assertEquals(vcoreErrorMsg, result.getVirtualCores(), 0);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -515,7 +515,8 @@ private Resource computeMaxAMResource() {
|
|||||||
getMaxShare().getVirtualCores()));
|
getMaxShare().getVirtualCores()));
|
||||||
}
|
}
|
||||||
|
|
||||||
return Resources.multiply(maxResource, maxAMShare);
|
// Round up to allow AM to run when there is only one vcore on the cluster
|
||||||
|
return Resources.multiplyAndRoundUp(maxResource, maxAMShare);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -659,15 +659,13 @@ public void testComputeMaxAMResource() throws IOException {
|
|||||||
// case, we use maxShare, since it is smaller than available resource.
|
// case, we use maxShare, since it is smaller than available resource.
|
||||||
assertEquals("QueueFSZeroWithMax's fair share should be zero",
|
assertEquals("QueueFSZeroWithMax's fair share should be zero",
|
||||||
0, queueFSZeroWithMax.getFairShare().getMemorySize());
|
0, queueFSZeroWithMax.getFairShare().getMemorySize());
|
||||||
|
Resource expectedAMResource = Resources.multiplyAndRoundUp(
|
||||||
|
queueFSZeroWithMax.getMaxShare(), queueFSZeroWithMax.getMaxAMShare());
|
||||||
assertEquals("QueueFSZeroWithMax's maximum AM resource should be "
|
assertEquals("QueueFSZeroWithMax's maximum AM resource should be "
|
||||||
+ "maxShare * maxAMShare",
|
+ "maxShare * maxAMShare", expectedAMResource.getMemorySize(),
|
||||||
(long)(queueFSZeroWithMax.getMaxShare().getMemorySize() *
|
|
||||||
queueFSZeroWithMax.getMaxAMShare()),
|
|
||||||
queueFSZeroWithMax.getMetrics().getMaxAMShareMB());
|
queueFSZeroWithMax.getMetrics().getMaxAMShareMB());
|
||||||
assertEquals("QueueFSZeroWithMax's maximum AM resource should be "
|
assertEquals("QueueFSZeroWithMax's maximum AM resource should be "
|
||||||
+ "maxShare * maxAMShare",
|
+ "maxShare * maxAMShare", expectedAMResource.getVirtualCores(),
|
||||||
(long)(queueFSZeroWithMax.getMaxShare().getVirtualCores() *
|
|
||||||
queueFSZeroWithMax.getMaxAMShare()),
|
|
||||||
queueFSZeroWithMax.getMetrics().getMaxAMShareVCores());
|
queueFSZeroWithMax.getMetrics().getMaxAMShareVCores());
|
||||||
assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to "
|
assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to "
|
||||||
+ "AM resource request",
|
+ "AM resource request",
|
||||||
@ -689,17 +687,19 @@ public void testComputeMaxAMResource() throws IOException {
|
|||||||
// the min(maxShare, available resource) to compute maxAMShare, in this
|
// the min(maxShare, available resource) to compute maxAMShare, in this
|
||||||
// case, we use available resource since it is smaller than the
|
// case, we use available resource since it is smaller than the
|
||||||
// default maxShare.
|
// default maxShare.
|
||||||
|
expectedAMResource = Resources.multiplyAndRoundUp(
|
||||||
|
Resources.createResource(memCapacity - amResource.getMemorySize(),
|
||||||
|
cpuCapacity - amResource.getVirtualCores()),
|
||||||
|
queueFSZeroWithAVL.getMaxAMShare());
|
||||||
assertEquals("QueueFSZeroWithAVL's fair share should be zero",
|
assertEquals("QueueFSZeroWithAVL's fair share should be zero",
|
||||||
0, queueFSZeroWithAVL.getFairShare().getMemorySize());
|
0, queueFSZeroWithAVL.getFairShare().getMemorySize());
|
||||||
assertEquals("QueueFSZeroWithAVL's maximum AM resource should be "
|
assertEquals("QueueFSZeroWithAVL's maximum AM resource should be "
|
||||||
+ " available resource * maxAMShare",
|
+ " available resource * maxAMShare",
|
||||||
(long) ((memCapacity - amResource.getMemorySize()) *
|
expectedAMResource.getMemorySize(),
|
||||||
queueFSZeroWithAVL.getMaxAMShare()),
|
|
||||||
queueFSZeroWithAVL.getMetrics().getMaxAMShareMB());
|
queueFSZeroWithAVL.getMetrics().getMaxAMShareMB());
|
||||||
assertEquals("QueueFSZeroWithAVL's maximum AM resource should be "
|
assertEquals("QueueFSZeroWithAVL's maximum AM resource should be "
|
||||||
+ " available resource * maxAMShare",
|
+ " available resource * maxAMShare",
|
||||||
(long) ((cpuCapacity - amResource.getVirtualCores()) *
|
expectedAMResource.getVirtualCores(),
|
||||||
queueFSZeroWithAVL.getMaxAMShare()),
|
|
||||||
queueFSZeroWithAVL.getMetrics().getMaxAMShareVCores());
|
queueFSZeroWithAVL.getMetrics().getMaxAMShareVCores());
|
||||||
assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to "
|
assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to "
|
||||||
+ "AM resource request",
|
+ "AM resource request",
|
||||||
@ -721,13 +721,13 @@ public void testComputeMaxAMResource() throws IOException {
|
|||||||
// fair share to compute maxAMShare
|
// fair share to compute maxAMShare
|
||||||
assertNotEquals("QueueFSNonZero's fair share shouldn't be zero",
|
assertNotEquals("QueueFSNonZero's fair share shouldn't be zero",
|
||||||
0, queueFSNonZero.getFairShare().getMemorySize());
|
0, queueFSNonZero.getFairShare().getMemorySize());
|
||||||
|
expectedAMResource = Resources.multiplyAndRoundUp(
|
||||||
|
queueFSNonZero.getFairShare(), queueFSNonZero.getMaxAMShare());
|
||||||
assertEquals("QueueFSNonZero's maximum AM resource should be "
|
assertEquals("QueueFSNonZero's maximum AM resource should be "
|
||||||
+ " fair share * maxAMShare",
|
+ " fair share * maxAMShare", expectedAMResource.getMemorySize(),
|
||||||
(long)(memCapacity * queueFSNonZero.getMaxAMShare()),
|
|
||||||
queueFSNonZero.getMetrics().getMaxAMShareMB());
|
queueFSNonZero.getMetrics().getMaxAMShareMB());
|
||||||
assertEquals("QueueFSNonZero's maximum AM resource should be "
|
assertEquals("QueueFSNonZero's maximum AM resource should be "
|
||||||
+ " fair share * maxAMShare",
|
+ " fair share * maxAMShare", expectedAMResource.getVirtualCores(),
|
||||||
(long)(cpuCapacity * queueFSNonZero.getMaxAMShare()),
|
|
||||||
queueFSNonZero.getMetrics().getMaxAMShareVCores());
|
queueFSNonZero.getMetrics().getMaxAMShareVCores());
|
||||||
assertEquals("QueueFSNonZero's AM resource usage should be the same to "
|
assertEquals("QueueFSNonZero's AM resource usage should be the same to "
|
||||||
+ "AM resource request",
|
+ "AM resource request",
|
||||||
|
Loading…
Reference in New Issue
Block a user