YARN-4418. AM Resource Limit per partition can be updated to ResourceUsage as well. (Sunil G via wangda)

This commit is contained in:
Wangda Tan 2015-12-14 11:24:30 -08:00
parent dfcbbddb09
commit 07b0fb996a
6 changed files with 88 additions and 12 deletions

View File

@ -616,6 +616,9 @@ Release 2.8.0 - UNRELEASED
YARN-4309. Add container launch related debug information to container logs
when a container fails. (Varun Vasudev via wangda)
YARN-4418. AM Resource Limit per partition can be updated to ResourceUsage as well.
(Sunil G via wangda)
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not

View File

@ -61,7 +61,7 @@ private enum ResourceType {
//CACHED_USED and CACHED_PENDING may be read by anyone, but must only
//be written by ordering policies
USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4),
CACHED_PENDING(5);
CACHED_PENDING(5), AMLIMIT(6);
private int idx;
@ -92,6 +92,7 @@ public String toString() {
sb.append("pending=" + resArr[1] + "%, ");
sb.append("am_used=" + resArr[2] + "%, ");
sb.append("reserved=" + resArr[3] + "%}");
sb.append("am_limit=" + resArr[6] + "%, ");
return sb.toString();
}
}
@ -106,11 +107,19 @@ public Resource getUsed() {
public Resource getUsed(String label) {
return _get(label, ResourceType.USED);
}
public Resource getCachedUsed() {
return _get(NL, ResourceType.CACHED_USED);
}
public Resource getCachedUsed(String label) {
return _get(label, ResourceType.CACHED_USED);
}
public Resource getCachedPending() {
return _get(NL, ResourceType.CACHED_PENDING);
}
public Resource getCachedPending(String label) {
return _get(label, ResourceType.CACHED_PENDING);
}
@ -149,15 +158,23 @@ public void copyAllUsed(ResourceUsage other) {
public void setUsed(String label, Resource res) {
_set(label, ResourceType.USED, res);
}
public void setCachedUsed(String label, Resource res) {
_set(label, ResourceType.CACHED_USED, res);
}
public void setCachedUsed(Resource res) {
_set(NL, ResourceType.CACHED_USED, res);
}
public void setCachedPending(String label, Resource res) {
_set(label, ResourceType.CACHED_PENDING, res);
}
public void setCachedPending(Resource res) {
_set(NL, ResourceType.CACHED_PENDING, res);
}
/*
* Pending
*/
@ -263,6 +280,41 @@ public void setAMUsed(String label, Resource res) {
_set(label, ResourceType.AMUSED, res);
}
/*
* AM-Resource Limit
*/
public Resource getAMLimit() {
return getAMLimit(NL);
}
public Resource getAMLimit(String label) {
return _get(label, ResourceType.AMLIMIT);
}
public void incAMLimit(String label, Resource res) {
_inc(label, ResourceType.AMLIMIT, res);
}
public void incAMLimit(Resource res) {
incAMLimit(NL, res);
}
public void decAMLimit(Resource res) {
decAMLimit(NL, res);
}
public void decAMLimit(String label, Resource res) {
_dec(label, ResourceType.AMLIMIT, res);
}
public void setAMLimit(Resource res) {
setAMLimit(NL, res);
}
public void setAMLimit(String label, Resource res) {
_set(label, ResourceType.AMLIMIT, res);
}
private static Resource normalize(Resource res) {
if (res == null) {
return Resources.none();

View File

@ -606,6 +606,7 @@ public synchronized Resource getAMResourceLimitPerPartition(
minimumAllocation);
metrics.setAMResouceLimit(amResouceLimit);
queueUsage.setAMLimit(nodePartition, amResouceLimit);
return amResouceLimit;
}

View File

@ -217,6 +217,11 @@ public void setAbsoluteMaximumCapacity(String label, float value) {
}
/* Absolute Maximum AM resource percentage Getter and Setter */
public float getMaxAMResourcePercentage() {
return _get(NL, CapacityType.MAX_AM_PERC);
}
public float getMaxAMResourcePercentage(String label) {
return _get(label, CapacityType.MAX_AM_PERC);
}
@ -225,6 +230,10 @@ public void setMaxAMResourcePercentage(String label, float value) {
_set(label, CapacityType.MAX_AM_PERC, value);
}
public void setMaxAMResourcePercentage(float value) {
_set(NL, CapacityType.MAX_AM_PERC, value);
}
/**
* Clear configurable fields, like
* (absolute)capacity/(absolute)maximum-capacity, this will be used by queue

View File

@ -37,8 +37,8 @@ public class TestResourceUsage {
@Parameterized.Parameters
public static Collection<String[]> getParameters() {
return Arrays.asList(new String[][] { { "Pending" }, { "Used" },
{ "Reserved" }, { "AMUsed" } });
return Arrays.asList(new String[][]{{"Pending"}, {"Used"}, {"Reserved"},
{"AMUsed"}, {"AMLimit"}, {"CachedUsed"}, {"CachedPending"}});
}
public TestResourceUsage(String suffix) {
@ -112,16 +112,26 @@ private void internalTestModifyAndRead(String label) throws Exception {
check(0, 0, res);
// Add 1,1 should returns 1,1
inc(usage, suffix, Resource.newInstance(1, 1), label);
check(1, 1, get(usage, suffix, label));
try {
inc(usage, suffix, Resource.newInstance(1, 1), label);
check(1, 1, get(usage, suffix, label));
} catch (NoSuchMethodException e) {
// Few operations need not have to be verified as some resources doesn't
// inc/dec apis exposed (For Eg: CachedUsed and CachedPending).
}
// Set 2,2
set(usage, suffix, Resource.newInstance(2, 2), label);
check(2, 2, get(usage, suffix, label));
// dec 2,2
dec(usage, suffix, Resource.newInstance(2, 2), label);
check(0, 0, get(usage, suffix, label));
try {
dec(usage, suffix, Resource.newInstance(2, 2), label);
check(0, 0, get(usage, suffix, label));
} catch (NoSuchMethodException e) {
// Few operations need not have to be verified, as some resources doesn't
// inc/dec apis exposed (For Eg: CachedUsed and CachedPending).
}
}
void check(int mem, int cpu, Resource res) {

View File

@ -43,7 +43,8 @@ public static Collection<String[]> getParameters() {
{ "UsedCapacity" },
{ "AbsoluteUsedCapacity" },
{ "MaximumCapacity" },
{ "AbsoluteMaximumCapacity" } });
{ "AbsoluteMaximumCapacity" },
{ "MaxAMResourcePercentage" } });
}
public TestQueueCapacities(String suffix) {