MAPREDUCE-3764. Fixed resource usage metrics for queues and users.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1238255 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy
parent
7ed0f74997
commit
7f23d72352
@ -621,6 +621,9 @@ Release 0.23.1 - Unreleased
|
||||
MAPREDUCE-3748. Changed a log in CapacityScheduler.nodeUpdate to debug.
|
||||
(ramya via acmurthy)
|
||||
|
||||
MAPREDUCE-3764. Fixed resource usage metrics for queues and users.
|
||||
(acmurthy)
|
||||
|
||||
Release 0.23.0 - 2011-11-01
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
||||
@ -51,20 +51,19 @@ public class QueueMetrics {
|
||||
@Metric("# of apps killed") MutableCounterInt appsKilled;
|
||||
@Metric("# of apps failed") MutableCounterInt appsFailed;
|
||||
|
||||
@Metric("Allocated memory in GiB") MutableGaugeInt allocatedGB;
|
||||
@Metric("Allocated memory in MB") MutableGaugeInt allocatedMB;
|
||||
@Metric("# of allocated containers") MutableGaugeInt allocatedContainers;
|
||||
@Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated;
|
||||
@Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased;
|
||||
@Metric("Available memory in GiB") MutableGaugeInt availableGB;
|
||||
@Metric("Pending memory allocation in GiB") MutableGaugeInt pendingGB;
|
||||
@Metric("Available memory in MB") MutableGaugeInt availableMB;
|
||||
@Metric("Pending memory allocation in MB") MutableGaugeInt pendingMB;
|
||||
@Metric("# of pending containers") MutableGaugeInt pendingContainers;
|
||||
@Metric("# of reserved memory in GiB") MutableGaugeInt reservedGB;
|
||||
@Metric("# of reserved memory in MB") MutableGaugeInt reservedMB;
|
||||
@Metric("# of reserved containers") MutableGaugeInt reservedContainers;
|
||||
@Metric("# of active users") MutableGaugeInt activeUsers;
|
||||
@Metric("# of active users") MutableGaugeInt activeApplications;
|
||||
|
||||
static final Logger LOG = LoggerFactory.getLogger(QueueMetrics.class);
|
||||
static final int GB = 1024; // resource.memory is in MB
|
||||
static final MetricsInfo RECORD_INFO = info("QueueMetrics",
|
||||
"Metrics for the resource scheduler");
|
||||
static final MetricsInfo QUEUE_INFO = info("Queue", "Metrics by queue");
|
||||
@ -183,7 +182,7 @@ public void finishApp(AppSchedulingInfo app,
|
||||
* @param limit resource limit
|
||||
*/
|
||||
public void setAvailableResourcesToQueue(Resource limit) {
|
||||
availableGB.set(limit.getMemory()/GB);
|
||||
availableMB.set(limit.getMemory());
|
||||
}
|
||||
|
||||
/**
|
||||
@ -219,7 +218,7 @@ public void incrPendingResources(String user, int containers, Resource res) {
|
||||
|
||||
private void _incrPendingResources(int containers, Resource res) {
|
||||
pendingContainers.incr(containers);
|
||||
pendingGB.incr(res.getMemory()/GB);
|
||||
pendingMB.incr(res.getMemory());
|
||||
}
|
||||
|
||||
public void decrPendingResources(String user, int containers, Resource res) {
|
||||
@ -235,13 +234,13 @@ public void decrPendingResources(String user, int containers, Resource res) {
|
||||
|
||||
private void _decrPendingResources(int containers, Resource res) {
|
||||
pendingContainers.decr(containers);
|
||||
pendingGB.decr(res.getMemory()/GB);
|
||||
pendingMB.decr(res.getMemory());
|
||||
}
|
||||
|
||||
public void allocateResources(String user, int containers, Resource res) {
|
||||
allocatedContainers.incr(containers);
|
||||
aggregateContainersAllocated.incr(containers);
|
||||
allocatedGB.incr(res.getMemory()/GB * containers);
|
||||
allocatedMB.incr(res.getMemory() * containers);
|
||||
_decrPendingResources(containers, multiply(res, containers));
|
||||
QueueMetrics userMetrics = getUserMetrics(user);
|
||||
if (userMetrics != null) {
|
||||
@ -255,7 +254,7 @@ public void allocateResources(String user, int containers, Resource res) {
|
||||
public void releaseResources(String user, int containers, Resource res) {
|
||||
allocatedContainers.decr(containers);
|
||||
aggregateContainersReleased.incr(containers);
|
||||
allocatedGB.decr(res.getMemory()/GB * containers);
|
||||
allocatedMB.decr(res.getMemory() * containers);
|
||||
QueueMetrics userMetrics = getUserMetrics(user);
|
||||
if (userMetrics != null) {
|
||||
userMetrics.releaseResources(user, containers, res);
|
||||
@ -267,7 +266,7 @@ public void releaseResources(String user, int containers, Resource res) {
|
||||
|
||||
public void reserveResource(String user, Resource res) {
|
||||
reservedContainers.incr();
|
||||
reservedGB.incr(res.getMemory()/GB);
|
||||
reservedMB.incr(res.getMemory());
|
||||
QueueMetrics userMetrics = getUserMetrics(user);
|
||||
if (userMetrics != null) {
|
||||
userMetrics.reserveResource(user, res);
|
||||
@ -279,7 +278,7 @@ public void reserveResource(String user, Resource res) {
|
||||
|
||||
public void unreserveResource(String user, Resource res) {
|
||||
reservedContainers.decr();
|
||||
reservedGB.decr(res.getMemory()/GB);
|
||||
reservedMB.decr(res.getMemory());
|
||||
QueueMetrics userMetrics = getUserMetrics(user);
|
||||
if (userMetrics != null) {
|
||||
userMetrics.unreserveResource(user, res);
|
||||
@ -343,28 +342,28 @@ public int getAppsFailed() {
|
||||
return appsFailed.value();
|
||||
}
|
||||
|
||||
public int getAllocatedGB() {
|
||||
return allocatedGB.value();
|
||||
public int getAllocatedMB() {
|
||||
return allocatedMB.value();
|
||||
}
|
||||
|
||||
public int getAllocatedContainers() {
|
||||
return allocatedContainers.value();
|
||||
}
|
||||
|
||||
public int getAvailableGB() {
|
||||
return availableGB.value();
|
||||
public int getAvailableMB() {
|
||||
return availableMB.value();
|
||||
}
|
||||
|
||||
public int getPendingGB() {
|
||||
return pendingGB.value();
|
||||
public int getPendingMB() {
|
||||
return pendingMB.value();
|
||||
}
|
||||
|
||||
public int getPendingContainers() {
|
||||
return pendingContainers.value();
|
||||
}
|
||||
|
||||
public int getReservedGB() {
|
||||
return reservedGB.value();
|
||||
public int getReservedMB() {
|
||||
return reservedMB.value();
|
||||
}
|
||||
|
||||
public int getReservedContainers() {
|
||||
|
||||
@ -31,8 +31,6 @@
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class ClusterMetricsInfo {
|
||||
|
||||
private static final long MB_IN_GB = 1024;
|
||||
|
||||
protected int appsSubmitted;
|
||||
protected long reservedMB;
|
||||
protected long availableMB;
|
||||
@ -55,9 +53,9 @@ public ClusterMetricsInfo(final ResourceManager rm, final RMContext rmContext) {
|
||||
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
|
||||
|
||||
this.appsSubmitted = metrics.getAppsSubmitted();
|
||||
this.reservedMB = metrics.getReservedGB() * MB_IN_GB;
|
||||
this.availableMB = metrics.getAvailableGB() * MB_IN_GB;
|
||||
this.allocatedMB = metrics.getAllocatedGB() * MB_IN_GB;
|
||||
this.reservedMB = metrics.getReservedMB();
|
||||
this.availableMB = metrics.getAvailableMB();
|
||||
this.allocatedMB = metrics.getAllocatedMB();
|
||||
this.containersAllocated = metrics.getAllocatedContainers();
|
||||
this.totalMB = availableMB + reservedMB + allocatedMB;
|
||||
this.activeNodes = clusterMetrics.getNumActiveNMs();
|
||||
|
||||
@ -31,8 +31,6 @@
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class UserMetricsInfo {
|
||||
|
||||
private static final long MB_IN_GB = 1024;
|
||||
|
||||
protected int appsSubmitted;
|
||||
protected int runningContainers;
|
||||
protected int pendingContainers;
|
||||
@ -60,9 +58,9 @@ public UserMetricsInfo(final ResourceManager rm, final RMContext rmContext,
|
||||
this.runningContainers = userMetrics.getAllocatedContainers();
|
||||
this.pendingContainers = userMetrics.getPendingContainers();
|
||||
this.reservedContainers = userMetrics.getReservedContainers();
|
||||
this.reservedMB = userMetrics.getReservedGB() * MB_IN_GB;
|
||||
this.pendingMB = userMetrics.getPendingGB() * MB_IN_GB;
|
||||
this.allocatedMB = userMetrics.getAllocatedGB() * MB_IN_GB;
|
||||
this.reservedMB = userMetrics.getReservedMB();
|
||||
this.pendingMB = userMetrics.getPendingMB();
|
||||
this.allocatedMB = userMetrics.getAllocatedMB();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -57,16 +57,16 @@ public class TestQueueMetrics {
|
||||
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB));
|
||||
// Available resources is set externally, as it depends on dynamic
|
||||
// configurable cluster/queue resources
|
||||
checkResources(queueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0);
|
||||
checkResources(queueSource, 0, 0, 0, 0, 100*GB, 15*GB, 5, 0, 0);
|
||||
|
||||
metrics.incrAppsRunning(user);
|
||||
checkApps(queueSource, 1, 0, 1, 0, 0, 0);
|
||||
|
||||
metrics.allocateResources(user, 3, Resources.createResource(2*GB));
|
||||
checkResources(queueSource, 6, 3, 3, 0, 100, 9, 2, 0, 0);
|
||||
checkResources(queueSource, 6*GB, 3, 3, 0, 100*GB, 9*GB, 2, 0, 0);
|
||||
|
||||
metrics.releaseResources(user, 1, Resources.createResource(2*GB));
|
||||
checkResources(queueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0);
|
||||
checkResources(queueSource, 4*GB, 2, 3, 1, 100*GB, 9*GB, 2, 0, 0);
|
||||
|
||||
metrics.finishApp(app, RMAppAttemptState.FINISHED);
|
||||
checkApps(queueSource, 1, 0, 0, 1, 0, 0);
|
||||
@ -92,20 +92,20 @@ public class TestQueueMetrics {
|
||||
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB));
|
||||
// Available resources is set externally, as it depends on dynamic
|
||||
// configurable cluster/queue resources
|
||||
checkResources(queueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0);
|
||||
checkResources(userSource, 0, 0, 0, 0, 10, 15, 5, 0, 0);
|
||||
checkResources(queueSource, 0, 0, 0, 0, 100*GB, 15*GB, 5, 0, 0);
|
||||
checkResources(userSource, 0, 0, 0, 0, 10*GB, 15*GB, 5, 0, 0);
|
||||
|
||||
metrics.incrAppsRunning(user);
|
||||
checkApps(queueSource, 1, 0, 1, 0, 0, 0);
|
||||
checkApps(userSource, 1, 0, 1, 0, 0, 0);
|
||||
|
||||
metrics.allocateResources(user, 3, Resources.createResource(2*GB));
|
||||
checkResources(queueSource, 6, 3, 3, 0, 100, 9, 2, 0, 0);
|
||||
checkResources(userSource, 6, 3, 3, 0, 10, 9, 2, 0, 0);
|
||||
checkResources(queueSource, 6*GB, 3, 3, 0, 100*GB, 9*GB, 2, 0, 0);
|
||||
checkResources(userSource, 6*GB, 3, 3, 0, 10*GB, 9*GB, 2, 0, 0);
|
||||
|
||||
metrics.releaseResources(user, 1, Resources.createResource(2*GB));
|
||||
checkResources(queueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0);
|
||||
checkResources(userSource, 4, 2, 3, 1, 10, 9, 2, 0, 0);
|
||||
checkResources(queueSource, 4*GB, 2, 3, 1, 100*GB, 9*GB, 2, 0, 0);
|
||||
checkResources(userSource, 4*GB, 2, 3, 1, 10*GB, 9*GB, 2, 0, 0);
|
||||
|
||||
metrics.finishApp(app, RMAppAttemptState.FINISHED);
|
||||
checkApps(queueSource, 1, 0, 0, 1, 0, 0);
|
||||
@ -141,10 +141,10 @@ public class TestQueueMetrics {
|
||||
parentMetrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB));
|
||||
metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB));
|
||||
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB));
|
||||
checkResources(queueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0);
|
||||
checkResources(parentQueueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0);
|
||||
checkResources(userSource, 0, 0, 0, 0, 10, 15, 5, 0, 0);
|
||||
checkResources(parentUserSource, 0, 0, 0, 0, 10, 15, 5, 0, 0);
|
||||
checkResources(queueSource, 0, 0, 0, 0, 100*GB, 15*GB, 5, 0, 0);
|
||||
checkResources(parentQueueSource, 0, 0, 0, 0, 100*GB, 15*GB, 5, 0, 0);
|
||||
checkResources(userSource, 0, 0, 0, 0, 10*GB, 15*GB, 5, 0, 0);
|
||||
checkResources(parentUserSource, 0, 0, 0, 0, 10*GB, 15*GB, 5, 0, 0);
|
||||
|
||||
metrics.incrAppsRunning(user);
|
||||
checkApps(queueSource, 1, 0, 1, 0, 0, 0);
|
||||
@ -154,17 +154,17 @@ public class TestQueueMetrics {
|
||||
metrics.reserveResource(user, Resources.createResource(3*GB));
|
||||
// Available resources is set externally, as it depends on dynamic
|
||||
// configurable cluster/queue resources
|
||||
checkResources(queueSource, 6, 3, 3, 0, 100, 9, 2, 3, 1);
|
||||
checkResources(parentQueueSource, 6, 3, 3, 0, 100, 9, 2, 3, 1);
|
||||
checkResources(userSource, 6, 3, 3, 0, 10, 9, 2, 3, 1);
|
||||
checkResources(parentUserSource, 6, 3, 3, 0, 10, 9, 2, 3, 1);
|
||||
checkResources(queueSource, 6*GB, 3, 3, 0, 100*GB, 9*GB, 2, 3*GB, 1);
|
||||
checkResources(parentQueueSource, 6*GB, 3, 3, 0, 100*GB, 9*GB, 2, 3*GB, 1);
|
||||
checkResources(userSource, 6*GB, 3, 3, 0, 10*GB, 9*GB, 2, 3*GB, 1);
|
||||
checkResources(parentUserSource, 6*GB, 3, 3, 0, 10*GB, 9*GB, 2, 3*GB, 1);
|
||||
|
||||
metrics.releaseResources(user, 1, Resources.createResource(2*GB));
|
||||
metrics.unreserveResource(user, Resources.createResource(3*GB));
|
||||
checkResources(queueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0);
|
||||
checkResources(parentQueueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0);
|
||||
checkResources(userSource, 4, 2, 3, 1, 10, 9, 2, 0, 0);
|
||||
checkResources(parentUserSource, 4, 2, 3, 1, 10, 9, 2, 0, 0);
|
||||
checkResources(queueSource, 4*GB, 2, 3, 1, 100*GB, 9*GB, 2, 0, 0);
|
||||
checkResources(parentQueueSource, 4*GB, 2, 3, 1, 100*GB, 9*GB, 2, 0, 0);
|
||||
checkResources(userSource, 4*GB, 2, 3, 1, 10*GB, 9*GB, 2, 0, 0);
|
||||
checkResources(parentUserSource, 4*GB, 2, 3, 1, 10*GB, 9*GB, 2, 0, 0);
|
||||
|
||||
metrics.finishApp(app, RMAppAttemptState.FINISHED);
|
||||
checkApps(queueSource, 1, 0, 0, 1, 0, 0);
|
||||
@ -184,18 +184,19 @@ public static void checkApps(MetricsSource source, int submitted, int pending,
|
||||
assertCounter("AppsKilled", killed, rb);
|
||||
}
|
||||
|
||||
public static void checkResources(MetricsSource source, int allocGB,
|
||||
int allocCtnrs, long aggreAllocCtnrs, long aggreReleasedCtnrs, int availGB, int pendingGB, int pendingCtnrs,
|
||||
int reservedGB, int reservedCtnrs) {
|
||||
public static void checkResources(MetricsSource source, int allocatedMB,
|
||||
int allocCtnrs, long aggreAllocCtnrs, long aggreReleasedCtnrs,
|
||||
int availableMB, int pendingMB, int pendingCtnrs,
|
||||
int reservedMB, int reservedCtnrs) {
|
||||
MetricsRecordBuilder rb = getMetrics(source);
|
||||
assertGauge("AllocatedGB", allocGB, rb);
|
||||
assertGauge("AllocatedMB", allocatedMB, rb);
|
||||
assertGauge("AllocatedContainers", allocCtnrs, rb);
|
||||
assertCounter("AggregateContainersAllocated", aggreAllocCtnrs, rb);
|
||||
assertCounter("AggregateContainersReleased", aggreReleasedCtnrs, rb);
|
||||
assertGauge("AvailableGB", availGB, rb);
|
||||
assertGauge("PendingGB", pendingGB, rb);
|
||||
assertGauge("AvailableMB", availableMB, rb);
|
||||
assertGauge("PendingMB", pendingMB, rb);
|
||||
assertGauge("PendingContainers", pendingCtnrs, rb);
|
||||
assertGauge("ReservedGB", reservedGB, rb);
|
||||
assertGauge("ReservedMB", reservedMB, rb);
|
||||
assertGauge("ReservedContainers", reservedCtnrs, rb);
|
||||
}
|
||||
|
||||
|
||||
@ -251,7 +251,7 @@ public void testSingleQueueOneUserMetrics() throws Exception {
|
||||
|
||||
// Only 1 container
|
||||
a.assignContainers(clusterResource, node_0);
|
||||
assertEquals(7, a.getMetrics().getAvailableGB());
|
||||
assertEquals(7*GB, a.getMetrics().getAvailableMB());
|
||||
}
|
||||
|
||||
|
||||
@ -307,9 +307,9 @@ public void testSingleQueueWithOneUser() throws Exception {
|
||||
assertEquals(1*GB, a.getUsedResources().getMemory());
|
||||
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
|
||||
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
|
||||
assertEquals(0, a.getMetrics().getReservedGB());
|
||||
assertEquals(1, a.getMetrics().getAllocatedGB());
|
||||
assertEquals(0, a.getMetrics().getAvailableGB());
|
||||
assertEquals(0*GB, a.getMetrics().getReservedMB());
|
||||
assertEquals(1*GB, a.getMetrics().getAllocatedMB());
|
||||
assertEquals(0*GB, a.getMetrics().getAvailableMB());
|
||||
|
||||
// Also 2nd -> minCapacity = 1024 since (.1 * 8G) < minAlloc, also
|
||||
// you can get one container more than user-limit
|
||||
@ -317,16 +317,16 @@ public void testSingleQueueWithOneUser() throws Exception {
|
||||
assertEquals(2*GB, a.getUsedResources().getMemory());
|
||||
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
|
||||
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
|
||||
assertEquals(0, a.getMetrics().getReservedGB());
|
||||
assertEquals(2, a.getMetrics().getAllocatedGB());
|
||||
assertEquals(0*GB, a.getMetrics().getReservedMB());
|
||||
assertEquals(2*GB, a.getMetrics().getAllocatedMB());
|
||||
|
||||
// Can't allocate 3rd due to user-limit
|
||||
a.assignContainers(clusterResource, node_0);
|
||||
assertEquals(2*GB, a.getUsedResources().getMemory());
|
||||
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
|
||||
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
|
||||
assertEquals(0, a.getMetrics().getReservedGB());
|
||||
assertEquals(2, a.getMetrics().getAllocatedGB());
|
||||
assertEquals(0*GB, a.getMetrics().getReservedMB());
|
||||
assertEquals(2*GB, a.getMetrics().getAllocatedMB());
|
||||
|
||||
// Bump up user-limit-factor, now allocate should work
|
||||
a.setUserLimitFactor(10);
|
||||
@ -334,16 +334,16 @@ public void testSingleQueueWithOneUser() throws Exception {
|
||||
assertEquals(3*GB, a.getUsedResources().getMemory());
|
||||
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
|
||||
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
|
||||
assertEquals(0, a.getMetrics().getReservedGB());
|
||||
assertEquals(3, a.getMetrics().getAllocatedGB());
|
||||
assertEquals(0*GB, a.getMetrics().getReservedMB());
|
||||
assertEquals(3*GB, a.getMetrics().getAllocatedMB());
|
||||
|
||||
// One more should work, for app_1, due to user-limit-factor
|
||||
a.assignContainers(clusterResource, node_0);
|
||||
assertEquals(4*GB, a.getUsedResources().getMemory());
|
||||
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
|
||||
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
|
||||
assertEquals(0, a.getMetrics().getReservedGB());
|
||||
assertEquals(4, a.getMetrics().getAllocatedGB());
|
||||
assertEquals(0*GB, a.getMetrics().getReservedMB());
|
||||
assertEquals(4*GB, a.getMetrics().getAllocatedMB());
|
||||
|
||||
// Test max-capacity
|
||||
// Now - no more allocs since we are at max-cap
|
||||
@ -352,8 +352,8 @@ public void testSingleQueueWithOneUser() throws Exception {
|
||||
assertEquals(4*GB, a.getUsedResources().getMemory());
|
||||
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
|
||||
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
|
||||
assertEquals(0, a.getMetrics().getReservedGB());
|
||||
assertEquals(4, a.getMetrics().getAllocatedGB());
|
||||
assertEquals(0*GB, a.getMetrics().getReservedMB());
|
||||
assertEquals(4*GB, a.getMetrics().getAllocatedMB());
|
||||
|
||||
// Release each container from app_0
|
||||
for (RMContainer rmContainer : app_0.getLiveContainers()) {
|
||||
@ -363,8 +363,8 @@ public void testSingleQueueWithOneUser() throws Exception {
|
||||
assertEquals(1*GB, a.getUsedResources().getMemory());
|
||||
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
|
||||
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
|
||||
assertEquals(0, a.getMetrics().getReservedGB());
|
||||
assertEquals(1, a.getMetrics().getAllocatedGB());
|
||||
assertEquals(0*GB, a.getMetrics().getReservedMB());
|
||||
assertEquals(1*GB, a.getMetrics().getAllocatedMB());
|
||||
|
||||
// Release each container from app_1
|
||||
for (RMContainer rmContainer : app_1.getLiveContainers()) {
|
||||
@ -374,9 +374,9 @@ public void testSingleQueueWithOneUser() throws Exception {
|
||||
assertEquals(0*GB, a.getUsedResources().getMemory());
|
||||
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
|
||||
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
|
||||
assertEquals(0, a.getMetrics().getReservedGB());
|
||||
assertEquals(0, a.getMetrics().getAllocatedGB());
|
||||
assertEquals(1, a.getMetrics().getAvailableGB());
|
||||
assertEquals(0*GB, a.getMetrics().getReservedMB());
|
||||
assertEquals(0*GB, a.getMetrics().getAllocatedMB());
|
||||
assertEquals(1*GB, a.getMetrics().getAvailableMB());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -700,9 +700,9 @@ public void testReservation() throws Exception {
|
||||
assertEquals(1*GB, a.getUsedResources().getMemory());
|
||||
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
|
||||
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
|
||||
assertEquals(0, a.getMetrics().getReservedGB());
|
||||
assertEquals(1, a.getMetrics().getAllocatedGB());
|
||||
assertEquals(0, a.getMetrics().getAvailableGB());
|
||||
assertEquals(0*GB, a.getMetrics().getReservedMB());
|
||||
assertEquals(1*GB, a.getMetrics().getAllocatedMB());
|
||||
assertEquals(0*GB, a.getMetrics().getAvailableMB());
|
||||
|
||||
// Also 2nd -> minCapacity = 1024 since (.1 * 8G) < minAlloc, also
|
||||
// you can get one container more than user-limit
|
||||
@ -710,8 +710,8 @@ public void testReservation() throws Exception {
|
||||
assertEquals(2*GB, a.getUsedResources().getMemory());
|
||||
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
|
||||
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
|
||||
assertEquals(0, a.getMetrics().getReservedGB());
|
||||
assertEquals(2, a.getMetrics().getAllocatedGB());
|
||||
assertEquals(0*GB, a.getMetrics().getReservedMB());
|
||||
assertEquals(2*GB, a.getMetrics().getAllocatedMB());
|
||||
|
||||
// Now, reservation should kick in for app_1
|
||||
a.assignContainers(clusterResource, node_0);
|
||||
@ -720,8 +720,8 @@ public void testReservation() throws Exception {
|
||||
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
|
||||
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
|
||||
assertEquals(2*GB, node_0.getUsedResource().getMemory());
|
||||
assertEquals(4, a.getMetrics().getReservedGB());
|
||||
assertEquals(2, a.getMetrics().getAllocatedGB());
|
||||
assertEquals(4*GB, a.getMetrics().getReservedMB());
|
||||
assertEquals(2*GB, a.getMetrics().getAllocatedMB());
|
||||
|
||||
// Now free 1 container from app_0 i.e. 1G
|
||||
a.completedContainer(clusterResource, app_0, node_0,
|
||||
@ -732,8 +732,8 @@ public void testReservation() throws Exception {
|
||||
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
|
||||
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
|
||||
assertEquals(1*GB, node_0.getUsedResource().getMemory());
|
||||
assertEquals(4, a.getMetrics().getReservedGB());
|
||||
assertEquals(1, a.getMetrics().getAllocatedGB());
|
||||
assertEquals(4*GB, a.getMetrics().getReservedMB());
|
||||
assertEquals(1*GB, a.getMetrics().getAllocatedMB());
|
||||
|
||||
// Now finish another container from app_0 and fulfill the reservation
|
||||
a.completedContainer(clusterResource, app_0, node_0,
|
||||
@ -744,8 +744,8 @@ public void testReservation() throws Exception {
|
||||
assertEquals(4*GB, app_1.getCurrentConsumption().getMemory());
|
||||
assertEquals(0*GB, app_1.getCurrentReservation().getMemory());
|
||||
assertEquals(4*GB, node_0.getUsedResource().getMemory());
|
||||
assertEquals(0, a.getMetrics().getReservedGB());
|
||||
assertEquals(4, a.getMetrics().getAllocatedGB());
|
||||
assertEquals(0*GB, a.getMetrics().getReservedMB());
|
||||
assertEquals(4*GB, a.getMetrics().getAllocatedMB());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@ -398,19 +398,19 @@ public void verifyClusterMetrics(int sub, int reservedMB, int availableMB,
|
||||
ResourceScheduler rs = rm.getResourceScheduler();
|
||||
QueueMetrics metrics = rs.getRootQueueMetrics();
|
||||
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
|
||||
final long MB_IN_GB = 1024;
|
||||
|
||||
long totalMBExpect = (metrics.getReservedGB() * MB_IN_GB)
|
||||
+ (metrics.getAvailableGB() * MB_IN_GB)
|
||||
+ (metrics.getAllocatedGB() * MB_IN_GB);
|
||||
long totalMBExpect =
|
||||
metrics.getReservedMB()+ metrics.getAvailableMB()
|
||||
+ metrics.getAllocatedMB();
|
||||
|
||||
assertEquals("appsSubmitted doesn't match", metrics.getAppsSubmitted(), sub);
|
||||
assertEquals("appsSubmitted doesn't match",
|
||||
metrics.getAppsSubmitted(), sub);
|
||||
assertEquals("reservedMB doesn't match",
|
||||
metrics.getReservedGB() * MB_IN_GB, reservedMB);
|
||||
assertEquals("availableMB doesn't match", metrics.getAvailableGB()
|
||||
* MB_IN_GB, availableMB);
|
||||
assertEquals("allocatedMB doesn't match", metrics.getAllocatedGB()
|
||||
* MB_IN_GB, allocMB);
|
||||
metrics.getReservedMB(), reservedMB);
|
||||
assertEquals("availableMB doesn't match",
|
||||
metrics.getAvailableMB(), availableMB);
|
||||
assertEquals("allocatedMB doesn't match",
|
||||
metrics.getAllocatedMB(), allocMB);
|
||||
assertEquals("containersAllocated doesn't match", 0, containersAlloc);
|
||||
assertEquals("totalMB doesn't match", totalMBExpect, totalMB);
|
||||
assertEquals(
|
||||
|
||||
Loading…
Reference in New Issue
Block a user