big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-3961. Expose pending, running and reserved containers of a queue in REST api and yarn top (adhoot via asuresh)

Browse Source
This commit is contained in:
Arun Suresh 2015-08-05 23:14:14 -07:00
parent cc71ad80e1
commit 154c9d2e42
9 changed files with 141 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -371,6 +371,9 @@ Release 2.8.0 - UNRELEASED
YARN-4004. container-executor should print output of docker logs if the docker
container exits with non-0 exit status. (Varun Vasudev via xgong)
YARN-3961. Expose pending, running and reserved containers of a queue in REST
api and yarn top (adhoot via asuresh)
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueStatistics.java
View File

@ -262,6 +262,42 @@ public static QueueStatistics newInstance(long submitted, long running,
*/
public abstract void setPendingVCores(long pendingVCores);
/**
* Get the number of pending containers.
* @return the number of pending containers.
*/
public abstract long getPendingContainers();
/**
* Set the number of pending containers.
* @param pendingContainers the pending containers.
*/
public abstract void setPendingContainers(long pendingContainers);
/**
* Get the number of allocated containers.
* @return the number of allocated containers.
*/
public abstract long getAllocatedContainers();
/**
* Set the number of allocated containers.
* @param allocatedContainers the allocated containers.
*/
public abstract void setAllocatedContainers(long allocatedContainers);
/**
* Get the number of reserved containers.
* @return the number of reserved containers.
*/
public abstract long getReservedContainers();
/**
* Set the number of reserved containers.
* @param reservedContainers the reserved containers.
*/
public abstract void setReservedContainers(long reservedContainers);
/**
* Get the reserved vcores
*

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -384,6 +384,9 @@ message QueueStatisticsProto {
optional int64 allocatedVCores = 13;
optional int64 pendingVCores = 14;
optional int64 reservedVCores = 15;
optional int64 allocatedContainers = 16;
optional int64 pendingContainers = 17;
optional int64 reservedContainers = 18;
}
message QueueInfoProto {

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
View File

@ -71,7 +71,7 @@ public class TopCLI extends YarnCLI {
private String SET_CURSOR_HOME = "\u001b[H";
private String CHANGE_BACKGROUND = "\u001b[7m";
private String RESET_BACKGROUND = "\u001b[0m";
private String SET_CURSOR_LINE_6_COLUMN_0 = "\u001b[6;0f";
private String SET_CURSOR_LINE_7_COLUMN_0 = "\u001b[7;0f";
// guava cache for getapplications call
protected Cache<GetApplicationsRequest, List<ApplicationReport>>
@ -331,6 +331,9 @@ private static class QueueMetrics {
long allocatedVCores;
long pendingVCores;
long reservedVCores;
long allocatedContainers;
long reservedContainers;
long pendingContainers;
}
private class KeyboardMonitor extends Thread {
@ -596,14 +599,14 @@ protected void setTerminalSequences() throws IOException,
String[] tput_cursor_home = { "tput", "cup", "0", "0" };
String[] tput_clear = { "tput", "clear" };
String[] tput_clear_line = { "tput", "el" };
String[] tput_set_cursor_line_6_column_0 = { "tput", "cup", "5", "0" };
String[] tput_set_cursor_line_7_column_0 = { "tput", "cup", "6", "0" };
String[] tput_change_background = { "tput", "smso" };
String[] tput_reset_background = { "tput", "rmso" };
SET_CURSOR_HOME = getCommandOutput(tput_cursor_home);
CLEAR = getCommandOutput(tput_clear);
CLEAR_LINE = getCommandOutput(tput_clear_line);
SET_CURSOR_LINE_6_COLUMN_0 =
getCommandOutput(tput_set_cursor_line_6_column_0);
SET_CURSOR_LINE_7_COLUMN_0 =
getCommandOutput(tput_set_cursor_line_7_column_0);
CHANGE_BACKGROUND = getCommandOutput(tput_change_background);
RESET_BACKGROUND = getCommandOutput(tput_reset_background);
}
@ -712,6 +715,9 @@ protected QueueMetrics getQueueMetrics() {
queueMetrics.allocatedVCores += stats.getAllocatedVCores();
queueMetrics.pendingVCores += stats.getPendingVCores();
queueMetrics.reservedVCores += stats.getReservedVCores();
queueMetrics.allocatedContainers += stats.getAllocatedContainers();
queueMetrics.pendingContainers += stats.getPendingContainers();
queueMetrics.reservedContainers += stats.getReservedContainers();
}
}
queueMetrics.availableMemoryGB = queueMetrics.availableMemoryGB / 1024;
@ -793,12 +799,18 @@ String getHeader(QueueMetrics queueMetrics, NodesInformation nodes) {
queueMetrics.availableVCores, queueMetrics.allocatedVCores,
queueMetrics.pendingVCores, queueMetrics.reservedVCores), terminalWidth,
true));
ret.append(CLEAR_LINE);
ret.append(limitLineLength(String.format(
"Queue(s) Containers: %d allocated, %d pending, %d reserved%n",
queueMetrics.allocatedContainers, queueMetrics.pendingContainers,
queueMetrics.reservedContainers), terminalWidth, true));
return ret.toString();
}
String getPrintableAppInformation(List<ApplicationInformation> appsInfo) {
StringBuilder ret = new StringBuilder();
int limit = terminalHeight - 8;
int limit = terminalHeight - 9;
List<String> columns = new ArrayList<>();
for (int i = 0; i < limit; ++i) {
ret.append(CLEAR_LINE);
@ -944,7 +956,7 @@ protected void showTopScreen() {
synchronized (lock) {
printHeader(header);
printApps(appsStr);
System.out.print(SET_CURSOR_LINE_6_COLUMN_0);
System.out.print(SET_CURSOR_LINE_7_COLUMN_0);
System.out.print(CLEAR_LINE);
}
}

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueStatisticsPBImpl.java
View File

@ -254,4 +254,40 @@ public void setReservedVCores(long reservedVCores) {
maybeInitBuilder();
builder.setReservedVCores(reservedVCores);
}
@Override
public long getPendingContainers() {
QueueStatisticsProtoOrBuilder p = viaProto ? proto : builder;
return (p.hasPendingContainers()) ? p.getPendingContainers() : -1;
}
@Override
public void setPendingContainers(long pendingContainers) {
maybeInitBuilder();
builder.setPendingContainers(pendingContainers);
}
@Override
public long getAllocatedContainers() {
QueueStatisticsProtoOrBuilder p = viaProto ? proto : builder;
return (p.hasAllocatedContainers()) ? p.getAllocatedContainers() : -1;
}
@Override
public void setAllocatedContainers(long allocatedContainers) {
maybeInitBuilder();
builder.setAllocatedContainers(allocatedContainers);
}
@Override
public long getReservedContainers() {
QueueStatisticsProtoOrBuilder p = viaProto ? proto : builder;
return (p.hasReservedContainers()) ? p.getReservedContainers() : -1;
}
@Override
public void setReservedContainers(long reservedContainers) {
maybeInitBuilder();
builder.setReservedContainers(reservedContainers);
}
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
View File

@ -323,6 +323,9 @@ public QueueStatistics getQueueStatistics() {
stats.setAllocatedVCores(getMetrics().getAllocatedVirtualCores());
stats.setPendingVCores(getMetrics().getPendingVirtualCores());
stats.setReservedVCores(getMetrics().getReservedVirtualCores());
stats.setPendingContainers(getMetrics().getPendingContainers());
stats.setAllocatedContainers(getMetrics().getAllocatedContainers());
stats.setReservedContainers(getMetrics().getReservedContainers());
return stats;
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
View File

@ -57,6 +57,9 @@ public class CapacitySchedulerQueueInfo {
protected ResourceInfo resourcesUsed;
private boolean hideReservationQueues = false;
protected ArrayList<String> nodeLabels = new ArrayList<String>();
protected long allocatedContainers;
protected long reservedContainers;
protected long pendingContainers;
CapacitySchedulerQueueInfo() {
};
@ -81,6 +84,9 @@ public class CapacitySchedulerQueueInfo {
absoluteUsedCapacity =
cap(qCapacities.getAbsoluteUsedCapacity(nodeLabel), 0f, 1f) * 100;
numApplications = q.getNumApplications();
allocatedContainers = q.getMetrics().getAllocatedContainers();
pendingContainers = q.getMetrics().getPendingContainers();
reservedContainers = q.getMetrics().getReservedContainers();
queueName = q.getQueueName();
state = q.getState();
resourcesUsed = new ResourceInfo(queueResourceUsage.getUsed(nodeLabel));
@ -124,6 +130,18 @@ public int getNumApplications() {
return numApplications;
}
public long getAllocatedContainers() {
return allocatedContainers;
}
public long getReservedContainers() {
return reservedContainers;
}
public long getPendingContainers() {
return pendingContainers;
}
public String getQueueName() {
return this.queueName;
}

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
View File

@ -57,7 +57,11 @@ public class FairSchedulerQueueInfo {
private ResourceInfo steadyFairResources;
private ResourceInfo fairResources;
private ResourceInfo clusterResources;
private long pendingContainers;
private long allocatedContainers;
private long reservedContainers;
private String queueName;
private String schedulingPolicy;
@ -95,6 +99,10 @@ public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) {
maxApps = allocConf.getQueueMaxApps(queueName);
pendingContainers = queue.getMetrics().getPendingContainers();
allocatedContainers = queue.getMetrics().getAllocatedContainers();
reservedContainers = queue.getMetrics().getReservedContainers();
if (allocConf.isReservable(queueName) &&
!allocConf.getShowReservationAsQueues(queueName)) {
return;
@ -103,6 +111,18 @@ public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) {
childQueues = getChildQueues(queue, scheduler);
}
public long getPendingContainers() {
return pendingContainers;
}
public long getAllocatedContainers() {
return allocatedContainers;
}
public long getReservedContainers() {
return reservedContainers;
}
protected FairSchedulerQueueInfoList getChildQueues(FSQueue queue,
FairScheduler scheduler) {
// Return null to omit 'childQueues' field from the return value of
@ -172,7 +192,7 @@ public String getQueueName() {
public ResourceInfo getUsedResources() {
return usedResources;
}
/**
* Returns the queue's min share in as a fraction of the entire
* cluster capacity.

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -349,10 +349,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity,
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
int numExpectedElements = 13;
int numExpectedElements = 16;
boolean isParentQueue = true;
if (!info.has("queues")) {
numExpectedElements = 25;
numExpectedElements = 28;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());