big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2072. RM/NM UIs and webservices are missing vcore information. (Nathan Roberts via tgraves)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1605162 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Thomas Graves 2014-06-24 19:34:34 +00:00
parent fdbff67856
commit 1f9a0fd927
23 changed files with 252 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -183,6 +183,9 @@ Release 2.5.0 - UNRELEASED
YARN-2109. Fix TestRM to work with both schedulers. (Anubhav Dhoot via kasha)
YARN-2072. RM/NM UIs and webservices are missing vcore information.
(Nathan Roberts via tgraves)
OPTIMIZATIONS
BUG FIXES

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java
View File

@ -27,4 +27,6 @@ public interface ResourceView {
long getPmemAllocatedForContainers();
boolean isPmemCheckEnabled();
long getVCoresAllocatedForContainers();
}

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
View File

@ -70,6 +70,8 @@ public class ContainersMonitorImpl extends AbstractService implements
private boolean pmemCheckEnabled;
private boolean vmemCheckEnabled;
private long maxVCoresAllottedForContainers;
private static final long UNKNOWN_MEMORY_LIMIT = -1L;
public ContainersMonitorImpl(ContainerExecutor exec,
@ -108,10 +110,16 @@ protected void serviceInit(Configuration conf) throws Exception {
YarnConfiguration.NM_PMEM_MB,
YarnConfiguration.DEFAULT_NM_PMEM_MB) * 1024 * 1024l;
long configuredVCoresForContainers = conf.getLong(
YarnConfiguration.NM_VCORES,
YarnConfiguration.DEFAULT_NM_VCORES);
// Setting these irrespective of whether checks are enabled. Required in
// the UI.
// ///////// Physical memory configuration //////
this.maxPmemAllottedForContainers = configuredPMemForContainers;
this.maxVCoresAllottedForContainers = configuredVCoresForContainers;
// ///////// Virtual memory configuration //////
float vmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO,
@ -518,6 +526,11 @@ public long getPmemAllocatedForContainers() {
return this.maxPmemAllottedForContainers;
}
@Override
public long getVCoresAllocatedForContainers() {
return this.maxVCoresAllottedForContainers;
}
/**
* Is the total virtual memory check enabled?
*

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
View File

@ -40,6 +40,9 @@ public class NodeManagerMetrics {
@Metric("Current # of allocated containers")
MutableGaugeInt allocatedContainers;
@Metric MutableGaugeInt availableGB;
@Metric("Current allocated Virtual Cores")
MutableGaugeInt allocatedVCores;
@Metric MutableGaugeInt availableVCores;
public static NodeManagerMetrics create() {
return create(DefaultMetricsSystem.instance());
@ -88,16 +91,21 @@ public void allocateContainer(Resource res) {
allocatedContainers.incr();
allocatedGB.incr(res.getMemory() / 1024);
availableGB.decr(res.getMemory() / 1024);
allocatedVCores.incr(res.getVirtualCores());
availableVCores.decr(res.getVirtualCores());
}
public void releaseContainer(Resource res) {
allocatedContainers.decr();
allocatedGB.decr(res.getMemory() / 1024);
availableGB.incr(res.getMemory() / 1024);
allocatedVCores.decr(res.getVirtualCores());
availableVCores.incr(res.getVirtualCores());
}
public void addResource(Resource res) {
availableGB.incr(res.getMemory() / 1024);
availableVCores.incr(res.getVirtualCores());
}
public int getRunningContainers() {

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
View File

@ -85,6 +85,7 @@ protected void render(Block html) {
._("Diagnostics", info.getDiagnostics())
._("User", info.getUser())
._("TotalMemoryNeeded", info.getMemoryNeeded())
._("TotalVCoresNeeded", info.getVCoresNeeded())
._("logs", info.getShortLogLink(), "Link to logs");
html._(InfoBlock.class);
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
View File

@ -73,6 +73,8 @@ protected void render(Block html) {
StringUtils.byteDesc(info.getTotalPmemAllocated() * BYTES_IN_MB))
._("Pmem enforcement enabled",
info.isVmemCheckEnabled())
._("Total VCores allocated for Containers",
String.valueOf(info.getTotalVCoresAllocated()))
._("NodeHealthyStatus",
info.getHealthStatus())
._("LastNodeHealthTime", new Date(

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
View File

@ -42,6 +42,7 @@ public class ContainerInfo {
protected String diagnostics;
protected String user;
protected long totalMemoryNeededMB;
protected long totalVCoresNeeded;
protected String containerLogsLink;
protected String nodeId;
@XmlTransient
@ -76,6 +77,7 @@ public ContainerInfo(final Context nmContext, final Container container,
Resource res = container.getResource();
if (res != null) {
this.totalMemoryNeededMB = res.getMemory();
this.totalVCoresNeeded = res.getVirtualCores();
}
this.containerLogsShortLink = ujoin("containerlogs", this.id,
container.getUser());
@ -130,4 +132,8 @@ public long getMemoryNeeded() {
return this.totalMemoryNeededMB;
}
public long getVCoresNeeded() {
return this.totalVCoresNeeded;
}
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java
View File

@ -36,6 +36,7 @@ public class NodeInfo {
protected String healthReport;
protected long totalVmemAllocatedContainersMB;
protected long totalPmemAllocatedContainersMB;
protected long totalVCoresAllocatedContainers;
protected boolean vmemCheckEnabled;
protected boolean pmemCheckEnabled;
protected long lastNodeUpdateTime;
@ -62,6 +63,8 @@ public NodeInfo(final Context context, final ResourceView resourceView) {
this.totalPmemAllocatedContainersMB = resourceView
.getPmemAllocatedForContainers() / BYTES_IN_MB;
this.pmemCheckEnabled = resourceView.isPmemCheckEnabled();
this.totalVCoresAllocatedContainers = resourceView
.getVCoresAllocatedForContainers();
this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy();
this.lastNodeUpdateTime = context.getNodeHealthStatus()
.getLastHealthReportTime();
@ -124,6 +127,10 @@ public long getTotalVmemAllocated() {
return this.totalVmemAllocatedContainersMB;
}
public long getTotalVCoresAllocated() {
return this.totalVCoresAllocatedContainers;
}
public boolean isVmemCheckEnabled() {
return this.vmemCheckEnabled;
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
View File

@ -31,8 +31,11 @@ public class TestNodeManagerMetrics {
NodeManagerMetrics metrics = NodeManagerMetrics.create();
Resource total = Records.newRecord(Resource.class);
total.setMemory(8*GiB);
total.setVirtualCores(16);
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(1*GiB);
resource.setVirtualCores(2);
metrics.addResource(total);
@ -57,12 +60,12 @@ public class TestNodeManagerMetrics {
metrics.initingContainer();
metrics.runningContainer();
checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6);
checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6, 4, 12);
}
private void checkMetrics(int launched, int completed, int failed, int killed,
int initing, int running, int allocatedGB,
int allocatedContainers, int availableGB) {
int allocatedContainers, int availableGB, int allocatedVCores, int availableVCores) {
MetricsRecordBuilder rb = getMetrics("NodeManagerMetrics");
assertCounter("ContainersLaunched", launched, rb);
assertCounter("ContainersCompleted", completed, rb);
@ -71,7 +74,10 @@ private void checkMetrics(int launched, int completed, int failed, int killed,
assertGauge("ContainersIniting", initing, rb);
assertGauge("ContainersRunning", running, rb);
assertGauge("AllocatedGB", allocatedGB, rb);
assertGauge("AllocatedVCores", allocatedVCores, rb);
assertGauge("AllocatedContainers", allocatedContainers, rb);
assertGauge("AvailableGB", availableGB, rb);
assertGauge("AvailableVCores",availableVCores, rb);
}
}

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
View File

@ -91,6 +91,10 @@ public long getPmemAllocatedForContainers() {
return 0;
}
@Override
public long getVCoresAllocatedForContainers() {
return 0;
}
@Override
public boolean isVmemCheckEnabled() {
return true;
}
@ -150,6 +154,10 @@ public long getPmemAllocatedForContainers() {
return 0;
}
@Override
public long getVCoresAllocatedForContainers() {
return 0;
}
@Override
public boolean isVmemCheckEnabled() {
return true;
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
View File

@ -124,6 +124,10 @@ public long getPmemAllocatedForContainers() {
return new Long("17179869184");
}
@Override
public long getVCoresAllocatedForContainers() {
return new Long("4000");
}
@Override
public boolean isVmemCheckEnabled() {
return true;
}
@ -375,6 +379,8 @@ public void verifyNodesXML(NodeList nodes) throws JSONException, Exception {
"totalVmemAllocatedContainersMB"),
WebServicesTestUtils.getXmlLong(element,
"totalPmemAllocatedContainersMB"),
WebServicesTestUtils.getXmlLong(element,
"totalVCoresAllocatedContainers"),
WebServicesTestUtils.getXmlBoolean(element, "vmemCheckEnabled"),
WebServicesTestUtils.getXmlBoolean(element, "pmemCheckEnabled"),
WebServicesTestUtils.getXmlLong(element, "lastNodeUpdateTime"),
@ -393,10 +399,11 @@ public void verifyNodesXML(NodeList nodes) throws JSONException, Exception {
public void verifyNodeInfo(JSONObject json) throws JSONException, Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("nodeInfo");
assertEquals("incorrect number of elements", 15, info.length());
assertEquals("incorrect number of elements", 16, info.length());
verifyNodeInfoGeneric(info.getString("id"), info.getString("healthReport"),
info.getLong("totalVmemAllocatedContainersMB"),
info.getLong("totalPmemAllocatedContainersMB"),
info.getLong("totalVCoresAllocatedContainers"),
info.getBoolean("vmemCheckEnabled"),
info.getBoolean("pmemCheckEnabled"),
info.getLong("lastNodeUpdateTime"), info.getBoolean("nodeHealthy"),
@ -410,6 +417,7 @@ public void verifyNodeInfo(JSONObject json) throws JSONException, Exception {
public void verifyNodeInfoGeneric(String id, String healthReport,
long totalVmemAllocatedContainersMB, long totalPmemAllocatedContainersMB,
long totalVCoresAllocatedContainers,
boolean vmemCheckEnabled, boolean pmemCheckEnabled,
long lastNodeUpdateTime, Boolean nodeHealthy, String nodeHostName,
String hadoopVersionBuiltOn, String hadoopBuildVersion,
@ -423,6 +431,8 @@ public void verifyNodeInfoGeneric(String id, String healthReport,
totalVmemAllocatedContainersMB);
assertEquals("totalPmemAllocatedContainersMB incorrect", 16384,
totalPmemAllocatedContainersMB);
assertEquals("totalVCoresAllocatedContainers incorrect", 4000,
totalVCoresAllocatedContainers);
assertEquals("vmemCheckEnabled incorrect", true, vmemCheckEnabled);
assertEquals("pmemCheckEnabled incorrect", true, pmemCheckEnabled);
assertTrue("lastNodeUpdateTime incorrect", lastNodeUpdateTime == nmContext

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
View File

@ -116,6 +116,12 @@ public long getPmemAllocatedForContainers() {
return new Long("17179869184");
}
@Override
public long getVCoresAllocatedForContainers() {
return new Long("4000");
}
@Override
public boolean isVmemCheckEnabled() {
return true;

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
View File

@ -106,6 +106,11 @@ public long getPmemAllocatedForContainers() {
return new Long("17179869184");
}
@Override
public long getVCoresAllocatedForContainers() {
return new Long("4000");
}
@Override
public boolean isVmemCheckEnabled() {
return true;
@ -461,24 +466,27 @@ public void verifyContainersInfoXML(NodeList nodes, Container cont)
WebServicesTestUtils.getXmlString(element, "diagnostics"),
WebServicesTestUtils.getXmlString(element, "nodeId"),
WebServicesTestUtils.getXmlInt(element, "totalMemoryNeededMB"),
WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"),
WebServicesTestUtils.getXmlString(element, "containerLogsLink"));
}
}
public void verifyNodeContainerInfo(JSONObject info, Container cont)
throws JSONException, Exception {
assertEquals("incorrect number of elements", 8, info.length());
assertEquals("incorrect number of elements", 9, info.length());
verifyNodeContainerInfoGeneric(cont, info.getString("id"),
info.getString("state"), info.getString("user"),
info.getInt("exitCode"), info.getString("diagnostics"),
info.getString("nodeId"), info.getInt("totalMemoryNeededMB"),
info.getInt("totalVCoresNeeded"),
info.getString("containerLogsLink"));
}
public void verifyNodeContainerInfoGeneric(Container cont, String id,
String state, String user, int exitCode, String diagnostics,
String nodeId, int totalMemoryNeededMB, String logsLink)
String nodeId, int totalMemoryNeededMB, int totalVCoresNeeded,
String logsLink)
throws JSONException, Exception {
WebServicesTestUtils.checkStringMatch("id", cont.getContainerId()
.toString(), id);
@ -495,6 +503,9 @@ public void verifyNodeContainerInfoGeneric(Container cont, String id,
assertEquals("totalMemoryNeededMB wrong",
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
totalMemoryNeededMB);
assertEquals("totalVCoresNeeded wrong",
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
totalVCoresNeeded);
String shortLink =
ujoin("containerlogs", cont.getContainerId().toString(),
cont.getUser());

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java
View File

@ -72,6 +72,9 @@ protected void render(Block html) {
th().$class("ui-state-default")._("Memory Used")._().
th().$class("ui-state-default")._("Memory Total")._().
th().$class("ui-state-default")._("Memory Reserved")._().
th().$class("ui-state-default")._("VCores Used")._().
th().$class("ui-state-default")._("VCores Total")._().
th().$class("ui-state-default")._("VCores Reserved")._().
th().$class("ui-state-default")._("Active Nodes")._().
th().$class("ui-state-default")._("Decommissioned Nodes")._().
th().$class("ui-state-default")._("Lost Nodes")._().
@ -94,6 +97,9 @@ protected void render(Block html) {
td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)).
td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())).
td(String.valueOf(clusterMetrics.getTotalVirtualCores())).
td(String.valueOf(clusterMetrics.getReservedVirtualCores())).
td().a(url("nodes"),String.valueOf(clusterMetrics.getActiveNodes()))._().
td().a(url("nodes/decommissioned"),String.valueOf(clusterMetrics.getDecommissionedNodes()))._().
td().a(url("nodes/lost"),String.valueOf(clusterMetrics.getLostNodes()))._().
@ -120,6 +126,9 @@ protected void render(Block html) {
th().$class("ui-state-default")._("Memory Used")._().
th().$class("ui-state-default")._("Memory Pending")._().
th().$class("ui-state-default")._("Memory Reserved")._().
th().$class("ui-state-default")._("VCores Used")._().
th().$class("ui-state-default")._("VCores Pending")._().
th().$class("ui-state-default")._("VCores Reserved")._().
_().
_().
tbody().$class("ui-widget-content").
@ -139,6 +148,9 @@ protected void render(Block html) {
td(StringUtils.byteDesc(userMetrics.getAllocatedMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(userMetrics.getPendingMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(userMetrics.getReservedMB() * BYTES_IN_MB)).
td(String.valueOf(userMetrics.getAllocatedVirtualCores())).
td(String.valueOf(userMetrics.getPendingVirtualCores())).
td(String.valueOf(userMetrics.getReservedVirtualCores())).
_().
_()._();

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
View File

@ -75,6 +75,8 @@ protected void render(Block html) {
th(".containers", "Containers").
th(".mem", "Mem Used").
th(".mem", "Mem Avail").
th(".vcores", "VCores Used").
th(".vcores", "VCores Avail").
th(".nodeManagerVersion", "Version").
_()._().
tbody();
@ -127,8 +129,10 @@ protected void render(Block html) {
td(String.valueOf(info.getNumContainers())).
td().br().$title(String.valueOf(usedMemory))._().
_(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
td().br().$title(String.valueOf(usedMemory))._().
td().br().$title(String.valueOf(availableMemory))._().
_(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
td(String.valueOf(info.getUsedVirtualCores())).
td(String.valueOf(info.getAvailableVirtualCores())).
td(ni.getNodeManagerVersion()).
_();
}

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
View File

@ -37,16 +37,21 @@ public class ClusterMetricsInfo {
protected int appsRunning;
protected int appsFailed;
protected int appsKilled;
protected long reservedMB;
protected long availableMB;
protected long allocatedMB;
protected long reservedVirtualCores;
protected long availableVirtualCores;
protected long allocatedVirtualCores;
protected int containersAllocated;
protected int containersReserved;
protected int containersPending;
protected long totalMB;
protected long totalVirtualCores;
protected int totalNodes;
protected int lostNodes;
protected int unhealthyNodes;
@ -68,16 +73,21 @@ public ClusterMetricsInfo(final ResourceManager rm, final RMContext rmContext) {
this.appsRunning = metrics.getAppsRunning();
this.appsFailed = metrics.getAppsFailed();
this.appsKilled = metrics.getAppsKilled();
this.reservedMB = metrics.getReservedMB();
this.availableMB = metrics.getAvailableMB();
this.allocatedMB = metrics.getAllocatedMB();
this.reservedVirtualCores = metrics.getReservedVirtualCores();
this.availableVirtualCores = metrics.getAvailableVirtualCores();
this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();
this.containersAllocated = metrics.getAllocatedContainers();
this.containersPending = metrics.getPendingContainers();
this.containersReserved = metrics.getReservedContainers();
this.totalMB = availableMB + allocatedMB;
this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
this.activeNodes = clusterMetrics.getNumActiveNMs();
this.lostNodes = clusterMetrics.getNumLostNMs();
this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
@ -123,6 +133,18 @@ public long getAllocatedMB() {
return this.allocatedMB;
}
public long getReservedVirtualCores() {
return this.reservedVirtualCores;
}
public long getAvailableVirtualCores() {
return this.availableVirtualCores;
}
public long getAllocatedVirtualCores() {
return this.allocatedVirtualCores;
}
public int getContainersAllocated() {
return this.containersAllocated;
}
@ -134,15 +156,19 @@ public int getReservedContainers() {
public int getPendingContainers() {
return this.containersPending;
}
public long getTotalMB() {
return this.totalMB;
}
public long getTotalVirtualCores() {
return this.totalVirtualCores;
}
public int getTotalNodes() {
return this.totalNodes;
}
public int getActiveNodes() {
return this.activeNodes;
}

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
View File

@ -43,6 +43,8 @@ public class NodeInfo {
protected int numContainers;
protected long usedMemoryMB;
protected long availMemoryMB;
protected long usedVirtualCores;
protected long availableVirtualCores;
public NodeInfo() {
} // JAXB needs this
@ -57,6 +59,8 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) {
this.numContainers = report.getNumContainers();
this.usedMemoryMB = report.getUsedResource().getMemory();
this.availMemoryMB = report.getAvailableResource().getMemory();
this.usedVirtualCores = report.getUsedResource().getVirtualCores();
this.availableVirtualCores = report.getAvailableResource().getVirtualCores();
}
this.id = id.toString();
this.rack = ni.getRackName();
@ -83,7 +87,7 @@ public String getNodeId() {
public String getNodeHTTPAddress() {
return this.nodeHTTPAddress;
}
public void setNodeHTTPAddress(String nodeHTTPAddress) {
this.nodeHTTPAddress = nodeHTTPAddress;
}
@ -112,4 +116,12 @@ public long getAvailableMemory() {
return this.availMemoryMB;
}
public long getUsedVirtualCores() {
return this.usedVirtualCores;
}
public long getAvailableVirtualCores() {
return this.availableVirtualCores;
}
}

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java
View File

@ -43,6 +43,9 @@ public class UserMetricsInfo {
protected long reservedMB;
protected long pendingMB;
protected long allocatedMB;
protected long reservedVirtualCores;
protected long pendingVirtualCores;
protected long allocatedVirtualCores;
@XmlTransient
protected boolean userMetricsAvailable;
@ -59,7 +62,7 @@ public UserMetricsInfo(final ResourceManager rm, final RMContext rmContext,
if (userMetrics != null) {
this.userMetricsAvailable = true;
this.appsSubmitted = userMetrics.getAppsSubmitted();
this.appsCompleted = metrics.getAppsCompleted();
this.appsPending = metrics.getAppsPending();
@ -70,10 +73,14 @@ public UserMetricsInfo(final ResourceManager rm, final RMContext rmContext,
this.runningContainers = userMetrics.getAllocatedContainers();
this.pendingContainers = userMetrics.getPendingContainers();
this.reservedContainers = userMetrics.getReservedContainers();
this.reservedMB = userMetrics.getReservedMB();
this.pendingMB = userMetrics.getPendingMB();
this.allocatedMB = userMetrics.getAllocatedMB();
this.reservedVirtualCores = userMetrics.getReservedVirtualCores();
this.pendingVirtualCores = userMetrics.getPendingVirtualCores();
this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores();
}
}
@ -117,6 +124,18 @@ public long getPendingMB() {
return this.pendingMB;
}
public long getReservedVirtualCores() {
return this.reservedVirtualCores;
}
public long getAllocatedVirtualCores() {
return this.allocatedVirtualCores;
}
public long getPendingVirtualCores() {
return this.pendingVirtualCores;
}
public int getReservedContainers() {
return this.reservedContainers;
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
View File

@ -48,8 +48,8 @@ public class TestNodesPage {
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
// future. In that case this value should be adjusted to the new value.
final int numberOfThInMetricsTable = 13;
final int numberOfActualTableHeaders = 10;
final int numberOfThInMetricsTable = 16;
final int numberOfActualTableHeaders = 12;
private Injector injector;

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
View File

@ -389,6 +389,10 @@ public void verifyClusterMetricsXML(String xml) throws JSONException,
WebServicesTestUtils.getXmlInt(element, "reservedMB"),
WebServicesTestUtils.getXmlInt(element, "availableMB"),
WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
WebServicesTestUtils.getXmlInt(element, "reservedVirtualCores"),
WebServicesTestUtils.getXmlInt(element, "availableVirtualCores"),
WebServicesTestUtils.getXmlInt(element, "allocatedVirtualCores"),
WebServicesTestUtils.getXmlInt(element, "totalVirtualCores"),
WebServicesTestUtils.getXmlInt(element, "containersAllocated"),
WebServicesTestUtils.getXmlInt(element, "totalMB"),
WebServicesTestUtils.getXmlInt(element, "totalNodes"),
@ -404,11 +408,13 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException,
Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
assertEquals("incorrect number of elements", 19, clusterinfo.length());
assertEquals("incorrect number of elements", 23, clusterinfo.length());
verifyClusterMetrics(
clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),
clusterinfo.getInt("allocatedMB"),
clusterinfo.getInt("reservedVirtualCores"), clusterinfo.getInt("availableVirtualCores"),
clusterinfo.getInt("allocatedVirtualCores"), clusterinfo.getInt("totalVirtualCores"),
clusterinfo.getInt("containersAllocated"),
clusterinfo.getInt("totalMB"), clusterinfo.getInt("totalNodes"),
clusterinfo.getInt("lostNodes"), clusterinfo.getInt("unhealthyNodes"),
@ -418,7 +424,9 @@ public void verifyClusterMetricsJSON(JSONObject json) throws JSONException,
public void verifyClusterMetrics(int submittedApps, int completedApps,
int reservedMB, int availableMB,
int allocMB, int containersAlloc, int totalMB, int totalNodes,
int allocMB, int reservedVirtualCores, int availableVirtualCores,
int allocVirtualCores, int totalVirtualCores,
int containersAlloc, int totalMB, int totalNodes,
int lostNodes, int unhealthyNodes, int decommissionedNodes,
int rebootedNodes, int activeNodes) throws JSONException, Exception {
@ -428,7 +436,8 @@ public void verifyClusterMetrics(int submittedApps, int completedApps,
long totalMBExpect =
metrics.getAvailableMB() + metrics.getAllocatedMB();
long totalVirtualCoresExpect =
metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores();
assertEquals("appsSubmitted doesn't match",
metrics.getAppsSubmitted(), submittedApps);
assertEquals("appsCompleted doesn't match",
@ -439,6 +448,12 @@ public void verifyClusterMetrics(int submittedApps, int completedApps,
metrics.getAvailableMB(), availableMB);
assertEquals("allocatedMB doesn't match",
metrics.getAllocatedMB(), allocMB);
assertEquals("reservedVirtualCores doesn't match",
metrics.getReservedVirtualCores(), reservedVirtualCores);
assertEquals("availableVirtualCores doesn't match",
metrics.getAvailableVirtualCores(), availableVirtualCores);
assertEquals("allocatedVirtualCores doesn't match",
totalVirtualCoresExpect, allocVirtualCores);
assertEquals("containersAllocated doesn't match", 0, containersAlloc);
assertEquals("totalMB doesn't match", totalMBExpect, totalMB);
assertEquals(

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
View File

@ -656,13 +656,15 @@ public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException,
WebServicesTestUtils.getXmlInt(element, "numContainers"),
WebServicesTestUtils.getXmlLong(element, "usedMemoryMB"),
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"),
WebServicesTestUtils.getXmlLong(element, "usedVirtualCores"),
WebServicesTestUtils.getXmlLong(element, "availableVirtualCores"),
WebServicesTestUtils.getXmlString(element, "version"));
}
}
public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
throws JSONException, Exception {
assertEquals("incorrect number of elements", 11, nodeInfo.length());
assertEquals("incorrect number of elements", 13, nodeInfo.length());
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
nodeInfo.getString("rack"),
@ -671,6 +673,7 @@ public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
nodeInfo.getLong("lastHealthUpdate"),
nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"),
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"),
nodeInfo.getLong("usedVirtualCores"), nodeInfo.getLong("availableVirtualCores"),
nodeInfo.getString("version"));
}
@ -678,7 +681,8 @@ public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
String id, String nodeHostName,
String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
int numContainers, long usedMemoryMB, long availMemoryMB, String version)
int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores,
long availVirtualCores, String version)
throws JSONException, Exception {
RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
@ -712,6 +716,10 @@ public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
.getUsedResource().getMemory(), usedMemoryMB);
assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report
.getAvailableResource().getMemory(), availMemoryMB);
assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report
.getUsedResource().getVirtualCores(), usedVirtualCores);
assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
.getAvailableResource().getVirtualCores(), availVirtualCores);
}
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm
View File

@ -62,6 +62,8 @@ NodeManager REST API's.
*---------------+--------------+-------------------------------+
| totalVmemAllocatedContainersMB | long | The amount of virtual memory allocated for use by containers in MB |
*---------------+--------------+-------------------------------+
| totalVCoresAllocatedContainers | long | The number of virtual cores allocated for use by containers |
*---------------+--------------+-------------------------------+
| lastNodeUpdateTime | long | The last timestamp at which the health report was received (in ms since epoch)|
*---------------+--------------+-------------------------------+
| healthReport | string | The diagnostic health report of the node |
@ -109,6 +111,7 @@ NodeManager REST API's.
"nodeManagerBuildVersion" : "0.23.1-SNAPSHOT from 1228355 by user1 source checksum 20647f76c36430e888cc7204826a445c",
"lastNodeUpdateTime" : 1326222266126,
"totalVmemAllocatedContainersMB" : 17203,
"totalVCoresAllocatedContainers" : 8,
"nodeHealthy" : true,
"healthReport" : "",
"totalPmemAllocatedContainersMB" : 8192,
@ -148,6 +151,7 @@ NodeManager REST API's.
<healthReport/>
<totalVmemAllocatedContainersMB>17203</totalVmemAllocatedContainersMB>
<totalPmemAllocatedContainersMB>8192</totalPmemAllocatedContainersMB>
<totalVCoresAllocatedContainers>8</totalVCoresAllocatedContainers>
<lastNodeUpdateTime>1326222386134</lastNodeUpdateTime>
<nodeHealthy>true</nodeHealthy>
<nodeManagerVersion>0.23.1-SNAPSHOT</nodeManagerVersion>
@ -446,6 +450,7 @@ NodeManager REST API's.
{
"nodeId" : "host.domain.com:8041",
"totalMemoryNeededMB" : 2048,
"totalVCoresNeeded" : 1,
"state" : "RUNNING",
"diagnostics" : "",
"containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000001/user1",
@ -456,6 +461,7 @@ NodeManager REST API's.
{
"nodeId" : "host.domain.com:8041",
"totalMemoryNeededMB" : 2048,
"totalVCoresNeeded" : 2,
"state" : "RUNNING",
"diagnostics" : "",
"containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000003/user1",
@ -498,6 +504,7 @@ NodeManager REST API's.
<diagnostics/>
<user>user1</user>
<totalMemoryNeededMB>2048</totalMemoryNeededMB>
<totalVCoresNeeded>1</totalVCoresNeeded>
<containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000001/user1</containerLogsLink>
<nodeId>host.domain.com:8041</nodeId>
</container>
@ -508,6 +515,7 @@ NodeManager REST API's.
<diagnostics>Container killed by the ApplicationMaster.</diagnostics>
<user>user1</user>
<totalMemoryNeededMB>2048</totalMemoryNeededMB>
<totalVCoresNeeded>2</totalVCoresNeeded>
<containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000003/user1</containerLogsLink>
<nodeId>host.domain.com:8041</nodeId>
</container>
@ -560,6 +568,8 @@ NodeManager REST API's.
*---------------+--------------+-------------------------------+
| totalMemoryNeededMB | long | Total amout of memory needed by the container (in MB) |
*---------------+--------------+-------------------------------+
| totalVCoresNeeded | long | Total number of virtual cores needed by the container |
*---------------+--------------+-------------------------------+
** Response Examples
@ -587,6 +597,7 @@ NodeManager REST API's.
"container" : {
"nodeId" : "host.domain.com:8041",
"totalMemoryNeededMB" : 2048,
"totalVCoresNeeded" : 1,
"state" : "RUNNING",
"diagnostics" : "",
"containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0007_01_000001/user1",
@ -626,6 +637,7 @@ NodeManager REST API's.
<diagnostics/>
<user>user1</user>
<totalMemoryNeededMB>2048</totalMemoryNeededMB>
<totalVCoresNeeded>1</totalVCoresNeeded>
<containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0007_01_000001/user1</containerLogsLink>
<nodeId>host.domain.com:8041</nodeId>
</container>

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
View File

@ -195,6 +195,14 @@ ResourceManager REST API's.
*---------------+--------------+-------------------------------+
| totalMB | long | The amount of total memory in MB |
*---------------+--------------+-------------------------------+
| reservedVirtualCores | long | The number of reserved virtual cores |
*---------------+--------------+-------------------------------+
| availableVirtualCores | long | The number of available virtual cores |
*---------------+--------------+-------------------------------+
| allocatedVirtualCores | long | The number of allocated virtual cores |
*---------------+--------------+-------------------------------+
| totalVirtualCores | long | The total number of virtual cores |
*---------------+--------------+-------------------------------+
| containersAllocated | int | The number of containers allocated |
*---------------+--------------+-------------------------------+
| containersReserved | int | The number of containers reserved |
@ -247,10 +255,14 @@ ResourceManager REST API's.
"reservedMB":0,
"availableMB":17408,
"allocatedMB":0,
"reservedVirtualCores":0,
"availableVirtualCores":7,
"allocatedVirtualCores":1,
"containersAllocated":0,
"containersReserved":0,
"containersPending":0,
"totalMB":17408,
"totalVirtualCores":8,
"totalNodes":1,
"lostNodes":0,
"unhealthyNodes":0,
@ -293,10 +305,14 @@ ResourceManager REST API's.
<reservedMB>0</reservedMB>
<availableMB>17408</availableMB>
<allocatedMB>0</allocatedMB>
<reservedVirtualCores>0</reservedVirtualCores>
<availableVirtualCores>7</availableVirtualCores>
<allocatedVirtualCores>1</allocatedVirtualCores>
<containersAllocated>0</containersAllocated>
<containersReserved>0</containersReserved>
<containersPending>0</containersPending>
<totalMB>17408</totalMB>
<totalVirtualCores>8</totalVirtualCores>
<totalNodes>1</totalNodes>
<lostNodes>0</lostNodes>
<unhealthyNodes>0</unhealthyNodes>
@ -2040,8 +2056,10 @@ Server: Jetty(6.1.26)
"lastHealthUpdate":1324056895432,
"healthReport":"Healthy",
"numContainers":0,
"usedMemoryMB":0
"availMemoryMB":8192
"usedMemoryMB":0,
"availMemoryMB":8192,
"usedVirtualCores":0,
"availableVirtualCores":8
},
{
"rack":"\/default-rack",
@ -2054,7 +2072,9 @@ Server: Jetty(6.1.26)
"healthReport":"Healthy",
"numContainers":0,
"usedMemoryMB":0,
"availMemoryMB":8192
"availMemoryMB":8192,
"usedVirtualCores":0,
"availableVirtualCores":8
}
]
}
@ -2096,6 +2116,8 @@ Server: Jetty(6.1.26)
<numContainers>0</numContainers>
<usedMemoryMB>0</usedMemoryMB>
<availMemoryMB>5120</availMemoryMB>
<usedVirtualCores>0</usedVirtualCores>
<availableVirtualCores>8</availableVirtualCores>
</node>
<node>
<rack>/default-rack</rack>
@ -2109,6 +2131,8 @@ Server: Jetty(6.1.26)
<numContainers>0</numContainers>
<usedMemoryMB>0</usedMemoryMB>
<availMemoryMB>5120</availMemoryMB>
<usedVirtualCores>0</usedVirtualCores>
<availableVirtualCores>8</availableVirtualCores>
</node>
</nodes>
+---+
@ -2159,10 +2183,14 @@ Server: Jetty(6.1.26)
*---------------+--------------+-------------------------------+
| lastHealthUpdate | long | The last time the node reported its health (in ms since epoch)|
*---------------+--------------+-------------------------------+
| usedMemoryMB | long | The total about of memory currently used on the node (in MB)|
| usedMemoryMB | long | The total amount of memory currently used on the node (in MB)|
*---------------+--------------+-------------------------------+
| availMemoryMB | long | The total amount of memory currently available on the node (in MB)|
*---------------+--------------+-------------------------------+
| usedVirtualCores | long | The total number of vCores currently used on the node |
*---------------+--------------+-------------------------------+
| availableVirtualCores | long | The total number of vCores available on the node |
*---------------+--------------+-------------------------------+
| numContainers | int | The total number of containers currently running on the node|
*---------------+--------------+-------------------------------+
@ -2201,7 +2229,9 @@ Server: Jetty(6.1.26)
"healthReport":"Healthy",
"numContainers":0,
"usedMemoryMB":0,
"availMemoryMB":5120
"availMemoryMB":5120,
"usedVirtualCores":0,
"availableVirtualCores":8
}
}
+---+
@ -2240,6 +2270,8 @@ Server: Jetty(6.1.26)
<numContainers>0</numContainers>
<usedMemoryMB>0</usedMemoryMB>
<availMemoryMB>5120</availMemoryMB>
<usedVirtualCores>0</usedVirtualCores>
<availableVirtualCores>5120</availableVirtualCores>
</node>
+---+