YARN-1199. Make NM/RM Versions Available (Mit Desai via jeagles)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1529003 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Turner Eagles
parent
dbdb8c6f1f
commit
7b687dda09
@ -120,6 +120,7 @@ public void testRMNMInfo() throws Exception {
|
||||
Assert.assertNotNull(n.get("NodeHTTPAddress"));
|
||||
Assert.assertNotNull(n.get("LastHealthUpdate"));
|
||||
Assert.assertNotNull(n.get("HealthReport"));
|
||||
Assert.assertNotNull(n.get("NodeManagerVersion"));
|
||||
Assert.assertNotNull(n.get("NumContainers"));
|
||||
Assert.assertEquals(
|
||||
n.get("NodeId") + ": Unexpected number of used containers",
|
||||
@ -156,6 +157,7 @@ public void testRMNMInfoMissmatch() throws Exception {
|
||||
Assert.assertNotNull(n.get("NodeHTTPAddress"));
|
||||
Assert.assertNotNull(n.get("LastHealthUpdate"));
|
||||
Assert.assertNotNull(n.get("HealthReport"));
|
||||
Assert.assertNotNull(n.get("NodeManagerVersion"));
|
||||
Assert.assertNull(n.get("NumContainers"));
|
||||
Assert.assertNull(n.get("UsedMemoryMB"));
|
||||
Assert.assertNull(n.get("AvailableMemoryMB"));
|
||||
|
||||
@ -147,6 +147,12 @@ public List<UpdatedContainerInfo> pullContainerUpdates() {
|
||||
list2));
|
||||
return list;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNodeManagerVersion() {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static RMNode newNodeInfo(String rackName, String hostName,
|
||||
|
||||
@ -138,4 +138,10 @@ List<UpdatedContainerInfo> getContainerUpdates() {
|
||||
return updates;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNodeManagerVersion() {
|
||||
// TODO Auto-generated method stub
|
||||
return null;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -46,6 +46,8 @@ Release 2.3.0 - UNRELEASED
|
||||
|
||||
YARN-425. coverage fix for yarn api (Aleksey Gorshkov via jeagles)
|
||||
|
||||
YARN-1199. Make NM/RM Versions Available (Mit Desai via jeagles)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
||||
@ -90,6 +90,8 @@ public String getLiveNodeManagers() {
|
||||
ni.getLastHealthReportTime());
|
||||
info.put("HealthReport",
|
||||
ni.getHealthReport());
|
||||
info.put("NodeManagerVersion",
|
||||
ni.getNodeManagerVersion());
|
||||
if(report != null) {
|
||||
info.put("NumContainers", report.getNumContainers());
|
||||
info.put("UsedMemoryMB", report.getUsedResource().getMemory());
|
||||
|
||||
@ -234,7 +234,7 @@ public RegisterNodeManagerResponse registerNodeManager(
|
||||
.getCurrentKey());
|
||||
|
||||
RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort,
|
||||
resolve(host), capability);
|
||||
resolve(host), capability, nodeManagerVersion);
|
||||
|
||||
RMNode oldNode = this.rmContext.getRMNodes().putIfAbsent(nodeId, rmNode);
|
||||
if (oldNode == null) {
|
||||
|
||||
@ -27,7 +27,6 @@
|
||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||
import org.apache.hadoop.yarn.api.records.NodeState;
|
||||
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
|
||||
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
|
||||
|
||||
/**
|
||||
* Node managers information on available resources
|
||||
@ -84,7 +83,13 @@ public interface RMNode {
|
||||
* @return the time of the latest health report received from this node.
|
||||
*/
|
||||
public long getLastHealthReportTime();
|
||||
|
||||
|
||||
/**
|
||||
* the node manager version of the node received as part of the
|
||||
* registration with the resource manager
|
||||
*/
|
||||
public String getNodeManagerVersion();
|
||||
|
||||
/**
|
||||
* the total available resource.
|
||||
* @return the total available resource.
|
||||
|
||||
@ -97,6 +97,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
|
||||
|
||||
private String healthReport;
|
||||
private long lastHealthReportTime;
|
||||
private String nodeManagerVersion;
|
||||
|
||||
/* set of containers that have just launched */
|
||||
private final Map<ContainerId, ContainerStatus> justLaunchedContainers =
|
||||
@ -172,7 +173,7 @@ RMNodeEventType.CLEANUP_CONTAINER, new CleanUpContainerTransition())
|
||||
RMNodeEvent> stateMachine;
|
||||
|
||||
public RMNodeImpl(NodeId nodeId, RMContext context, String hostName,
|
||||
int cmPort, int httpPort, Node node, Resource capability) {
|
||||
int cmPort, int httpPort, Node node, Resource capability, String nodeManagerVersion) {
|
||||
this.nodeId = nodeId;
|
||||
this.context = context;
|
||||
this.hostName = hostName;
|
||||
@ -184,6 +185,7 @@ public RMNodeImpl(NodeId nodeId, RMContext context, String hostName,
|
||||
this.node = node;
|
||||
this.healthReport = "Healthy";
|
||||
this.lastHealthReportTime = System.currentTimeMillis();
|
||||
this.nodeManagerVersion = nodeManagerVersion;
|
||||
|
||||
this.latestNodeHeartBeatResponse.setResponseId(0);
|
||||
|
||||
@ -288,6 +290,11 @@ public long getLastHealthReportTime() {
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNodeManagerVersion() {
|
||||
return nodeManagerVersion;
|
||||
}
|
||||
|
||||
@Override
|
||||
public NodeState getState() {
|
||||
this.readLock.lock();
|
||||
|
||||
@ -76,6 +76,7 @@ protected void render(Block html) {
|
||||
th(".containers", "Containers").
|
||||
th(".mem", "Mem Used").
|
||||
th(".mem", "Mem Avail").
|
||||
th(".nodeManagerVersion", "Version").
|
||||
_()._().
|
||||
tbody();
|
||||
NodeState stateFilter = null;
|
||||
@ -129,6 +130,7 @@ protected void render(Block html) {
|
||||
_(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
|
||||
td().br().$title(String.valueOf(usedMemory))._().
|
||||
_(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
|
||||
td(ni.getNodeManagerVersion()).
|
||||
_();
|
||||
}
|
||||
tbody._()._();
|
||||
|
||||
@ -38,6 +38,7 @@ public class NodeInfo {
|
||||
protected String nodeHostName;
|
||||
protected String nodeHTTPAddress;
|
||||
protected long lastHealthUpdate;
|
||||
protected String version;
|
||||
protected String healthReport;
|
||||
protected int numContainers;
|
||||
protected long usedMemoryMB;
|
||||
@ -64,6 +65,7 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) {
|
||||
this.nodeHTTPAddress = ni.getHttpAddress();
|
||||
this.lastHealthUpdate = ni.getLastHealthReportTime();
|
||||
this.healthReport = String.valueOf(ni.getHealthReport());
|
||||
this.version = ni.getNodeManagerVersion();
|
||||
}
|
||||
|
||||
public String getRack() {
|
||||
@ -90,6 +92,10 @@ public long getLastHealthUpdate() {
|
||||
return this.lastHealthUpdate;
|
||||
}
|
||||
|
||||
public String getVersion() {
|
||||
return this.version;
|
||||
}
|
||||
|
||||
public String getHealthReport() {
|
||||
return this.healthReport;
|
||||
}
|
||||
|
||||
@ -40,6 +40,7 @@
|
||||
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
|
||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||
import org.apache.hadoop.yarn.util.Records;
|
||||
import org.apache.hadoop.yarn.util.YarnVersionInfo;
|
||||
|
||||
public class MockNM {
|
||||
|
||||
@ -51,6 +52,7 @@ public class MockNM {
|
||||
private final int httpPort = 2;
|
||||
private MasterKey currentContainerTokenMasterKey;
|
||||
private MasterKey currentNMTokenMasterKey;
|
||||
private String version;
|
||||
|
||||
public MockNM(String nodeIdStr, int memory, ResourceTrackerService resourceTracker) {
|
||||
// scale vcores based on the requested memory
|
||||
@ -61,10 +63,16 @@ public MockNM(String nodeIdStr, int memory, ResourceTrackerService resourceTrack
|
||||
}
|
||||
|
||||
public MockNM(String nodeIdStr, int memory, int vcores,
|
||||
ResourceTrackerService resourceTracker) {
|
||||
ResourceTrackerService resourceTracker) {
|
||||
this(nodeIdStr, memory, vcores, resourceTracker, YarnVersionInfo.getVersion());
|
||||
}
|
||||
|
||||
public MockNM(String nodeIdStr, int memory, int vcores,
|
||||
ResourceTrackerService resourceTracker, String version) {
|
||||
this.memory = memory;
|
||||
this.vCores = vcores;
|
||||
this.resourceTracker = resourceTracker;
|
||||
this.version = version;
|
||||
String[] splits = nodeIdStr.split(":");
|
||||
nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1]));
|
||||
}
|
||||
@ -96,6 +104,7 @@ public RegisterNodeManagerResponse registerNode() throws Exception {
|
||||
req.setHttpPort(httpPort);
|
||||
Resource resource = BuilderUtils.newResource(memory, vCores);
|
||||
req.setResource(resource);
|
||||
req.setNMVersion(version);
|
||||
RegisterNodeManagerResponse registrationResponse =
|
||||
resourceTracker.registerNodeManager(req);
|
||||
this.currentContainerTokenMasterKey =
|
||||
|
||||
@ -183,6 +183,11 @@ public NodeHeartbeatResponse getLastNodeHeartBeatResponse() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getNodeManagerVersion() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<UpdatedContainerInfo> pullContainerUpdates() {
|
||||
return new ArrayList<UpdatedContainerInfo>();
|
||||
|
||||
@ -110,7 +110,7 @@ public Void answer(InvocationOnMock invocation) throws Throwable {
|
||||
new TestSchedulerEventDispatcher());
|
||||
|
||||
NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
|
||||
node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null);
|
||||
node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
|
||||
|
||||
}
|
||||
|
||||
@ -166,7 +166,7 @@ public void testContainerUpdate() throws InterruptedException{
|
||||
node.handle(new RMNodeEvent(null,RMNodeEventType.STARTED));
|
||||
|
||||
NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
|
||||
RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null);
|
||||
RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
|
||||
node2.handle(new RMNodeEvent(null,RMNodeEventType.STARTED));
|
||||
|
||||
ContainerId completedContainerIdFromNode1 = BuilderUtils.newContainerId(
|
||||
@ -432,7 +432,7 @@ public void testUpdateHeartbeatResponseForCleanup() {
|
||||
private RMNodeImpl getRunningNode() {
|
||||
NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
|
||||
RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
|
||||
null, null);
|
||||
null, null, null);
|
||||
node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.STARTED));
|
||||
Assert.assertEquals(NodeState.RUNNING, node.getState());
|
||||
return node;
|
||||
|
||||
@ -49,7 +49,7 @@ public class TestNodesPage {
|
||||
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
|
||||
// future. In that case this value should be adjusted to the new value.
|
||||
final int numberOfThInMetricsTable = 13;
|
||||
final int numberOfActualTableHeaders = 9;
|
||||
final int numberOfActualTableHeaders = 10;
|
||||
|
||||
private Injector injector;
|
||||
|
||||
|
||||
@ -655,13 +655,14 @@ public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException,
|
||||
WebServicesTestUtils.getXmlString(element, "healthReport"),
|
||||
WebServicesTestUtils.getXmlInt(element, "numContainers"),
|
||||
WebServicesTestUtils.getXmlLong(element, "usedMemoryMB"),
|
||||
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"));
|
||||
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"),
|
||||
WebServicesTestUtils.getXmlString(element, "version"));
|
||||
}
|
||||
}
|
||||
|
||||
public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
|
||||
throws JSONException, Exception {
|
||||
assertEquals("incorrect number of elements", 10, nodeInfo.length());
|
||||
assertEquals("incorrect number of elements", 11, nodeInfo.length());
|
||||
|
||||
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
|
||||
nodeInfo.getString("rack"),
|
||||
@ -669,14 +670,15 @@ public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
|
||||
nodeInfo.getString("nodeHTTPAddress"),
|
||||
nodeInfo.getLong("lastHealthUpdate"),
|
||||
nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"),
|
||||
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"));
|
||||
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"),
|
||||
nodeInfo.getString("version"));
|
||||
|
||||
}
|
||||
|
||||
public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
|
||||
String id, String nodeHostName,
|
||||
String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
|
||||
int numContainers, long usedMemoryMB, long availMemoryMB)
|
||||
int numContainers, long usedMemoryMB, long availMemoryMB, String version)
|
||||
throws JSONException, Exception {
|
||||
|
||||
RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
|
||||
@ -695,6 +697,8 @@ public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
|
||||
+ nm.getHttpPort();
|
||||
WebServicesTestUtils.checkStringMatch("nodeHTTPAddress",
|
||||
expectedHttpAddress, nodeHTTPAddress);
|
||||
WebServicesTestUtils.checkStringMatch("version",
|
||||
node.getNodeManagerVersion(), version);
|
||||
|
||||
long expectedHealthUpdate = node.getLastHealthReportTime();
|
||||
assertEquals("lastHealthUpdate doesn't match, got: " + lastHealthUpdate
|
||||
|
||||
Loading…
Reference in New Issue
Block a user