From 9b81cb0537e5b731581e6a375bf0a59abf61c359 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Tue, 23 Jan 2018 17:09:58 +0530 Subject: [PATCH] YARN-7779. Display allocation tags in RM web UI and expose same through REST API. Contributed by Weiwei Yang. --- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 6 ++ .../yarn/sls/scheduler/RMNodeWrapper.java | 6 ++ .../server/resourcemanager/rmnode/RMNode.java | 7 ++ .../resourcemanager/rmnode/RMNodeImpl.java | 6 ++ .../constraint/AllocationTagsManager.java | 11 +++ .../resourcemanager/webapp/NodesPage.java | 3 + .../webapp/dao/AllocationTagInfo.java | 56 ++++++++++++++ .../webapp/dao/AllocationTagsInfo.java | 59 ++++++++++++++ .../resourcemanager/webapp/dao/NodeInfo.java | 15 ++++ .../server/resourcemanager/MockNodes.java | 6 ++ .../resourcemanager/webapp/TestNodesPage.java | 4 +- .../webapp/TestRMWebServicesNodes.java | 77 ++++++++++++++++++- 12 files changed, 253 insertions(+), 3 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AllocationTagInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AllocationTagsInfo.java diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java index 1016ce1679..0c99139d54 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -212,6 +213,11 @@ public Integer getDecommissioningTimeout() { return null; } + @Override + public Map getAllocationTagsWithCount() { + return null; + } + @Override public Resource getPhysicalResource() { return null; diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java index fdad826994..92f9b0fbeb 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java @@ -37,6 +37,7 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; @Private @@ -202,6 +203,11 @@ public Integer getDecommissioningTimeout() { return null; } + @Override + public Map getAllocationTagsWithCount() { + return node.getAllocationTagsWithCount(); + } + @Override public Resource getPhysicalResource() { return null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java index a5615ef06e..872f2a6ef8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java @@ -20,6 +20,7 @@ import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.hadoop.net.Node; @@ -182,4 +183,10 @@ public interface RMNode { * @return the decommissioning timeout in second. */ Integer getDecommissioningTimeout(); + + /** + * Get the allocation tags and their counts associated with this node. + * @return a map of each allocation tag and its count. + */ + Map getAllocationTagsWithCount(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index da54eb93a2..4fc2d8a4a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -1529,4 +1529,10 @@ public void setUntrackedTimeStamp(long ts) { public Integer getDecommissioningTimeout() { return decommissioningTimeout; } + + @Override + public Map getAllocationTagsWithCount() { + return context.getAllocationTagsManager() + .getAllocationTagsWithCount(getNodeID()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java index 7ad5e8c372..42a78c98ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java @@ -548,4 +548,15 @@ public long getRackCardinalityByOp(String rack, ApplicationId applicationId, readLock.unlock(); } } + + /** + * Returns a map whose key is the allocation tag and value is the + * count of allocations with this tag. + * + * @param nodeId + * @return allocation tag to count mapping + */ + public Map getAllocationTagsWithCount(NodeId nodeId) { + return globalNodeMapping.getTypeToTagsWithCount().get(nodeId); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java index d0e384da34..3e78cf4f40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java @@ -81,12 +81,14 @@ protected void render(Block html) { if (!this.opportunisticContainersEnabled) { trbody.th(".containers", "Containers") + .th(".allocationTags", "Allocation Tags") .th(".mem", "Mem Used") .th(".mem", "Mem Avail") .th(".vcores", "VCores Used") .th(".vcores", "VCores Avail"); } else { trbody.th(".containers", "Running Containers (G)") + .th(".allocationTags", "Allocation Tags") .th(".mem", "Mem Used (G)") .th(".mem", "Mem Avail (G)") .th(".vcores", "VCores Used (G)") @@ -167,6 +169,7 @@ protected void render(Block html) { .append(Times.format(info.getLastHealthUpdate())).append("\",\"") .append(info.getHealthReport()).append("\",\"") .append(String.valueOf(info.getNumContainers())).append("\",\"") + .append(info.getAllocationTagsSummary()).append("\",\"") .append("
").append(StringUtils.byteDesc(usedMemory * BYTES_IN_MB)) .append("\",\"").append("