big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8092. Expose Node Attributes info via RM nodes REST API. Contributed by Weiwei Yang.

Browse Source
This commit is contained in:
Sunil G 2018-04-01 18:07:07 +05:30
parent 440ff7f563
commit 89b3ebd11e
9 changed files with 197 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
View File

@ -32,6 +32,7 @@
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
@ -219,6 +220,18 @@ public Map<String, Long> getAllocationTagsWithCount() {
return null;
}
@Override
public void setNodeAttributes(String prefix,
Set<NodeAttribute> nodeAttributes) {
}
@Override
public Map<String, Set<NodeAttribute>> getAllNodeAttributes() {
return null;
}
@Override
public RMContext getRMContext() {
return null;

12
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
View File

@ -24,6 +24,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
@ -206,6 +207,17 @@ public Integer getDecommissioningTimeout() {
public Map<String, Long> getAllocationTagsWithCount() {
return node.getAllocationTagsWithCount();
}
@Override
public void setNodeAttributes(String prefix,
Set<NodeAttribute> nodeAttributes) {
node.setNodeAttributes(prefix, nodeAttributes);
}
@Override
public Map<String, Set<NodeAttribute>> getAllNodeAttributes() {
return node.getAllNodeAttributes();
}
@Override
public RMContext getRMContext() {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
View File

@ -673,6 +673,10 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
this.rmContext.getNodeAttributesManager()
.replaceNodeAttributes(NodeAttribute.PREFIX_DISTRIBUTED,
ImmutableMap.of(nodeId.getHost(), nodeAttributes));
// Update node attributes to RMNode
rmNode.setNodeAttributes(NodeAttribute.PREFIX_DISTRIBUTED,
nodeAttributes);
}
}

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
View File

@ -31,6 +31,7 @@
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@ -195,4 +196,16 @@ public interface RMNode {
* @return the RM context associated with this RM node.
*/
RMContext getRMContext();
/**
* Sets node attributes per prefix.
* @param prefix node attribute prefix
* @param nodeAttributes node attributes
*/
void setNodeAttributes(String prefix, Set<NodeAttribute> nodeAttributes);
/**
* @return all node attributes grouped by their prefix as a map.
*/
Map<String, Set<NodeAttribute>> getAllNodeAttributes();
}

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
View File

@ -50,6 +50,7 @@
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
@ -185,6 +186,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private NodeHeartbeatResponse latestNodeHeartBeatResponse = recordFactory
.newRecordInstance(NodeHeartbeatResponse.class);
// Node attributes, store by prefix
private Map<String, Set<NodeAttribute>> nodeAttributes = new HashMap<>();
private static final StateMachineFactory<RMNodeImpl,
NodeState,
RMNodeEventType,
@ -1546,4 +1550,15 @@ public Map<String, Long> getAllocationTagsWithCount() {
public RMContext getRMContext() {
return this.context;
}
@Override
public void setNodeAttributes(String prefix,
Set<NodeAttribute> nodeAttributeSet) {
this.nodeAttributes.put(prefix, nodeAttributeSet);
}
@Override
public Map<String, Set<NodeAttribute>> getAllNodeAttributes() {
return this.nodeAttributes;
}
}

65
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java Normal file
View File

@ -0,0 +1,65 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
/**
* DAO for node an attribute record.
*/
@XmlRootElement(name = "nodeAttributeInfo")
@XmlAccessorType(XmlAccessType.FIELD)
public class NodeAttributeInfo {
private String prefix;
private String name;
private String type;
private String value;
public NodeAttributeInfo() {
// JAXB needs this
}
public NodeAttributeInfo(NodeAttribute nodeAttribute) {
this.prefix = nodeAttribute.getAttributePrefix();
this.name = nodeAttribute.getAttributeName();
this.type = nodeAttribute.getAttributeType().toString();
this.value = nodeAttribute.getAttributeValue();
}
public String getPrefix() {
return prefix;
}
public String getName() {
return name;
}
public String getType() {
return type;
}
public String getValue() {
return value;
}
}

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributesInfo.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
/**
* DAO for a list of node attributes info.
*/
@XmlRootElement(name = "nodeAttributesInfo")
@XmlAccessorType(XmlAccessType.FIELD)
public class NodeAttributesInfo {
@XmlElement(name = "nodeAttributeInfo")
private ArrayList<NodeAttributeInfo> nodeAttributesInfo =
new ArrayList<>();
public NodeAttributesInfo() {
// JAXB needs this
}
public void addNodeAttributeInfo(NodeAttributeInfo attributeInfo) {
this.nodeAttributesInfo.add(attributeInfo);
}
public ArrayList<NodeAttributeInfo> getNodeAttributesInfo() {
return nodeAttributesInfo;
}
}

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
View File

@ -27,6 +27,7 @@
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
@ -62,6 +63,7 @@ public class NodeInfo {
protected ResourceUtilizationInfo resourceUtilization;
protected ResourceInfo usedResource;
protected ResourceInfo availableResource;
protected NodeAttributesInfo nodeAttributesInfo;
public NodeInfo() {
} // JAXB needs this
@ -113,6 +115,19 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) {
Collections.sort(nodeLabels);
}
// add attributes
Map<String, Set<NodeAttribute>> nodeAttributes =
ni.getAllNodeAttributes();
nodeAttributesInfo = new NodeAttributesInfo();
if (nodeAttributes != null) {
for (Set<NodeAttribute> attrs : nodeAttributes.values()) {
for (NodeAttribute attribute : attrs) {
NodeAttributeInfo info = new NodeAttributeInfo(attribute);
this.nodeAttributesInfo.addNodeAttributeInfo(info);
}
}
}
// add allocation tags
allocationTags = new AllocationTagsInfo();
Map<String, Long> allocationTagsInfo = ni.getAllocationTagsWithCount();

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
View File

@ -28,6 +28,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
@ -284,6 +285,16 @@ public Integer getDecommissioningTimeout() {
public Map<String, Long> getAllocationTagsWithCount() {
return null;
}
public void setNodeAttributes(String prefix,
Set<NodeAttribute> nodeAttributes) {
}
@Override
public Map<String, Set<NodeAttribute>> getAllNodeAttributes() {
return null;
}
@Override
public RMContext getRMContext() {