big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-6680. Avoid locking overhead for NO_LABEL lookups. Contributed by Daryn Sharp

Browse Source
This commit is contained in:
Naganarasimha 2017-06-19 22:42:45 +05:30
parent e9c2aa1bc3
commit ee89ac84e6
4 changed files with 26 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
View File

@ -99,6 +99,8 @@ public class CommonNodeLabelsManager extends AbstractService {
protected ConcurrentMap<String, Host> nodeCollections =
new ConcurrentHashMap<String, Host>();
protected RMNodeLabel noNodeLabel;
protected final ReadLock readLock;
protected final WriteLock writeLock;
@ -225,7 +227,8 @@ protected void serviceInit(Configuration conf) throws Exception {
isCentralizedNodeLabelConfiguration =
YarnConfiguration.isCentralizedNodeLabelConfiguration(conf);
labelCollections.put(NO_LABEL, new RMNodeLabel(NO_LABEL));
noNodeLabel = new RMNodeLabel(NO_LABEL);
labelCollections.put(NO_LABEL, noNodeLabel);
}
/**
@ -947,6 +950,9 @@ public List<NodeLabel> getClusterNodeLabels() {
}
public boolean isExclusiveNodeLabel(String nodeLabel) throws IOException {
if (nodeLabel.equals(NO_LABEL)) {
return noNodeLabel.getIsExclusive();
}
try {
readLock.lock();
RMNodeLabel label = labelCollections.get(nodeLabel);

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
View File

@ -519,12 +519,16 @@ private void updateResourceMappings(Map<String, Host> before,
public Resource getResourceByLabel(String label, Resource clusterResource) {
label = normalizeLabel(label);
if (label.equals(NO_LABEL)) {
return noNodeLabel.getResource();
}
try {
readLock.lock();
if (null == labelCollections.get(label)) {
RMNodeLabel nodeLabel = labelCollections.get(label);
if (nodeLabel == null) {
return Resources.none();
}
return labelCollections.get(label).getResource();
return nodeLabel.getResource();
} finally {
readLock.unlock();
}

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
View File

@ -28,7 +28,6 @@
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.util.resource.Resources;
/**
@ -46,6 +45,7 @@ public class ResourceUsage {
private Map<String, UsageByLabel> usages;
// short for no-label :)
private static final String NL = CommonNodeLabelsManager.NO_LABEL;
private final UsageByLabel usageNoLabel;
public ResourceUsage() {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
@ -53,7 +53,8 @@ public ResourceUsage() {
writeLock = lock.writeLock();
usages = new HashMap<String, UsageByLabel>();
usages.put(NL, new UsageByLabel(NL));
usageNoLabel = new UsageByLabel(NL);
usages.put(NL, usageNoLabel);
}
// Usage enum here to make implement cleaner
@ -323,10 +324,9 @@ private static Resource normalize(Resource res) {
}
private Resource _get(String label, ResourceType type) {
if (label == null) {
label = RMNodeLabelsManager.NO_LABEL;
if (label == null || label.equals(NL)) {
return normalize(usageNoLabel.resArr[type.idx]);
}
try {
readLock.lock();
UsageByLabel usage = usages.get(label);
@ -362,8 +362,8 @@ public Resource getAllUsed() {
}
private UsageByLabel getAndAddIfMissing(String label) {
if (label == null) {
label = RMNodeLabelsManager.NO_LABEL;
if (label == null || label.equals(NL)) {
return usageNoLabel;
}
if (!usages.containsKey(label)) {
UsageByLabel u = new UsageByLabel(label);

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
View File

@ -52,6 +52,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.MockAsm;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@ -229,12 +230,15 @@ public static CapacityScheduler mockCapacityScheduler() throws IOException {
setupQueueConfiguration(conf);
CapacityScheduler cs = new CapacityScheduler();
cs.setConf(new YarnConfiguration());
YarnConfiguration yarnConf = new YarnConfiguration();
cs.setConf(yarnConf);
RMContext rmContext = new RMContextImpl(null, null, null, null, null,
null, new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM(), null);
rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
RMNodeLabelsManager labelManager = new NullRMNodeLabelsManager();
labelManager.init(yarnConf);
rmContext.setNodeLabelManager(labelManager);
cs.setRMContext(rmContext);
cs.init(conf);
return cs;