big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9394. Use new API of RackResolver to get better performance. Contributed by Lantao Jin.

Browse Source
This commit is contained in:
Weiwei Yang 2019-04-04 18:05:01 +08:00
parent a92806d05a
commit 945e8c6064
2 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
View File

@ -40,6 +40,7 @@
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@ -763,10 +764,11 @@ public synchronized List<? extends Collection<T>> getMatchingRequests(
private Set<String> resolveRacks(List<String> nodes) { private Set<String> resolveRacks(List<String> nodes) {
Set<String> racks = new HashSet<String>(); Set<String> racks = new HashSet<String>();
if (nodes != null) { if (nodes != null) {
for (String node : nodes) { List<Node> tmpList = RackResolver.resolve(nodes);
for (Node node : tmpList) {
String rack = node.getNetworkLocation();
// Ensure node requests are accompanied by requests for // Ensure node requests are accompanied by requests for
// corresponding rack // corresponding rack
String rack = RackResolver.resolve(node).getNetworkLocation();
if (rack == null) { if (rack == null) {
LOG.warn("Failed to resolve rack for node " + node + "."); LOG.warn("Failed to resolve rack for node " + node + ".");
} else { } else {
@ -774,7 +776,6 @@ private Set<String> resolveRacks(List<String> nodes) {
} }
} }
} }
return racks; return racks;
} }

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
View File

@ -20,7 +20,7 @@
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import java.util.Arrays; import java.util.Collections;
import java.util.List; import java.util.List;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@ -254,7 +254,7 @@ private static class MyResolver implements DNSToSwitchMapping {
@Override @Override
public List<String> resolve(List<String> names) { public List<String> resolve(List<String> names) {
return Arrays.asList("/rack1"); return Collections.nCopies(names.size(), "/rack1");
} }
@Override @Override