big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-4611. Fix scheduler load simulator to support multi-layer network

Browse Source 
location. Contributed by Ming Ma.
This commit is contained in:
Xuan 2016-01-20 15:41:50 -08:00
parent 890a2ebd1a
commit 1708a4cd23
4 changed files with 18 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java
View File

@ -32,6 +32,7 @@
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.tools.rumen.JobTraceReader; import org.apache.hadoop.tools.rumen.JobTraceReader;
import org.apache.hadoop.tools.rumen.LoggedJob; import org.apache.hadoop.tools.rumen.LoggedJob;
import org.apache.hadoop.tools.rumen.LoggedTask; import org.apache.hadoop.tools.rumen.LoggedTask;
@ -43,9 +44,15 @@
@Unstable @Unstable
public class SLSUtils { public class SLSUtils {
// hostname includes the network path and the host name. for example
// "/default-rack/hostFoo" or "/coreSwitchA/TORSwitchB/hostBar".
// the function returns two Strings, the first element is the network
// location without "/", the second element is the host name. for example,
// {"default-rack", "hostFoo"} or "coreSwitchA/TORSwitchB", "hostBar"
public static String[] getRackHostName(String hostname) { public static String[] getRackHostName(String hostname) {
hostname = hostname.substring(1); NodeBase node = new NodeBase(hostname);
return hostname.split("/"); return new String[] {node.getNetworkLocation().substring(1),
node.getName()};
} }
/** /**

2
hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java
View File

@ -51,7 +51,7 @@ public void setup() {
public void testNMSimulator() throws Exception { public void testNMSimulator() throws Exception {
// Register one node // Register one node
NMSimulator node1 = new NMSimulator(); NMSimulator node1 = new NMSimulator();
node1.init("rack1/node1", GB * 10, 10, 0, 1000, rm); node1.init("/rack1/node1", GB * 10, 10, 0, 1000, rm);
node1.middleStep(); node1.middleStep();
int numClusterNodes = rm.getResourceScheduler().getNumClusterNodes(); int numClusterNodes = rm.getResourceScheduler().getNumClusterNodes();

5
hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/utils/TestSLSUtils.java
View File

@ -29,6 +29,11 @@ public void testGetRackHostname() {
String rackHostname[] = SLSUtils.getRackHostName(str); String rackHostname[] = SLSUtils.getRackHostName(str);
Assert.assertEquals(rackHostname[0], "rack1"); Assert.assertEquals(rackHostname[0], "rack1");
Assert.assertEquals(rackHostname[1], "node1"); Assert.assertEquals(rackHostname[1], "node1");
str = "/rackA/rackB/node1";
rackHostname = SLSUtils.getRackHostName(str);
Assert.assertEquals(rackHostname[0], "rackA/rackB");
Assert.assertEquals(rackHostname[1], "node1");
} }
} }

3
hadoop-yarn-project/CHANGES.txt
View File

@ -131,6 +131,9 @@ Release 2.9.0 - UNRELEASED
YARN-3446. FairScheduler headroom calculation should exclude nodes in the YARN-3446. FairScheduler headroom calculation should exclude nodes in the
blacklist. (Zhihai Xu via kasha) blacklist. (Zhihai Xu via kasha)
YARN-4611. Fix scheduler load simulator to support multi-layer network
location. (Ming Ma via xgong)
Release 2.8.0 - UNRELEASED Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES