big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-1912. ResourceLocalizer started without any jvm memory control.

Browse Source 
Contributed by Masatake Iwasaki
This commit is contained in:
Xuan 2015-05-08 20:01:21 -07:00
parent 333f9a896d
commit 6471d18bc7
6 changed files with 33 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -215,6 +215,9 @@ Release 2.8.0 - UNRELEASED
YARN-3271. FairScheduler: Move tests related to max-runnable-apps from
TestFairScheduler to TestAppRunnability. (nijel via kasha)
YARN-1912. ResourceLocalizer started without any jvm memory control.
(Masatake Iwasaki via xgong)
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -1004,6 +1004,13 @@ private static void addDeprecatedKeys() {
public static final String NM_HEALTH_CHECK_SCRIPT_OPTS =
NM_PREFIX + "health-checker.script.opts";
/** The JVM options used on forking ContainerLocalizer process
by container executor. */
public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY =
NM_PREFIX + "container-localizer.java.opts";
public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT =
"-Xmx256m";
/** The Docker image name(For DockerContainerExecutor).*/
public static final String NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME =
NM_PREFIX + "docker-container-executor.image-name";

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
View File

@ -244,6 +244,7 @@ public void startLocalizer(Path nmPrivateContainerTokensPath,
if (javaLibPath != null) {
command.add("-Djava.library.path=" + javaLibPath);
}
command.addAll(ContainerLocalizer.getJavaOpts(getConf()));
buildMainArgs(command, user, appId, locId, nmAddr, localDirs);
String[] commandArray = command.toArray(new String[command.size()]);
ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
View File

@ -697,6 +697,7 @@ public void startLocalizer(Path nmPrivateContainerTokens,
if (javaLibPath != null) {
command.add("-Djava.library.path=" + javaLibPath);
}
command.addAll(ContainerLocalizer.getJavaOpts(getConf()));
ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
localDirs);

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
View File

@ -55,6 +55,7 @@
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@ -313,6 +314,16 @@ private LocalizerStatus createStatus() throws InterruptedException {
return status;
}
/**
* Returns the JVM options to to launch the resource localizer.
* @param conf the configuration properties to launch the resource localizer.
*/
public static List<String> getJavaOpts(Configuration conf) {
String opts = conf.get(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY,
YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT);
return Arrays.asList(opts.split(" "));
}
/**
* Adds the ContainerLocalizer arguments for a @{link ShellCommandExecutor},
* as expected by ContainerLocalizer.main

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
View File

@ -187,19 +187,20 @@ public void testStartLocalizer() throws IOException {
try {
mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler);
List<String> result=readMockParams();
Assert.assertEquals(result.size(), 17);
Assert.assertEquals(result.size(), 18);
Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER);
Assert.assertEquals(result.get(1), "test");
Assert.assertEquals(result.get(2), "0" );
Assert.assertEquals(result.get(3),"application_0" );
Assert.assertEquals(result.get(4), "/bin/nmPrivateCTokensPath");
Assert.assertEquals(result.get(8), "-classpath" );
Assert.assertEquals(result.get(11),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" );
Assert.assertEquals(result.get(12), "test");
Assert.assertEquals(result.get(13), "application_0");
Assert.assertEquals(result.get(14),"12345" );
Assert.assertEquals(result.get(15),"localhost" );
Assert.assertEquals(result.get(16),"8040" );
Assert.assertEquals(result.get(11), "-Xmx256m" );
Assert.assertEquals(result.get(12),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" );
Assert.assertEquals(result.get(13), "test");
Assert.assertEquals(result.get(14), "application_0");
Assert.assertEquals(result.get(15),"12345" );
Assert.assertEquals(result.get(16),"localhost" );
Assert.assertEquals(result.get(17),"8040" );
} catch (InterruptedException e) {
LOG.error("Error:"+e.getMessage(),e);