diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 05ae1444ba..8f07b794b3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -215,6 +215,9 @@ Release 2.8.0 - UNRELEASED YARN-3271. FairScheduler: Move tests related to max-runnable-apps from TestFairScheduler to TestAppRunnability. (nijel via kasha) + YARN-1912. ResourceLocalizer started without any jvm memory control. + (Masatake Iwasaki via xgong) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 0851f3c554..48a75c0882 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1004,6 +1004,13 @@ private static void addDeprecatedKeys() { public static final String NM_HEALTH_CHECK_SCRIPT_OPTS = NM_PREFIX + "health-checker.script.opts"; + /** The JVM options used on forking ContainerLocalizer process + by container executor. */ + public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY = + NM_PREFIX + "container-localizer.java.opts"; + public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT = + "-Xmx256m"; + /** The Docker image name(For DockerContainerExecutor).*/ public static final String NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME = NM_PREFIX + "docker-container-executor.image-name"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index f8da958543..16068d7461 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -244,6 +244,7 @@ public void startLocalizer(Path nmPrivateContainerTokensPath, if (javaLibPath != null) { command.add("-Djava.library.path=" + javaLibPath); } + command.addAll(ContainerLocalizer.getJavaOpts(getConf())); buildMainArgs(command, user, appId, locId, nmAddr, localDirs); String[] commandArray = command.toArray(new String[command.size()]); ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index b7bec5f0cf..619b845c8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -697,7 +697,8 @@ public void startLocalizer(Path nmPrivateContainerTokens, if (javaLibPath != null) { command.add("-Djava.library.path=" + javaLibPath); } - + command.addAll(ContainerLocalizer.getJavaOpts(getConf())); + ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, localDirs); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 32e355318d..f82f894ca7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -312,7 +313,17 @@ private LocalizerStatus createStatus() throws InterruptedException { status.addAllResources(currentResources); return status; } - + + /** + * Returns the JVM options to to launch the resource localizer. + * @param conf the configuration properties to launch the resource localizer. + */ + public static List getJavaOpts(Configuration conf) { + String opts = conf.get(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY, + YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT); + return Arrays.asList(opts.split(" ")); + } + /** * Adds the ContainerLocalizer arguments for a @{link ShellCommandExecutor}, * as expected by ContainerLocalizer.main diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index 98ab8e0631..dce2cd3da1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -187,19 +187,20 @@ public void testStartLocalizer() throws IOException { try { mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler); List result=readMockParams(); - Assert.assertEquals(result.size(), 17); + Assert.assertEquals(result.size(), 18); Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER); Assert.assertEquals(result.get(1), "test"); Assert.assertEquals(result.get(2), "0" ); Assert.assertEquals(result.get(3),"application_0" ); Assert.assertEquals(result.get(4), "/bin/nmPrivateCTokensPath"); Assert.assertEquals(result.get(8), "-classpath" ); - Assert.assertEquals(result.get(11),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" ); - Assert.assertEquals(result.get(12), "test"); - Assert.assertEquals(result.get(13), "application_0"); - Assert.assertEquals(result.get(14),"12345" ); - Assert.assertEquals(result.get(15),"localhost" ); - Assert.assertEquals(result.get(16),"8040" ); + Assert.assertEquals(result.get(11), "-Xmx256m" ); + Assert.assertEquals(result.get(12),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" ); + Assert.assertEquals(result.get(13), "test"); + Assert.assertEquals(result.get(14), "application_0"); + Assert.assertEquals(result.get(15),"12345" ); + Assert.assertEquals(result.get(16),"localhost" ); + Assert.assertEquals(result.get(17),"8040" ); } catch (InterruptedException e) { LOG.error("Error:"+e.getMessage(),e);