big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11630. Passing admin Java options to container localizers (#6349)

Browse Source
This commit is contained in:
Peter Szucs 2023-12-14 16:52:57 +01:00 committed by GitHub
parent 1498a8685d
commit 16edb4ac96
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 92 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -2232,6 +2232,13 @@ public static boolean isAclEnabled(Configuration conf) {
public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT =
"-Xmx256m";
/** The admin JVM options used on forking ContainerLocalizer process
by container executor. */
public static final String NM_CONTAINER_LOCALIZER_ADMIN_JAVA_OPTS_KEY =
NM_PREFIX + "container-localizer.admin.java.opts";
public static final String NM_CONTAINER_LOCALIZER_ADMIN_JAVA_OPTS_DEFAULT = "";
/*
* Flag to indicate whether JDK17's required add-exports flags should be added to
* container localizers regardless of the user specified JAVA_OPTS.

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -1488,6 +1488,15 @@
<value>-Xmx256m</value>
</property>
<property>
<description>
The admin JVM options used on forking ContainerLocalizer process
by the container executor.
</description>
<name>yarn.nodemanager.container-localizer.admin.java.opts</name>
<value></value>
</property>
<property>
<name>yarn.nodemanager.container-localizer.java.opts.add-exports-as-default</name>
<value>true</value>

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
View File

@ -47,6 +47,8 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
@ -406,16 +408,25 @@ private LocalizerStatus createStatus() throws InterruptedException {
* @param conf the configuration properties to launch the resource localizer.
*/
public static List<String> getJavaOpts(Configuration conf) {
String opts = conf.get(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY,
String adminOpts = conf.get(YarnConfiguration.NM_CONTAINER_LOCALIZER_ADMIN_JAVA_OPTS_KEY,
YarnConfiguration.NM_CONTAINER_LOCALIZER_ADMIN_JAVA_OPTS_DEFAULT);
String userOpts = conf.get(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY,
YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT);
boolean isExtraJDK17OptionsConfigured =
conf.getBoolean(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_KEY,
YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_ADD_EXPORTS_DEFAULT);
if (Shell.isJavaVersionAtLeast(17) && isExtraJDK17OptionsConfigured) {
opts = opts.trim().concat(" " + ADDITIONAL_JDK17_PLUS_OPTIONS);
userOpts = userOpts.trim().concat(" " + ADDITIONAL_JDK17_PLUS_OPTIONS);
}
return Arrays.asList(opts.split(" "));
List<String> adminOptionList = Arrays.asList(adminOpts.split("\\s+"));
List<String> userOptionList = Arrays.asList(userOpts.split("\\s+"));
return Stream.concat(adminOptionList.stream(), userOptionList.stream())
.filter(s -> !s.isEmpty())
.collect(Collectors.toList());
}
/**

62
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
View File

@ -737,4 +737,66 @@ public void testDefaultJavaOptionsWhenExtraJDK17OptionsAreNotConfigured() throws
}
Assert.assertTrue(javaOpts.contains("-Xmx256m"));
}
@Test
public void testAdminOptionsPrecedeUserDefinedJavaOptions() throws Exception {
ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
Configuration conf = new Configuration();
conf.setStrings(YarnConfiguration.NM_CONTAINER_LOCALIZER_ADMIN_JAVA_OPTS_KEY,
"adminOption1 adminOption2");
conf.setStrings(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY,
" userOption1 userOption2");
List<String> javaOpts = localizer.getJavaOpts(conf);
Assert.assertEquals(4, javaOpts.size());
Assert.assertTrue(javaOpts.get(0).equals("adminOption1"));
Assert.assertTrue(javaOpts.get(1).equals("adminOption2"));
Assert.assertTrue(javaOpts.get(2).equals("userOption1"));
Assert.assertTrue(javaOpts.get(3).equals("userOption2"));
}
@Test
public void testAdminOptionsPrecedeDefaultUserOptions() throws Exception {
ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
Configuration conf = new Configuration();
conf.setStrings(YarnConfiguration.NM_CONTAINER_LOCALIZER_ADMIN_JAVA_OPTS_KEY,
"adminOption1 adminOption2");
List<String> javaOpts = localizer.getJavaOpts(conf);
Assert.assertEquals(3, javaOpts.size());
Assert.assertTrue(javaOpts.get(0).equals("adminOption1"));
Assert.assertTrue(javaOpts.get(1).equals("adminOption2"));
Assert.assertTrue(javaOpts.get(2).equals("-Xmx256m"));
}
@Test
public void testUserOptionsWhenAdminOptionsAreNotDefined() throws Exception {
ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
Configuration conf = new Configuration();
conf.setStrings(YarnConfiguration.NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY,
"userOption1 userOption2");
List<String> javaOpts = localizer.getJavaOpts(conf);
Assert.assertEquals(2, javaOpts.size());
Assert.assertTrue(javaOpts.get(0).equals("userOption1"));
Assert.assertTrue(javaOpts.get(1).equals("userOption2"));
}
@Test
public void testJavaOptionsWithoutDefinedAdminOrUserOptions() throws Exception {
ContainerLocalizerWrapper wrapper = new ContainerLocalizerWrapper();
ContainerLocalizer localizer = wrapper.setupContainerLocalizerForTest();
Configuration conf = new Configuration();
List<String> javaOpts = localizer.getJavaOpts(conf);
Assert.assertEquals(1, javaOpts.size());
Assert.assertTrue(javaOpts.get(0).equals("-Xmx256m"));
}
}