From d8923cdbf1567aee10a54f144fef734d1465ebed Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Mon, 27 Nov 2017 11:47:11 -0800 Subject: [PATCH] YARN-7363. ContainerLocalizer don't have a valid log4j config in case of Linux container executor. (Contributed by Yufei Gu) --- .../hadoop/yarn/conf/YarnConfiguration.java | 6 +++ .../src/main/resources/yarn-default.xml | 8 ++++ .../nodemanager/LinuxContainerExecutor.java | 28 ++++++++++- .../WindowsSecureContainerExecutor.java | 2 +- .../localizer/ContainerLocalizer.java | 46 ++++++++++++++++++- .../TestLinuxContainerExecutorWithMocks.java | 19 +++++--- 6 files changed, 98 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ead99779fb..c1024ea9db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1675,6 +1675,12 @@ public static boolean isAclEnabled(Configuration conf) { public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT = "-Xmx256m"; + /** The log level of container localizer process. */ + public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL= + NM_PREFIX + "container-localizer.log.level"; + public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT = + "INFO"; + /** Prefix for runtime configuration constants. */ public static final String LINUX_CONTAINER_RUNTIME_PREFIX = NM_PREFIX + "runtime.linux."; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 12cb90273f..dd9c6bdc25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1163,6 +1163,14 @@ -Xmx256m + + + The log level for container localizer while it is an independent process. + + yarn.nodemanager.container-localizer.log.level + INFO + + Where to store container logs. An application's localized log directory diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index e8c46a2060..eaf664f2f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.exceptions.ConfigurationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; @@ -382,6 +383,10 @@ public void startLocalizer(LocalizerStartContext ctx) List localizerArgs = new ArrayList<>(); buildMainArgs(localizerArgs, user, appId, locId, nmAddr, localDirs); + + Path containerLogDir = getContainerLogDir(dirsHandler, appId, locId); + localizerArgs = replaceWithContainerLogDir(localizerArgs, containerLogDir); + initializeContainerOp.appendArgs(localizerArgs); try { @@ -402,6 +407,27 @@ public void startLocalizer(LocalizerStartContext ctx) } } + private List replaceWithContainerLogDir(List commands, + Path containerLogDir) { + List newCmds = new ArrayList<>(commands.size()); + + for (String item : commands) { + newCmds.add(item.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR, + containerLogDir.toString())); + } + + return newCmds; + } + + private Path getContainerLogDir(LocalDirsHandlerService dirsHandler, + String appId, String containerId) throws IOException { + String relativeContainerLogDir = ContainerLaunch + .getRelativeContainerLogDir(appId, containerId); + + return dirsHandler.getLogPathForWrite(relativeContainerLogDir, + false); + } + /** * Set up the {@link ContainerLocalizer}. * @@ -417,7 +443,7 @@ public void startLocalizer(LocalizerStartContext ctx) public void buildMainArgs(List command, String user, String appId, String locId, InetSocketAddress nmAddr, List localDirs) { ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, - localDirs); + localDirs, super.getConf()); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index 7c124c3700..e9344727cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -702,7 +702,7 @@ public void startLocalizer(LocalizerStartContext ctx) throws IOException, command.addAll(ContainerLocalizer.getJavaOpts(getConf())); ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, - localDirs); + localDirs, super.getConf()); String cmdLine = StringUtils.join(command, " "); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 2378c454e7..541911a0b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -63,6 +63,8 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.SerializedException; @@ -408,8 +410,12 @@ public static List getJavaOpts(Configuration conf) { */ public static void buildMainArgs(List command, String user, String appId, String locId, - InetSocketAddress nmAddr, List localDirs) { - + InetSocketAddress nmAddr, List localDirs, Configuration conf) { + + String logLevel = conf.get(YarnConfiguration. + NM_CONTAINER_LOCALIZER_LOG_LEVEL, + YarnConfiguration.NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT); + addLog4jSystemProperties(logLevel, command); command.add(ContainerLocalizer.class.getName()); command.add(user); command.add(appId); @@ -421,6 +427,16 @@ public static void buildMainArgs(List command, } } + private static void addLog4jSystemProperties( + String logLevel, List command) { + command.add("-Dlog4j.configuration=container-log4j.properties"); + command.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" + + ApplicationConstants.LOG_DIR_EXPANSION_VAR); + command.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=0"); + command.add("-Dhadoop.root.logger=" + logLevel + ",CLA"); + command.add("-Dhadoop.root.logfile=container-localizer-syslog"); + } + public static void main(String[] argv) throws Throwable { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); int nRet = 0; @@ -431,6 +447,7 @@ public static void main(String[] argv) throws Throwable { // MKDIR $x/$user/appcache/$appid/filecache // LOAD $x/$user/appcache/$appid/appTokens try { + createLogDir(); String user = argv[0]; String appId = argv[1]; String locId = argv[2]; @@ -466,6 +483,31 @@ public static void main(String[] argv) throws Throwable { } } + /** + * Create the log directory, if the directory exists, make sure its permission + * is 750. + */ + private static void createLogDir() { + FileContext localFs; + try { + localFs = FileContext.getLocalFSFileContext(new Configuration()); + + String logDir = System.getProperty( + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR); + + if (logDir != null && !logDir.trim().isEmpty()) { + Path containerLogPath = new Path(logDir); + FsPermission containerLogDirPerm= new FsPermission((short)0750); + localFs.mkdir(containerLogPath, containerLogDirPerm, true); + // set permission again to make sure the permission is correct + // in case the directory is already there. + localFs.setPermission(containerLogPath, containerLogDirPerm); + } + } catch (IOException e) { + throw new YarnRuntimeException("Unable to create the log dir", e); + } + } + private static void initDirs(Configuration conf, String user, String appId, FileContext lfs, List localDirs) throws IOException { if (null == localDirs || 0 == localDirs.size()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index e1d9c69444..9a5665658a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -269,7 +269,7 @@ public void testStartLocalizer() throws IOException { .build()); List result=readMockParams(); - Assert.assertEquals(result.size(), 19); + Assert.assertEquals(result.size(), 24); Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER); Assert.assertEquals(result.get(1), "test"); Assert.assertEquals(result.get(2), "0" ); @@ -277,12 +277,17 @@ public void testStartLocalizer() throws IOException { Assert.assertEquals(result.get(4), "/bin/nmPrivateCTokensPath"); Assert.assertEquals(result.get(8), "-classpath" ); Assert.assertEquals(result.get(11), "-Xmx256m" ); - Assert.assertEquals(result.get(12),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" ); - Assert.assertEquals(result.get(13), "test"); - Assert.assertEquals(result.get(14), "application_0"); - Assert.assertEquals(result.get(15),"12345" ); - Assert.assertEquals(result.get(16),"localhost" ); - Assert.assertEquals(result.get(17),"8040" ); + Assert.assertEquals(result.get(12), "-Dlog4j.configuration=container-log4j.properties" ); + Assert.assertEquals(result.get(13), "-Dyarn.app.container.log.dir=${yarn.log.dir}/userlogs/application_0/12345"); + Assert.assertEquals(result.get(14), "-Dyarn.app.container.log.filesize=0"); + Assert.assertEquals(result.get(15), "-Dhadoop.root.logger=INFO,CLA"); + Assert.assertEquals(result.get(16), "-Dhadoop.root.logfile=container-localizer-syslog"); + Assert.assertEquals(result.get(17),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" ); + Assert.assertEquals(result.get(18), "test"); + Assert.assertEquals(result.get(19), "application_0"); + Assert.assertEquals(result.get(20),"12345" ); + Assert.assertEquals(result.get(21),"localhost" ); + Assert.assertEquals(result.get(22),"8040" ); } catch (InterruptedException e) { LOG.error("Error:"+e.getMessage(),e);