From cf4a678ce946cecd035677bce756794ef286d795 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Wed, 15 Mar 2023 09:46:17 -0700 Subject: [PATCH] HADOOP-18649. CLA and CRLA appenders to be replaced with RFA (#5448) --- .../hadoop/mapreduce/v2/util/MRApps.java | 5 +- .../src/main/resources/mapred-default.xml | 15 +- .../hadoop/yarn/ContainerLogAppender.java | 129 ------------------ .../yarn/ContainerRollingLogAppender.java | 76 ----------- .../hadoop/yarn/TestContainerLogAppender.java | 48 ------- .../main/resources/container-log4j.properties | 39 +++--- 6 files changed, 24 insertions(+), 288 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index a3ccfd72d8..72dd48b09c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -60,8 +60,6 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.util.ApplicationClassLoader; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.ContainerLogAppender; -import org.apache.hadoop.yarn.ContainerRollingLogAppender; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -588,8 +586,7 @@ public static String getChildLogLevel(Configuration conf, boolean isMap) { /** * Add the JVM system properties necessary to configure - * {@link ContainerLogAppender} or - * {@link ContainerRollingLogAppender}. + * {@link org.apache.log4j.RollingFileAppender}. * * @param task for map/reduce, or null for app master * @param vargs the argument list to append to diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 7e1b49c925..7aadd52109 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -840,11 +840,8 @@ yarn.app.mapreduce.task.container.log.backups 0 Number of backup files for task logs when using - ContainerRollingLogAppender (CRLA). See - org.apache.log4j.RollingFileAppender.maxBackupIndex. By default, - ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA - is enabled for tasks when both mapreduce.task.userlog.limit.kb and - yarn.app.mapreduce.task.container.log.backups are greater than zero. + RollingFileAppender (RFA). See + org.apache.log4j.RollingFileAppender.maxBackupIndex. @@ -852,12 +849,8 @@ yarn.app.mapreduce.am.container.log.backups 0 Number of backup files for the ApplicationMaster logs when using - ContainerRollingLogAppender (CRLA). See - org.apache.log4j.RollingFileAppender.maxBackupIndex. By default, - ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA - is enabled for the ApplicationMaster when both - yarn.app.mapreduce.am.container.log.limit.kb and - yarn.app.mapreduce.am.container.log.backups are greater than zero. + RollingFileAppender (RFA). See + org.apache.log4j.RollingFileAppender.maxBackupIndex. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java deleted file mode 100644 index 09efe41e0c..0000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn; - -import java.io.File; -import java.io.Flushable; -import java.util.ArrayDeque; -import java.util.Deque; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.log4j.FileAppender; -import org.apache.log4j.spi.LoggingEvent; - -/** - * A simple log4j-appender for container's logs. - */ -@Public -@Unstable -public class ContainerLogAppender extends FileAppender - implements Flushable { - - private String containerLogDir; - private String containerLogFile; - private int maxEvents; - private Deque eventBuffer; - private boolean closed = false; - - @Override - public synchronized void activateOptions() { - if (maxEvents > 0) { - this.eventBuffer = new ArrayDeque<>(); - } - setFile(new File(this.containerLogDir, containerLogFile).toString()); - setAppend(true); - super.activateOptions(); - } - - @Override - public synchronized void append(LoggingEvent event) { - if (closed) { - return; - } - if (eventBuffer != null) { - if (eventBuffer.size() == maxEvents) { - eventBuffer.removeFirst(); - } - eventBuffer.addLast(event); - } else { - super.append(event); - } - } - - @Override - public void flush() { - if (qw != null) { - qw.flush(); - } - } - - @Override - public synchronized void close() { - if (!closed) { - closed = true; - if (eventBuffer != null) { - for (LoggingEvent event : eventBuffer) { - super.append(event); - } - // let garbage collection do its work - eventBuffer = null; - } - super.close(); - } - } - - /** - * Getter/Setter methods for log4j. - * - * @return containerLogDir. - */ - - public String getContainerLogDir() { - return this.containerLogDir; - } - - public void setContainerLogDir(String containerLogDir) { - this.containerLogDir = containerLogDir; - } - - public String getContainerLogFile() { - return containerLogFile; - } - - public void setContainerLogFile(String containerLogFile) { - this.containerLogFile = containerLogFile; - } - - private static final long EVENT_SIZE = 100; - - public long getTotalLogFileSize() { - return maxEvents * EVENT_SIZE; - } - - /** - * Setter so that log4j can configure it from the - * configuration(log4j.properties). - * - * @param logSize log size. - */ - public void setTotalLogFileSize(long logSize) { - maxEvents = (int)(logSize / EVENT_SIZE); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java deleted file mode 100644 index f0e00fc194..0000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.log4j.RollingFileAppender; - -import java.io.File; -import java.io.Flushable; - -/** - * A simple log4j-appender for container's logs. - * - */ -@Public -@Unstable -public class ContainerRollingLogAppender extends RollingFileAppender - implements Flushable { - private String containerLogDir; - private String containerLogFile; - - @Override - public void activateOptions() { - synchronized (this) { - setFile(new File(this.containerLogDir, containerLogFile).toString()); - setAppend(true); - super.activateOptions(); - } - } - - @Override - public void flush() { - if (qw != null) { - qw.flush(); - } - } - - /** - * Getter/Setter methods for log4j. - * - * @return containerLogDir. - */ - - public String getContainerLogDir() { - return this.containerLogDir; - } - - public void setContainerLogDir(String containerLogDir) { - this.containerLogDir = containerLogDir; - } - - public String getContainerLogFile() { - return containerLogFile; - } - - public void setContainerLogFile(String containerLogFile) { - this.containerLogFile = containerLogFile; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java deleted file mode 100644 index 26acfd7bad..0000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn; - -import org.junit.jupiter.api.Test; - -import org.apache.log4j.Logger; -import org.apache.log4j.PatternLayout; - -public class TestContainerLogAppender { - - @Test - void testAppendInClose() throws Exception { - final ContainerLogAppender claAppender = new ContainerLogAppender(); - claAppender.setName("testCLA"); - claAppender.setLayout(new PatternLayout("%-5p [%t]: %m%n")); - claAppender.setContainerLogDir("target/testAppendInClose/logDir"); - claAppender.setContainerLogFile("syslog"); - claAppender.setTotalLogFileSize(1000); - claAppender.activateOptions(); - final Logger claLog = Logger.getLogger("testAppendInClose-catergory"); - claLog.setAdditivity(false); - claLog.addAppender(claAppender); - claLog.info(new Object() { - public String toString() { - claLog.info("message1"); - return "return message1"; - } - }); - claAppender.close(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties index 678e3a74c8..c5371c6d9e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties @@ -26,36 +26,35 @@ log4j.threshold=ALL # #Default values -yarn.app.container.log.dir=null -yarn.app.container.log.filesize=100 +yarn.app.container.log.filesize=100MB +yarn.app.container.log.backups=1 +yarn.app.mapreduce.shuffle.log.backups=1 -log4j.appender.CLA=org.apache.hadoop.yarn.ContainerLogAppender -log4j.appender.CLA.containerLogDir=${yarn.app.container.log.dir} -log4j.appender.CLA.containerLogFile=${hadoop.root.logfile} -log4j.appender.CLA.totalLogFileSize=${yarn.app.container.log.filesize} +log4j.appender.CLA=org.apache.log4j.RollingFileAppender +log4j.appender.CLA.File=${yarn.app.container.log.dir}/${hadoop.root.logfile} +log4j.appender.CLA.MaxFileSize=${yarn.app.container.log.filesize} +log4j.appender.CLA.MaxBackupIndex=${yarn.app.container.log.backups} log4j.appender.CLA.layout=org.apache.log4j.PatternLayout log4j.appender.CLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n -log4j.appender.CRLA=org.apache.hadoop.yarn.ContainerRollingLogAppender -log4j.appender.CRLA.containerLogDir=${yarn.app.container.log.dir} -log4j.appender.CRLA.containerLogFile=${hadoop.root.logfile} -log4j.appender.CRLA.maximumFileSize=${yarn.app.container.log.filesize} -log4j.appender.CRLA.maxBackupIndex=${yarn.app.container.log.backups} +log4j.appender.CRLA=org.apache.log4j.RollingFileAppender +log4j.appender.CRLA.File=${yarn.app.container.log.dir}/${hadoop.root.logfile} +log4j.appender.CRLA.MaxFileSize=${yarn.app.container.log.filesize} +log4j.appender.CRLA.MaxBackupIndex=${yarn.app.container.log.backups} log4j.appender.CRLA.layout=org.apache.log4j.PatternLayout log4j.appender.CRLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n -log4j.appender.shuffleCLA=org.apache.hadoop.yarn.ContainerLogAppender -log4j.appender.shuffleCLA.containerLogDir=${yarn.app.container.log.dir} -log4j.appender.shuffleCLA.containerLogFile=${yarn.app.mapreduce.shuffle.logfile} -log4j.appender.shuffleCLA.totalLogFileSize=${yarn.app.mapreduce.shuffle.log.filesize} +log4j.appender.shuffleCLA=org.apache.log4j.RollingFileAppender +log4j.appender.shuffleCLA.File=${yarn.app.container.log.dir}/${yarn.app.mapreduce.shuffle.logfile} +log4j.appender.shuffleCLA.MaxFileSize=${yarn.app.mapreduce.shuffle.log.filesize} +log4j.appender.shuffleCLA.MaxBackupIndex=${yarn.app.mapreduce.shuffle.log.backups} log4j.appender.shuffleCLA.layout=org.apache.log4j.PatternLayout log4j.appender.shuffleCLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n -log4j.appender.shuffleCRLA=org.apache.hadoop.yarn.ContainerRollingLogAppender -log4j.appender.shuffleCRLA.containerLogDir=${yarn.app.container.log.dir} -log4j.appender.shuffleCRLA.containerLogFile=${yarn.app.mapreduce.shuffle.logfile} -log4j.appender.shuffleCRLA.maximumFileSize=${yarn.app.mapreduce.shuffle.log.filesize} -log4j.appender.shuffleCRLA.maxBackupIndex=${yarn.app.mapreduce.shuffle.log.backups} +log4j.appender.shuffleCRLA=org.apache.log4j.RollingFileAppender +log4j.appender.shuffleCRLA.File=${yarn.app.container.log.dir}/${yarn.app.mapreduce.shuffle.logfile} +log4j.appender.shuffleCRLA.MaxFileSize=${yarn.app.mapreduce.shuffle.log.filesize} +log4j.appender.shuffleCRLA.MaxBackupIndex=${yarn.app.mapreduce.shuffle.log.backups} log4j.appender.shuffleCRLA.layout=org.apache.log4j.PatternLayout log4j.appender.shuffleCRLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n