From bc5458bbd4e6829bc02de57671c017fbdac06849 Mon Sep 17 00:00:00 2001 From: Neil Date: Fri, 11 Dec 2020 05:47:00 +0800 Subject: [PATCH] HDFS-15720 namenode audit async logger should add some log4j config (#2532) (cherry picked from commit 9bd3c9bc50672ebdd7f12b9a2235aec37b2a07cf) --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++++ .../hdfs/server/namenode/FSNamesystem.java | 12 +++++++++-- .../src/main/resources/hdfs-default.xml | 21 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 5264799927..8888b08087 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -664,6 +664,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final boolean DFS_NAMENODE_AUDIT_LOG_TOKEN_TRACKING_ID_DEFAULT = false; public static final String DFS_NAMENODE_AUDIT_LOG_ASYNC_KEY = "dfs.namenode.audit.log.async"; public static final boolean DFS_NAMENODE_AUDIT_LOG_ASYNC_DEFAULT = false; + public static final String DFS_NAMENODE_AUDIT_LOG_ASYNC_BLOCKING_KEY = "dfs.namenode.audit.log.async.blocking"; + public static final boolean DFS_NAMENODE_AUDIT_LOG_ASYNC_BLOCKING_DEFAULT = true; + public static final String DFS_NAMENODE_AUDIT_LOG_ASYNC_BUFFER_SIZE_KEY = "dfs.namenode.audit.log.async.buffer.size"; + public static final int DFS_NAMENODE_AUDIT_LOG_ASYNC_BUFFER_SIZE_DEFAULT = 128; public static final String DFS_NAMENODE_AUDIT_LOG_DEBUG_CMDLIST = "dfs.namenode.audit.log.debug.cmdlist"; public static final String DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_KEY = "dfs.namenode.metrics.logger.period.seconds"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 7249e3ddf1..ad7a0dbf8b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -815,7 +815,7 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException { if (conf.getBoolean(DFS_NAMENODE_AUDIT_LOG_ASYNC_KEY, DFS_NAMENODE_AUDIT_LOG_ASYNC_DEFAULT)) { LOG.info("Enabling async auditlog"); - enableAsyncAuditLog(); + enableAsyncAuditLog(conf); } fsLock = new FSNamesystemLock(conf, detailedLockHoldTimeMetrics); cond = fsLock.newWriteLockCondition(); @@ -8484,7 +8484,7 @@ public void logAuditMessage(String message) { } } - private static void enableAsyncAuditLog() { + private static void enableAsyncAuditLog(Configuration conf) { if (!(auditLog instanceof Log4JLogger)) { LOG.warn("Log4j is required to enable async auditlog"); return; @@ -8495,6 +8495,14 @@ private static void enableAsyncAuditLog() { // failsafe against trying to async it more than once if (!appenders.isEmpty() && !(appenders.get(0) instanceof AsyncAppender)) { AsyncAppender asyncAppender = new AsyncAppender(); + asyncAppender.setBlocking(conf.getBoolean( + DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_ASYNC_BLOCKING_KEY, + DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_ASYNC_BLOCKING_DEFAULT + )); + asyncAppender.setBufferSize(conf.getInt( + DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_ASYNC_BUFFER_SIZE_KEY, + DFSConfigKeys.DFS_NAMENODE_AUDIT_LOG_ASYNC_BUFFER_SIZE_DEFAULT + )); // change logger to have an async appender containing all the // previously configured appenders for (Appender appender : appenders) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index a33522163e..ca1618f47a 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -4808,6 +4808,27 @@ + + dfs.namenode.audit.log.async.blocking + true + + Only used when enables asynchronous audit log. Sets whether audit log async + appender should wait if there is no space available in the event buffer or + immediately return. Default value is true. + + + + + dfs.namenode.audit.log.async.buffer.size + 128 + + Only used when enables asynchronous audit log. Sets the number of audit + logs allowed in the event buffer before the calling thread is blocked + (if dfs.namenode.audit.log.async.blocking is true) or until logs are + summarized and discarded. Default value is 128. + + + dfs.namenode.audit.log.token.tracking.id false