big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15124. Crashing bugs in NameNode when using a valid configuration for . Contributed by Ctest.

Browse Source
This commit is contained in:
Ayush Saxena 2020-02-27 22:19:35 +05:30
parent 57aa048516
commit cd2c6b1aac
3 changed files with 32 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -1110,6 +1110,7 @@ private List<AuditLogger> initAuditLoggers(Configuration conf) {
Collection<String> alClasses = Collection<String> alClasses =
conf.getTrimmedStringCollection(DFS_NAMENODE_AUDIT_LOGGERS_KEY); conf.getTrimmedStringCollection(DFS_NAMENODE_AUDIT_LOGGERS_KEY);
List<AuditLogger> auditLoggers = Lists.newArrayList(); List<AuditLogger> auditLoggers = Lists.newArrayList();
boolean topAuditLoggerAdded = false;
if (alClasses != null && !alClasses.isEmpty()) { if (alClasses != null && !alClasses.isEmpty()) {
for (String className : alClasses) { for (String className : alClasses) {
try { try {
@ -1118,9 +1119,16 @@ private List<AuditLogger> initAuditLoggers(Configuration conf) {
logger = new FSNamesystemAuditLogger(); logger = new FSNamesystemAuditLogger();
} else { } else {
logger = (AuditLogger) Class.forName(className).newInstance(); logger = (AuditLogger) Class.forName(className).newInstance();
if (TopAuditLogger.class.getName().equals(
logger.getClass().getName())) {
topAuditLoggerAdded = true;
}
} }
logger.initialize(conf); logger.initialize(conf);
auditLoggers.add(logger); auditLoggers.add(logger);
} catch (InstantiationException e) {
LOG.error("{} instantiation failed.", className, e);
throw new RuntimeException(e);
} catch (RuntimeException re) { } catch (RuntimeException re) {
throw re; throw re;
} catch (Exception e) { } catch (Exception e) {
@ -1137,7 +1145,7 @@ private List<AuditLogger> initAuditLoggers(Configuration conf) {
} }
// Add audit logger to calculate top users // Add audit logger to calculate top users
if (topConf.isEnabled) { if (topConf.isEnabled && !topAuditLoggerAdded) {
topMetrics = new TopMetrics(conf, topConf.nntopReportingPeriodsMs); topMetrics = new TopMetrics(conf, topConf.nntopReportingPeriodsMs);
if (DefaultMetricsSystem.instance().getSource( if (DefaultMetricsSystem.instance().getSource(
TOPMETRICS_METRICS_SOURCE_NAME) == null) { TOPMETRICS_METRICS_SOURCE_NAME) == null) {

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/TopAuditLogger.java
View File

@ -23,12 +23,16 @@
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.namenode.AuditLogger; import org.apache.hadoop.hdfs.server.namenode.AuditLogger;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics; import org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics;
import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME;
/** /**
* An {@link AuditLogger} that sends logged data directly to the metrics * An {@link AuditLogger} that sends logged data directly to the metrics
* systems. It is used when the top service is used directly by the name node * systems. It is used when the top service is used directly by the name node
@ -39,6 +43,17 @@ public class TopAuditLogger implements AuditLogger {
private final TopMetrics topMetrics; private final TopMetrics topMetrics;
public TopAuditLogger() {
Configuration conf = new HdfsConfiguration();
TopConf topConf = new TopConf(conf);
this.topMetrics = new TopMetrics(conf, topConf.nntopReportingPeriodsMs);
if (DefaultMetricsSystem.instance().getSource(
TOPMETRICS_METRICS_SOURCE_NAME) == null) {
DefaultMetricsSystem.instance().register(TOPMETRICS_METRICS_SOURCE_NAME,
"Top N operations by user", topMetrics);
}
}
public TopAuditLogger(TopMetrics topMetrics) { public TopAuditLogger(TopMetrics topMetrics) {
Preconditions.checkNotNull(topMetrics, "Cannot init with a null " + Preconditions.checkNotNull(topMetrics, "Cannot init with a null " +
"TopMetrics"); "TopMetrics");

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
View File

@ -303,6 +303,14 @@ public void testInitAuditLoggers() throws IOException {
.or(instanceOf(TopAuditLogger.class)) .or(instanceOf(TopAuditLogger.class))
.or(instanceOf(DummyAuditLogger.class))); .or(instanceOf(DummyAuditLogger.class)));
} }
// Test Configuring TopAuditLogger.
conf.set(DFSConfigKeys.DFS_NAMENODE_AUDIT_LOGGERS_KEY,
"org.apache.hadoop.hdfs.server.namenode.top.TopAuditLogger");
fsn = new FSNamesystem(conf, fsImage);
auditLoggers = fsn.getAuditLoggers();
assertEquals(1, auditLoggers.size());
assertThat(auditLoggers.get(0), instanceOf(TopAuditLogger.class));
} }
static class DummyAuditLogger implements AuditLogger { static class DummyAuditLogger implements AuditLogger {