big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "HDFS-12502. nntop should support a category based on FilesInGetListingOps."

Browse Source 
This reverts commit 60bfee270e.
This commit is contained in:
Zhe Zhang 2017-10-24 15:06:50 -07:00
parent 67e7673750
commit 17cd8d0c17
3 changed files with 10 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -3673,11 +3673,6 @@ DirectoryListing getListing(String src, byte[] startAfter,
readUnlock(operationName); readUnlock(operationName);
} }
logAuditEvent(true, operationName, src); logAuditEvent(true, operationName, src);
if (topConf.isEnabled && isAuditEnabled() && isExternalInvocation()
&& dl != null && Server.getRemoteUser() != null) {
topMetrics.reportFilesInGetListing(Server.getRemoteUser().toString(),
dl.getPartialListing().length);
}
return dl; return dl;
} }

30
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/metrics/TopMetrics.java
View File

@ -70,14 +70,6 @@ public class TopMetrics implements MetricsSource {
public static final Logger LOG = LoggerFactory.getLogger(TopMetrics.class); public static final Logger LOG = LoggerFactory.getLogger(TopMetrics.class);
public static final String TOPMETRICS_METRICS_SOURCE_NAME = public static final String TOPMETRICS_METRICS_SOURCE_NAME =
"NNTopUserOpCounts"; "NNTopUserOpCounts";
/**
* In addition to counts of different RPC calls, NNTop also reports top
* users listing large directories (measured by the number of files involved
* in listing operations from the user). This is important because the CPU
* and GC overhead of a listing operation grows linearly with the number of
* files involved. This category in NNTop is {@link #FILES_IN_GETLISTING}.
*/
public static final String FILES_IN_GETLISTING = "filesInGetListing";
private final boolean isMetricsSourceEnabled; private final boolean isMetricsSourceEnabled;
private static void logConf(Configuration conf) { private static void logConf(Configuration conf) {
@ -131,30 +123,22 @@ public List<TopWindow> getTopWindows() {
public void report(boolean succeeded, String userName, InetAddress addr, public void report(boolean succeeded, String userName, InetAddress addr,
String cmd, String src, String dst, FileStatus status) { String cmd, String src, String dst, FileStatus status) {
// currently nntop only makes use of the username and the command // currently nntop only makes use of the username and the command
report(userName, cmd, 1); report(userName, cmd);
} }
public void reportFilesInGetListing(String userName, int numFiles) { public void report(String userName, String cmd) {
report(userName, FILES_IN_GETLISTING, numFiles);
}
public void report(String userName, String cmd, int delta) {
long currTime = Time.monotonicNow(); long currTime = Time.monotonicNow();
report(currTime, userName, cmd, delta); report(currTime, userName, cmd);
} }
public void report(long currTime, String userName, String cmd, int delta) { public void report(long currTime, String userName, String cmd) {
LOG.debug("a metric is reported: cmd: {} user: {}", cmd, userName); LOG.debug("a metric is reported: cmd: {} user: {}", cmd, userName);
userName = UserGroupInformation.trimLoginMethod(userName); userName = UserGroupInformation.trimLoginMethod(userName);
for (RollingWindowManager rollingWindowManager : rollingWindowManagers for (RollingWindowManager rollingWindowManager : rollingWindowManagers
.values()) { .values()) {
rollingWindowManager.recordMetric(currTime, cmd, userName, delta); rollingWindowManager.recordMetric(currTime, cmd, userName, 1);
// Increase the number of all RPC calls by the user, unless the report rollingWindowManager.recordMetric(currTime,
// is for the number of files in a listing operation. TopConf.ALL_CMDS, userName, 1);
if (!cmd.equals(FILES_IN_GETLISTING)) {
rollingWindowManager.recordMetric(currTime,
TopConf.ALL_CMDS, userName, delta);
}
} }
} }

11
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestTopMetrics.java
View File

@ -26,7 +26,6 @@
import org.junit.Test; import org.junit.Test;
import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME; import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.TOPMETRICS_METRICS_SOURCE_NAME;
import static org.apache.hadoop.hdfs.server.namenode.top.metrics.TopMetrics.FILES_IN_GETLISTING;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.mockito.Mockito.times; import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
@ -42,10 +41,9 @@ public void testPresence() {
TopMetrics topMetrics = new TopMetrics(conf, TopMetrics topMetrics = new TopMetrics(conf,
topConf.nntopReportingPeriodsMs); topConf.nntopReportingPeriodsMs);
// Dummy command // Dummy command
topMetrics.report("test", "listStatus", 1); topMetrics.report("test", "listStatus");
topMetrics.report("test", "listStatus", 1); topMetrics.report("test", "listStatus");
topMetrics.report("test", "listStatus", 1); topMetrics.report("test", "listStatus");
topMetrics.report("test", FILES_IN_GETLISTING, 1000);
MetricsRecordBuilder rb = getMetrics(topMetrics); MetricsRecordBuilder rb = getMetrics(topMetrics);
MetricsCollector mc = rb.parent(); MetricsCollector mc = rb.parent();
@ -61,8 +59,5 @@ public void testPresence() {
verify(rb, times(3)).addCounter(Interns.info("op=listStatus." + verify(rb, times(3)).addCounter(Interns.info("op=listStatus." +
"user=test.count", "Total operations performed by user"), 3L); "user=test.count", "Total operations performed by user"), 3L);
verify(rb, times(3)).addCounter(Interns.info("op=" + FILES_IN_GETLISTING +
".user=test.count", "Total operations performed by user"), 1000L);
} }
} }