big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-7491. Add incremental blockreport latency to DN metrics. Contributed by Ming Ma.

Browse Source
This commit is contained in:
cnauroth 2015-03-11 14:11:19 -07:00
parent 344d7cb6dc
commit fb34f45727
5 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
View File

@ -303,6 +303,8 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
| `HeartbeatsAvgTime` | Average heartbeat time in milliseconds |
| `BlockReportsNumOps` | Total number of block report operations |
| `BlockReportsAvgTime` | Average time of block report operations in milliseconds |
| `IncrementalBlockReportsNumOps` | Total number of incremental block report operations |
| `IncrementalBlockReportsAvgTime` | Average time of incremental block report operations in milliseconds |
| `CacheReportsNumOps` | Total number of cache report operations |
| `CacheReportsAvgTime` | Average time of cache report operations in milliseconds |
| `PacketAckRoundTripTimeNanosNumOps` | Total number of ack round trip |

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -740,6 +740,9 @@ Release 2.7.0 - UNRELEASED
HDFS-6806. HDFS Rolling upgrade document should mention the versions
available. (J.Andreina via aajisaka)
HDFS-7491. Add incremental blockreport latency to DN metrics.
(Ming Ma via cnauroth)
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
View File

@ -291,12 +291,14 @@ private void reportReceivedDeletedBlocks() throws IOException {
// Send incremental block reports to the Namenode outside the lock
boolean success = false;
final long startTime = Time.monotonicNow();
try {
bpNamenode.blockReceivedAndDeleted(bpRegistration,
bpos.getBlockPoolId(),
reports.toArray(new StorageReceivedDeletedBlocks[reports.size()]));
success = true;
} finally {
dn.getMetrics().addIncrementalBlockReport(Time.monotonicNow()-startTime);
if (!success) {
synchronized (pendingIncrementalBRperStorage) {
for (StorageReceivedDeletedBlocks report : reports) {

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
View File

@ -107,6 +107,7 @@ public class DataNodeMetrics {
@Metric MutableRate replaceBlockOp;
@Metric MutableRate heartbeats;
@Metric MutableRate blockReports;
@Metric MutableRate incrementalBlockReports;
@Metric MutableRate cacheReports;
@Metric MutableRate packetAckRoundTripTimeNanos;
final MutableQuantiles[] packetAckRoundTripTimeNanosQuantiles;
@ -201,6 +202,10 @@ public void addBlockReport(long latency) {
blockReports.add(latency);
}
public void addIncrementalBlockReport(long latency) {
incrementalBlockReports.add(latency);
}
public void addCacheReport(long latency) {
cacheReports.add(latency);
}

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java
View File

@ -72,6 +72,8 @@ public void testDataNodeMetrics() throws Exception {
DataNode datanode = datanodes.get(0);
MetricsRecordBuilder rb = getMetrics(datanode.getMetrics().name());
assertCounter("BytesWritten", LONG_FILE_LEN, rb);
assertTrue("Expected non-zero number of incremental block reports",
getLongCounter("IncrementalBlockReportsNumOps", rb) > 0);
} finally {
if (cluster != null) {cluster.shutdown();}
}