big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12320. Add quantiles for transactions batched in Journal sync. Contributed by Hanisha Koneru.

Browse Source
This commit is contained in:
Anu Engineer 2017-09-23 10:34:13 -07:00
parent 36fb90c939
commit 423ebcaf51
2 changed files with 28 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
View File

@ -268,6 +268,24 @@ The server-side metrics for a journal from the JournalNode's perspective. Each m
| `Syncs3600s90thPercentileLatencyMicros` | The 90th percentile of sync latency in microseconds (1 hour granularity) | | `Syncs3600s90thPercentileLatencyMicros` | The 90th percentile of sync latency in microseconds (1 hour granularity) |
| `Syncs3600s95thPercentileLatencyMicros` | The 95th percentile of sync latency in microseconds (1 hour granularity) | | `Syncs3600s95thPercentileLatencyMicros` | The 95th percentile of sync latency in microseconds (1 hour granularity) |
| `Syncs3600s99thPercentileLatencyMicros` | The 99th percentile of sync latency in microseconds (1 hour granularity) | | `Syncs3600s99thPercentileLatencyMicros` | The 99th percentile of sync latency in microseconds (1 hour granularity) |
| `NumTransactionsBatchedInSync60sNumOps` | Number of times transactions were batched in sync operation (1 minute granularity) |
| `NumTransactionsBatchedInSync60s50thPercentileLatencyMicros` | The 50th percentile of transactions batched in sync count (1 minute granularity) |
| `NumTransactionsBatchedInSync60s75thPercentileLatencyMicros` | The 75th percentile of transactions batched in sync count (1 minute granularity) |
| `NumTransactionsBatchedInSync60s90thPercentileLatencyMicros` | The 90th percentile of transactions batched in sync count (1 minute granularity) |
| `NumTransactionsBatchedInSync60s95thPercentileLatencyMicros` | The 95th percentile of transactions batched in sync count (1 minute granularity) |
| `NumTransactionsBatchedInSync60s99thPercentileLatencyMicros` | The 99th percentile of transactions batched in sync count (1 minute granularity) |
| `NumTransactionsBatchedInSync300sNumOps` | Number of times transactions were batched in sync operation (5 minutes granularity) |
| `NumTransactionsBatchedInSync300s50thPercentileLatencyMicros` | The 50th percentile of transactions batched in sync count (5 minutes granularity) |
| `NumTransactionsBatchedInSync300s75thPercentileLatencyMicros` | The 75th percentile of transactions batched in sync count (5 minutes granularity) |
| `NumTransactionsBatchedInSync300s90thPercentileLatencyMicros` | The 90th percentile of transactions batched in sync count (5 minutes granularity) |
| `NumTransactionsBatchedInSync300s95thPercentileLatencyMicros` | The 95th percentile of transactions batched in sync count (5 minutes granularity) |
| `NumTransactionsBatchedInSync300s99thPercentileLatencyMicros` | The 99th percentile of transactions batched in sync count (5 minutes granularity) |
| `NumTransactionsBatchedInSync3600sNumOps` | Number of times transactions were batched in sync operation (1 hour granularity) |
| `NumTransactionsBatchedInSync3600s50thPercentileLatencyMicros` | The 50th percentile of transactions batched in sync count (1 hour granularity) |
| `NumTransactionsBatchedInSync3600s75thPercentileLatencyMicros` | The 75th percentile of transactions batched in sync count (1 hour granularity) |
| `NumTransactionsBatchedInSync3600s90thPercentileLatencyMicros` | The 90th percentile of transactions batched in sync count (1 hour granularity) |
| `NumTransactionsBatchedInSync3600s95thPercentileLatencyMicros` | The 95th percentile of transactions batched in sync count (1 hour granularity) |
| `NumTransactionsBatchedInSync3600s99thPercentileLatencyMicros` | The 99th percentile of transactions batched in sync count (1 hour granularity) |
| `BatchesWritten` | Total number of batches written since startup | | `BatchesWritten` | Total number of batches written since startup |
| `TxnsWritten` | Total number of transactions written since startup | | `TxnsWritten` | Total number of transactions written since startup |
| `BytesWritten` | Total number of bytes written since startup | | `BytesWritten` | Total number of bytes written since startup |

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
View File

@ -115,6 +115,8 @@ public long totalFileOps(){
final MutableQuantiles[] syncsQuantiles; final MutableQuantiles[] syncsQuantiles;
@Metric("Journal transactions batched in sync") @Metric("Journal transactions batched in sync")
MutableCounterLong transactionsBatchedInSync; MutableCounterLong transactionsBatchedInSync;
@Metric("Journal transactions batched in sync")
final MutableQuantiles[] numTransactionsBatchedInSync;
@Metric("Number of blockReports from individual storages") @Metric("Number of blockReports from individual storages")
MutableRate storageBlockReport; MutableRate storageBlockReport;
final MutableQuantiles[] storageBlockReportQuantiles; final MutableQuantiles[] storageBlockReportQuantiles;
@ -148,6 +150,7 @@ public long totalFileOps(){
final int len = intervals.length; final int len = intervals.length;
syncsQuantiles = new MutableQuantiles[len]; syncsQuantiles = new MutableQuantiles[len];
numTransactionsBatchedInSync = new MutableQuantiles[len];
storageBlockReportQuantiles = new MutableQuantiles[len]; storageBlockReportQuantiles = new MutableQuantiles[len];
cacheReportQuantiles = new MutableQuantiles[len]; cacheReportQuantiles = new MutableQuantiles[len];
generateEDEKTimeQuantiles = new MutableQuantiles[len]; generateEDEKTimeQuantiles = new MutableQuantiles[len];
@ -159,6 +162,10 @@ public long totalFileOps(){
syncsQuantiles[i] = registry.newQuantiles( syncsQuantiles[i] = registry.newQuantiles(
"syncs" + interval + "s", "syncs" + interval + "s",
"Journal syncs", "ops", "latency", interval); "Journal syncs", "ops", "latency", interval);
numTransactionsBatchedInSync[i] = registry.newQuantiles(
"numTransactionsBatchedInSync" + interval + "s",
"Number of Transactions batched in sync", "ops",
"count", interval);
storageBlockReportQuantiles[i] = registry.newQuantiles( storageBlockReportQuantiles[i] = registry.newQuantiles(
"storageBlockReport" + interval + "s", "storageBlockReport" + interval + "s",
"Storage block report", "ops", "latency", interval); "Storage block report", "ops", "latency", interval);
@ -304,6 +311,9 @@ public void addTransaction(long latency) {
public void incrTransactionsBatchedInSync(long count) { public void incrTransactionsBatchedInSync(long count) {
transactionsBatchedInSync.incr(count); transactionsBatchedInSync.incr(count);
for (MutableQuantiles q : numTransactionsBatchedInSync) {
q.add(count);
}
} }
public void incSuccessfulReReplications() { public void incSuccessfulReReplications() {