big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16089. Add metric EcReconstructionValidateTimeMillis for StripedBlockReconstructor (#3146)

Browse Source
This commit is contained in:
litao 2021-06-29 18:15:12 +08:00 committed by GitHub
parent 93a1685073
commit 95454d821c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 14 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
View File

@ -135,9 +135,16 @@ private void reconstructTargets(int toReconstructLen) throws IOException {
resetBuffers(inputs); resetBuffers(inputs);
DataNodeFaultInjector.get().badDecoding(outputs); DataNodeFaultInjector.get().badDecoding(outputs);
long start = Time.monotonicNow();
try { try {
getValidator().validate(inputs, erasedIndices, outputs); getValidator().validate(inputs, erasedIndices, outputs);
long validateEnd = Time.monotonicNow();
getDatanode().getMetrics().incrECReconstructionValidateTime(
validateEnd - start);
} catch (InvalidDecodingException e) { } catch (InvalidDecodingException e) {
long validateFailedEnd = Time.monotonicNow();
getDatanode().getMetrics().incrECReconstructionValidateTime(
validateFailedEnd - start);
getDatanode().getMetrics().incrECInvalidReconstructionTasks(); getDatanode().getMetrics().incrECInvalidReconstructionTasks();
throw e; throw e;
} }

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
View File

@ -168,6 +168,8 @@ public class DataNodeMetrics {
private MutableCounterLong ecReconstructionDecodingTimeMillis; private MutableCounterLong ecReconstructionDecodingTimeMillis;
@Metric("Milliseconds spent on write by erasure coding worker") @Metric("Milliseconds spent on write by erasure coding worker")
private MutableCounterLong ecReconstructionWriteTimeMillis; private MutableCounterLong ecReconstructionWriteTimeMillis;
@Metric("Milliseconds spent on validating by erasure coding worker")
private MutableCounterLong ecReconstructionValidateTimeMillis;
@Metric("Sum of all BPServiceActors command queue length") @Metric("Sum of all BPServiceActors command queue length")
private MutableCounterLong sumOfActorCommandQueueLength; private MutableCounterLong sumOfActorCommandQueueLength;
@Metric("Num of processed commands of all BPServiceActors") @Metric("Num of processed commands of all BPServiceActors")
@ -629,6 +631,10 @@ public void incrECReconstructionDecodingTime(long millis) {
ecReconstructionDecodingTimeMillis.incr(millis); ecReconstructionDecodingTimeMillis.incr(millis);
} }
public void incrECReconstructionValidateTime(long millis) {
ecReconstructionValidateTimeMillis.incr(millis);
}
public DataNodeUsageReport getDNUsageReport(long timeSinceLastReport) { public DataNodeUsageReport getDNUsageReport(long timeSinceLastReport) {
return dnUsageReportUtil.getUsageReport(bytesWritten.value(), bytesRead return dnUsageReportUtil.getUsageReport(bytesWritten.value(), bytesRead
.value(), totalWriteTime.value(), totalReadTime.value(), .value(), totalWriteTime.value(), totalReadTime.value(),

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
View File

@ -364,7 +364,7 @@ public void testCountLiveReplicas() throws Exception {
} }
} }
@Test(timeout=120000) // 1 min timeout @Test(timeout=120000) // 2 min timeout
public void testReconstructionWork() throws Exception { public void testReconstructionWork() throws Exception {
Configuration conf = new HdfsConfiguration(); Configuration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0); conf.setLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 0);