big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-9674. The HTrace span for OpWriteBlock should record the maxWriteToDisk time. Contributed by Colin McCabe.

Browse Source 
Change-Id: I9bf3f3bcd57f5880189ad7c160f3dd66f97d904b
This commit is contained in:
Zhe Zhang 2016-01-21 13:25:42 -08:00
parent 2a867355df
commit b4a05c1fd5
2 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -941,6 +941,9 @@ Release 2.9.0 - UNRELEASED
HDFS-9542. Move BlockIdManager from FSNamesystem to BlockManager. (jing9) HDFS-9542. Move BlockIdManager from FSNamesystem to BlockManager. (jing9)
HDFS-9674. The HTrace span for OpWriteBlock should record the maxWriteToDisk
time. (cmccabe via zhz)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
View File

@ -58,6 +58,8 @@
import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Time;
import org.apache.htrace.core.Span;
import org.apache.htrace.core.Tracer;
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED; import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.POSIX_FADV_DONTNEED;
import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.SYNC_FILE_RANGE_WRITE; import static org.apache.hadoop.io.nativeio.NativeIO.POSIX.SYNC_FILE_RANGE_WRITE;
@ -136,6 +138,7 @@ class BlockReceiver implements Closeable {
private long lastResponseTime = 0; private long lastResponseTime = 0;
private boolean isReplaceBlock = false; private boolean isReplaceBlock = false;
private DataOutputStream replyOut = null; private DataOutputStream replyOut = null;
private long maxWriteToDiskMs = 0;
private boolean pinning; private boolean pinning;
private long lastSentTime; private long lastSentTime;
@ -302,6 +305,11 @@ String getStorageUuid() {
*/ */
@Override @Override
public void close() throws IOException { public void close() throws IOException {
Span span = Tracer.getCurrentSpan();
if (span != null) {
span.addKVAnnotation("maxWriteToDiskMs",
Long.toString(maxWriteToDiskMs));
}
packetReceiver.close(); packetReceiver.close();
IOException ioe = null; IOException ioe = null;
@ -697,6 +705,9 @@ private int receivePacket() throws IOException {
long begin = Time.monotonicNow(); long begin = Time.monotonicNow();
out.write(dataBuf.array(), startByteToDisk, numBytesToDisk); out.write(dataBuf.array(), startByteToDisk, numBytesToDisk);
long duration = Time.monotonicNow() - begin; long duration = Time.monotonicNow() - begin;
if (duration > maxWriteToDiskMs) {
maxWriteToDiskMs = duration;
}
if (duration > datanodeSlowLogThresholdMs) { if (duration > datanodeSlowLogThresholdMs) {
LOG.warn("Slow BlockReceiver write data to disk cost:" + duration LOG.warn("Slow BlockReceiver write data to disk cost:" + duration
+ "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)"); + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)");