big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[HDFS-16971] Add read metrics for remote reads in FileSystem Statistics #5534 (#5536)

Browse Source
This commit is contained in:
Melissa You 2023-04-13 09:07:42 -07:00 committed by GitHub
parent 06f9bdffa6
commit 2b60d0c1f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 64 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
View File

@ -3942,6 +3942,7 @@ public static class StatisticsData {
private volatile long bytesReadDistanceOfThreeOrFour;
private volatile long bytesReadDistanceOfFiveOrLarger;
private volatile long bytesReadErasureCoded;
private volatile long remoteReadTimeMS;
/**
* Add another StatisticsData object to this one.
@ -3959,6 +3960,7 @@ void add(StatisticsData other) {
this.bytesReadDistanceOfFiveOrLarger +=
other.bytesReadDistanceOfFiveOrLarger;
this.bytesReadErasureCoded += other.bytesReadErasureCoded;
this.remoteReadTimeMS += other.remoteReadTimeMS;
}
/**
@ -3977,6 +3979,7 @@ void negate() {
this.bytesReadDistanceOfFiveOrLarger =
-this.bytesReadDistanceOfFiveOrLarger;
this.bytesReadErasureCoded = -this.bytesReadErasureCoded;
this.remoteReadTimeMS = -this.remoteReadTimeMS;
}
@Override
@ -4025,6 +4028,10 @@ public long getBytesReadDistanceOfFiveOrLarger() {
public long getBytesReadErasureCoded() {
return bytesReadErasureCoded;
}
public long getRemoteReadTimeMS() {
return remoteReadTimeMS;
}
}
private interface StatisticsAggregator<T> {
@ -4252,6 +4259,14 @@ public void incrementBytesReadByDistance(int distance, long newBytes) {
}
}
/**
* Increment the time taken to read bytes from remote in the statistics.
* @param durationMS time taken in ms to read bytes from remote
*/
public void increaseRemoteReadTime(final long durationMS) {
getThreadStatistics().remoteReadTimeMS += durationMS;
}
/**
* Apply the given aggregator to all StatisticsData objects associated with
* this Statistics object.
@ -4399,6 +4414,25 @@ public long getBytesReadByDistance(int distance) {
return bytesRead;
}
/**
* Get total time taken in ms for bytes read from remote.
* @return time taken in ms for remote bytes read.
*/
public long getRemoteReadTime() {
return visitAll(new StatisticsAggregator<Long>() {
private long remoteReadTimeMS = 0;
@Override
public void accept(StatisticsData data) {
remoteReadTimeMS += data.remoteReadTimeMS;
}
public Long aggregate() {
return remoteReadTimeMS;
}
});
}
/**
* Get all statistics data.
* MR or other frameworks can use the method to get all statistics at once.

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java
View File

@ -47,7 +47,8 @@ public class FileSystemStorageStatistics extends StorageStatistics {
"bytesReadDistanceOfOneOrTwo",
"bytesReadDistanceOfThreeOrFour",
"bytesReadDistanceOfFiveOrLarger",
"bytesReadErasureCoded"
"bytesReadErasureCoded",
"remoteReadTimeMS"
};
private static class LongStatisticIterator
@ -107,6 +108,8 @@ private static Long fetch(StatisticsData data, String key) {
return data.getBytesReadDistanceOfFiveOrLarger();
case "bytesReadErasureCoded":
return data.getBytesReadErasureCoded();
case "remoteReadTimeMS":
return data.getRemoteReadTimeMS();
default:
return null;
}

6
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemStorageStatistics.java
View File

@ -52,7 +52,8 @@ public class TestFileSystemStorageStatistics {
"bytesReadDistanceOfOneOrTwo",
"bytesReadDistanceOfThreeOrFour",
"bytesReadDistanceOfFiveOrLarger",
"bytesReadErasureCoded"
"bytesReadErasureCoded",
"remoteReadTimeMS"
};
private FileSystem.Statistics statistics =
@ -74,6 +75,7 @@ public void setup() {
statistics.incrementBytesReadByDistance(1, RandomUtils.nextInt(0, 100));
statistics.incrementBytesReadByDistance(3, RandomUtils.nextInt(0, 100));
statistics.incrementBytesReadErasureCoded(RandomUtils.nextInt(0, 100));
statistics.increaseRemoteReadTime(RandomUtils.nextInt(0, 100));
}
@Test
@ -128,6 +130,8 @@ private long getStatisticsValue(String name) {
return statistics.getBytesReadByDistance(5);
case "bytesReadErasureCoded":
return statistics.getBytesReadErasureCoded();
case "remoteReadTimeMS":
return statistics.getRemoteReadTime();
default:
return 0;
}

10
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
View File

@ -3090,10 +3090,14 @@ public Peer newConnectedPeer(InetSocketAddress addr,
}
}
void updateFileSystemReadStats(int distance, int nRead) {
void updateFileSystemReadStats(int distance, int readBytes, long readTimeMS) {
if (stats != null) {
stats.incrementBytesRead(nRead);
stats.incrementBytesReadByDistance(distance, nRead);
stats.incrementBytesRead(readBytes);
stats.incrementBytesReadByDistance(distance, readBytes);
if (distance > 0) {
//remote read
stats.increaseRemoteReadTime(readTimeMS);
}
}
}

9
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
View File

@ -851,8 +851,9 @@ protected synchronized int readWithStrategy(ReaderStrategy strategy)
locatedBlocks.getFileLength() - pos);
}
}
long beginReadMS = Time.monotonicNow();
int result = readBuffer(strategy, realLen, corruptedBlocks);
long readTimeMS = Time.monotonicNow() - beginReadMS;
if (result >= 0) {
pos += result;
} else {
@ -861,7 +862,7 @@ protected synchronized int readWithStrategy(ReaderStrategy strategy)
}
updateReadStatistics(readStatistics, result, blockReader);
dfsClient.updateFileSystemReadStats(blockReader.getNetworkDistance(),
result);
result, readTimeMS);
if (readStatistics.getBlockType() == BlockType.STRIPED) {
dfsClient.updateFileSystemECReadStats(result);
}
@ -1184,6 +1185,7 @@ void actualGetFromOneDataNode(final DNAddrPair datanode, final long startInBlk,
ByteBuffer tmp = buf.duplicate();
tmp.limit(tmp.position() + len);
tmp = tmp.slice();
long beginReadMS = Time.monotonicNow();
int nread = 0;
int ret;
while (true) {
@ -1193,11 +1195,12 @@ void actualGetFromOneDataNode(final DNAddrPair datanode, final long startInBlk,
}
nread += ret;
}
long readTimeMS = Time.monotonicNow() - beginReadMS;
buf.position(buf.position() + nread);
IOUtilsClient.updateReadStatistics(readStatistics, nread, reader);
dfsClient.updateFileSystemReadStats(
reader.getNetworkDistance(), nread);
reader.getNetworkDistance(), nread, readTimeMS);
if (readStatistics.getBlockType() == BlockType.STRIPED) {
dfsClient.updateFileSystemECReadStats(nread);
}

6
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
View File

@ -331,15 +331,17 @@ private void readOneStripe(CorruptedBlocks corruptedBlocks)
* its ThreadLocal.
*
* @param stats striped read stats
* @param readTimeMS read time metrics in ms
*
*/
void updateReadStats(final StripedBlockUtil.BlockReadStats stats) {
void updateReadStats(final StripedBlockUtil.BlockReadStats stats, long readTimeMS) {
if (stats == null) {
return;
}
updateReadStatistics(readStatistics, stats.getBytesRead(),
stats.isShortCircuit(), stats.getNetworkDistance());
dfsClient.updateFileSystemReadStats(stats.getNetworkDistance(),
stats.getBytesRead());
stats.getBytesRead(), readTimeMS);
assert readStatistics.getBlockType() == BlockType.STRIPED;
dfsClient.updateFileSystemECReadStats(stats.getBytesRead());
}

5
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java
View File

@ -351,9 +351,12 @@ void readStripe() throws IOException {
// first read failure
while (!futures.isEmpty()) {
try {
long beginReadMS = Time.monotonicNow();
StripingChunkReadResult r = StripedBlockUtil
.getNextCompletedStripedRead(service, futures, 0);
dfsStripedInputStream.updateReadStats(r.getReadStats());
long readTimeMS = Time.monotonicNow() - beginReadMS;
dfsStripedInputStream.updateReadStats(r.getReadStats(), readTimeMS);
DFSClient.LOG.debug("Read task returned: {}, for stripe {}",
r, alignedStripe);
StripingChunk returnedChunk = alignedStripe.chunks[r.index];