big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15369. Refactor method VolumeScanner#runLoop(). Contributed by Yang Yun.

Browse Source
This commit is contained in:
Ayush Saxena 2020-05-24 18:03:41 +05:30
parent f4901d0778
commit f43a152b97
1 changed files with 47 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
View File

@ -483,6 +483,50 @@ static boolean calculateShouldScan(String storageId, long targetBytesPerSec,
return shouldScan;
}
/**
* Get next block and check if it's needed to scan.
*
* @return the candidate block.
*/
ExtendedBlock getNextBlockToScan() {
ExtendedBlock block;
try {
block = curBlockIter.nextBlock();
} catch (IOException e) {
// There was an error listing the next block in the volume. This is a
// serious issue.
LOG.warn("{}: nextBlock error on {}", this, curBlockIter);
// On the next loop iteration, curBlockIter#eof will be set to true, and
// we will pick a different block iterator.
return null;
}
if (block == null) {
// The BlockIterator is at EOF.
LOG.info("{}: finished scanning block pool {}",
this, curBlockIter.getBlockPoolId());
saveBlockIterator(curBlockIter);
return null;
} else if (conf.skipRecentAccessed) {
// Check the access time of block file to avoid scanning recently
// changed blocks, reducing disk IO.
try {
BlockLocalPathInfo blockLocalPathInfo =
volume.getDataset().getBlockLocalPathInfo(block);
BasicFileAttributes attr = Files.readAttributes(
new File(blockLocalPathInfo.getBlockPath()).toPath(),
BasicFileAttributes.class);
if (System.currentTimeMillis() - attr.lastAccessTime().
to(TimeUnit.MILLISECONDS) < conf.scanPeriodMs) {
return null;
}
} catch (IOException ioe) {
LOG.debug("Failed to get access time of block {}",
block, ioe);
}
}
return block;
}
/**
* Run an iteration of the VolumeScanner loop.
*
@ -507,10 +551,10 @@ private long runLoop(ExtendedBlock suspectBlock) {
return 30000L;
}
// Find a usable block pool to scan.
if (suspectBlock != null) {
block = suspectBlock;
} else {
// Find a usable block pool to scan.
if ((curBlockIter == null) || curBlockIter.atEnd()) {
long timeout = findNextUsableBlockIter();
if (timeout > 0) {
@ -528,40 +572,9 @@ private long runLoop(ExtendedBlock suspectBlock) {
}
return 0L;
}
try {
block = curBlockIter.nextBlock();
} catch (IOException e) {
// There was an error listing the next block in the volume. This is a
// serious issue.
LOG.warn("{}: nextBlock error on {}", this, curBlockIter);
// On the next loop iteration, curBlockIter#eof will be set to true, and
// we will pick a different block iterator.
return 0L;
}
block = getNextBlockToScan();
if (block == null) {
// The BlockIterator is at EOF.
LOG.info("{}: finished scanning block pool {}",
this, curBlockIter.getBlockPoolId());
saveBlockIterator(curBlockIter);
return 0;
} else if (conf.skipRecentAccessed) {
// Check the access time of block file to avoid scanning recently
// changed blocks, reducing disk IO.
try {
BlockLocalPathInfo blockLocalPathInfo =
volume.getDataset().getBlockLocalPathInfo(block);
BasicFileAttributes attr = Files.readAttributes(
new File(blockLocalPathInfo.getBlockPath()).toPath(),
BasicFileAttributes.class);
if (System.currentTimeMillis() - attr.lastAccessTime().
to(TimeUnit.MILLISECONDS) < conf.scanPeriodMs) {
return 0;
}
} catch (IOException ioe) {
LOG.debug("Failed to get access time of block {}",
block, ioe);
}
return 0L;
}
}
if (curBlockIter != null) {