big-data/hadoop
big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

Revert "HDFS-14476. lock too long when fix inconsistent blocks between disk and in-memory. Contributed by Sean Chow."

Browse Source 
This reverts commit b58a35f374a9a750fddc2fc92e7f7a7ae8a4d3a4.
This commit is contained in:
Wei-Chiu Chuang 2019-08-21 08:59:49 -07:00
parent 3ad1fcfc8b
commit 57f7370174
1 changed files with 1 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
View File

@ -66,7 +66,7 @@ public class DirectoryScanner implements Runnable {
LoggerFactory.getLogger(DirectoryScanner.class); LoggerFactory.getLogger(DirectoryScanner.class);
private static final int DEFAULT_MAP_SIZE = 32768; private static final int DEFAULT_MAP_SIZE = 32768;
private static final int RECONCILE_BLOCKS_BATCH_SIZE = 1000;
private final FsDatasetSpi<?> dataset; private final FsDatasetSpi<?> dataset;
private final ExecutorService reportCompileThreadPool; private final ExecutorService reportCompileThreadPool;
private final ScheduledExecutorService masterThread; private final ScheduledExecutorService masterThread;
@ -424,23 +424,10 @@ public class DirectoryScanner implements Runnable {
*/ */
@VisibleForTesting @VisibleForTesting
public void reconcile() throws IOException { public void reconcile() throws IOException {
LOG.debug("reconcile start DirectoryScanning");
scan(); scan();
// HDFS-14476: run checkAndUpadte with batch to avoid holding the lock too
// long
int loopCount = 0;
for (final Map.Entry<String, ScanInfo> entry : diffs.getEntries()) { for (final Map.Entry<String, ScanInfo> entry : diffs.getEntries()) {
dataset.checkAndUpdate(entry.getKey(), entry.getValue()); dataset.checkAndUpdate(entry.getKey(), entry.getValue());
if (loopCount % RECONCILE_BLOCKS_BATCH_SIZE == 0) {
try {
Thread.sleep(2000);
} catch (InterruptedException e) {
// do nothing
}
}
loopCount++;
} }
if (!retainDiffs) { if (!retainDiffs) {