big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15406. Improve the speed of Datanode Block Scan. Contributed by hemanthboyina

Browse Source
This commit is contained in:
S O'Donnell 2020-06-18 12:26:22 +01:00
parent 9cbd76cc77
commit 123777823e
2 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
View File

@ -133,6 +133,7 @@ public class FsVolumeImpl implements FsVolumeSpi {
protected volatile long configuredCapacity; protected volatile long configuredCapacity;
private final FileIoProvider fileIoProvider; private final FileIoProvider fileIoProvider;
private final DataNodeVolumeMetrics metrics; private final DataNodeVolumeMetrics metrics;
private URI baseURI;
/** /**
* Per-volume worker pool that processes new blocks to cache. * Per-volume worker pool that processes new blocks to cache.
@ -182,6 +183,7 @@ public class FsVolumeImpl implements FsVolumeSpi {
File parent = currentDir.getParentFile(); File parent = currentDir.getParentFile();
cacheExecutor = initializeCacheExecutor(parent); cacheExecutor = initializeCacheExecutor(parent);
this.metrics = DataNodeVolumeMetrics.create(conf, parent.getPath()); this.metrics = DataNodeVolumeMetrics.create(conf, parent.getPath());
this.baseURI = new File(currentDir.getParent()).toURI();
} else { } else {
cacheExecutor = null; cacheExecutor = null;
this.metrics = null; this.metrics = null;
@ -506,7 +508,7 @@ BlockPoolSlice getBlockPoolSlice(String bpid) throws IOException {
@Override @Override
public URI getBaseURI() { public URI getBaseURI() {
return new File(currentDir.getParent()).toURI(); return baseURI;
} }
@Override @Override

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/DataNodeTestUtils.java
View File

@ -239,7 +239,7 @@ public static FsVolumeImpl getVolume(DataNode dn, File basePath) throws
try (FsDatasetSpi.FsVolumeReferences volumes = dn.getFSDataset() try (FsDatasetSpi.FsVolumeReferences volumes = dn.getFSDataset()
.getFsVolumeReferences()) { .getFsVolumeReferences()) {
for (FsVolumeSpi vol : volumes) { for (FsVolumeSpi vol : volumes) {
if (vol.getBaseURI().equals(basePath.toURI())) { if (new File(vol.getBaseURI()).equals(basePath)) {
return (FsVolumeImpl) vol; return (FsVolumeImpl) vol;
} }
} }