big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15039. Cache meta file length of FinalizedReplica to reduce call File.length(). Contributed by Yang Yun.

Browse Source
This commit is contained in:
Wei-Chiu Chuang 2020-03-12 19:23:12 -07:00
parent 0695f7a538
commit 20903f72b4
2 changed files with 45 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/FinalizedReplica.java
View File

@ -30,6 +30,7 @@
*/ */
public class FinalizedReplica extends LocalReplica { public class FinalizedReplica extends LocalReplica {
private byte[] lastPartialChunkChecksum; private byte[] lastPartialChunkChecksum;
private int metaLength = -1;
/** /**
* Constructor. * Constructor.
* @param blockId block id * @param blockId block id
@ -144,6 +145,14 @@ public ReplicaRecoveryInfo createInfo() {
" does not support createInfo"); " does not support createInfo");
} }
@Override
public long getMetadataLength() {
if (metaLength < 0) {
metaLength = (int)super.getMetadataLength();
}
return metaLength;
}
public byte[] getLastPartialChunkChecksum() { public byte[] getLastPartialChunkChecksum() {
return lastPartialChunkChecksum; return lastPartialChunkChecksum;
} }

36
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
View File

@ -20,6 +20,8 @@
import com.google.common.base.Supplier; import com.google.common.base.Supplier;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import java.io.FileInputStream;
import java.io.OutputStream;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
@ -987,4 +989,38 @@ public void testReplicaCacheFileToOtherPlace() throws IOException {
assertTrue(f.exists()); assertTrue(f.exists());
} }
} }
@Test
public void testGetMetadataLengthOfFinalizedReplica() throws IOException {
FsVolumeImpl fsv1 = Mockito.mock(FsVolumeImpl.class);
File blockDir = new File(BASE_DIR,"testFinalizedReplica/block");
if (!blockDir.exists()) {
assertTrue(blockDir.mkdirs());
}
long blockID = 1;
long genStamp = 2;
File metaFile = new File(blockDir,Block.BLOCK_FILE_PREFIX +
blockID + "_" + genStamp + Block.METADATA_EXTENSION);
// create meta file on disk
OutputStream os = new FileOutputStream(metaFile);
os.write("TEST_META_SIZE".getBytes());
os.close();
long fileLength = metaFile.length();
ReplicaInfo replica = new FinalizedReplica(
blockID, 2, genStamp, fsv1, blockDir);
long metaLength = replica.getMetadataLength();
assertEquals(fileLength, metaLength);
// Delete the meta file on disks, make sure we still can get the length
// from cached meta size.
metaFile.delete();
metaLength = replica.getMetadataLength();
assertEquals(fileLength, metaLength);
if (!blockDir.exists()) {
assertTrue(blockDir.delete());
}
}
} }