big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-7217. Better batching of IBRs. Contributed by Kihwal Lee.

Browse Source
This commit is contained in:
Kihwal Lee 2014-10-09 12:07:51 -05:00
parent 8d7c54967d
commit db71bb54bc
4 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -660,6 +660,8 @@ Release 2.6.0 - UNRELEASED
HDFS-7169. Add SE_BAD_FIELD to findbugsExcludeFile.xml. (szetszwo) HDFS-7169. Add SE_BAD_FIELD to findbugsExcludeFile.xml. (szetszwo)
HDFS-7217. Better batching of IBRs. (kihwal)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang) HDFS-6690. Deduplicate xattr names in memory. (wang)

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
View File

@ -236,7 +236,7 @@ void notifyNamenodeReceivedBlock(
delHint); delHint);
for (BPServiceActor actor : bpServices) { for (BPServiceActor actor : bpServices) {
actor.notifyNamenodeBlockImmediately(bInfo, storageUuid); actor.notifyNamenodeBlock(bInfo, storageUuid, true);
} }
} }
@ -264,7 +264,7 @@ void notifyNamenodeReceivingBlock(ExtendedBlock block, String storageUuid) {
block.getLocalBlock(), BlockStatus.RECEIVING_BLOCK, null); block.getLocalBlock(), BlockStatus.RECEIVING_BLOCK, null);
for (BPServiceActor actor : bpServices) { for (BPServiceActor actor : bpServices) {
actor.notifyNamenodeBlockImmediately(bInfo, storageUuid); actor.notifyNamenodeBlock(bInfo, storageUuid, false);
} }
} }

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
View File

@ -368,15 +368,19 @@ void addPendingReplicationBlockInfo(ReceivedDeletedBlockInfo bInfo,
* till namenode is informed before responding with success to the * till namenode is informed before responding with success to the
* client? For now we don't. * client? For now we don't.
*/ */
void notifyNamenodeBlockImmediately( void notifyNamenodeBlock(ReceivedDeletedBlockInfo bInfo,
ReceivedDeletedBlockInfo bInfo, String storageUuid) { String storageUuid, boolean now) {
synchronized (pendingIncrementalBRperStorage) { synchronized (pendingIncrementalBRperStorage) {
addPendingReplicationBlockInfo( addPendingReplicationBlockInfo(
bInfo, dn.getFSDataset().getStorage(storageUuid)); bInfo, dn.getFSDataset().getStorage(storageUuid));
sendImmediateIBR = true; sendImmediateIBR = true;
// If now is true, the report is sent right away.
// Otherwise, it will be sent out in the next heartbeat.
if (now) {
pendingIncrementalBRperStorage.notifyAll(); pendingIncrementalBRperStorage.notifyAll();
} }
} }
}
void notifyNamenodeDeletedBlock( void notifyNamenodeDeletedBlock(
ReceivedDeletedBlockInfo bInfo, String storageUuid) { ReceivedDeletedBlockInfo bInfo, String storageUuid) {

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBlockReports.java
View File

@ -84,7 +84,7 @@ private static Block getDummyBlock() {
private void injectBlockReceived() { private void injectBlockReceived() {
ReceivedDeletedBlockInfo rdbi = new ReceivedDeletedBlockInfo( ReceivedDeletedBlockInfo rdbi = new ReceivedDeletedBlockInfo(
getDummyBlock(), BlockStatus.RECEIVED_BLOCK, null); getDummyBlock(), BlockStatus.RECEIVED_BLOCK, null);
actor.notifyNamenodeBlockImmediately(rdbi, storageUuid); actor.notifyNamenodeBlock(rdbi, storageUuid, true);
} }
/** /**