big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8684. Erasure Coding: fix some block number calculation for striped block. (yliu)

Browse Source
This commit is contained in:
yliu 2015-07-06 19:14:18 +08:00
parent 0b7af27b9a
commit ee01a09500
3 changed files with 9 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
View File

@ -325,4 +325,7 @@
multiple times. (Kai Sasaki via szetszwo)
HDFS-8468. 2 RPC calls for every file read in DFSClient#open(..) resulting in
double Audit log entries (vinayakumarb)
double Audit log entries (vinayakumarb)
HDFS-8684. Erasure Coding: fix some block number calculation for striped
block. (yliu)

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -3109,12 +3109,12 @@ private void chooseExcessReplicates(
bc.getStoragePolicyID());
final List<StorageType> excessTypes = storagePolicy.chooseExcess(
replication, DatanodeStorageInfo.toStorageTypes(nonExcess));
if (!storedBlock.isStriped()) {
chooseExcessReplicasContiguous(bc, nonExcess, storedBlock,
replication, addedNode, delNodeHint, excessTypes);
} else {
if (storedBlock.isStriped()) {
chooseExcessReplicasStriped(bc, nonExcess, storedBlock, delNodeHint,
excessTypes);
} else {
chooseExcessReplicasContiguous(bc, nonExcess, storedBlock,
replication, addedNode, delNodeHint, excessTypes);
}
}
@ -3191,9 +3191,6 @@ private void chooseExcessReplicasStriped(BlockCollection bc,
assert storedBlock instanceof BlockInfoStriped;
BlockInfoStriped sblk = (BlockInfoStriped) storedBlock;
short groupSize = sblk.getTotalBlockNum();
if (nonExcess.size() <= groupSize) {
return;
}
BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(true);
List<DatanodeStorageInfo> empty = new ArrayList<>(0);

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
View File

@ -809,13 +809,9 @@ public final long computeFileSize(boolean includesLastUcBlock,
if (!includesLastUcBlock) {
size = 0;
} else if (usePreferredBlockSize4LastUcBlock) {
// Striped blocks keeps block group which counts
// (data blocks num + parity blocks num). When you
// count actual used size by BlockInfoStripedUC must
// be multiplied by these blocks number.
BlockInfoStripedUnderConstruction blockInfoStripedUC
= (BlockInfoStripedUnderConstruction) blockInfos[last];
size = getPreferredBlockSize() * blockInfoStripedUC.getTotalBlockNum();
size = getPreferredBlockSize() * blockInfoStripedUC.getDataBlockNum();
}
}
//sum other blocks