big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8702. Erasure coding: update BlockManager.blockHasEnoughRacks(..) logic for striped block. Contributed by Kai Sasaki.

Browse Source
This commit is contained in:
Jing Zhao 2015-07-14 10:55:58 -07:00
parent b1e6429a6b
commit 6ff957be88
2 changed files with 44 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
View File

@ -347,3 +347,6 @@
HDFS-8669. Erasure Coding: handle missing internal block locations in HDFS-8669. Erasure Coding: handle missing internal block locations in
DFSStripedInputStream. (jing9) DFSStripedInputStream. (jing9)
HDFS-8702. Erasure coding: update BlockManager.blockHasEnoughRacks(..) logic
for striped block. (Kai Sasaki via jing9)

43
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -3819,14 +3819,53 @@ private int invalidateWorkForOneNode(DatanodeInfo dn) {
return toInvalidate.size(); return toInvalidate.size();
} }
// TODO: update the enough rack logic for striped blocks
boolean blockHasEnoughRacks(BlockInfo storedBlock, int expectedStorageNum) { boolean blockHasEnoughRacks(BlockInfo storedBlock, int expectedStorageNum) {
if (!this.shouldCheckForEnoughRacks) { if (!this.shouldCheckForEnoughRacks) {
return true; return true;
} }
boolean enoughRacks = false;
Collection<DatanodeDescriptor> corruptNodes = Collection<DatanodeDescriptor> corruptNodes =
corruptReplicas.getNodes(storedBlock); corruptReplicas.getNodes(storedBlock);
if (storedBlock.isStriped()) {
return blockHasEnoughRacksStriped(storedBlock, corruptNodes);
} else {
return blockHashEnoughRacksContiguous(storedBlock, expectedStorageNum,
corruptNodes);
}
}
/**
* Verify whether given striped block is distributed through enough racks.
* As dicussed in HDFS-7613, ec file requires racks at least as many as
* the number of data block number.
*/
boolean blockHasEnoughRacksStriped(BlockInfo storedBlock,
Collection<DatanodeDescriptor> corruptNodes) {
if (!datanodeManager.hasClusterEverBeenMultiRack()) {
return true;
}
boolean enoughRacks = false;
Set<String> rackNameSet = new HashSet<>();
int dataBlockNum = ((BlockInfoStriped)storedBlock).getRealDataBlockNum();
for (DatanodeStorageInfo storage : blocksMap.getStorages(storedBlock)) {
final DatanodeDescriptor cur = storage.getDatanodeDescriptor();
if (!cur.isDecommissionInProgress() && !cur.isDecommissioned()) {
if ((corruptNodes == null) || !corruptNodes.contains(cur)) {
String rackNameNew = cur.getNetworkLocation();
rackNameSet.add(rackNameNew);
if (rackNameSet.size() >= dataBlockNum) {
enoughRacks = true;
break;
}
}
}
}
return enoughRacks;
}
boolean blockHashEnoughRacksContiguous(BlockInfo storedBlock,
int expectedStorageNum, Collection<DatanodeDescriptor> corruptNodes) {
boolean enoughRacks = false;
String rackName = null; String rackName = null;
for(DatanodeStorageInfo storage : blocksMap.getStorages(storedBlock)) { for(DatanodeStorageInfo storage : blocksMap.getStorages(storedBlock)) {
final DatanodeDescriptor cur = storage.getDatanodeDescriptor(); final DatanodeDescriptor cur = storage.getDatanodeDescriptor();