big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-9036. In BlockPlacementPolicyWithNodeGroup#chooseLocalStorage , random node is selected eventhough fallbackToLocalRack is true. (Contributed by J.Andreina)

Browse Source
This commit is contained in:
Vinayakumar B 2015-09-12 17:40:16 +05:30
parent d8455479b8
commit c715650385
3 changed files with 29 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -1315,6 +1315,10 @@ Release 2.8.0 - UNRELEASED
HDFS-8581. ContentSummary on / skips further counts on yielding lock HDFS-8581. ContentSummary on / skips further counts on yielding lock
(J.Andreina via vinayakumarb) (J.Andreina via vinayakumarb)
HDFS-9036. In BlockPlacementPolicyWithNodeGroup#chooseLocalStorage , random
node is selected eventhough fallbackToLocalRack is true.
(J.Andreina via vinayakumarb)
Release 2.7.2 - UNRELEASED Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
View File

@ -31,7 +31,7 @@
* for placing block replicas on environment with node-group layer. * for placing block replicas on environment with node-group layer.
* The replica placement strategy is adjusted to: * The replica placement strategy is adjusted to:
* If the writer is on a datanode, the 1st replica is placed on the local * If the writer is on a datanode, the 1st replica is placed on the local
* node (or local node-group), otherwise a random datanode. * node(or local node-group or on local rack), otherwise a random datanode.
* The 2nd replica is placed on a datanode that is on a different rack with 1st * The 2nd replica is placed on a datanode that is on a different rack with 1st
* replica node. * replica node.
* The 3rd replica is placed on a datanode which is on a different node-group * The 3rd replica is placed on a datanode which is on a different node-group
@ -165,7 +165,7 @@ protected void chooseRemoteRack(int numOfReplicas,
/* choose one node from the nodegroup that <i>localMachine</i> is on. /* choose one node from the nodegroup that <i>localMachine</i> is on.
* if no such node is available, choose one node from the nodegroup where * if no such node is available, choose one node from the nodegroup where
* a second replica is on. * a second replica is on.
* if still no such node is available, choose a random node in the cluster. * if still no such node is available, return null.
* @return the chosen node * @return the chosen node
*/ */
private DatanodeStorageInfo chooseLocalNodeGroup( private DatanodeStorageInfo chooseLocalNodeGroup(
@ -195,14 +195,12 @@ private DatanodeStorageInfo chooseLocalNodeGroup(
excludedNodes, blocksize, maxNodesPerRack, results, excludedNodes, blocksize, maxNodesPerRack, results,
avoidStaleNodes, storageTypes); avoidStaleNodes, storageTypes);
} catch(NotEnoughReplicasException e2) { } catch(NotEnoughReplicasException e2) {
//otherwise randomly choose one from the network //otherwise return null
return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, return null;
maxNodesPerRack, results, avoidStaleNodes, storageTypes);
} }
} else { } else {
//otherwise randomly choose one from the network //otherwise return null
return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize, return null;
maxNodesPerRack, results, avoidStaleNodes, storageTypes);
} }
} }
} }

19
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
View File

@ -486,6 +486,25 @@ public void testChooseTarget5() throws Exception {
verifyNoTwoTargetsOnSameNodeGroup(targets); verifyNoTwoTargetsOnSameNodeGroup(targets);
} }
/**
* In this testcase, client is dataNodes[7], but it is not qualified
* to be chosen. And there is no other node available on client Node group.
* So the 1st replica should be placed on client local rack dataNodes[6]
* @throws Exception
*/
@Test
public void testChooseTargetForLocalStorage() throws Exception {
updateHeartbeatWithUsage(dataNodes[7],
2* HdfsServerConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
(HdfsServerConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L,
0L, 0L, 0, 0); // no space
DatanodeStorageInfo[] targets;
targets = chooseTarget(1, dataNodes[7]);
assertEquals(targets.length, 1);
assertTrue(targets[0].getDatanodeDescriptor().equals(dataNodes[6]));
}
/** /**
* This testcase tests re-replication, when dataNodes[0] is already chosen. * This testcase tests re-replication, when dataNodes[0] is already chosen.
* So the 1st replica can be placed on random rack. * So the 1st replica can be placed on random rack.