big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11121. Add assertions to BlockInfo#addStorage to protect from breaking reportedBlock-blockGroup mapping. Contributed by Takanobu Asanuma.

Browse Source
This commit is contained in:
Wei-Chiu Chuang 2017-01-21 05:56:22 +08:00
parent d79c645b2e
commit 355b907c09
4 changed files with 45 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockType;
@ -55,6 +56,9 @@ private int ensureCapacity(int num) {
@Override
boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock) {
Preconditions.checkArgument(this.getBlockId() == reportedBlock.getBlockId(),
"reported blk_%s is different from stored blk_%s",
reportedBlock.getBlockId(), this.getBlockId());
// find the last null node
int lastNode = ensureCapacity(1);
setStorageInfo(lastNode, storage);

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockType;
@ -114,6 +115,12 @@ private int findSlot() {
@Override
boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock) {
Preconditions.checkArgument(BlockIdManager.isStripedBlockID(
reportedBlock.getBlockId()), "reportedBlock is not striped");
Preconditions.checkArgument(BlockIdManager.convertToStripedID(
reportedBlock.getBlockId()) == this.getBlockId(),
"reported blk_%s does not belong to the group of stored blk_%s",
reportedBlock.getBlockId(), this.getBlockId());
int blockIndex = BlockIdManager.getBlockIndex(reportedBlock);
int index = blockIndex;
DatanodeStorageInfo old = getStorageInfo(index);

20
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
View File

@ -23,6 +23,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.junit.Assert;
@ -54,7 +55,8 @@ public void testIsDeleted() {
public void testAddStorage() throws Exception {
BlockInfo blockInfo = new BlockInfoContiguous((short) 3);
final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo("storageID", "127.0.0.1");
final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo(
"storageID", "127.0.0.1");
boolean added = blockInfo.addStorage(storage, blockInfo);
@ -66,8 +68,10 @@ public void testAddStorage() throws Exception {
public void testReplaceStorage() throws Exception {
// Create two dummy storages.
final DatanodeStorageInfo storage1 = DFSTestUtil.createDatanodeStorageInfo("storageID1", "127.0.0.1");
final DatanodeStorageInfo storage2 = new DatanodeStorageInfo(storage1.getDatanodeDescriptor(), new DatanodeStorage("storageID2"));
final DatanodeStorageInfo storage1 = DFSTestUtil.createDatanodeStorageInfo(
"storageID1", "127.0.0.1");
final DatanodeStorageInfo storage2 = new DatanodeStorageInfo(
storage1.getDatanodeDescriptor(), new DatanodeStorage("storageID2"));
final int NUM_BLOCKS = 10;
BlockInfo[] blockInfos = new BlockInfo[NUM_BLOCKS];
@ -83,4 +87,14 @@ public void testReplaceStorage() throws Exception {
Assert.assertThat(added, is(false));
Assert.assertThat(blockInfos[NUM_BLOCKS/2].getStorageInfo(0), is(storage2));
}
@Test(expected=IllegalArgumentException.class)
public void testAddStorageWithDifferentBlock() throws Exception {
BlockInfo blockInfo1 = new BlockInfoContiguous(new Block(1000L), (short) 3);
BlockInfo blockInfo2 = new BlockInfoContiguous(new Block(1001L), (short) 3);
final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo(
"storageID", "127.0.0.1");
blockInfo1.addStorage(storage, blockInfo2);
}
}

17
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
View File

@ -220,4 +220,21 @@ public void testWrite() {
assertEquals(byteBuffer.array().length, byteStream.toByteArray().length);
assertArrayEquals(byteBuffer.array(), byteStream.toByteArray());
}
@Test(expected=IllegalArgumentException.class)
public void testAddStorageWithReplicatedBlock() {
DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo(
"storageID", "127.0.0.1");
BlockInfo replica = new BlockInfoContiguous(new Block(1000L), (short) 3);
info.addStorage(storage, replica);
}
@Test(expected=IllegalArgumentException.class)
public void testAddStorageWithDifferentBlockGroup() {
DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo(
"storageID", "127.0.0.1");
BlockInfo diffGroup = new BlockInfoStriped(new Block(BASE_ID + 100),
testECPolicy);
info.addStorage(storage, diffGroup);
}
}