big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8145. Fix the editlog corruption exposed by failed TestAddStripedBlocks. Contributed by Jing Zhao.

Browse Source
This commit is contained in:
Jing Zhao 2015-04-17 18:13:47 -07:00 committed by Zhe Zhang
parent ab76e1fe36
commit f6e1160ef1
8 changed files with 29 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
View File

@ -244,13 +244,6 @@ public int numNodes() {
return num;
}
@Override
public void write(DataOutput out) throws IOException {
out.writeShort(dataBlockNum);
out.writeShort(parityBlockNum);
super.write(out);
}
/**
* Convert a complete block to an under construction block.
* @return BlockInfoUnderConstruction - an under construction block.

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java
View File

@ -54,10 +54,6 @@ public ErasureCodingZoneManager(FSDirectory dir) {
this.dir = dir;
}
boolean getECPolicy(INodesInPath iip) throws IOException {
return getECSchema(iip) != null;
}
ECSchema getECSchema(INodesInPath iip) throws IOException {
ECZoneInfo ecZoneInfo = getECZoneInfo(iip);
return ecZoneInfo == null ? null : ecZoneInfo.getSchema();
@ -109,7 +105,7 @@ XAttr createErasureCodingZone(String src, ECSchema schema)
throw new IOException("Attempt to create an erasure coding zone " +
"for a file.");
}
if (getECPolicy(srcIIP)) {
if (getECSchema(srcIIP) != null) {
throw new IOException("Directory " + src + " is already in an " +
"erasure coding zone.");
}
@ -132,8 +128,10 @@ XAttr createErasureCodingZone(String src, ECSchema schema)
void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src)
throws IOException {
assert dir.hasReadLock();
if (getECPolicy(srcIIP)
!= getECPolicy(dstIIP)) {
final ECSchema srcSchema = getECSchema(srcIIP);
final ECSchema dstSchema = getECSchema(dstIIP);
if ((srcSchema != null && !srcSchema.equals(dstSchema)) ||
(dstSchema != null && !dstSchema.equals(srcSchema))) {
throw new IOException(
src + " can't be moved because the source and destination have " +
"different erasure coding policies.");

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
View File

@ -1237,7 +1237,7 @@ XAttr createErasureCodingZone(String src, ECSchema schema)
}
}
public boolean getECPolicy(INodesInPath iip) throws IOException {
public boolean isInECZone(INodesInPath iip) throws IOException {
return getECSchema(iip) != null;
}

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
View File

@ -40,6 +40,7 @@
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
@ -416,7 +417,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
newFile.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
// TODO whether the file is striped should later be retrieved from iip
updateBlocks(fsDir, addCloseOp, iip, newFile, fsDir.getECPolicy(iip));
updateBlocks(fsDir, addCloseOp, iip, newFile, fsDir.isInECZone(iip));
break;
}
case OP_CLOSE: {
@ -437,7 +438,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
file.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
// TODO whether the file is striped should later be retrieved from iip
updateBlocks(fsDir, addCloseOp, iip, file, fsDir.getECPolicy(iip));
updateBlocks(fsDir, addCloseOp, iip, file, fsDir.isInECZone(iip));
// Now close the file
if (!file.isUnderConstruction() &&
@ -496,7 +497,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
// Update in-memory data structures
// TODO whether the file is striped should later be retrieved from iip
updateBlocks(fsDir, updateOp, iip, oldFile, fsDir.getECPolicy(iip));
updateBlocks(fsDir, updateOp, iip, oldFile, fsDir.isInECZone(iip));
if (toAddRetryCache) {
fsNamesys.addCacheEntry(updateOp.rpcClientId, updateOp.rpcCallId);
@ -514,7 +515,7 @@ private long applyEditLogOp(FSEditLogOp op, FSDirectory fsDir,
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
// add the new block to the INodeFile
// TODO whether the file is striped should later be retrieved from iip
addNewBlock(addBlockOp, oldFile, fsDir.getECPolicy(iip));
addNewBlock(addBlockOp, oldFile, fsDir.isInECZone(iip));
break;
}
case OP_SET_REPLICATION: {
@ -1079,8 +1080,9 @@ private void updateBlocks(FSDirectory fsDir, BlockListUpdatingOp op,
// is only executed when loading edits written by prior
// versions of Hadoop. Current versions always log
// OP_ADD operations as each block is allocated.
newBI = new BlockInfoContiguous(newBlock,
file.getPreferredBlockReplication());
newBI = isStriped ? new BlockInfoStriped(newBlock,
HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS) :
new BlockInfoContiguous(newBlock, file.getPreferredBlockReplication());
}
fsNamesys.getBlockManager().addBlockCollectionWithCheck(newBI, file);
file.addBlock(newBI);

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
View File

@ -771,10 +771,8 @@ INode loadINode(final byte[] localName, boolean isSnapshotINode,
if (isStriped) {
blocks = new Block[numBlocks];
for (int j = 0; j < numBlocks; j++) {
short dataBlockNum = in.readShort();
short parityBlockNum = in.readShort();
blocks[j] = new BlockInfoStriped(new Block(),
dataBlockNum, parityBlockNum);
HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS);
blocks[j].readFields(in);
}
} else {

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
View File

@ -32,6 +32,7 @@
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
@ -139,17 +140,15 @@ static INodeFile readINodeUnderConstruction(
blocksStriped = new BlockInfoStriped[numBlocks];
int i = 0;
for (; i < numBlocks - 1; i++) {
short dataBlockNum = in.readShort();
short parityBlockNum = in.readShort();
blocksStriped[i] = new BlockInfoStriped(new Block(), dataBlockNum,
parityBlockNum);
blocksStriped[i] = new BlockInfoStriped(new Block(),
HdfsConstants.NUM_DATA_BLOCKS,
HdfsConstants.NUM_PARITY_BLOCKS);
blocksStriped[i].readFields(in);
}
if (numBlocks > 0) {
short dataBlockNum = in.readShort();
short parityBlockNum = in.readShort();
blocksStriped[i] = new BlockInfoStripedUnderConstruction(new Block(),
dataBlockNum, parityBlockNum, BlockUCState.UNDER_CONSTRUCTION, null);
HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS,
BlockUCState.UNDER_CONSTRUCTION, null);
blocksStriped[i].readFields(in);
}
} else {

23
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
View File

@ -21,7 +21,6 @@
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.internal.util.reflection.Whitebox;
@ -43,12 +42,8 @@ public class TestBlockInfoStriped {
private static final int TOTAL_NUM_BLOCKS = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
private static final long BASE_ID = -1600;
private static final Block baseBlock = new Block(BASE_ID);
private BlockInfoStriped info;
@Before
public void setup() {
info = new BlockInfoStriped(baseBlock, NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS);
}
private final BlockInfoStriped info = new BlockInfoStriped(baseBlock,
NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS);
private Block[] createReportedBlocks(int num) {
Block[] blocks = new Block[num];
@ -230,17 +225,14 @@ public void testWrite() {
long blkID = 1;
long numBytes = 1;
long generationStamp = 1;
short dataBlockNum = 6;
short parityBlockNum = 3;
ByteBuffer byteBuffer = ByteBuffer.allocate(Long.SIZE/Byte.SIZE*3
+ Short.SIZE/Byte.SIZE*2);
byteBuffer.putShort(dataBlockNum).putShort(parityBlockNum)
.putLong(blkID).putLong(numBytes).putLong(generationStamp);
ByteBuffer byteBuffer = ByteBuffer.allocate(Long.SIZE / Byte.SIZE * 3);
byteBuffer.putLong(blkID).putLong(numBytes).putLong(generationStamp);
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
DataOutput out = new DataOutputStream(byteStream);
BlockInfoStriped blk = new BlockInfoStriped(new Block(1,1,1),
(short)6,(short)3);
BlockInfoStriped blk = new BlockInfoStriped(new Block(blkID, numBytes,
generationStamp), NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS);
try {
blk.write(out);
} catch(Exception ex) {
@ -249,5 +241,4 @@ public void testWrite() {
assertEquals(byteBuffer.array().length, byteStream.toByteArray().length);
assertArrayEquals(byteBuffer.array(), byteStream.toByteArray());
}
}

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
View File

@ -158,7 +158,7 @@ private void testSaveAndLoadStripedINodeFile(FSNamesystem fsn, Configuration con
for (int i = 0; i < stripedBlks.length; i++) {
stripedBlks[i] = new BlockInfoStriped(
new Block(stripedBlkId + i, preferredBlockSize, timestamp),
(short) 6, (short) 3);
HdfsConstants.NUM_DATA_BLOCKS, HdfsConstants.NUM_PARITY_BLOCKS);
file.getStripedBlocksFeature().addBlock(stripedBlks[i]);
}