big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8585. Erasure Coding: Remove dataBlockNum and parityBlockNum from StripedBlockProto. Contributed by Yi Liu.

Browse Source
This commit is contained in:
Jing Zhao 2015-06-12 14:48:53 -07:00
parent 98d340745b
commit 683332b36d
7 changed files with 23 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
View File

@ -296,3 +296,6 @@
HDFS-8450. Erasure Coding: Consolidate erasure coding zone related
implementation into a single class (Rakesh R via vinayakumarb)
HDFS-8585. Erasure Coding: Remove dataBlockNum and parityBlockNum from
StripedBlockProto. (Yi Liu via jing9)

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
View File

@ -183,7 +183,6 @@
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypesProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StripedBlockProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsResponseProto;
@ -195,7 +194,6 @@
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
@ -444,20 +442,6 @@ public static Block convert(BlockProto b) {
return new Block(b.getBlockId(), b.getNumBytes(), b.getGenStamp());
}
public static BlockInfoStriped convert(StripedBlockProto p, ECSchema schema) {
return new BlockInfoStriped(convert(p.getBlock()), schema);
}
public static StripedBlockProto convert(BlockInfoStriped blk) {
BlockProto bp = BlockProto.newBuilder().setBlockId(blk.getBlockId())
.setGenStamp(blk.getGenerationStamp()).setNumBytes(blk.getNumBytes())
.build();
return StripedBlockProto.newBuilder()
.setDataBlockNum(blk.getDataBlockNum())
.setParityBlockNum(blk.getParityBlockNum())
.setBlock(bp).build();
}
public static BlockWithLocationsProto convert(BlockWithLocations blk) {
BlockWithLocationsProto.Builder builder = BlockWithLocationsProto
.newBuilder().setBlock(convert(blk.getBlock()))

23
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
View File

@ -25,7 +25,6 @@
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -42,7 +41,6 @@
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StripedBlockProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
@ -330,10 +328,14 @@ private INodeFile loadINodeFile(INodeSection.INode n) {
short replication = (short) f.getReplication();
LoaderContext state = parent.getLoaderContext();
BlockInfoContiguous[] blocks = new BlockInfoContiguous[bp.size()];
for (int i = 0, e = bp.size(); i < e; ++i) {
blocks[i] = new BlockInfoContiguous(PBHelper.convert(bp.get(i)), replication);
BlockInfoContiguous[] blocks = null;
if (!f.hasStripedBlocks()) {
blocks = new BlockInfoContiguous[bp.size()];
for (int i = 0, e = bp.size(); i < e; ++i) {
blocks[i] = new BlockInfoContiguous(PBHelper.convert(bp.get(i)), replication);
}
}
final PermissionStatus permissions = loadPermission(f.getPermission(),
parent.getLoaderContext().getStringTable());
@ -357,10 +359,9 @@ private INodeFile loadINodeFile(INodeSection.INode n) {
if (f.hasStripedBlocks()) {
// TODO: HDFS-7859
ECSchema schema = ErasureCodingSchemaManager.getSystemDefaultSchema();
StripedBlocksFeature sb = f.getStripedBlocks();
stripeFeature = file.addStripedBlocksFeature();
for (StripedBlockProto sp : sb.getBlocksList()) {
stripeFeature.addBlock(PBHelper.convert(sp, schema));
for (BlockProto b : bp) {
stripeFeature.addBlock(new BlockInfoStriped(PBHelper.convert(b), schema));
}
}
@ -658,14 +659,14 @@ private void save(OutputStream out, INodeFile n) throws IOException {
FileWithStripedBlocksFeature sb = n.getStripedBlocksFeature();
if (sb != null) {
StripedBlocksFeature.Builder builder =
StripedBlocksFeature.newBuilder();
BlockInfoStriped[] sblocks = sb.getBlocks();
if (sblocks != null) {
for (BlockInfoStriped sblk : sblocks) {
builder.addBlocks(PBHelper.convert(sblk));
b.addBlocks(PBHelper.convert(sblk));
}
}
StripedBlocksFeature.Builder builder =
StripedBlocksFeature.newBuilder();
b.setStripedBlocks(builder.build());
}

20
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
View File

@ -41,15 +41,12 @@
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StripedBlockProto;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf;
import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.hdfs.web.JsonUtil;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.LimitInputStream;
@ -485,21 +482,8 @@ private long lookup(String path) throws IOException {
static long getFileSize(FsImageProto.INodeSection.INodeFile f) {
long size = 0;
if (f.hasStripedBlocks()) {
List<StripedBlockProto> blocksList = f.getStripedBlocks().getBlocksList();
// Get total of actual data block size
for (StripedBlockProto p : blocksList) {
// Total usage by this striped blocks should be the total of data
// blocks and parity blocks
size += StripedBlockUtil.spaceConsumedByStripedBlock(p.getBlock()
.getNumBytes(), p.getDataBlockNum(), p.getParityBlockNum(),
HdfsConstants.BLOCK_STRIPED_CELL_SIZE);
}
} else {
for (HdfsProtos.BlockProto p : f.getBlocksList()) {
size += p.getNumBytes();
}
for (HdfsProtos.BlockProto p : f.getBlocksList()) {
size += p.getNumBytes();
}
return size;
}

2
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
View File

@ -93,7 +93,7 @@ message INodeSection {
}
message StripedBlocksFeature {
repeated StripedBlockProto blocks = 1;
// store striped blocks related information
}
message AclFeatureProto {

10
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto
View File

@ -514,16 +514,6 @@ message BlockProto {
optional uint64 numBytes = 3 [default = 0];
}
/**
* Striped block information. Besides the basic information for a block,
* it also contains the number of data/parity blocks.
*/
message StripedBlockProto {
required BlockProto block = 1;
optional uint32 dataBlockNum = 2;
optional uint32 parityBlockNum = 3;
}
/**
* Block and datanodes where is it located
*/

14
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
View File

@ -39,7 +39,6 @@
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@ -136,28 +135,25 @@ private void testFileSize(int numBytes) throws IOException,
}
FSImageLoader loader = FSImageLoader.load(orgFsimage.getAbsolutePath());
String fileStatus = loader.getFileStatus("/eczone/striped");
long expectedSpaceConsumed = StripedBlockUtil.spaceConsumedByStripedBlock(
bytes.length, HdfsConstants.NUM_DATA_BLOCKS,
HdfsConstants.NUM_PARITY_BLOCKS, HdfsConstants.BLOCK_STRIPED_CELL_SIZE);
long expectedFileSize = bytes.length;
// Verify space consumed present in BlockInfoStriped
FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
assertTrue("Invalid block size", fileNode.getBlocks().length > 0);
long actualSpaceConsumed = 0;
long actualFileSize = 0;
for (BlockInfo blockInfo : fileNode.getBlocks()) {
assertTrue("Didn't find block striped information",
blockInfo instanceof BlockInfoStriped);
BlockInfoStriped b = (BlockInfoStriped) blockInfo;
actualSpaceConsumed += b.spaceConsumed();
actualFileSize += blockInfo.getNumBytes();
}
assertEquals("Wrongly computed file size contains striped blocks",
expectedSpaceConsumed, actualSpaceConsumed);
expectedFileSize, actualFileSize);
// Verify space consumed present in filestatus
String EXPECTED_FILE_SIZE = "\"length\":"
+ String.valueOf(expectedSpaceConsumed);
+ String.valueOf(expectedFileSize);
assertTrue(
"Wrongly computed file size contains striped blocks, file status:"
+ fileStatus + ". Expected file size is : " + EXPECTED_FILE_SIZE,