diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 596bbcfdd2..145494f810 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -155,3 +155,6 @@ HDFS-8308. Erasure Coding: NameNode may get blocked in waitForLoadingFSImage() when loading editlog. (jing9) + + HDFS-7949. WebImageViewer need support file size calculation with striped + blocks. (Rakesh R via Zhe Zhang) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java index 23e3153b57..f0e52e370b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java @@ -19,9 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; - -import java.io.DataOutput; -import java.io.IOException; +import org.apache.hadoop.hdfs.util.StripedBlockUtil; import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE; @@ -203,28 +201,9 @@ public class BlockInfoStriped extends BlockInfo { // In case striped blocks, total usage by this striped blocks should // be the total of data blocks and parity blocks because // `getNumBytes` is the total of actual data block size. - - // 0. Calculate the total bytes per stripes - long numBytesPerStripe = dataBlockNum * BLOCK_STRIPED_CELL_SIZE; - if (getNumBytes() % numBytesPerStripe == 0) { - return getNumBytes() / dataBlockNum * getTotalBlockNum(); + return StripedBlockUtil.spaceConsumedByStripedBlock(getNumBytes(), + dataBlockNum, parityBlockNum, BLOCK_STRIPED_CELL_SIZE); } - // 1. Calculate the number of stripes in this block group. - long numStripes = (getNumBytes() - 1) / numBytesPerStripe + 1; - // 2. Calculate the parity cell length in the last stripe. Note that the - // size of parity cells should equal the size of the first cell, if it - // is not full. - long lastStripeParityCellLen = Math.min(getNumBytes() % numBytesPerStripe, - BLOCK_STRIPED_CELL_SIZE); - // 3. Total consumed space is the total of - // - The total of the full cells of data blocks and parity blocks. - // - The remaining of data block which does not make a stripe. - // - The last parity block cells. These size should be same - // to the first cell in this stripe. - return getTotalBlockNum() * (BLOCK_STRIPED_CELL_SIZE * (numStripes - 1)) - + getNumBytes() % numBytesPerStripe - + lastStripeParityCellLen * parityBlockNum; - } @Override public final boolean isStriped() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java index 351ff03224..42f6c0be27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.tools.offlineImageViewer; import java.io.BufferedInputStream; -import java.io.EOFException; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; @@ -42,12 +41,15 @@ import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StripedBlockProto; import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode; import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf; import org.apache.hadoop.hdfs.server.namenode.FSImageUtil; import org.apache.hadoop.hdfs.server.namenode.FsImageProto; import org.apache.hadoop.hdfs.server.namenode.INodeId; +import org.apache.hadoop.hdfs.util.StripedBlockUtil; import org.apache.hadoop.hdfs.web.JsonUtil; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.LimitInputStream; @@ -483,8 +485,21 @@ class FSImageLoader { static long getFileSize(FsImageProto.INodeSection.INodeFile f) { long size = 0; - for (HdfsProtos.BlockProto p : f.getBlocksList()) { - size += p.getNumBytes(); + if (f.hasStripedBlocks()) { + List blocksList = f.getStripedBlocks().getBlocksList(); + // Get total of actual data block size + for (StripedBlockProto p : blocksList) { + // Total usage by this striped blocks should be the total of data + // blocks and parity blocks + size += StripedBlockUtil.spaceConsumedByStripedBlock(p.getBlock() + .getNumBytes(), p.getDataBlockNum(), p.getParityBlockNum(), + HdfsConstants.BLOCK_STRIPED_CELL_SIZE); + } + } else { + for (HdfsProtos.BlockProto p : f.getBlocksList()) { + size += p.getNumBytes(); + } + } return size; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java index cb6d39a692..b18e36f0fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java @@ -231,6 +231,28 @@ public class StripedBlockUtil { } } + /** + * Get the total usage of the striped blocks, which is the total of data + * blocks and parity blocks + * + * @param numDataBlkBytes + * Size of the block group only counting data blocks + * @param dataBlkNum + * The number of data blocks + * @param parityBlkNum + * The number of parity blocks + * @param cellSize + * The size of a striping cell + * @return The total usage of data blocks and parity blocks + */ + public static long spaceConsumedByStripedBlock(long numDataBlkBytes, + int dataBlkNum, int parityBlkNum, int cellSize) { + int parityIndex = dataBlkNum + 1; + long numParityBlkBytes = getInternalBlockLength(numDataBlkBytes, cellSize, + dataBlkNum, parityIndex) * parityBlkNum; + return numDataBlkBytes + numParityBlkBytes; + } + /** * This class represents the portion of I/O associated with each block in the * striped block group. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java new file mode 100644 index 0000000000..f3ef39a80c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools.offlineImageViewer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; +import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped; +import org.apache.hadoop.hdfs.server.namenode.FSDirectory; +import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.hdfs.server.namenode.INodeFile; +import org.apache.hadoop.hdfs.util.StripedBlockUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestOfflineImageViewerWithStripedBlocks { + private static int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS; + private static int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS; + + private static MiniDFSCluster cluster; + private static DistributedFileSystem fs; + private static final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE; + private static final int stripesPerBlock = 3; + private static final int blockSize = cellSize * stripesPerBlock; + + @BeforeClass + public static void setup() throws IOException { + int numDNs = dataBlocks + parityBlocks + 2; + Configuration conf = new Configuration(); + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); + cluster.waitActive(); + cluster.getFileSystem().getClient().createErasureCodingZone("/", null); + fs = cluster.getFileSystem(); + Path eczone = new Path("/eczone"); + fs.mkdirs(eczone); + } + + @AfterClass + public static void tearDown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test(timeout = 60000) + public void testFileEqualToOneStripe() throws Exception { + int numBytes = cellSize; + testFileSize(numBytes); + } + + @Test(timeout = 60000) + public void testFileLessThanOneStripe() throws Exception { + int numBytes = cellSize - 100; + testFileSize(numBytes); + } + + @Test(timeout = 60000) + public void testFileHavingMultipleBlocks() throws Exception { + int numBytes = blockSize * 3; + testFileSize(numBytes); + } + + @Test(timeout = 60000) + public void testFileLargerThanABlockGroup1() throws IOException { + testFileSize(blockSize * dataBlocks + cellSize + 123); + } + + @Test(timeout = 60000) + public void testFileLargerThanABlockGroup2() throws IOException { + testFileSize(blockSize * dataBlocks * 3 + cellSize * dataBlocks + cellSize + + 123); + } + + @Test(timeout = 60000) + public void testFileFullBlockGroup() throws IOException { + testFileSize(blockSize * dataBlocks); + } + + @Test(timeout = 60000) + public void testFileMoreThanOneStripe() throws Exception { + int numBytes = blockSize + blockSize / 2; + testFileSize(numBytes); + } + + private void testFileSize(int numBytes) throws IOException, + UnresolvedLinkException, SnapshotAccessControlException { + fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + File orgFsimage = null; + Path file = new Path("/eczone/striped"); + FSDataOutputStream out = fs.create(file, true); + byte[] bytes = DFSTestUtil.generateSequentialBytes(0, numBytes); + out.write(bytes); + out.close(); + + // Write results to the fsimage file + fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false); + fs.saveNamespace(); + + // Determine location of fsimage file + orgFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil + .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0)); + if (orgFsimage == null) { + throw new RuntimeException("Didn't generate or can't find fsimage"); + } + FSImageLoader loader = FSImageLoader.load(orgFsimage.getAbsolutePath()); + String fileStatus = loader.getFileStatus("/eczone/striped"); + long expectedSpaceConsumed = StripedBlockUtil.spaceConsumedByStripedBlock( + bytes.length, HdfsConstants.NUM_DATA_BLOCKS, + HdfsConstants.NUM_PARITY_BLOCKS, HdfsConstants.BLOCK_STRIPED_CELL_SIZE); + + // Verify space consumed present in BlockInfoStriped + FSDirectory fsdir = cluster.getNamesystem().getFSDirectory(); + INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile(); + assertTrue("Invalid block size", fileNode.getBlocks().length > 0); + long actualSpaceConsumed = 0; + for (BlockInfo blockInfo : fileNode.getBlocks()) { + assertTrue("Didn't find block striped information", + blockInfo instanceof BlockInfoStriped); + BlockInfoStriped b = (BlockInfoStriped) blockInfo; + actualSpaceConsumed += b.spaceConsumed(); + } + + assertEquals("Wrongly computed file size contains striped blocks", + expectedSpaceConsumed, actualSpaceConsumed); + + // Verify space consumed present in filestatus + String EXPECTED_FILE_SIZE = "\"length\":" + + String.valueOf(expectedSpaceConsumed); + assertTrue( + "Wrongly computed file size contains striped blocks, file status:" + + fileStatus + ". Expected file size is : " + EXPECTED_FILE_SIZE, + fileStatus.contains(EXPECTED_FILE_SIZE)); + } +}