HDFS-7949. WebImageViewer need support file size calculation with striped blocks. Contributed by Rakesh R.

hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt

@@ -155,3 +155,6 @@
 
     HDFS-8308. Erasure Coding: NameNode may get blocked in waitForLoadingFSImage()
     when loading editlog. (jing9)
+
+    HDFS-7949. WebImageViewer need support file size calculation with striped 
+    blocks. (Rakesh R via Zhe Zhang)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java

@@ -19,9 +19,7 @@
 
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
-
-import java.io.DataOutput;
-import java.io.IOException;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
 
 import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
 
@@ -203,28 +201,9 @@ public long spaceConsumed() {
     // In case striped blocks, total usage by this striped blocks should
     // be the total of data blocks and parity blocks because
     // `getNumBytes` is the total of actual data block size.
-
-    // 0. Calculate the total bytes per stripes <Num Bytes per Stripes>
-    long numBytesPerStripe = dataBlockNum * BLOCK_STRIPED_CELL_SIZE;
-    if (getNumBytes() % numBytesPerStripe == 0) {
-      return getNumBytes() / dataBlockNum * getTotalBlockNum();
+    return StripedBlockUtil.spaceConsumedByStripedBlock(getNumBytes(),
+        dataBlockNum, parityBlockNum, BLOCK_STRIPED_CELL_SIZE);
     }
-    // 1. Calculate the number of stripes in this block group. <Num Stripes>
-    long numStripes = (getNumBytes() - 1) / numBytesPerStripe + 1;
-    // 2. Calculate the parity cell length in the last stripe. Note that the
-    //    size of parity cells should equal the size of the first cell, if it
-    //    is not full. <Last Stripe Parity Cell Length>
-    long lastStripeParityCellLen = Math.min(getNumBytes() % numBytesPerStripe,
-        BLOCK_STRIPED_CELL_SIZE);
-    // 3. Total consumed space is the total of
-    //     - The total of the full cells of data blocks and parity blocks.
-    //     - The remaining of data block which does not make a stripe.
-    //     - The last parity block cells. These size should be same
-    //       to the first cell in this stripe.
-    return getTotalBlockNum() * (BLOCK_STRIPED_CELL_SIZE * (numStripes - 1))
-        + getNumBytes() % numBytesPerStripe
-        + lastStripeParityCellLen * parityBlockNum;
-  }
 
   @Override
   public final boolean isStriped() {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java

@@ -18,7 +18,6 @@
 package org.apache.hadoop.hdfs.tools.offlineImageViewer;
 
 import java.io.BufferedInputStream;
-import java.io.EOFException;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -42,12 +41,15 @@
 import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StripedBlockProto;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf;
 import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
 import org.apache.hadoop.hdfs.server.namenode.INodeId;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
 import org.apache.hadoop.hdfs.web.JsonUtil;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.LimitInputStream;
@@ -483,8 +485,21 @@ private long lookup(String path) throws IOException {
 
   static long getFileSize(FsImageProto.INodeSection.INodeFile f) {
     long size = 0;
-    for (HdfsProtos.BlockProto p : f.getBlocksList()) {
-      size += p.getNumBytes();
+    if (f.hasStripedBlocks()) {
+      List<StripedBlockProto> blocksList = f.getStripedBlocks().getBlocksList();
+      // Get total of actual data block size
+      for (StripedBlockProto p : blocksList) {
+        // Total usage by this striped blocks should be the total of data
+        // blocks and parity blocks
+        size += StripedBlockUtil.spaceConsumedByStripedBlock(p.getBlock()
+            .getNumBytes(), p.getDataBlockNum(), p.getParityBlockNum(),
+            HdfsConstants.BLOCK_STRIPED_CELL_SIZE);
+      }
+    } else {
+      for (HdfsProtos.BlockProto p : f.getBlocksList()) {
+        size += p.getNumBytes();
+      }
+
     }
     return size;
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java

@@ -231,6 +231,28 @@ public static StripedReadResult getNextCompletedStripedRead(
     }
   }
 
+  /**
+   * Get the total usage of the striped blocks, which is the total of data
+   * blocks and parity blocks
+   *
+   * @param numDataBlkBytes
+   *          Size of the block group only counting data blocks
+   * @param dataBlkNum
+   *          The number of data blocks
+   * @param parityBlkNum
+   *          The number of parity blocks
+   * @param cellSize
+   *          The size of a striping cell
+   * @return The total usage of data blocks and parity blocks
+   */
+  public static long spaceConsumedByStripedBlock(long numDataBlkBytes,
+      int dataBlkNum, int parityBlkNum, int cellSize) {
+    int parityIndex = dataBlkNum + 1;
+    long numParityBlkBytes = getInternalBlockLength(numDataBlkBytes, cellSize,
+        dataBlkNum, parityIndex) * parityBlkNum;
+    return numDataBlkBytes + numParityBlkBytes;
+  }
+
   /**
    * This class represents the portion of I/O associated with each block in the
    * striped block group.

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java

@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestOfflineImageViewerWithStripedBlocks {
+  private static int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS;
+  private static int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS;
+
+  private static MiniDFSCluster cluster;
+  private static DistributedFileSystem fs;
+  private static final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+  private static final int stripesPerBlock = 3;
+  private static final int blockSize = cellSize * stripesPerBlock;
+
+  @BeforeClass
+  public static void setup() throws IOException {
+    int numDNs = dataBlocks + parityBlocks + 2;
+    Configuration conf = new Configuration();
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+    cluster.waitActive();
+    cluster.getFileSystem().getClient().createErasureCodingZone("/", null);
+    fs = cluster.getFileSystem();
+    Path eczone = new Path("/eczone");
+    fs.mkdirs(eczone);
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+  }
+
+  @Test(timeout = 60000)
+  public void testFileEqualToOneStripe() throws Exception {
+    int numBytes = cellSize;
+    testFileSize(numBytes);
+  }
+
+  @Test(timeout = 60000)
+  public void testFileLessThanOneStripe() throws Exception {
+    int numBytes = cellSize - 100;
+    testFileSize(numBytes);
+  }
+
+  @Test(timeout = 60000)
+  public void testFileHavingMultipleBlocks() throws Exception {
+    int numBytes = blockSize * 3;
+    testFileSize(numBytes);
+  }
+
+  @Test(timeout = 60000)
+  public void testFileLargerThanABlockGroup1() throws IOException {
+    testFileSize(blockSize * dataBlocks + cellSize + 123);
+  }
+
+  @Test(timeout = 60000)
+  public void testFileLargerThanABlockGroup2() throws IOException {
+    testFileSize(blockSize * dataBlocks * 3 + cellSize * dataBlocks + cellSize
+        + 123);
+  }
+
+  @Test(timeout = 60000)
+  public void testFileFullBlockGroup() throws IOException {
+    testFileSize(blockSize * dataBlocks);
+  }
+
+  @Test(timeout = 60000)
+  public void testFileMoreThanOneStripe() throws Exception {
+    int numBytes = blockSize + blockSize / 2;
+    testFileSize(numBytes);
+  }
+
+  private void testFileSize(int numBytes) throws IOException,
+      UnresolvedLinkException, SnapshotAccessControlException {
+    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+    File orgFsimage = null;
+    Path file = new Path("/eczone/striped");
+    FSDataOutputStream out = fs.create(file, true);
+    byte[] bytes = DFSTestUtil.generateSequentialBytes(0, numBytes);
+    out.write(bytes);
+    out.close();
+
+    // Write results to the fsimage file
+    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false);
+    fs.saveNamespace();
+
+    // Determine location of fsimage file
+    orgFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil
+        .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0));
+    if (orgFsimage == null) {
+      throw new RuntimeException("Didn't generate or can't find fsimage");
+    }
+    FSImageLoader loader = FSImageLoader.load(orgFsimage.getAbsolutePath());
+    String fileStatus = loader.getFileStatus("/eczone/striped");
+    long expectedSpaceConsumed = StripedBlockUtil.spaceConsumedByStripedBlock(
+        bytes.length, HdfsConstants.NUM_DATA_BLOCKS,
+        HdfsConstants.NUM_PARITY_BLOCKS, HdfsConstants.BLOCK_STRIPED_CELL_SIZE);
+
+    // Verify space consumed present in BlockInfoStriped
+    FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
+    INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
+    assertTrue("Invalid block size", fileNode.getBlocks().length > 0);
+    long actualSpaceConsumed = 0;
+    for (BlockInfo blockInfo : fileNode.getBlocks()) {
+      assertTrue("Didn't find block striped information",
+          blockInfo instanceof BlockInfoStriped);
+      BlockInfoStriped b = (BlockInfoStriped) blockInfo;
+      actualSpaceConsumed += b.spaceConsumed();
+    }
+
+    assertEquals("Wrongly computed file size contains striped blocks",
+        expectedSpaceConsumed, actualSpaceConsumed);
+
+    // Verify space consumed present in filestatus
+    String EXPECTED_FILE_SIZE = "\"length\":"
+        + String.valueOf(expectedSpaceConsumed);
+    assertTrue(
+        "Wrongly computed file size contains striped blocks, file status:"
+            + fileStatus + ". Expected file size is : " + EXPECTED_FILE_SIZE,
+        fileStatus.contains(EXPECTED_FILE_SIZE));
+  }
+}