big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-7937. Erasure Coding: INodeFile quota computation unit tests. Contributed by Kai Sasaki.

Browse Source
This commit is contained in:
Jing Zhao 2015-04-17 18:07:07 -07:00 committed by Zhe Zhang
parent cd458c38a0
commit ab76e1fe36
2 changed files with 250 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
View File

@ -203,8 +203,27 @@ public long spaceConsumed() {
// In case striped blocks, total usage by this striped blocks should
// be the total of data blocks and parity blocks because
// `getNumBytes` is the total of actual data block size.
return ((getNumBytes() - 1) / (dataBlockNum * BLOCK_STRIPED_CELL_SIZE) + 1)
* BLOCK_STRIPED_CELL_SIZE * parityBlockNum + getNumBytes();
// 0. Calculate the total bytes per stripes <Num Bytes per Stripes>
long numBytesPerStripe = dataBlockNum * BLOCK_STRIPED_CELL_SIZE;
if (getNumBytes() % numBytesPerStripe == 0) {
return getNumBytes() / dataBlockNum * getTotalBlockNum();
}
// 1. Calculate the number of stripes in this block group. <Num Stripes>
long numStripes = (getNumBytes() - 1) / numBytesPerStripe + 1;
// 2. Calculate the parity cell length in the last stripe. Note that the
// size of parity cells should equal the size of the first cell, if it
// is not full. <Last Stripe Parity Cell Length>
long lastStripeParityCellLen = Math.min(getNumBytes() % numBytesPerStripe,
BLOCK_STRIPED_CELL_SIZE);
// 3. Total consumed space is the total of
// - The total of the full cells of data blocks and parity blocks.
// - The remaining of data block which does not make a stripe.
// - The last parity block cells. These size should be same
// to the first cell in this stripe.
return getTotalBlockNum() * (BLOCK_STRIPED_CELL_SIZE * (numStripes - 1))
+ getNumBytes() % numBytesPerStripe
+ lastStripeParityCellLen * parityBlockNum;
}
@Override

229
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java Normal file
View File

@ -0,0 +1,229 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.junit.Test;
/**
* This class tests INodeFile with striped feature.
*/
public class TestStripedINodeFile {
public static final Log LOG = LogFactory.getLog(TestINodeFile.class);
private static final PermissionStatus perm = new PermissionStatus(
"userName", null, FsPermission.getDefault());
private static INodeFile createStripedINodeFile() {
return new INodeFile(INodeId.GRANDFATHER_INODE_ID, null, perm, 0L, 0L,
null, (short)0, 1024L, HdfsConstants.COLD_STORAGE_POLICY_ID);
}
@Test
public void testBlockStripedFeature()
throws IOException, InterruptedException{
INodeFile inf = createStripedINodeFile();
inf.addStripedBlocksFeature();
assertTrue(inf.isStriped());
}
@Test
public void testBlockStripedTotalBlockCount() {
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
Block blk = new Block(1);
BlockInfoStriped blockInfoStriped
= new BlockInfoStriped(blk,
(short)defaultSchema.getNumDataUnits(),
(short)defaultSchema.getNumParityUnits());
assertEquals(9, blockInfoStriped.getTotalBlockNum());
}
@Test
public void testBlockStripedLength()
throws IOException, InterruptedException {
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
INodeFile inf = createStripedINodeFile();
inf.addStripedBlocksFeature();
Block blk = new Block(1);
BlockInfoStriped blockInfoStriped
= new BlockInfoStriped(blk,
(short)defaultSchema.getNumDataUnits(),
(short)defaultSchema.getNumParityUnits());
inf.addBlock(blockInfoStriped);
assertEquals(1, inf.getBlocks().length);
}
@Test
public void testBlockStripedConsumedSpace()
throws IOException, InterruptedException {
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
INodeFile inf = createStripedINodeFile();
inf.addStripedBlocksFeature();
Block blk = new Block(1);
BlockInfoStriped blockInfoStriped
= new BlockInfoStriped(blk,
(short)defaultSchema.getNumDataUnits(),
(short)defaultSchema.getNumParityUnits());
blockInfoStriped.setNumBytes(1);
inf.addBlock(blockInfoStriped);
// 0. Calculate the total bytes per stripes <Num Bytes per Stripes>
// 1. Calculate the number of stripes in this block group. <Num Stripes>
// 2. Calculate the last remaining length which does not make a stripe. <Last Stripe Length>
// 3. Total consumed space is the total of
// a. The total of the full cells of data blocks and parity blocks.
// b. The remaining of data block which does not make a stripe.
// c. The last parity block cells. These size should be same
// to the first cell in this stripe.
// So the total consumed space is the sum of
// a. <Cell Size> * (<Num Stripes> - 1) * <Total Block Num> = 0
// b. <Num Bytes> % <Num Bytes per Stripes> = 1
// c. <Last Stripe Length> * <Parity Block Num> = 1 * 3
assertEquals(4, inf.storagespaceConsumedWithStriped());
assertEquals(4, inf.storagespaceConsumed());
}
@Test
public void testMultipleBlockStripedConsumedSpace()
throws IOException, InterruptedException {
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
INodeFile inf = createStripedINodeFile();
inf.addStripedBlocksFeature();
Block blk1 = new Block(1);
BlockInfoStriped blockInfoStriped1
= new BlockInfoStriped(blk1,
(short)defaultSchema.getNumDataUnits(),
(short)defaultSchema.getNumParityUnits());
blockInfoStriped1.setNumBytes(1);
Block blk2 = new Block(2);
BlockInfoStriped blockInfoStriped2
= new BlockInfoStriped(blk2,
(short)defaultSchema.getNumDataUnits(),
(short)defaultSchema.getNumParityUnits());
blockInfoStriped2.setNumBytes(1);
inf.addBlock(blockInfoStriped1);
inf.addBlock(blockInfoStriped2);
// This is the double size of one block in above case.
assertEquals(4 * 2, inf.storagespaceConsumedWithStriped());
assertEquals(4 * 2, inf.storagespaceConsumed());
}
@Test
public void testBlockStripedFileSize()
throws IOException, InterruptedException {
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
INodeFile inf = createStripedINodeFile();
inf.addStripedBlocksFeature();
Block blk = new Block(1);
BlockInfoStriped blockInfoStriped
= new BlockInfoStriped(blk,
(short)defaultSchema.getNumDataUnits(),
(short)defaultSchema.getNumParityUnits());
blockInfoStriped.setNumBytes(100);
inf.addBlock(blockInfoStriped);
// Compute file size should return actual data
// size which is retained by this file.
assertEquals(100, inf.computeFileSize());
assertEquals(100, inf.computeFileSize(false, false));
}
@Test
public void testBlockStripedUCFileSize()
throws IOException, InterruptedException {
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
INodeFile inf = createStripedINodeFile();
inf.addStripedBlocksFeature();
Block blk = new Block(1);
BlockInfoStripedUnderConstruction bInfoStripedUC
= new BlockInfoStripedUnderConstruction(blk,
(short)defaultSchema.getNumDataUnits(),
(short)defaultSchema.getNumParityUnits());
bInfoStripedUC.setNumBytes(100);
inf.addBlock(bInfoStripedUC);
assertEquals(100, inf.computeFileSize());
assertEquals(0, inf.computeFileSize(false, false));
}
@Test
public void testBlockStripedComputeQuotaUsage()
throws IOException, InterruptedException {
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
INodeFile inf = createStripedINodeFile();
inf.addStripedBlocksFeature();
Block blk = new Block(1);
BlockInfoStriped blockInfoStriped
= new BlockInfoStriped(blk,
(short)defaultSchema.getNumDataUnits(),
(short)defaultSchema.getNumParityUnits());
blockInfoStriped.setNumBytes(100);
inf.addBlock(blockInfoStriped);
BlockStoragePolicySuite suite =
BlockStoragePolicySuite.createDefaultSuite();
QuotaCounts counts =
inf.computeQuotaUsageWithStriped(suite,
new QuotaCounts.Builder().build());
assertEquals(1, counts.getNameSpace());
// The total consumed space is the sum of
// a. <Cell Size> * (<Num Stripes> - 1) * <Total Block Num> = 0
// b. <Num Bytes> % <Num Bytes per Stripes> = 100
// c. <Last Stripe Length> * <Parity Block Num> = 100 * 3
assertEquals(400, counts.getStorageSpace());
}
@Test
public void testBlockStripedUCComputeQuotaUsage()
throws IOException, InterruptedException {
ECSchema defaultSchema = ECSchemaManager.getSystemDefaultSchema();
INodeFile inf = createStripedINodeFile();
inf.addStripedBlocksFeature();
Block blk = new Block(1);
BlockInfoStripedUnderConstruction bInfoStripedUC
= new BlockInfoStripedUnderConstruction(blk,
(short)defaultSchema.getNumDataUnits(),
(short)defaultSchema.getNumParityUnits());
bInfoStripedUC.setNumBytes(100);
inf.addBlock(bInfoStripedUC);
BlockStoragePolicySuite suite
= BlockStoragePolicySuite.createDefaultSuite();
QuotaCounts counts
= inf.computeQuotaUsageWithStriped(suite,
new QuotaCounts.Builder().build());
assertEquals(1024, inf.getPreferredBlockSize());
assertEquals(1, counts.getNameSpace());
// Consumed space in the case of BlockInfoStripedUC can be calculated
// by using preferred block size. This is 1024 and total block num
// is 9(= 3 + 6). Consumed storage space should be 1024 * 9 = 9216.
assertEquals(9216, counts.getStorageSpace());
}
}