big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8191. Fix byte to integer casting in SimulatedFSDataset#simulatedByte. Contributed by Zhe Zhang.

Browse Source
This commit is contained in:
Andrew Wang 2015-04-24 11:54:25 -07:00
parent cf6c8a1b4e
commit c7d9ad68e3
3 changed files with 54 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -557,6 +557,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8231. StackTrace displayed at client while QuotaByStorageType exceeds HDFS-8231. StackTrace displayed at client while QuotaByStorageType exceeds
(J.Andreina and Xiaoyu Yao via vinayakumarb) (J.Andreina and Xiaoyu Yao via vinayakumarb)
HDFS-8191. Fix byte to integer casting in SimulatedFSDataset#simulatedByte.
(Zhe Zhang via wang)
Release 2.7.1 - UNRELEASED Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

10
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
View File

@ -80,6 +80,7 @@
* Note the synchronization is coarse grained - it is at each method. * Note the synchronization is coarse grained - it is at each method.
*/ */
public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
public final static int BYTE_MASK = 0xff;
static class Factory extends FsDatasetSpi.Factory<SimulatedFSDataset> { static class Factory extends FsDatasetSpi.Factory<SimulatedFSDataset> {
@Override @Override
public SimulatedFSDataset newInstance(DataNode datanode, public SimulatedFSDataset newInstance(DataNode datanode,
@ -99,8 +100,8 @@ public static void setFactory(Configuration conf) {
} }
public static byte simulatedByte(Block b, long offsetInBlk) { public static byte simulatedByte(Block b, long offsetInBlk) {
byte firstByte = (byte) (b.getBlockId() % Byte.MAX_VALUE); byte firstByte = (byte) (b.getBlockId() & BYTE_MASK);
return (byte) ((firstByte + offsetInBlk) % Byte.MAX_VALUE); return (byte) ((firstByte + offsetInBlk) & BYTE_MASK);
} }
public static final String CONFIG_PROPERTY_CAPACITY = public static final String CONFIG_PROPERTY_CAPACITY =
@ -1028,12 +1029,13 @@ long getLength() {
@Override @Override
public int read() throws IOException { public int read() throws IOException {
if (currentPos >= length) if (currentPos >= length) {
return -1; return -1;
}
if (data !=null) { if (data !=null) {
return data[currentPos++]; return data[currentPos++];
} else { } else {
return simulatedByte(theBlock, currentPos++); return simulatedByte(theBlock, currentPos++) & BYTE_MASK;
} }
} }

62
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
View File

@ -33,6 +33,7 @@
import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.SequentialBlockIdGenerator;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory;
@ -48,6 +49,7 @@ public class TestSimulatedFSDataset {
static final String bpid = "BP-TEST"; static final String bpid = "BP-TEST";
static final int NUMBLOCKS = 20; static final int NUMBLOCKS = 20;
static final int BLOCK_LENGTH_MULTIPLIER = 79; static final int BLOCK_LENGTH_MULTIPLIER = 79;
static final long FIRST_BLK_ID = 1;
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
@ -55,15 +57,25 @@ public void setUp() throws Exception {
SimulatedFSDataset.setFactory(conf); SimulatedFSDataset.setFactory(conf);
} }
long blockIdToLen(long blkid) { static long blockIdToLen(long blkid) {
return blkid * BLOCK_LENGTH_MULTIPLIER; return blkid * BLOCK_LENGTH_MULTIPLIER;
} }
int addSomeBlocks(SimulatedFSDataset fsdataset, int startingBlockId) static int addSomeBlocks(SimulatedFSDataset fsdataset) throws IOException {
throws IOException { return addSomeBlocks(fsdataset, false);
}
static int addSomeBlocks(SimulatedFSDataset fsdataset,
boolean negativeBlkID) throws IOException {
return addSomeBlocks(fsdataset, FIRST_BLK_ID, negativeBlkID);
}
static int addSomeBlocks(SimulatedFSDataset fsdataset, long startingBlockId,
boolean negativeBlkID) throws IOException {
int bytesAdded = 0; int bytesAdded = 0;
for (int i = startingBlockId; i < startingBlockId+NUMBLOCKS; ++i) { for (long i = startingBlockId; i < startingBlockId+NUMBLOCKS; ++i) {
ExtendedBlock b = new ExtendedBlock(bpid, i, 0, 0); long blkID = negativeBlkID ? i * -1 : i;
ExtendedBlock b = new ExtendedBlock(bpid, blkID, 0, 0);
// we pass expected len as zero, - fsdataset should use the sizeof actual // we pass expected len as zero, - fsdataset should use the sizeof actual
// data written // data written
ReplicaInPipelineInterface bInfo = fsdataset.createRbw( ReplicaInPipelineInterface bInfo = fsdataset.createRbw(
@ -87,8 +99,16 @@ int addSomeBlocks(SimulatedFSDataset fsdataset, int startingBlockId)
} }
return bytesAdded; return bytesAdded;
} }
int addSomeBlocks(SimulatedFSDataset fsdataset ) throws IOException {
return addSomeBlocks(fsdataset, 1); static void readSomeBlocks(SimulatedFSDataset fsdataset,
boolean negativeBlkID) throws IOException {
for (long i = FIRST_BLK_ID; i <= NUMBLOCKS; ++i) {
long blkID = negativeBlkID ? i * -1 : i;
ExtendedBlock b = new ExtendedBlock(bpid, blkID, 0, 0);
assertTrue(fsdataset.isValidBlock(b));
assertEquals(blockIdToLen(i), fsdataset.getLength(b));
checkBlockDataAndSize(fsdataset, b, blockIdToLen(i));
}
} }
@Test @Test
@ -107,7 +127,7 @@ public void testFSDatasetFactory() {
@Test @Test
public void testGetMetaData() throws IOException { public void testGetMetaData() throws IOException {
final SimulatedFSDataset fsdataset = getSimulatedFSDataset(); final SimulatedFSDataset fsdataset = getSimulatedFSDataset();
ExtendedBlock b = new ExtendedBlock(bpid, 1, 5, 0); ExtendedBlock b = new ExtendedBlock(bpid, FIRST_BLK_ID, 5, 0);
try { try {
assertTrue(fsdataset.getMetaDataInputStream(b) == null); assertTrue(fsdataset.getMetaDataInputStream(b) == null);
assertTrue("Expected an IO exception", false); assertTrue("Expected an IO exception", false);
@ -115,7 +135,7 @@ public void testGetMetaData() throws IOException {
// ok - as expected // ok - as expected
} }
addSomeBlocks(fsdataset); // Only need to add one but .... addSomeBlocks(fsdataset); // Only need to add one but ....
b = new ExtendedBlock(bpid, 1, 0, 0); b = new ExtendedBlock(bpid, FIRST_BLK_ID, 0, 0);
InputStream metaInput = fsdataset.getMetaDataInputStream(b); InputStream metaInput = fsdataset.getMetaDataInputStream(b);
DataInputStream metaDataInput = new DataInputStream(metaInput); DataInputStream metaDataInput = new DataInputStream(metaInput);
short version = metaDataInput.readShort(); short version = metaDataInput.readShort();
@ -138,14 +158,14 @@ public void testStorageUsage() throws IOException {
void checkBlockDataAndSize(SimulatedFSDataset fsdataset, ExtendedBlock b, static void checkBlockDataAndSize(SimulatedFSDataset fsdataset,
long expectedLen) throws IOException { ExtendedBlock b, long expectedLen) throws IOException {
InputStream input = fsdataset.getBlockInputStream(b); InputStream input = fsdataset.getBlockInputStream(b);
long lengthRead = 0; long lengthRead = 0;
int data; int data;
while ((data = input.read()) != -1) { while ((data = input.read()) != -1) {
assertEquals(SimulatedFSDataset.simulatedByte(b.getLocalBlock(), assertEquals(SimulatedFSDataset.simulatedByte(b.getLocalBlock(),
lengthRead), data); lengthRead), (byte) (data & SimulatedFSDataset.BYTE_MASK));
lengthRead++; lengthRead++;
} }
assertEquals(expectedLen, lengthRead); assertEquals(expectedLen, lengthRead);
@ -153,14 +173,14 @@ void checkBlockDataAndSize(SimulatedFSDataset fsdataset, ExtendedBlock b,
@Test @Test
public void testWriteRead() throws IOException { public void testWriteRead() throws IOException {
final SimulatedFSDataset fsdataset = getSimulatedFSDataset(); testWriteRead(false);
addSomeBlocks(fsdataset); testWriteRead(true);
for (int i=1; i <= NUMBLOCKS; ++i) {
ExtendedBlock b = new ExtendedBlock(bpid, i, 0, 0);
assertTrue(fsdataset.isValidBlock(b));
assertEquals(blockIdToLen(i), fsdataset.getLength(b));
checkBlockDataAndSize(fsdataset, b, blockIdToLen(i));
} }
private void testWriteRead(boolean negativeBlkID) throws IOException {
final SimulatedFSDataset fsdataset = getSimulatedFSDataset();
addSomeBlocks(fsdataset, negativeBlkID);
readSomeBlocks(fsdataset, negativeBlkID);
} }
@Test @Test
@ -225,7 +245,7 @@ public void testInjectionNonEmpty() throws IOException {
SimulatedFSDataset sfsdataset = getSimulatedFSDataset(); SimulatedFSDataset sfsdataset = getSimulatedFSDataset();
// Add come blocks whose block ids do not conflict with // Add come blocks whose block ids do not conflict with
// the ones we are going to inject. // the ones we are going to inject.
bytesAdded += addSomeBlocks(sfsdataset, NUMBLOCKS+1); bytesAdded += addSomeBlocks(sfsdataset, NUMBLOCKS+1, false);
sfsdataset.getBlockReport(bpid); sfsdataset.getBlockReport(bpid);
assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks()); assertEquals(NUMBLOCKS, blockReport.getNumberOfBlocks());
sfsdataset.getBlockReport(bpid); sfsdataset.getBlockReport(bpid);
@ -283,7 +303,7 @@ public void checkInvalidBlock(ExtendedBlock b) {
@Test @Test
public void testInValidBlocks() throws IOException { public void testInValidBlocks() throws IOException {
final SimulatedFSDataset fsdataset = getSimulatedFSDataset(); final SimulatedFSDataset fsdataset = getSimulatedFSDataset();
ExtendedBlock b = new ExtendedBlock(bpid, 1, 5, 0); ExtendedBlock b = new ExtendedBlock(bpid, FIRST_BLK_ID, 5, 0);
checkInvalidBlock(b); checkInvalidBlock(b);
// Now check invlaid after adding some blocks // Now check invlaid after adding some blocks