From dc735b286bb656903df49aee776d22ee0c61f860 Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Thu, 4 Jan 2018 10:16:40 -0800 Subject: [PATCH] HDFS-12860. StripedBlockUtil#getRangesInternalBlocks throws exception for the block group size larger than 2GB. (Contributed by Lei (Eddy) Xu) --- .../hadoop/hdfs/util/StripedBlockUtil.java | 63 ++++++++++++++----- .../hdfs/util/TestStripedBlockUtil.java | 41 +++++++++++- 2 files changed, 86 insertions(+), 18 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java index 9e245766bf..9bad45dfb8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java @@ -396,7 +396,9 @@ private static StripingCell[] getStripingCellsOfByteRange( long rangeStartInBlockGroup, long rangeEndInBlockGroup) { Preconditions.checkArgument( rangeStartInBlockGroup <= rangeEndInBlockGroup && - rangeEndInBlockGroup < blockGroup.getBlockSize()); + rangeEndInBlockGroup < blockGroup.getBlockSize(), + "start=%s end=%s blockSize=%s", rangeStartInBlockGroup, + rangeEndInBlockGroup, blockGroup.getBlockSize()); long len = rangeEndInBlockGroup - rangeStartInBlockGroup + 1; int firstCellIdxInBG = (int) (rangeStartInBlockGroup / cellSize); int lastCellIdxInBG = (int) (rangeEndInBlockGroup / cellSize); @@ -578,28 +580,39 @@ private static void prepareAllZeroChunks(LocatedStripedBlock blockGroup, public static class StripingCell { final ErasureCodingPolicy ecPolicy; /** Logical order in a block group, used when doing I/O to a block group. */ - final int idxInBlkGroup; - final int idxInInternalBlk; - final int idxInStripe; + private final long idxInBlkGroup; + private final long idxInInternalBlk; + private final int idxInStripe; /** * When a logical byte range is mapped to a set of cells, it might * partially overlap with the first and last cells. This field and the * {@link #size} variable represent the start offset and size of the * overlap. */ - final int offset; - final int size; + private final long offset; + private final int size; - StripingCell(ErasureCodingPolicy ecPolicy, int cellSize, int idxInBlkGroup, - int offset) { + StripingCell(ErasureCodingPolicy ecPolicy, int cellSize, long idxInBlkGroup, + long offset) { this.ecPolicy = ecPolicy; this.idxInBlkGroup = idxInBlkGroup; this.idxInInternalBlk = idxInBlkGroup / ecPolicy.getNumDataUnits(); - this.idxInStripe = idxInBlkGroup - - this.idxInInternalBlk * ecPolicy.getNumDataUnits(); + this.idxInStripe = (int)(idxInBlkGroup - + this.idxInInternalBlk * ecPolicy.getNumDataUnits()); this.offset = offset; this.size = cellSize; } + + int getIdxInStripe() { + return idxInStripe; + } + + @Override + public String toString() { + return String.format("StripingCell(idxInBlkGroup=%d, " + + "idxInInternalBlk=%d, idxInStrip=%d, offset=%d, size=%d)", + idxInBlkGroup, idxInInternalBlk, idxInStripe, offset, size); + } } /** @@ -646,7 +659,9 @@ public static class AlignedStripe { public int missingChunksNum = 0; public AlignedStripe(long offsetInBlock, long length, int width) { - Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0); + Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0, + "OffsetInBlock(%s) and length(%s) must be non-negative", + offsetInBlock, length); this.range = new VerticalRange(offsetInBlock, length); this.chunks = new StripingChunk[width]; } @@ -665,9 +680,9 @@ public long getSpanInBlock() { @Override public String toString() { - return "Offset=" + range.offsetInBlock + ", length=" + range.spanInBlock + - ", fetchedChunksNum=" + fetchedChunksNum + - ", missingChunksNum=" + missingChunksNum; + return "AlignedStripe(Offset=" + range.offsetInBlock + ", length=" + + range.spanInBlock + ", fetchedChunksNum=" + fetchedChunksNum + + ", missingChunksNum=" + missingChunksNum + ")"; } } @@ -698,7 +713,9 @@ public static class VerticalRange { public long spanInBlock; public VerticalRange(long offsetInBlock, long length) { - Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0); + Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0, + "OffsetInBlock(%s) and length(%s) must be non-negative", + offsetInBlock, length); this.offsetInBlock = offsetInBlock; this.spanInBlock = length; } @@ -707,6 +724,12 @@ public VerticalRange(long offsetInBlock, long length) { public boolean include(long pos) { return pos >= offsetInBlock && pos < offsetInBlock + spanInBlock; } + + @Override + public String toString() { + return String.format("VerticalRange(offsetInBlock=%d, spanInBlock=%d)", + this.offsetInBlock, this.spanInBlock); + } } /** @@ -880,7 +903,9 @@ public static class StripeRange { final long length; public StripeRange(long offsetInBlock, long length) { - Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0); + Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0, + "Offset(%s) and length(%s) must be non-negative", offsetInBlock, + length); this.offsetInBlock = offsetInBlock; this.length = length; } @@ -892,6 +917,12 @@ public boolean include(long pos) { public long getLength() { return length; } + + @Override + public String toString() { + return String.format("StripeRange(offsetInBlock=%d, length=%d)", + offsetInBlock, length); + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java index 5de63eb4eb..fe6bb09bad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java @@ -40,6 +40,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Need to cover the following combinations: @@ -127,7 +128,7 @@ private byte hashIntToByte(int i) { return (byte) (((i + 13) * 29) & BYTE_MASK); } - private LocatedStripedBlock createDummyLocatedBlock(int bgSize) { + private LocatedStripedBlock createDummyLocatedBlock(long bgSize) { final long blockGroupID = -1048576; DatanodeInfo[] locs = new DatanodeInfo[groupSize]; String[] storageIDs = new String[groupSize]; @@ -160,7 +161,7 @@ private byte[][] createInternalBlkBuffers(int bgSize) { Preconditions.checkState(done % cellSize == 0); StripingCell cell = new StripingCell(ecPolicy, cellSize, done / cellSize, 0); - int idxInStripe = cell.idxInStripe; + int idxInStripe = cell.getIdxInStripe(); int size = Math.min(cellSize, bgSize - done); for (int i = 0; i < size; i++) { bufs[idxInStripe][pos[idxInStripe] + i] = hashIntToByte(done + i); @@ -283,4 +284,40 @@ public void testDivideByteRangeIntoStripes() { } } } + + /** + * Test dividing a byte range that located above the 2GB range, which is + * {@link Integer#MAX_VALUE}. + * + * HDFS-12860 occurs when {@link VerticalRange#offsetInBlock} is larger than + * {@link Integer#MAX_VALUE} + * + * Take RS-6-3-1024k EC policy as example: + *
  • cellSize = 1MB
  • + *
  • The first {@link VerticalRange#offsetInBlock} that is larger than + * {@link Integer#MAX_VALUE} is Math.ceilInteger.MAX_VALUE / cellSize = 2048 + *
  • + *
  • The first offset in block group that causes HDFS-12860 is: + * 2048 * cellSize * dataBlocks (6)
  • + */ + @Test + public void testDivideOneStripeLargeBlockSize() { + ByteBuffer buffer = ByteBuffer.allocate(stripeSize); + + // This offset will cause overflow before HDFS-12860. + long offsetInInternalBlk = Integer.MAX_VALUE / cellSize + 10; + long rangeStartInBlockGroup = offsetInInternalBlk * dataBlocks * cellSize; + long rangeEndInBlockGroup = rangeStartInBlockGroup + + dataBlocks / 2 * cellSize - 1; + // each block is 4GB, each block group has 4GB * (6 + 3) = 36GB. + long blockGroupSize = 4096L * cellSize * groupSize; + LocatedStripedBlock blockGroup = createDummyLocatedBlock(blockGroupSize); + AlignedStripe[] stripes = StripedBlockUtil.divideOneStripe(ecPolicy, + cellSize, blockGroup, rangeStartInBlockGroup, rangeEndInBlockGroup, + buffer); + long offset = offsetInInternalBlk * cellSize; + assertTrue(offset > Integer.MAX_VALUE); + assertEquals(offset, stripes[0].range.offsetInBlock); + assertEquals(1, stripes.length); + } }