big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16272. Fix int overflow in computing safe length during EC block recovery (#3548)

Browse Source
This commit is contained in:
daimin 2021-10-18 18:55:55 +08:00 committed by GitHub
parent 280ae1c0a9
commit 5337bebcc5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 21 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
View File

@ -245,8 +245,7 @@ public static long getSafeLength(ErasureCodingPolicy ecPolicy,
Arrays.sort(cpy); Arrays.sort(cpy);
// full stripe is a stripe has at least dataBlkNum full cells. // full stripe is a stripe has at least dataBlkNum full cells.
// lastFullStripeIdx is the index of the last full stripe. // lastFullStripeIdx is the index of the last full stripe.
int lastFullStripeIdx = long lastFullStripeIdx = cpy[cpy.length - dataBlkNum] / cellSize;
(int) (cpy[cpy.length - dataBlkNum] / cellSize);
return lastFullStripeIdx * stripeSize; // return the safeLength return lastFullStripeIdx * stripeSize; // return the safeLength
// TODO: Include lastFullStripeIdx+1 stripe in safeLength, if there exists // TODO: Include lastFullStripeIdx+1 stripe in safeLength, if there exists
// such a stripe (and it must be partial). // such a stripe (and it must be partial).
@ -271,9 +270,9 @@ private static int lastCellSize(int size, int cellSize, int numDataBlocks,
*/ */
public static long offsetInBlkToOffsetInBG(int cellSize, int dataBlkNum, public static long offsetInBlkToOffsetInBG(int cellSize, int dataBlkNum,
long offsetInBlk, int idxInBlockGroup) { long offsetInBlk, int idxInBlockGroup) {
int cellIdxInBlk = (int) (offsetInBlk / cellSize); long cellIdxInBlk = offsetInBlk / cellSize;
return cellIdxInBlk * cellSize * dataBlkNum // n full stripes before offset return cellIdxInBlk * cellSize * dataBlkNum // n full stripes before offset
+ idxInBlockGroup * cellSize // m full cells before offset + (long)idxInBlockGroup * cellSize // m full cells before offset
+ offsetInBlk % cellSize; // partial cell + offsetInBlk % cellSize; // partial cell
} }

18
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java
View File

@ -188,6 +188,24 @@ public void testLeaseRecovery() throws Exception {
} }
} }
@Test
public void testSafeLength() {
checkSafeLength(0, 0); // Length of: 0
checkSafeLength(1024 * 1024, 6291456L); // Length of: 1 MiB
checkSafeLength(64 * 1024 * 1024, 402653184L); // Length of: 64 MiB
checkSafeLength(189729792, 1132462080L); // Length of: 189729792
checkSafeLength(256 * 1024 * 1024, 1610612736L); // Length of: 256 MiB
checkSafeLength(517399040, 3101687808L); // Length of: 517399040
checkSafeLength(1024 * 1024 * 1024, 6442450944L); // Length of: 1 GiB
}
private void checkSafeLength(int blockLength, long expectedSafeLength) {
int[] blockLengths = new int[]{blockLength, blockLength, blockLength, blockLength,
blockLength, blockLength};
long safeLength = new BlockLengths(ecPolicy, blockLengths).getSafeLength();
Assert.assertEquals(expectedSafeLength, safeLength);
}
private void runTest(int[] blockLengths, long safeLength) throws Exception { private void runTest(int[] blockLengths, long safeLength) throws Exception {
writePartialBlocks(blockLengths); writePartialBlocks(blockLengths);