big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18355. Update previous index properly while validating overlapping ranges. (#4647)

Browse Source 
part of HADOOP-18103.

Contributed By: Mukund Thakur
This commit is contained in:
Mukund Thakur 2022-08-04 04:08:04 +05:30 committed by GitHub
parent c5ec727435
commit 66dec9d322
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 33 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java
View File

@ -210,6 +210,7 @@ public static List<? extends FileRange> validateNonOverlappingAndReturnSortedRan
if (sortedRanges[i].getOffset() < prev.getOffset() + prev.getLength()) { if (sortedRanges[i].getOffset() < prev.getOffset() + prev.getLength()) {
throw new UnsupportedOperationException("Overlapping ranges are not supported"); throw new UnsupportedOperationException("Overlapping ranges are not supported");
} }
prev = sortedRanges[i];
} }
return Arrays.asList(sortedRanges); return Arrays.asList(sortedRanges);
} }

32
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java
View File

@ -35,6 +35,8 @@
import org.apache.hadoop.test.HadoopTestBase; import org.apache.hadoop.test.HadoopTestBase;
import static org.apache.hadoop.fs.VectoredReadUtils.sortRanges; import static org.apache.hadoop.fs.VectoredReadUtils.sortRanges;
import static org.apache.hadoop.fs.VectoredReadUtils.validateNonOverlappingAndReturnSortedRanges;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
import static org.apache.hadoop.test.MoreAsserts.assertFutureCompletedSuccessfully; import static org.apache.hadoop.test.MoreAsserts.assertFutureCompletedSuccessfully;
import static org.apache.hadoop.test.MoreAsserts.assertFutureFailedExceptionally; import static org.apache.hadoop.test.MoreAsserts.assertFutureFailedExceptionally;
@ -231,6 +233,36 @@ public void testSortAndMergeMoreCases() throws Exception {
} }
@Test
public void testValidateOverlappingRanges() throws Exception {
List<FileRange> input = Arrays.asList(
FileRange.createFileRange(100, 100),
FileRange.createFileRange(200, 100),
FileRange.createFileRange(250, 100)
);
intercept(UnsupportedOperationException.class,
() -> validateNonOverlappingAndReturnSortedRanges(input));
List<FileRange> input1 = Arrays.asList(
FileRange.createFileRange(100, 100),
FileRange.createFileRange(500, 100),
FileRange.createFileRange(1000, 100),
FileRange.createFileRange(1000, 100)
);
intercept(UnsupportedOperationException.class,
() -> validateNonOverlappingAndReturnSortedRanges(input1));
List<FileRange> input2 = Arrays.asList(
FileRange.createFileRange(100, 100),
FileRange.createFileRange(200, 100),
FileRange.createFileRange(300, 100)
);
// consecutive ranges should pass.
validateNonOverlappingAndReturnSortedRanges(input2);
}
@Test @Test
public void testMaxSizeZeroDisablesMering() throws Exception { public void testMaxSizeZeroDisablesMering() throws Exception {
List<FileRange> randomRanges = Arrays.asList( List<FileRange> randomRanges = Arrays.asList(