HADOOP-19253. Google GCS compilation fails due to VectorIO changes (#7002)

Fixes a compilation failure caused by HADOOP-19098

Restore original sortRanges() method signature,
  FileRange[] sortRanges(List<? extends FileRange>)

This ensures that google GCS connector will compile again.
It has also been marked as Stable so it is left alone

The version returning List<? extends FileRange>
has been renamed sortRangeList()

Contributed by Steve Loughran
This commit is contained in:
Steve Loughran 2024-08-19 19:54:47 +01:00
parent df08e0de41
commit 2fd7cf53fa
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
2 changed files with 44 additions and 8 deletions

View File

@ -308,7 +308,7 @@ public static List<? extends FileRange> validateAndSortRanges(
validateRangeRequest(input.get(0)); validateRangeRequest(input.get(0));
sortedRanges = input; sortedRanges = input;
} else { } else {
sortedRanges = sortRanges(input); sortedRanges = sortRangeList(input);
FileRange prev = null; FileRange prev = null;
for (final FileRange current : sortedRanges) { for (final FileRange current : sortedRanges) {
validateRangeRequest(current); validateRangeRequest(current);
@ -341,12 +341,25 @@ public static List<? extends FileRange> validateAndSortRanges(
* @param input input ranges. * @param input input ranges.
* @return a new list of the ranges, sorted by offset. * @return a new list of the ranges, sorted by offset.
*/ */
public static List<? extends FileRange> sortRanges(List<? extends FileRange> input) { public static List<? extends FileRange> sortRangeList(List<? extends FileRange> input) {
final List<? extends FileRange> l = new ArrayList<>(input); final List<? extends FileRange> l = new ArrayList<>(input);
l.sort(Comparator.comparingLong(FileRange::getOffset)); l.sort(Comparator.comparingLong(FileRange::getOffset));
return l; return l;
} }
/**
* Sort the input ranges by offset; no validation is done.
* <p>
* This method is used externally and must be retained with
* the signature unchanged.
* @param input input ranges.
* @return a new list of the ranges, sorted by offset.
*/
@InterfaceStability.Stable
public static FileRange[] sortRanges(List<? extends FileRange> input) {
return sortRangeList(input).toArray(new FileRange[0]);
}
/** /**
* Merge sorted ranges to optimize the access from the underlying file * Merge sorted ranges to optimize the access from the underlying file
* system. * system.

View File

@ -23,6 +23,7 @@
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.IntBuffer; import java.nio.IntBuffer;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
@ -47,6 +48,7 @@
import static org.apache.hadoop.fs.VectoredReadUtils.mergeSortedRanges; import static org.apache.hadoop.fs.VectoredReadUtils.mergeSortedRanges;
import static org.apache.hadoop.fs.VectoredReadUtils.readRangeFrom; import static org.apache.hadoop.fs.VectoredReadUtils.readRangeFrom;
import static org.apache.hadoop.fs.VectoredReadUtils.readVectored; import static org.apache.hadoop.fs.VectoredReadUtils.readVectored;
import static org.apache.hadoop.fs.VectoredReadUtils.sortRangeList;
import static org.apache.hadoop.fs.VectoredReadUtils.sortRanges; import static org.apache.hadoop.fs.VectoredReadUtils.sortRanges;
import static org.apache.hadoop.fs.VectoredReadUtils.validateAndSortRanges; import static org.apache.hadoop.fs.VectoredReadUtils.validateAndSortRanges;
import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@ -196,7 +198,7 @@ public void testSortAndMerge() {
); );
assertIsNotOrderedDisjoint(input, 100, 800); assertIsNotOrderedDisjoint(input, 100, 800);
final List<CombinedFileRange> outputList = mergeSortedRanges( final List<CombinedFileRange> outputList = mergeSortedRanges(
sortRanges(input), 100, 1001, 2500); sortRangeList(input), 100, 1001, 2500);
assertRangeListSize(outputList, 1); assertRangeListSize(outputList, 1);
CombinedFileRange output = outputList.get(0); CombinedFileRange output = outputList.get(0);
@ -208,7 +210,7 @@ public void testSortAndMerge() {
// the minSeek doesn't allow the first two to merge // the minSeek doesn't allow the first two to merge
assertIsNotOrderedDisjoint(input, 100, 100); assertIsNotOrderedDisjoint(input, 100, 100);
final List<CombinedFileRange> list2 = mergeSortedRanges( final List<CombinedFileRange> list2 = mergeSortedRanges(
sortRanges(input), sortRangeList(input),
100, 1000, 2100); 100, 1000, 2100);
assertRangeListSize(list2, 2); assertRangeListSize(list2, 2);
assertRangeElement(list2, 0, 1000, 100); assertRangeElement(list2, 0, 1000, 100);
@ -219,7 +221,7 @@ public void testSortAndMerge() {
// the maxSize doesn't allow the third range to merge // the maxSize doesn't allow the third range to merge
assertIsNotOrderedDisjoint(input, 100, 800); assertIsNotOrderedDisjoint(input, 100, 800);
final List<CombinedFileRange> list3 = mergeSortedRanges( final List<CombinedFileRange> list3 = mergeSortedRanges(
sortRanges(input), sortRangeList(input),
100, 1001, 2099); 100, 1001, 2099);
assertRangeListSize(list3, 2); assertRangeListSize(list3, 2);
CombinedFileRange range0 = list3.get(0); CombinedFileRange range0 = list3.get(0);
@ -240,7 +242,7 @@ public void testSortAndMerge() {
// test the round up and round down (the maxSize doesn't allow any merges) // test the round up and round down (the maxSize doesn't allow any merges)
assertIsNotOrderedDisjoint(input, 16, 700); assertIsNotOrderedDisjoint(input, 16, 700);
final List<CombinedFileRange> list4 = mergeSortedRanges( final List<CombinedFileRange> list4 = mergeSortedRanges(
sortRanges(input), sortRangeList(input),
16, 1001, 100); 16, 1001, 100);
assertRangeListSize(list4, 3); assertRangeListSize(list4, 3);
// range[992,1104) // range[992,1104)
@ -273,6 +275,27 @@ private static <ELEMENT extends FileRange> void assertFileRange(
.isEqualTo(length); .isEqualTo(length);
} }
/**
* Verify that {@link VectoredReadUtils#sortRanges(List)}
* returns an array matching the list sort ranges.
*/
@Test
public void testArraySortRange() throws Throwable {
List<FileRange> input = asList(
createFileRange(3000, 100, "1"),
createFileRange(2100, 100, null),
createFileRange(1000, 100, "3")
);
final FileRange[] rangeArray = sortRanges(input);
final List<? extends FileRange> rangeList = sortRangeList(input);
Assertions.assertThat(rangeArray)
.describedAs("range array from sortRanges()")
.isSortedAccordingTo(Comparator.comparingLong(FileRange::getOffset));
Assertions.assertThat(rangeList.toArray(new FileRange[0]))
.describedAs("range from sortRangeList()")
.isEqualTo(rangeArray);
}
/** /**
* Assert that a file range satisfies the conditions. * Assert that a file range satisfies the conditions.
* @param range range to validate * @param range range to validate
@ -399,7 +422,7 @@ public void testSortAndMergeMoreCases() throws Exception {
); );
assertIsNotOrderedDisjoint(input, 100, 800); assertIsNotOrderedDisjoint(input, 100, 800);
List<CombinedFileRange> outputList = mergeSortedRanges( List<CombinedFileRange> outputList = mergeSortedRanges(
sortRanges(input), 1, 1001, 2500); sortRangeList(input), 1, 1001, 2500);
Assertions.assertThat(outputList) Assertions.assertThat(outputList)
.describedAs("merged range size") .describedAs("merged range size")
.hasSize(1); .hasSize(1);
@ -411,7 +434,7 @@ public void testSortAndMergeMoreCases() throws Exception {
assertOrderedDisjoint(outputList, 1, 800); assertOrderedDisjoint(outputList, 1, 800);
outputList = mergeSortedRanges( outputList = mergeSortedRanges(
sortRanges(input), 100, 1001, 2500); sortRangeList(input), 100, 1001, 2500);
assertRangeListSize(outputList, 1); assertRangeListSize(outputList, 1);
output = outputList.get(0); output = outputList.get(0);