HADOOP-19253. Google GCS compilation fails due to VectorIO changes (#7002)
Fixes a compilation failure caused by HADOOP-19098 Restore original sortRanges() method signature, FileRange[] sortRanges(List<? extends FileRange>) This ensures that google GCS connector will compile again. It has also been marked as Stable so it is left alone The version returning List<? extends FileRange> has been renamed sortRangeList() Contributed by Steve Loughran
This commit is contained in:
parent
df08e0de41
commit
2fd7cf53fa
@ -308,7 +308,7 @@ public static List<? extends FileRange> validateAndSortRanges(
|
|||||||
validateRangeRequest(input.get(0));
|
validateRangeRequest(input.get(0));
|
||||||
sortedRanges = input;
|
sortedRanges = input;
|
||||||
} else {
|
} else {
|
||||||
sortedRanges = sortRanges(input);
|
sortedRanges = sortRangeList(input);
|
||||||
FileRange prev = null;
|
FileRange prev = null;
|
||||||
for (final FileRange current : sortedRanges) {
|
for (final FileRange current : sortedRanges) {
|
||||||
validateRangeRequest(current);
|
validateRangeRequest(current);
|
||||||
@ -341,12 +341,25 @@ public static List<? extends FileRange> validateAndSortRanges(
|
|||||||
* @param input input ranges.
|
* @param input input ranges.
|
||||||
* @return a new list of the ranges, sorted by offset.
|
* @return a new list of the ranges, sorted by offset.
|
||||||
*/
|
*/
|
||||||
public static List<? extends FileRange> sortRanges(List<? extends FileRange> input) {
|
public static List<? extends FileRange> sortRangeList(List<? extends FileRange> input) {
|
||||||
final List<? extends FileRange> l = new ArrayList<>(input);
|
final List<? extends FileRange> l = new ArrayList<>(input);
|
||||||
l.sort(Comparator.comparingLong(FileRange::getOffset));
|
l.sort(Comparator.comparingLong(FileRange::getOffset));
|
||||||
return l;
|
return l;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sort the input ranges by offset; no validation is done.
|
||||||
|
* <p>
|
||||||
|
* This method is used externally and must be retained with
|
||||||
|
* the signature unchanged.
|
||||||
|
* @param input input ranges.
|
||||||
|
* @return a new list of the ranges, sorted by offset.
|
||||||
|
*/
|
||||||
|
@InterfaceStability.Stable
|
||||||
|
public static FileRange[] sortRanges(List<? extends FileRange> input) {
|
||||||
|
return sortRangeList(input).toArray(new FileRange[0]);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Merge sorted ranges to optimize the access from the underlying file
|
* Merge sorted ranges to optimize the access from the underlying file
|
||||||
* system.
|
* system.
|
||||||
|
@ -23,6 +23,7 @@
|
|||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
import java.nio.IntBuffer;
|
import java.nio.IntBuffer;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
import java.util.Comparator;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.concurrent.CompletableFuture;
|
import java.util.concurrent.CompletableFuture;
|
||||||
@ -47,6 +48,7 @@
|
|||||||
import static org.apache.hadoop.fs.VectoredReadUtils.mergeSortedRanges;
|
import static org.apache.hadoop.fs.VectoredReadUtils.mergeSortedRanges;
|
||||||
import static org.apache.hadoop.fs.VectoredReadUtils.readRangeFrom;
|
import static org.apache.hadoop.fs.VectoredReadUtils.readRangeFrom;
|
||||||
import static org.apache.hadoop.fs.VectoredReadUtils.readVectored;
|
import static org.apache.hadoop.fs.VectoredReadUtils.readVectored;
|
||||||
|
import static org.apache.hadoop.fs.VectoredReadUtils.sortRangeList;
|
||||||
import static org.apache.hadoop.fs.VectoredReadUtils.sortRanges;
|
import static org.apache.hadoop.fs.VectoredReadUtils.sortRanges;
|
||||||
import static org.apache.hadoop.fs.VectoredReadUtils.validateAndSortRanges;
|
import static org.apache.hadoop.fs.VectoredReadUtils.validateAndSortRanges;
|
||||||
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
||||||
@ -196,7 +198,7 @@ public void testSortAndMerge() {
|
|||||||
);
|
);
|
||||||
assertIsNotOrderedDisjoint(input, 100, 800);
|
assertIsNotOrderedDisjoint(input, 100, 800);
|
||||||
final List<CombinedFileRange> outputList = mergeSortedRanges(
|
final List<CombinedFileRange> outputList = mergeSortedRanges(
|
||||||
sortRanges(input), 100, 1001, 2500);
|
sortRangeList(input), 100, 1001, 2500);
|
||||||
|
|
||||||
assertRangeListSize(outputList, 1);
|
assertRangeListSize(outputList, 1);
|
||||||
CombinedFileRange output = outputList.get(0);
|
CombinedFileRange output = outputList.get(0);
|
||||||
@ -208,7 +210,7 @@ public void testSortAndMerge() {
|
|||||||
// the minSeek doesn't allow the first two to merge
|
// the minSeek doesn't allow the first two to merge
|
||||||
assertIsNotOrderedDisjoint(input, 100, 100);
|
assertIsNotOrderedDisjoint(input, 100, 100);
|
||||||
final List<CombinedFileRange> list2 = mergeSortedRanges(
|
final List<CombinedFileRange> list2 = mergeSortedRanges(
|
||||||
sortRanges(input),
|
sortRangeList(input),
|
||||||
100, 1000, 2100);
|
100, 1000, 2100);
|
||||||
assertRangeListSize(list2, 2);
|
assertRangeListSize(list2, 2);
|
||||||
assertRangeElement(list2, 0, 1000, 100);
|
assertRangeElement(list2, 0, 1000, 100);
|
||||||
@ -219,7 +221,7 @@ public void testSortAndMerge() {
|
|||||||
// the maxSize doesn't allow the third range to merge
|
// the maxSize doesn't allow the third range to merge
|
||||||
assertIsNotOrderedDisjoint(input, 100, 800);
|
assertIsNotOrderedDisjoint(input, 100, 800);
|
||||||
final List<CombinedFileRange> list3 = mergeSortedRanges(
|
final List<CombinedFileRange> list3 = mergeSortedRanges(
|
||||||
sortRanges(input),
|
sortRangeList(input),
|
||||||
100, 1001, 2099);
|
100, 1001, 2099);
|
||||||
assertRangeListSize(list3, 2);
|
assertRangeListSize(list3, 2);
|
||||||
CombinedFileRange range0 = list3.get(0);
|
CombinedFileRange range0 = list3.get(0);
|
||||||
@ -240,7 +242,7 @@ public void testSortAndMerge() {
|
|||||||
// test the round up and round down (the maxSize doesn't allow any merges)
|
// test the round up and round down (the maxSize doesn't allow any merges)
|
||||||
assertIsNotOrderedDisjoint(input, 16, 700);
|
assertIsNotOrderedDisjoint(input, 16, 700);
|
||||||
final List<CombinedFileRange> list4 = mergeSortedRanges(
|
final List<CombinedFileRange> list4 = mergeSortedRanges(
|
||||||
sortRanges(input),
|
sortRangeList(input),
|
||||||
16, 1001, 100);
|
16, 1001, 100);
|
||||||
assertRangeListSize(list4, 3);
|
assertRangeListSize(list4, 3);
|
||||||
// range[992,1104)
|
// range[992,1104)
|
||||||
@ -273,6 +275,27 @@ private static <ELEMENT extends FileRange> void assertFileRange(
|
|||||||
.isEqualTo(length);
|
.isEqualTo(length);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verify that {@link VectoredReadUtils#sortRanges(List)}
|
||||||
|
* returns an array matching the list sort ranges.
|
||||||
|
*/
|
||||||
|
@Test
|
||||||
|
public void testArraySortRange() throws Throwable {
|
||||||
|
List<FileRange> input = asList(
|
||||||
|
createFileRange(3000, 100, "1"),
|
||||||
|
createFileRange(2100, 100, null),
|
||||||
|
createFileRange(1000, 100, "3")
|
||||||
|
);
|
||||||
|
final FileRange[] rangeArray = sortRanges(input);
|
||||||
|
final List<? extends FileRange> rangeList = sortRangeList(input);
|
||||||
|
Assertions.assertThat(rangeArray)
|
||||||
|
.describedAs("range array from sortRanges()")
|
||||||
|
.isSortedAccordingTo(Comparator.comparingLong(FileRange::getOffset));
|
||||||
|
Assertions.assertThat(rangeList.toArray(new FileRange[0]))
|
||||||
|
.describedAs("range from sortRangeList()")
|
||||||
|
.isEqualTo(rangeArray);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Assert that a file range satisfies the conditions.
|
* Assert that a file range satisfies the conditions.
|
||||||
* @param range range to validate
|
* @param range range to validate
|
||||||
@ -399,7 +422,7 @@ public void testSortAndMergeMoreCases() throws Exception {
|
|||||||
);
|
);
|
||||||
assertIsNotOrderedDisjoint(input, 100, 800);
|
assertIsNotOrderedDisjoint(input, 100, 800);
|
||||||
List<CombinedFileRange> outputList = mergeSortedRanges(
|
List<CombinedFileRange> outputList = mergeSortedRanges(
|
||||||
sortRanges(input), 1, 1001, 2500);
|
sortRangeList(input), 1, 1001, 2500);
|
||||||
Assertions.assertThat(outputList)
|
Assertions.assertThat(outputList)
|
||||||
.describedAs("merged range size")
|
.describedAs("merged range size")
|
||||||
.hasSize(1);
|
.hasSize(1);
|
||||||
@ -411,7 +434,7 @@ public void testSortAndMergeMoreCases() throws Exception {
|
|||||||
assertOrderedDisjoint(outputList, 1, 800);
|
assertOrderedDisjoint(outputList, 1, 800);
|
||||||
|
|
||||||
outputList = mergeSortedRanges(
|
outputList = mergeSortedRanges(
|
||||||
sortRanges(input), 100, 1001, 2500);
|
sortRangeList(input), 100, 1001, 2500);
|
||||||
assertRangeListSize(outputList, 1);
|
assertRangeListSize(outputList, 1);
|
||||||
|
|
||||||
output = outputList.get(0);
|
output = outputList.get(0);
|
||||||
|
Loading…
Reference in New Issue
Block a user