HADOOP-19204. VectorIO regression: empty ranges are now rejected (#6887)

- restore old outcome: no-op
- test this
- update spec

This is a critical fix for vector IO and MUST be cherrypicked to all branches with
that feature

Contributed by Steve Loughran
This commit is contained in:
Steve Loughran 2024-06-19 12:05:24 +01:00 committed by GitHub
parent 1e6411c9ec
commit 56c8aa5f1c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 22 additions and 6 deletions

View File

@ -294,7 +294,14 @@ public static List<? extends FileRange> validateAndSortRanges(
final Optional<Long> fileLength) throws EOFException {
requireNonNull(input, "Null input list");
checkArgument(!input.isEmpty(), "Empty input list");
if (input.isEmpty()) {
// this may seem a pathological case, but it was valid
// before and somehow Spark can call it through parquet.
LOG.debug("Empty input list");
return input;
}
final List<? extends FileRange> sortedRanges;
if (input.size() == 1) {

View File

@ -474,7 +474,6 @@ No empty lists.
```python
if ranges = null raise NullPointerException
if ranges.len() = 0 raise IllegalArgumentException
if allocate = null raise NullPointerException
```

View File

@ -340,6 +340,17 @@ public void testConsecutiveRanges() throws Exception {
}
}
@Test
public void testEmptyRanges() throws Exception {
List<FileRange> fileRanges = new ArrayList<>();
try (FSDataInputStream in = openVectorFile()) {
in.readVectored(fileRanges, allocate);
Assertions.assertThat(fileRanges)
.describedAs("Empty ranges must stay empty")
.isEmpty();
}
}
/**
* Test to validate EOF ranges.
* <p>

View File

@ -702,12 +702,11 @@ private static Stream mockStreamWithReadFully() throws IOException {
}
/**
* Empty ranges cannot be sorted.
* Empty ranges are allowed.
*/
@Test
public void testEmptyRangesRaisesIllegalArgument() throws Throwable {
intercept(IllegalArgumentException.class,
() -> validateAndSortRanges(Collections.emptyList(), Optional.empty()));
public void testEmptyRangesAllowed() throws Throwable {
validateAndSortRanges(Collections.emptyList(), Optional.empty());
}
/**