HADOOP-19204. VectorIO regression: empty ranges are now rejected (#6887)
- restore old outcome: no-op - test this - update spec This is a critical fix for vector IO and MUST be cherrypicked to all branches with that feature Contributed by Steve Loughran
This commit is contained in:
parent
1e6411c9ec
commit
56c8aa5f1c
@ -294,7 +294,14 @@ public static List<? extends FileRange> validateAndSortRanges(
|
|||||||
final Optional<Long> fileLength) throws EOFException {
|
final Optional<Long> fileLength) throws EOFException {
|
||||||
|
|
||||||
requireNonNull(input, "Null input list");
|
requireNonNull(input, "Null input list");
|
||||||
checkArgument(!input.isEmpty(), "Empty input list");
|
|
||||||
|
if (input.isEmpty()) {
|
||||||
|
// this may seem a pathological case, but it was valid
|
||||||
|
// before and somehow Spark can call it through parquet.
|
||||||
|
LOG.debug("Empty input list");
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
|
||||||
final List<? extends FileRange> sortedRanges;
|
final List<? extends FileRange> sortedRanges;
|
||||||
|
|
||||||
if (input.size() == 1) {
|
if (input.size() == 1) {
|
||||||
|
@ -474,7 +474,6 @@ No empty lists.
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
if ranges = null raise NullPointerException
|
if ranges = null raise NullPointerException
|
||||||
if ranges.len() = 0 raise IllegalArgumentException
|
|
||||||
if allocate = null raise NullPointerException
|
if allocate = null raise NullPointerException
|
||||||
```
|
```
|
||||||
|
|
||||||
|
@ -340,6 +340,17 @@ public void testConsecutiveRanges() throws Exception {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testEmptyRanges() throws Exception {
|
||||||
|
List<FileRange> fileRanges = new ArrayList<>();
|
||||||
|
try (FSDataInputStream in = openVectorFile()) {
|
||||||
|
in.readVectored(fileRanges, allocate);
|
||||||
|
Assertions.assertThat(fileRanges)
|
||||||
|
.describedAs("Empty ranges must stay empty")
|
||||||
|
.isEmpty();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Test to validate EOF ranges.
|
* Test to validate EOF ranges.
|
||||||
* <p>
|
* <p>
|
||||||
|
@ -702,12 +702,11 @@ private static Stream mockStreamWithReadFully() throws IOException {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Empty ranges cannot be sorted.
|
* Empty ranges are allowed.
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void testEmptyRangesRaisesIllegalArgument() throws Throwable {
|
public void testEmptyRangesAllowed() throws Throwable {
|
||||||
intercept(IllegalArgumentException.class,
|
validateAndSortRanges(Collections.emptyList(), Optional.empty());
|
||||||
() -> validateAndSortRanges(Collections.emptyList(), Optional.empty()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
Loading…
Reference in New Issue
Block a user