big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17181. Handle transient stream read failures in FileSystem contract tests (#2286)

Browse Source 
Contributed by Steve Loughran.

* Fixes AbstractContractSeekTest test to use readFully
* Doesn't do this to AbstractContractUnbufferTest test as it changes the test too much.
Instead just notes in the error that this may be transient

The issue is that read(buffer) doesn't guarantee that the buffer is filled, only that it will
read up to a point, and that may be just be the amount of data left in the TCP packet.
readFully corrects for this, but using it in the unbuffer test runs the risk that what
is tested for in terms of unbuffering doesn't actually get validated.
This commit is contained in:
Steve Loughran 2020-09-09 11:58:26 +01:00 committed by GitHub
parent aba4a506d6
commit 2029556dbb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractSeekTest.java
View File

@ -317,7 +317,7 @@ public void testPositionedBulkReadDoesntChangePosition() throws Throwable {
int v = 256; int v = 256;
byte[] readBuffer = new byte[v]; byte[] readBuffer = new byte[v];
assertEquals(v, instream.read(128, readBuffer, 0, v)); instream.readFully(128, readBuffer, 0, v);
//have gone back //have gone back
assertEquals(40000, instream.getPos()); assertEquals(40000, instream.getPos());
//content is the same too //content is the same too
@ -572,8 +572,7 @@ public void testReadSmallFile() throws Throwable {
// now read the entire file in one go // now read the entire file in one go
byte[] fullFile = new byte[TEST_FILE_LEN]; byte[] fullFile = new byte[TEST_FILE_LEN];
assertEquals(TEST_FILE_LEN, instream.readFully(0, fullFile, 0, fullFile.length);
instream.read(0, fullFile, 0, fullFile.length));
assertEquals(0, instream.getPos()); assertEquals(0, instream.getPos());
// now read past the end of the file // now read past the end of the file

3
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractUnbufferTest.java
View File

@ -137,7 +137,8 @@ protected void validateFileContents(FSDataInputStream stream, int length,
throws IOException { throws IOException {
byte[] streamData = new byte[length]; byte[] streamData = new byte[length];
assertEquals("failed to read expected number of bytes from " assertEquals("failed to read expected number of bytes from "
+ "stream", length, stream.read(streamData)); + "stream. This may be transient",
length, stream.read(streamData));
byte[] validateFileBytes; byte[] validateFileBytes;
if (startIndex == 0 && length == fileBytes.length) { if (startIndex == 0 && length == fileBytes.length) {
validateFileBytes = fileBytes; validateFileBytes = fileBytes;