big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18084. ABFS: Add testfilePath while verifying test contents are read correctly (#3903)

Browse Source 
Contributed by: Anmol Asrani
This commit is contained in:
Anmol Asrani 2022-01-19 15:43:13 +05:30 committed by GitHub
parent db2c3200e6
commit 7c97c0f969
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 19 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java
View File

@ -81,7 +81,7 @@ protected void testWithNoOptimization(final FileSystem fs,
}
assertEquals(length, bytesRead);
assertContentReadCorrectly(fileContent,
(int) (seekPos + totalBytesRead - length), length, buffer);
(int) (seekPos + totalBytesRead - length), length, buffer, testFilePath);
assertTrue(abfsInputStream.getFCursor() >= seekPos + totalBytesRead);
assertTrue(abfsInputStream.getFCursorAfterLastRead() >= seekPos + totalBytesRead);
@ -133,7 +133,7 @@ private void testExceptionInOptimization(final FileSystem fs,
actualLength = length - delta;
}
assertEquals(bytesRead, actualLength);
assertContentReadCorrectly(fileContent, seekPos, (int) actualLength, buffer);
assertContentReadCorrectly(fileContent, seekPos, (int) actualLength, buffer, testFilePath);
assertEquals(fileContent.length, abfsInputStream.getFCursor());
assertEquals(fileContent.length, abfsInputStream.getFCursorAfterLastRead());
assertEquals(actualLength, abfsInputStream.getBCursor());
@ -200,24 +200,24 @@ protected Map<String, Long> getInstrumentationMap(FileSystem fs)
}
protected void assertContentReadCorrectly(byte[] actualFileContent, int from,
int len, byte[] contentRead) {
int len, byte[] contentRead, Path testFilePath) {
for (int i = 0; i < len; i++) {
assertEquals(contentRead[i], actualFileContent[i + from]);
assertEquals("The test file path is " + testFilePath, contentRead[i], actualFileContent[i + from]);
}
}
protected void assertBuffersAreNotEqual(byte[] actualContent,
byte[] contentRead, AbfsConfiguration conf) {
assertBufferEquality(actualContent, contentRead, conf, false);
byte[] contentRead, AbfsConfiguration conf, Path testFilePath) {
assertBufferEquality(actualContent, contentRead, conf, false, testFilePath);
}
protected void assertBuffersAreEqual(byte[] actualContent, byte[] contentRead,
AbfsConfiguration conf) {
assertBufferEquality(actualContent, contentRead, conf, true);
AbfsConfiguration conf, Path testFilePath) {
assertBufferEquality(actualContent, contentRead, conf, true, testFilePath);
}
private void assertBufferEquality(byte[] actualContent, byte[] contentRead,
AbfsConfiguration conf, boolean assertEqual) {
AbfsConfiguration conf, boolean assertEqual, Path testFilePath) {
int bufferSize = conf.getReadBufferSize();
int actualContentSize = actualContent.length;
int n = (actualContentSize < bufferSize) ? actualContentSize : bufferSize;
@ -228,9 +228,9 @@ private void assertBufferEquality(byte[] actualContent, byte[] contentRead,
}
}
if (assertEqual) {
assertEquals(n, matches);
assertEquals("The test file path is " + testFilePath, n, matches);
} else {
assertNotEquals(n, matches);
assertNotEquals("The test file path is " + testFilePath, n, matches);
}
}

6
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamReadFooter.java
View File

@ -235,14 +235,14 @@ private void seekReadAndTest(final FileSystem fs, final Path testFilePath,
assertEquals(expectedBCurson, abfsInputStream.getBCursor());
assertEquals(actualLength, bytesRead);
// Verify user-content read
assertContentReadCorrectly(fileContent, seekPos, (int) actualLength, buffer);
assertContentReadCorrectly(fileContent, seekPos, (int) actualLength, buffer, testFilePath);
// Verify data read to AbfsInputStream buffer
int from = seekPos;
if (optimizationOn) {
from = (int) max(0, actualContentLength - bufferSize);
}
assertContentReadCorrectly(fileContent, from, (int) abfsInputStream.getLimit(),
abfsInputStream.getBuffer());
abfsInputStream.getBuffer(), testFilePath);
}
}
@ -280,7 +280,7 @@ private void testPartialReadWithNoData(final FileSystem fs,
byte[] buffer = new byte[length];
int bytesRead = iStream.read(buffer, 0, length);
assertEquals(length, bytesRead);
assertContentReadCorrectly(fileContent, seekPos, length, buffer);
assertContentReadCorrectly(fileContent, seekPos, length, buffer, testFilePath);
assertEquals(fileContent.length, abfsInputStream.getFCursor());
assertEquals(length, abfsInputStream.getBCursor());
assertTrue(abfsInputStream.getLimit() >= length);

10
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamSmallFileReads.java
View File

@ -189,7 +189,7 @@ private void seekReadAndTest(FileSystem fs, Path testFilePath, int seekPos,
byte[] buffer = new byte[length];
int bytesRead = iStream.read(buffer, 0, length);
assertEquals(bytesRead, length);
assertContentReadCorrectly(fileContent, seekPos, length, buffer);
assertContentReadCorrectly(fileContent, seekPos, length, buffer, testFilePath);
AbfsInputStream abfsInputStream = (AbfsInputStream) iStream
.getWrappedStream();
@ -199,16 +199,16 @@ private void seekReadAndTest(FileSystem fs, Path testFilePath, int seekPos,
int expectedLimit, expectedFCursor;
int expectedBCursor;
if (conf.readSmallFilesCompletely() && smallFile) {
assertBuffersAreEqual(fileContent, abfsInputStream.getBuffer(), conf);
assertBuffersAreEqual(fileContent, abfsInputStream.getBuffer(), conf, testFilePath);
expectedFCursor = fileContentLength;
expectedLimit = fileContentLength;
expectedBCursor = seekPos + length;
} else {
if ((seekPos == 0)) {
assertBuffersAreEqual(fileContent, abfsInputStream.getBuffer(), conf);
assertBuffersAreEqual(fileContent, abfsInputStream.getBuffer(), conf, testFilePath);
} else {
assertBuffersAreNotEqual(fileContent, abfsInputStream.getBuffer(),
conf);
conf, testFilePath);
}
expectedBCursor = length;
expectedFCursor = (fileContentLength < (seekPos + readBufferSize))
@ -260,7 +260,7 @@ private void partialReadWithNoData(final FileSystem fs,
byte[] buffer = new byte[length];
int bytesRead = iStream.read(buffer, 0, length);
assertEquals(bytesRead, length);
assertContentReadCorrectly(fileContent, seekPos, length, buffer);
assertContentReadCorrectly(fileContent, seekPos, length, buffer, testFilePath);
assertEquals(fileContent.length, abfsInputStream.getFCursor());
assertEquals(fileContent.length,
abfsInputStream.getFCursorAfterLastRead());