big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18084. ABFS: Add testfilePath while verifying test contents are read correctly (#3903)

Browse Source 
Contributed by: Anmol Asrani

Change-Id: I6e71bf349a74032f453398c7ae66f9c3305be190
This commit is contained in:
Anmol Asrani 2022-01-19 15:43:13 +05:30 committed by Steve Loughran
parent f9c0bc094a
commit 9b221b9599
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
3 changed files with 19 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStream.java
View File

@ -81,7 +81,7 @@ protected void testWithNoOptimization(final FileSystem fs,
} }
assertEquals(length, bytesRead); assertEquals(length, bytesRead);
assertContentReadCorrectly(fileContent, assertContentReadCorrectly(fileContent,
(int) (seekPos + totalBytesRead - length), length, buffer); (int) (seekPos + totalBytesRead - length), length, buffer, testFilePath);
assertTrue(abfsInputStream.getFCursor() >= seekPos + totalBytesRead); assertTrue(abfsInputStream.getFCursor() >= seekPos + totalBytesRead);
assertTrue(abfsInputStream.getFCursorAfterLastRead() >= seekPos + totalBytesRead); assertTrue(abfsInputStream.getFCursorAfterLastRead() >= seekPos + totalBytesRead);
@ -133,7 +133,7 @@ private void testExceptionInOptimization(final FileSystem fs,
actualLength = length - delta; actualLength = length - delta;
} }
assertEquals(bytesRead, actualLength); assertEquals(bytesRead, actualLength);
assertContentReadCorrectly(fileContent, seekPos, (int) actualLength, buffer); assertContentReadCorrectly(fileContent, seekPos, (int) actualLength, buffer, testFilePath);
assertEquals(fileContent.length, abfsInputStream.getFCursor()); assertEquals(fileContent.length, abfsInputStream.getFCursor());
assertEquals(fileContent.length, abfsInputStream.getFCursorAfterLastRead()); assertEquals(fileContent.length, abfsInputStream.getFCursorAfterLastRead());
assertEquals(actualLength, abfsInputStream.getBCursor()); assertEquals(actualLength, abfsInputStream.getBCursor());
@ -200,24 +200,24 @@ protected Map<String, Long> getInstrumentationMap(FileSystem fs)
} }
protected void assertContentReadCorrectly(byte[] actualFileContent, int from, protected void assertContentReadCorrectly(byte[] actualFileContent, int from,
int len, byte[] contentRead) { int len, byte[] contentRead, Path testFilePath) {
for (int i = 0; i < len; i++) { for (int i = 0; i < len; i++) {
assertEquals(contentRead[i], actualFileContent[i + from]); assertEquals("The test file path is " + testFilePath, contentRead[i], actualFileContent[i + from]);
} }
} }
protected void assertBuffersAreNotEqual(byte[] actualContent, protected void assertBuffersAreNotEqual(byte[] actualContent,
byte[] contentRead, AbfsConfiguration conf) { byte[] contentRead, AbfsConfiguration conf, Path testFilePath) {
assertBufferEquality(actualContent, contentRead, conf, false); assertBufferEquality(actualContent, contentRead, conf, false, testFilePath);
} }
protected void assertBuffersAreEqual(byte[] actualContent, byte[] contentRead, protected void assertBuffersAreEqual(byte[] actualContent, byte[] contentRead,
AbfsConfiguration conf) { AbfsConfiguration conf, Path testFilePath) {
assertBufferEquality(actualContent, contentRead, conf, true); assertBufferEquality(actualContent, contentRead, conf, true, testFilePath);
} }
private void assertBufferEquality(byte[] actualContent, byte[] contentRead, private void assertBufferEquality(byte[] actualContent, byte[] contentRead,
AbfsConfiguration conf, boolean assertEqual) { AbfsConfiguration conf, boolean assertEqual, Path testFilePath) {
int bufferSize = conf.getReadBufferSize(); int bufferSize = conf.getReadBufferSize();
int actualContentSize = actualContent.length; int actualContentSize = actualContent.length;
int n = (actualContentSize < bufferSize) ? actualContentSize : bufferSize; int n = (actualContentSize < bufferSize) ? actualContentSize : bufferSize;
@ -228,9 +228,9 @@ private void assertBufferEquality(byte[] actualContent, byte[] contentRead,
} }
} }
if (assertEqual) { if (assertEqual) {
assertEquals(n, matches); assertEquals("The test file path is " + testFilePath, n, matches);
} else { } else {
assertNotEquals(n, matches); assertNotEquals("The test file path is " + testFilePath, n, matches);
} }
} }

6
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamReadFooter.java
View File

@ -235,14 +235,14 @@ private void seekReadAndTest(final FileSystem fs, final Path testFilePath,
assertEquals(expectedBCurson, abfsInputStream.getBCursor()); assertEquals(expectedBCurson, abfsInputStream.getBCursor());
assertEquals(actualLength, bytesRead); assertEquals(actualLength, bytesRead);
// Verify user-content read // Verify user-content read
assertContentReadCorrectly(fileContent, seekPos, (int) actualLength, buffer); assertContentReadCorrectly(fileContent, seekPos, (int) actualLength, buffer, testFilePath);
// Verify data read to AbfsInputStream buffer // Verify data read to AbfsInputStream buffer
int from = seekPos; int from = seekPos;
if (optimizationOn) { if (optimizationOn) {
from = (int) max(0, actualContentLength - bufferSize); from = (int) max(0, actualContentLength - bufferSize);
} }
assertContentReadCorrectly(fileContent, from, (int) abfsInputStream.getLimit(), assertContentReadCorrectly(fileContent, from, (int) abfsInputStream.getLimit(),
abfsInputStream.getBuffer()); abfsInputStream.getBuffer(), testFilePath);
} }
} }
@ -280,7 +280,7 @@ private void testPartialReadWithNoData(final FileSystem fs,
byte[] buffer = new byte[length]; byte[] buffer = new byte[length];
int bytesRead = iStream.read(buffer, 0, length); int bytesRead = iStream.read(buffer, 0, length);
assertEquals(length, bytesRead); assertEquals(length, bytesRead);
assertContentReadCorrectly(fileContent, seekPos, length, buffer); assertContentReadCorrectly(fileContent, seekPos, length, buffer, testFilePath);
assertEquals(fileContent.length, abfsInputStream.getFCursor()); assertEquals(fileContent.length, abfsInputStream.getFCursor());
assertEquals(length, abfsInputStream.getBCursor()); assertEquals(length, abfsInputStream.getBCursor());
assertTrue(abfsInputStream.getLimit() >= length); assertTrue(abfsInputStream.getLimit() >= length);

10
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsInputStreamSmallFileReads.java
View File

@ -189,7 +189,7 @@ private void seekReadAndTest(FileSystem fs, Path testFilePath, int seekPos,
byte[] buffer = new byte[length]; byte[] buffer = new byte[length];
int bytesRead = iStream.read(buffer, 0, length); int bytesRead = iStream.read(buffer, 0, length);
assertEquals(bytesRead, length); assertEquals(bytesRead, length);
assertContentReadCorrectly(fileContent, seekPos, length, buffer); assertContentReadCorrectly(fileContent, seekPos, length, buffer, testFilePath);
AbfsInputStream abfsInputStream = (AbfsInputStream) iStream AbfsInputStream abfsInputStream = (AbfsInputStream) iStream
.getWrappedStream(); .getWrappedStream();
@ -199,16 +199,16 @@ private void seekReadAndTest(FileSystem fs, Path testFilePath, int seekPos,
int expectedLimit, expectedFCursor; int expectedLimit, expectedFCursor;
int expectedBCursor; int expectedBCursor;
if (conf.readSmallFilesCompletely() && smallFile) { if (conf.readSmallFilesCompletely() && smallFile) {
assertBuffersAreEqual(fileContent, abfsInputStream.getBuffer(), conf); assertBuffersAreEqual(fileContent, abfsInputStream.getBuffer(), conf, testFilePath);
expectedFCursor = fileContentLength; expectedFCursor = fileContentLength;
expectedLimit = fileContentLength; expectedLimit = fileContentLength;
expectedBCursor = seekPos + length; expectedBCursor = seekPos + length;
} else { } else {
if ((seekPos == 0)) { if ((seekPos == 0)) {
assertBuffersAreEqual(fileContent, abfsInputStream.getBuffer(), conf); assertBuffersAreEqual(fileContent, abfsInputStream.getBuffer(), conf, testFilePath);
} else { } else {
assertBuffersAreNotEqual(fileContent, abfsInputStream.getBuffer(), assertBuffersAreNotEqual(fileContent, abfsInputStream.getBuffer(),
conf); conf, testFilePath);
} }
expectedBCursor = length; expectedBCursor = length;
expectedFCursor = (fileContentLength < (seekPos + readBufferSize)) expectedFCursor = (fileContentLength < (seekPos + readBufferSize))
@ -260,7 +260,7 @@ private void partialReadWithNoData(final FileSystem fs,
byte[] buffer = new byte[length]; byte[] buffer = new byte[length];
int bytesRead = iStream.read(buffer, 0, length); int bytesRead = iStream.read(buffer, 0, length);
assertEquals(bytesRead, length); assertEquals(bytesRead, length);
assertContentReadCorrectly(fileContent, seekPos, length, buffer); assertContentReadCorrectly(fileContent, seekPos, length, buffer, testFilePath);
assertEquals(fileContent.length, abfsInputStream.getFCursor()); assertEquals(fileContent.length, abfsInputStream.getFCursor());
assertEquals(fileContent.length, assertEquals(fileContent.length,
abfsInputStream.getFCursorAfterLastRead()); abfsInputStream.getFCursorAfterLastRead());