From 77dfd7ed246555f4859509baa299391a3ca59451 Mon Sep 17 00:00:00 2001 From: HarshitGupta11 <50410275+HarshitGupta11@users.noreply.github.com> Date: Tue, 29 Nov 2022 20:21:22 +0530 Subject: [PATCH] HADOOP-18530. ChecksumFileSystem::readVectored might return byte buffers not positioned at 0 (#5168) Contributed by Harshit Gupta --- .../main/java/org/apache/hadoop/fs/VectoredReadUtils.java | 7 +++++++ .../java/org/apache/hadoop/fs/TestVectoredReadUtils.java | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java index 50cab7dc4c..cf1b1ef969 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java @@ -307,9 +307,16 @@ public static ByteBuffer sliceTo(ByteBuffer readData, long readOffset, FileRange request) { int offsetChange = (int) (request.getOffset() - readOffset); int requestLength = request.getLength(); + // Create a new buffer that is backed by the original contents + // The buffer will have position 0 and the same limit as the original one readData = readData.slice(); + // Change the offset and the limit of the buffer as the reader wants to see + // only relevant data readData.position(offsetChange); readData.limit(offsetChange + requestLength); + // Create a new buffer after the limit change so that only that portion of the data is + // returned to the reader. + readData = readData.slice(); return readData; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java index fdfa8f6eb6..e964d23f4b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestVectoredReadUtils.java @@ -61,6 +61,9 @@ public void testSliceTo() { .describedAs("Slicing on the same offset shouldn't " + "create a new buffer") .isEqualTo(slice); + Assertions.assertThat(slice.position()) + .describedAs("Slicing should return buffers starting from position 0") + .isEqualTo(0); // try slicing a range final int offset = 100; @@ -77,6 +80,9 @@ public void testSliceTo() { .describedAs("Slicing should use the same underlying " + "data") .isEqualTo(slice.array()); + Assertions.assertThat(slice.position()) + .describedAs("Slicing should return buffers starting from position 0") + .isEqualTo(0); // test the contents of the slice intBuffer = slice.asIntBuffer(); for(int i=0; i < sliceLength / Integer.BYTES; ++i) {