big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-7446. Fix NegativeArraySizeException in IFile's readRawValue method (#5895)

Browse Source
This commit is contained in:
Peter Szucs 2023-07-28 14:16:03 +02:00 committed by GitHub
parent 87c036efb3
commit aae5527c9a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFile.java
View File

@ -58,6 +58,7 @@
public class IFile { public class IFile {
private static final Logger LOG = LoggerFactory.getLogger(IFile.class); private static final Logger LOG = LoggerFactory.getLogger(IFile.class);
public static final int EOF_MARKER = -1; // End of File Marker public static final int EOF_MARKER = -1; // End of File Marker
private static final int ARRAY_MAX_SIZE = Integer.MAX_VALUE - 8;
/** /**
* <code>IFile.Writer</code> to write out intermediate map-outputs. * <code>IFile.Writer</code> to write out intermediate map-outputs.
@ -152,7 +153,7 @@ public void close() throws IOException {
// Write EOF_MARKER for key/value length // Write EOF_MARKER for key/value length
WritableUtils.writeVInt(out, EOF_MARKER); WritableUtils.writeVInt(out, EOF_MARKER);
WritableUtils.writeVInt(out, EOF_MARKER); WritableUtils.writeVInt(out, EOF_MARKER);
decompressedBytesWritten += 2 * WritableUtils.getVIntSize(EOF_MARKER); decompressedBytesWritten += (long) 2 * WritableUtils.getVIntSize(EOF_MARKER);
//Flush the stream //Flush the stream
out.flush(); out.flush();
@ -219,7 +220,7 @@ public void append(K key, V value) throws IOException {
buffer.reset(); buffer.reset();
// Update bytes written // Update bytes written
decompressedBytesWritten += keyLength + valueLength + decompressedBytesWritten += (long) keyLength + valueLength +
WritableUtils.getVIntSize(keyLength) + WritableUtils.getVIntSize(keyLength) +
WritableUtils.getVIntSize(valueLength); WritableUtils.getVIntSize(valueLength);
++numRecordsWritten; ++numRecordsWritten;
@ -245,7 +246,7 @@ public void append(DataInputBuffer key, DataInputBuffer value)
out.write(value.getData(), value.getPosition(), valueLength); out.write(value.getData(), value.getPosition(), valueLength);
// Update bytes written // Update bytes written
decompressedBytesWritten += keyLength + valueLength + decompressedBytesWritten += (long) keyLength + valueLength +
WritableUtils.getVIntSize(keyLength) + WritableUtils.getVIntSize(keyLength) +
WritableUtils.getVIntSize(valueLength); WritableUtils.getVIntSize(valueLength);
++numRecordsWritten; ++numRecordsWritten;
@ -394,7 +395,7 @@ protected boolean positionToNextRecord(DataInput dIn) throws IOException {
// Read key and value lengths // Read key and value lengths
currentKeyLength = WritableUtils.readVInt(dIn); currentKeyLength = WritableUtils.readVInt(dIn);
currentValueLength = WritableUtils.readVInt(dIn); currentValueLength = WritableUtils.readVInt(dIn);
bytesRead += WritableUtils.getVIntSize(currentKeyLength) + bytesRead += (long) WritableUtils.getVIntSize(currentKeyLength) +
WritableUtils.getVIntSize(currentValueLength); WritableUtils.getVIntSize(currentValueLength);
// Check for EOF // Check for EOF
@ -433,8 +434,10 @@ public boolean nextRawKey(DataInputBuffer key) throws IOException {
} }
public void nextRawValue(DataInputBuffer value) throws IOException { public void nextRawValue(DataInputBuffer value) throws IOException {
final int targetSize = currentValueLength << 1;
final byte[] valBytes = (value.getData().length < currentValueLength) final byte[] valBytes = (value.getData().length < currentValueLength)
? new byte[currentValueLength << 1] ? new byte[targetSize < 0 ? ARRAY_MAX_SIZE : targetSize]
: value.getData(); : value.getData();
int i = readData(valBytes, 0, currentValueLength); int i = readData(valBytes, 0, currentValueLength);
if (i != currentValueLength) { if (i != currentValueLength) {