big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-7441. Fix race condition in closing FadvisedFileRegion. Contributed by Benjamin Teke

Browse Source
This commit is contained in:
Szilard Nemeth 2023-06-23 14:40:03 -04:00
parent eb88b9ff21
commit 1c15987ee3
1 changed files with 58 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
View File

@ -41,6 +41,7 @@ public class FadvisedFileRegion extends DefaultFileRegion {
private static final Logger LOG = private static final Logger LOG =
LoggerFactory.getLogger(FadvisedFileRegion.class); LoggerFactory.getLogger(FadvisedFileRegion.class);
private final Object closeLock = new Object();
private final boolean manageOsCache; private final boolean manageOsCache;
private final int readaheadLength; private final int readaheadLength;
private final ReadaheadPool readaheadPool; private final ReadaheadPool readaheadPool;
@ -52,7 +53,7 @@ public class FadvisedFileRegion extends DefaultFileRegion {
private final boolean shuffleTransferToAllowed; private final boolean shuffleTransferToAllowed;
private final FileChannel fileChannel; private final FileChannel fileChannel;
private ReadaheadRequest readaheadRequest; private volatile ReadaheadRequest readaheadRequest;
public FadvisedFileRegion(RandomAccessFile file, long position, long count, public FadvisedFileRegion(RandomAccessFile file, long position, long count,
boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool, boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
@ -74,6 +75,8 @@ public FadvisedFileRegion(RandomAccessFile file, long position, long count,
@Override @Override
public long transferTo(WritableByteChannel target, long position) public long transferTo(WritableByteChannel target, long position)
throws IOException { throws IOException {
synchronized (closeLock) {
if (fd.valid()) {
if (readaheadPool != null && readaheadLength > 0) { if (readaheadPool != null && readaheadLength > 0) {
readaheadRequest = readaheadPool.readaheadStream(identifier, fd, readaheadRequest = readaheadPool.readaheadStream(identifier, fd,
position() + position, readaheadLength, position() + position, readaheadLength,
@ -85,6 +88,11 @@ public long transferTo(WritableByteChannel target, long position)
} else { } else {
return customShuffleTransfer(target, position); return customShuffleTransfer(target, position);
} }
} else {
return 0L;
}
}
} }
/** /**
@ -147,23 +155,29 @@ long customShuffleTransfer(WritableByteChannel target, long position)
@Override @Override
protected void deallocate() { protected void deallocate() {
synchronized (closeLock) {
if (readaheadRequest != null) { if (readaheadRequest != null) {
readaheadRequest.cancel(); readaheadRequest.cancel();
readaheadRequest = null;
} }
super.deallocate(); super.deallocate();
} }
}
/** /**
* Call when the transfer completes successfully so we can advise the OS that * Call when the transfer completes successfully so we can advise the OS that
* we don't need the region to be cached anymore. * we don't need the region to be cached anymore.
*/ */
public void transferSuccessful() { public void transferSuccessful() {
if (manageOsCache && count() > 0) { synchronized (closeLock) {
if (fd.valid() && manageOsCache && count() > 0) {
try { try {
NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier, NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
fd, position(), count(), POSIX_FADV_DONTNEED); fd, position(), count(), POSIX_FADV_DONTNEED);
} catch (Throwable t) { } catch (Throwable t) {
LOG.warn("Failed to manage OS cache for " + identifier, t); LOG.warn("Failed to manage OS cache for " + identifier +
" fd " + fd, t);
}
} }
} }
} }