big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-7441. Fix race condition in closing FadvisedFileRegion. Contributed by Benjamin Teke

Browse Source
This commit is contained in:
Szilard Nemeth 2023-06-23 14:40:03 -04:00
parent eb88b9ff21
commit 1c15987ee3
1 changed files with 58 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java
View File

@ -41,6 +41,7 @@ public class FadvisedFileRegion extends DefaultFileRegion {
private static final Logger LOG =
LoggerFactory.getLogger(FadvisedFileRegion.class);
private final Object closeLock = new Object();
private final boolean manageOsCache;
private final int readaheadLength;
private final ReadaheadPool readaheadPool;
@ -52,7 +53,7 @@ public class FadvisedFileRegion extends DefaultFileRegion {
private final boolean shuffleTransferToAllowed;
private final FileChannel fileChannel;
private ReadaheadRequest readaheadRequest;
private volatile ReadaheadRequest readaheadRequest;
public FadvisedFileRegion(RandomAccessFile file, long position, long count,
boolean manageOsCache, int readaheadLength, ReadaheadPool readaheadPool,
@ -74,6 +75,8 @@ public FadvisedFileRegion(RandomAccessFile file, long position, long count,
@Override
public long transferTo(WritableByteChannel target, long position)
throws IOException {
synchronized (closeLock) {
if (fd.valid()) {
if (readaheadPool != null && readaheadLength > 0) {
readaheadRequest = readaheadPool.readaheadStream(identifier, fd,
position() + position, readaheadLength,
@ -85,6 +88,11 @@ public long transferTo(WritableByteChannel target, long position)
} else {
return customShuffleTransfer(target, position);
}
} else {
return 0L;
}
}
}
/**
@ -147,23 +155,29 @@ long customShuffleTransfer(WritableByteChannel target, long position)
@Override
protected void deallocate() {
synchronized (closeLock) {
if (readaheadRequest != null) {
readaheadRequest.cancel();
readaheadRequest = null;
}
super.deallocate();
}
}
/**
* Call when the transfer completes successfully so we can advise the OS that
* we don't need the region to be cached anymore.
*/
public void transferSuccessful() {
if (manageOsCache && count() > 0) {
synchronized (closeLock) {
if (fd.valid() && manageOsCache && count() > 0) {
try {
NativeIO.POSIX.getCacheManipulator().posixFadviseIfPossible(identifier,
fd, position(), count(), POSIX_FADV_DONTNEED);
} catch (Throwable t) {
LOG.warn("Failed to manage OS cache for " + identifier, t);
LOG.warn("Failed to manage OS cache for " + identifier +
" fd " + fd, t);
}
}
}
}