big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10207. CLOSE_WAIT socket connection leaks during rendering of (corrupted) aggregated logs on the JobHistoryServer Web UI. Contributed by Siddharth Ahuja

Browse Source
This commit is contained in:
Szilard Nemeth 2020-04-07 17:03:17 +02:00
parent 45362a9f4c
commit bffb43b00e
2 changed files with 29 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
View File

@ -579,13 +579,17 @@ public static class LogReader {
public LogReader(Configuration conf, Path remoteAppLogFile) public LogReader(Configuration conf, Path remoteAppLogFile)
throws IOException { throws IOException {
FileContext fileContext = try {
FileContext.getFileContext(remoteAppLogFile.toUri(), conf); FileContext fileContext =
this.fsDataIStream = fileContext.open(remoteAppLogFile); FileContext.getFileContext(remoteAppLogFile.toUri(), conf);
reader = this.fsDataIStream = fileContext.open(remoteAppLogFile);
new TFile.Reader(this.fsDataIStream, fileContext.getFileStatus( reader = new TFile.Reader(this.fsDataIStream,
remoteAppLogFile).getLen(), conf); fileContext.getFileStatus(remoteAppLogFile).getLen(), conf);
this.scanner = reader.createScanner(); this.scanner = reader.createScanner();
} catch (IOException ioe) {
close();
throw new IOException("Error in creating LogReader", ioe);
}
} }
private boolean atBeginning = true; private boolean atBeginning = true;

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
View File

@ -33,6 +33,10 @@
import java.io.StringWriter; import java.io.StringWriter;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.io.Writer; import java.io.Writer;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.concurrent.CountDownLatch; import java.util.concurrent.CountDownLatch;
@ -121,6 +125,20 @@ public void testForCorruptedAggregatedLogs() throws Exception {
Assert.fail("Aggregated logs are corrupted."); Assert.fail("Aggregated logs are corrupted.");
} }
} }
//Append some corrupted text to the end of the aggregated file.
URI logUri = URI.create("file:///" + remoteAppLogFile.toUri().toString());
Files.write(Paths.get(logUri),
"corrupt_text".getBytes(), StandardOpenOption.APPEND);
try {
// Trying to read a corrupted log file created above should cause
// log reading to fail below with an IOException.
logReader = new LogReader(conf, remoteAppLogFile);
Assert.fail("Expect IOException from reading corrupt aggregated logs.");
} catch (IOException ioe) {
DataInputStream dIS = logReader.next(rLogKey);
Assert.assertNull("Input stream not available for reading", dIS);
}
} }
private void writeSrcFileAndALog(Path srcFilePath, String fileName, final long length, private void writeSrcFileAndALog(Path srcFilePath, String fileName, final long length,