big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5075. DistCp leaks input file handles since ThrottledInputStream does not close the wrapped InputStream. Contributed by Chris Nauroth

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1458741 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2013-03-20 10:40:14 +00:00
parent c92b59fd8d
commit 718f0f92a9
4 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -250,6 +250,9 @@ Release 2.0.4-alpha - UNRELEASED
BUG FIXES
MAPREDUCE-5075. DistCp leaks input file handles since ThrottledInputStream
does not close the wrapped InputStream. (Chris Nauroth via szetszwo)
Release 2.0.3-alpha - 2013-02-06
INCOMPATIBLE CHANGES

9
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
View File

@ -124,7 +124,7 @@ private long copyToTmpFile(Path tmpTargetPath, FileSystem targetFS,
tmpTargetPath, true, BUFFER_SIZE,
getReplicationFactor(fileAttributes, sourceFileStatus, targetFS, tmpTargetPath),
getBlockSize(fileAttributes, sourceFileStatus, targetFS, tmpTargetPath), context));
return copyBytes(sourceFileStatus, outStream, BUFFER_SIZE, true, context);
return copyBytes(sourceFileStatus, outStream, BUFFER_SIZE, context);
}
private void compareFileLengths(FileStatus sourceFileStatus, Path target,
@ -170,8 +170,8 @@ private Path getTmpFile(Path target, Mapper.Context context) {
}
private long copyBytes(FileStatus sourceFileStatus, OutputStream outStream,
int bufferSize, boolean mustCloseStream,
Mapper.Context context) throws IOException {
int bufferSize, Mapper.Context context)
throws IOException {
Path source = sourceFileStatus.getPath();
byte buf[] = new byte[bufferSize];
ThrottledInputStream inStream = null;
@ -187,8 +187,7 @@ private long copyBytes(FileStatus sourceFileStatus, OutputStream outStream,
bytesRead = inStream.read(buf);
}
} finally {
if (mustCloseStream)
IOUtils.cleanup(LOG, outStream, inStream);
IOUtils.cleanup(LOG, outStream, inStream);
}
return totalBytesRead;

5
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/ThrottledInputStream.java
View File

@ -52,6 +52,11 @@ public ThrottledInputStream(InputStream rawStream, long maxBytesPerSec) {
this.maxBytesPerSec = maxBytesPerSec;
}
@Override
public void close() throws IOException {
rawStream.close();
}
/** @inheritDoc */
@Override
public int read() throws IOException {

2
hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestIntegration.java
View File

@ -101,7 +101,7 @@ private void caseSingleFileTargetFile(boolean sync) {
try {
addEntries(listFile, "singlefile1/file1");
createFiles("singlefile1/file1", target.toString());
createFiles("singlefile1/file1", "target");
runTest(listFile, target, sync);