big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-4133. MR over viewfs is broken (John George via bobby)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1329403 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans 2012-04-23 19:44:30 +00:00
parent 796e1a48ac
commit 80ab78c217
4 changed files with 9 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-mapreduce-project/CHANGES.txt
View File

@ -395,6 +395,7 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4190. Improve web UI for task attempts userlog link (Tom Graves MAPREDUCE-4190. Improve web UI for task attempts userlog link (Tom Graves
via bobby) via bobby)
MAPREDUCE-4133. MR over viewfs is broken (John George via bobby)
Release 0.23.2 - UNRELEASED Release 0.23.2 - UNRELEASED

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
View File

@ -827,7 +827,7 @@ private void makeUberDecision(long dataInputLength) {
|| (sysMaxReduces == 1); || (sysMaxReduces == 1);
long sysMaxBytes = conf.getLong(MRJobConfig.JOB_UBERTASK_MAXBYTES, long sysMaxBytes = conf.getLong(MRJobConfig.JOB_UBERTASK_MAXBYTES,
fs.getDefaultBlockSize()); // FIXME: this is wrong; get FS from fs.getDefaultBlockSize(this.remoteJobSubmitDir)); // FIXME: this is wrong; get FS from
// [File?]InputFormat and default block size // [File?]InputFormat and default block size
// from that // from that

7
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
View File

@ -128,7 +128,7 @@ private void listSubtree(FileStatus rootStatus,
if (rootStatus.isFile()) { if (rootStatus.isFile()) {
nrFiles++; nrFiles++;
// For a regular file generate <fName,offset> pairs // For a regular file generate <fName,offset> pairs
long blockSize = fs.getDefaultBlockSize(); long blockSize = fs.getDefaultBlockSize(rootFile);
long fileLength = rootStatus.getLen(); long fileLength = rootStatus.getLen();
for(long offset = 0; offset < fileLength; offset += blockSize) for(long offset = 0; offset < fileLength; offset += blockSize)
writer.append(new Text(rootFile.toString()), new LongWritable(offset)); writer.append(new Text(rootFile.toString()), new LongWritable(offset));
@ -160,15 +160,16 @@ public Object doIO(Reporter reporter,
) throws IOException { ) throws IOException {
// open file // open file
FSDataInputStream in = null; FSDataInputStream in = null;
Path p = new Path(name);
try { try {
in = fs.open(new Path(name)); in = fs.open(p);
} catch(IOException e) { } catch(IOException e) {
return name + "@(missing)"; return name + "@(missing)";
} }
in.seek(offset); in.seek(offset);
long actualSize = 0; long actualSize = 0;
try { try {
long blockSize = fs.getDefaultBlockSize(); long blockSize = fs.getDefaultBlockSize(p);
reporter.setStatus("reading " + name + "@" + reporter.setStatus("reading " + name + "@" +
offset + "/" + blockSize); offset + "/" + blockSize);
for( int curSize = bufferSize; for( int curSize = bufferSize;

6
hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
View File

@ -287,7 +287,7 @@ public void setupEventWriter(JobID jobId, JobConf jobConf)
FSDataOutputStream out = logDirFs.create(logFile, FSDataOutputStream out = logDirFs.create(logFile,
new FsPermission(JobHistory.HISTORY_FILE_PERMISSION), new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
true, defaultBufferSize, true, defaultBufferSize,
logDirFs.getDefaultReplication(), logDirFs.getDefaultReplication(logFile),
jobHistoryBlockSize, null); jobHistoryBlockSize, null);
EventWriter writer = new EventWriter(out); EventWriter writer = new EventWriter(out);
@ -306,8 +306,8 @@ public void setupEventWriter(JobID jobId, JobConf jobConf)
jobFileOut = logDirFs.create(logDirConfPath, jobFileOut = logDirFs.create(logDirConfPath,
new FsPermission(JobHistory.HISTORY_FILE_PERMISSION), new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
true, defaultBufferSize, true, defaultBufferSize,
logDirFs.getDefaultReplication(), logDirFs.getDefaultReplication(logDirConfPath),
logDirFs.getDefaultBlockSize(), null); logDirFs.getDefaultBlockSize(logDirConfPath), null);
jobConf.writeXml(jobFileOut); jobConf.writeXml(jobFileOut);
jobFileOut.close(); jobFileOut.close();
} }