big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-13163 Reuse pre-computed filestatus in Distcp-CopyMapper (Rajesh Balamohan via stevel)

Browse Source
This commit is contained in:
Steve Loughran 2016-05-17 12:59:59 +01:00
parent b357930526
commit c69a649257
1 changed files with 2 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyMapper.java
View File

@ -182,7 +182,7 @@ public void map(Text relPath, CopyListingFileStatus sourceFileStatus,
return; return;
} }
FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target); FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target, targetStatus);
if (action == FileAction.SKIP) { if (action == FileAction.SKIP) {
LOG.info("Skipping copy of " + sourceCurrStatus.getPath() LOG.info("Skipping copy of " + sourceCurrStatus.getPath()
+ " to " + target); + " to " + target);
@ -270,13 +270,7 @@ private static void incrementCounter(Context context, Counter counter,
} }
private FileAction checkUpdate(FileSystem sourceFS, FileStatus source, private FileAction checkUpdate(FileSystem sourceFS, FileStatus source,
Path target) throws IOException { Path target, FileStatus targetFileStatus) throws IOException {
final FileStatus targetFileStatus;
try {
targetFileStatus = targetFS.getFileStatus(target);
} catch (FileNotFoundException e) {
return FileAction.OVERWRITE;
}
if (targetFileStatus != null && !overWrite) { if (targetFileStatus != null && !overWrite) {
if (canSkip(sourceFS, source, targetFileStatus)) { if (canSkip(sourceFS, source, targetFileStatus)) {
return FileAction.SKIP; return FileAction.SKIP;