big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6121. JobResourceUpdater#compareFs() doesn't handle HA namespaces (rchiang via rkanter)

Browse Source
This commit is contained in:
Robert Kanter 2015-06-30 16:46:41 -07:00
parent aaafa0b2ee
commit 3a72bfd082
2 changed files with 4 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -518,6 +518,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6413. TestLocalJobSubmission is failing with unknown host MAPREDUCE-6413. TestLocalJobSubmission is failing with unknown host
(zhihai xu via jlowe) (zhihai xu via jlowe)
MAPREDUCE-6121. JobResourceUpdater#compareFs() doesn't handle HA namespaces
(rchiang via rkanter)
Release 2.7.2 - UNRELEASED Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

38
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
View File

@ -194,7 +194,7 @@ private Path copyRemoteFiles(Path parentDir, Path originalPath,
FileSystem remoteFs = null; FileSystem remoteFs = null;
remoteFs = originalPath.getFileSystem(conf); remoteFs = originalPath.getFileSystem(conf);
if (compareFs(remoteFs, jtFs)) { if (FileUtil.compareFs(remoteFs, jtFs)) {
return originalPath; return originalPath;
} }
// this might have name collisions. copy will throw an exception // this might have name collisions. copy will throw an exception
@ -205,42 +205,6 @@ private Path copyRemoteFiles(Path parentDir, Path originalPath,
return newPath; return newPath;
} }
/*
* see if two file systems are the same or not.
*/
private boolean compareFs(FileSystem srcFs, FileSystem destFs) {
URI srcUri = srcFs.getUri();
URI dstUri = destFs.getUri();
if (srcUri.getScheme() == null) {
return false;
}
if (!srcUri.getScheme().equals(dstUri.getScheme())) {
return false;
}
String srcHost = srcUri.getHost();
String dstHost = dstUri.getHost();
if ((srcHost != null) && (dstHost != null)) {
try {
srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
} catch (UnknownHostException ue) {
return false;
}
if (!srcHost.equals(dstHost)) {
return false;
}
} else if (srcHost == null && dstHost != null) {
return false;
} else if (srcHost != null && dstHost == null) {
return false;
}
// check for ports
if (srcUri.getPort() != dstUri.getPort()) {
return false;
}
return true;
}
private void copyJar(Path originalJarPath, Path submitJarFile, private void copyJar(Path originalJarPath, Path submitJarFile,
short replication) throws IOException { short replication) throws IOException {
jtFs.copyFromLocalFile(originalJarPath, submitJarFile); jtFs.copyFromLocalFile(originalJarPath, submitJarFile);