big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2720. Windows: Wildcard classpath variables not expanded against resources contained in archives. Contributed by Craig Welch.

Browse Source
This commit is contained in:
cnauroth 2014-10-21 12:33:21 -07:00
parent 4e134a02a4
commit 6637e3cf95
6 changed files with 28 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
View File

@ -1212,10 +1212,11 @@ public static String[] list(File dir) throws IOException {
* @param pwd Path to working directory to save jar
* @param callerEnv Map<String, String> caller's environment variables to use
* for expansion
* @return String absolute path to new jar
* @return String[] with absolute path to new jar in position 0 and
* unexpanded wild card entry path in position 1
* @throws IOException if there is an I/O error while writing the jar file
*/
public static String createJarWithClassPath(String inputClassPath, Path pwd,
public static String[] createJarWithClassPath(String inputClassPath, Path pwd,
Map<String, String> callerEnv) throws IOException {
// Replace environment variables, case-insensitive on Windows
@SuppressWarnings("unchecked")
@ -1235,6 +1236,7 @@ public static String createJarWithClassPath(String inputClassPath, Path pwd,
LOG.debug("mkdirs false for " + workingDir + ", execution will continue");
}
StringBuilder unexpandedWildcardClasspath = new StringBuilder();
// Append all entries
List<String> classPathEntryList = new ArrayList<String>(
classPathEntries.length);
@ -1243,16 +1245,22 @@ public static String createJarWithClassPath(String inputClassPath, Path pwd,
continue;
}
if (classPathEntry.endsWith("*")) {
boolean foundWildCardJar = false;
// Append all jars that match the wildcard
Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}");
FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util()
.globStatus(globPath);
if (wildcardJars != null) {
for (FileStatus wildcardJar: wildcardJars) {
foundWildCardJar = true;
classPathEntryList.add(wildcardJar.getPath().toUri().toURL()
.toExternalForm());
}
}
if (!foundWildCardJar) {
unexpandedWildcardClasspath.append(File.pathSeparator);
unexpandedWildcardClasspath.append(classPathEntry);
}
} else {
// Append just this entry
File fileCpEntry = null;
@ -1300,7 +1308,8 @@ public static String createJarWithClassPath(String inputClassPath, Path pwd,
} finally {
IOUtils.cleanup(LOG, jos, bos, fos);
}
return classPathJar.getCanonicalPath();
String[] jarCp = {classPathJar.getCanonicalPath(),
unexpandedWildcardClasspath.toString()};
return jarCp;
}
}

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java
View File

@ -94,7 +94,7 @@ public static void main(String[] args) {
final String tmpJarPath;
try {
tmpJarPath = FileUtil.createJarWithClassPath(classPath, workingDir,
System.getenv());
System.getenv())[0];
} catch (IOException e) {
terminate(1, "I/O error creating jar: " + e.getMessage());
return;

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
View File

@ -1036,8 +1036,10 @@ public void testCreateJarWithClassPath() throws Exception {
List<String> classPaths = Arrays.asList("", "cp1.jar", "cp2.jar", wildcardPath,
"cp3.jar", nonExistentSubdir);
String inputClassPath = StringUtils.join(File.pathSeparator, classPaths);
String classPathJar = FileUtil.createJarWithClassPath(inputClassPath,
String[] jarCp = FileUtil.createJarWithClassPath(inputClassPath + File.pathSeparator + "unexpandedwildcard/*",
new Path(tmp.getCanonicalPath()), System.getenv());
String classPathJar = jarCp[0];
assertNotEquals("Unexpanded wildcard was not placed in extra classpath", jarCp[1].indexOf("unexpanded"), -1);
// verify classpath by reading manifest from jar file
JarFile jarFile = null;

3
hadoop-yarn-project/CHANGES.txt
View File

@ -708,6 +708,9 @@ Release 2.6.0 - UNRELEASED
YARN-2717. Avoided duplicate logging when container logs are not found. (Xuan
Gong via zjshen)
YARN-2720. Windows: Wildcard classpath variables not expanded against
resources contained in archives. (Craig Welch via cnauroth)
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
View File

@ -148,10 +148,12 @@ public void startLocalizer(Path nmPrivateContainerTokens,
// Passing CLASSPATH explicitly is *way* too long for command line.
String classPath = System.getProperty("java.class.path");
Map<String, String> env = new HashMap<String, String>(System.getenv());
String classPathJar = FileUtil.createJarWithClassPath(classPath, appStorageDir, env);
String[] jarCp = FileUtil.createJarWithClassPath(classPath, appStorageDir, env);
String classPathJar = jarCp[0];
localizeClasspathJar(new Path(classPathJar), user);
String replacementClassPath = classPathJar + jarCp[1];
command.add("-classpath");
command.add(classPathJar);
command.add(replacementClassPath);
String javaLibPath = System.getProperty("java.library.path");
if (javaLibPath != null) {

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
View File

@ -764,11 +764,13 @@ public void sanitizeEnv(Map<String, String> environment, Path pwd,
System.getenv());
mergedEnv.putAll(environment);
String classPathJar = FileUtil.createJarWithClassPath(
String[] jarCp = FileUtil.createJarWithClassPath(
newClassPath.toString(), pwd, mergedEnv);
String classPathJar = jarCp[0];
// In a secure cluster the classpath jar must be localized to grant access
this.exec.localizeClasspathJar(new Path(classPathJar), container.getUser());
environment.put(Environment.CLASSPATH.name(), classPathJar);
String replacementClassPath = classPathJar + jarCp[1];
environment.put(Environment.CLASSPATH.name(), replacementClassPath);
}
}
// put AuxiliaryService data to environment