diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index 2b0529356e..0848cb1cd4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -1212,10 +1212,11 @@ public static String[] list(File dir) throws IOException { * @param pwd Path to working directory to save jar * @param callerEnv Map caller's environment variables to use * for expansion - * @return String absolute path to new jar + * @return String[] with absolute path to new jar in position 0 and + * unexpanded wild card entry path in position 1 * @throws IOException if there is an I/O error while writing the jar file */ - public static String createJarWithClassPath(String inputClassPath, Path pwd, + public static String[] createJarWithClassPath(String inputClassPath, Path pwd, Map callerEnv) throws IOException { // Replace environment variables, case-insensitive on Windows @SuppressWarnings("unchecked") @@ -1235,6 +1236,7 @@ public static String createJarWithClassPath(String inputClassPath, Path pwd, LOG.debug("mkdirs false for " + workingDir + ", execution will continue"); } + StringBuilder unexpandedWildcardClasspath = new StringBuilder(); // Append all entries List classPathEntryList = new ArrayList( classPathEntries.length); @@ -1243,16 +1245,22 @@ public static String createJarWithClassPath(String inputClassPath, Path pwd, continue; } if (classPathEntry.endsWith("*")) { + boolean foundWildCardJar = false; // Append all jars that match the wildcard Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}"); FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util() .globStatus(globPath); if (wildcardJars != null) { for (FileStatus wildcardJar: wildcardJars) { + foundWildCardJar = true; classPathEntryList.add(wildcardJar.getPath().toUri().toURL() .toExternalForm()); } } + if (!foundWildCardJar) { + unexpandedWildcardClasspath.append(File.pathSeparator); + unexpandedWildcardClasspath.append(classPathEntry); + } } else { // Append just this entry File fileCpEntry = null; @@ -1300,7 +1308,8 @@ public static String createJarWithClassPath(String inputClassPath, Path pwd, } finally { IOUtils.cleanup(LOG, jos, bos, fos); } - - return classPathJar.getCanonicalPath(); + String[] jarCp = {classPathJar.getCanonicalPath(), + unexpandedWildcardClasspath.toString()}; + return jarCp; } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java index 0d3df2d0ce..29ffbe1943 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Classpath.java @@ -94,7 +94,7 @@ public static void main(String[] args) { final String tmpJarPath; try { tmpJarPath = FileUtil.createJarWithClassPath(classPath, workingDir, - System.getenv()); + System.getenv())[0]; } catch (IOException e) { terminate(1, "I/O error creating jar: " + e.getMessage()); return; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index 2a039887c0..41794b8531 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -1036,8 +1036,10 @@ public void testCreateJarWithClassPath() throws Exception { List classPaths = Arrays.asList("", "cp1.jar", "cp2.jar", wildcardPath, "cp3.jar", nonExistentSubdir); String inputClassPath = StringUtils.join(File.pathSeparator, classPaths); - String classPathJar = FileUtil.createJarWithClassPath(inputClassPath, + String[] jarCp = FileUtil.createJarWithClassPath(inputClassPath + File.pathSeparator + "unexpandedwildcard/*", new Path(tmp.getCanonicalPath()), System.getenv()); + String classPathJar = jarCp[0]; + assertNotEquals("Unexpanded wildcard was not placed in extra classpath", jarCp[1].indexOf("unexpanded"), -1); // verify classpath by reading manifest from jar file JarFile jarFile = null; diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index af056b36cd..06c0eab75e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -708,6 +708,9 @@ Release 2.6.0 - UNRELEASED YARN-2717. Avoided duplicate logging when container logs are not found. (Xuan Gong via zjshen) + YARN-2720. Windows: Wildcard classpath variables not expanded against + resources contained in archives. (Craig Welch via cnauroth) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index fc6c64f530..578a9a2042 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -148,10 +148,12 @@ public void startLocalizer(Path nmPrivateContainerTokens, // Passing CLASSPATH explicitly is *way* too long for command line. String classPath = System.getProperty("java.class.path"); Map env = new HashMap(System.getenv()); - String classPathJar = FileUtil.createJarWithClassPath(classPath, appStorageDir, env); + String[] jarCp = FileUtil.createJarWithClassPath(classPath, appStorageDir, env); + String classPathJar = jarCp[0]; localizeClasspathJar(new Path(classPathJar), user); + String replacementClassPath = classPathJar + jarCp[1]; command.add("-classpath"); - command.add(classPathJar); + command.add(replacementClassPath); String javaLibPath = System.getProperty("java.library.path"); if (javaLibPath != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index f87ed6a125..30abef5528 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -764,11 +764,13 @@ public void sanitizeEnv(Map environment, Path pwd, System.getenv()); mergedEnv.putAll(environment); - String classPathJar = FileUtil.createJarWithClassPath( + String[] jarCp = FileUtil.createJarWithClassPath( newClassPath.toString(), pwd, mergedEnv); + String classPathJar = jarCp[0]; // In a secure cluster the classpath jar must be localized to grant access this.exec.localizeClasspathJar(new Path(classPathJar), container.getUser()); - environment.put(Environment.CLASSPATH.name(), classPathJar); + String replacementClassPath = classPathJar + jarCp[1]; + environment.put(Environment.CLASSPATH.name(), replacementClassPath); } } // put AuxiliaryService data to environment