diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java index 88a90c6e6f..84bb234767 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java @@ -106,10 +106,12 @@ private PathData(FileSystem fs, String pathString) throws IOException { /** * Validates the given Windows path. - * Throws IOException on failure. * @param pathString a String of the path suppliued by the user. + * @return true if the URI scheme was not present in the pathString but + * inferred; false, otherwise. + * @throws IOException if anything goes wrong */ - private void ValidateWindowsPath(String pathString) + private static boolean checkIfSchemeInferredFromPath(String pathString) throws IOException { if (windowsNonUriAbsolutePath1.matcher(pathString).find()) { @@ -118,23 +120,21 @@ private void ValidateWindowsPath(String pathString) throw new IOException("Invalid path string " + pathString); } - inferredSchemeFromPath = true; - return; + return true; } // Is it a forward slash-separated absolute path? if (windowsNonUriAbsolutePath2.matcher(pathString).find()) { - inferredSchemeFromPath = true; - return; + return true; } // Does it look like a URI? If so then just leave it alone. if (potentialUri.matcher(pathString).find()) { - return; + return false; } // Looks like a relative path on Windows. - return; + return false; } /** @@ -153,7 +153,7 @@ private PathData(FileSystem fs, String pathString, FileStatus stat) setStat(stat); if (Path.WINDOWS) { - ValidateWindowsPath(pathString); + inferredSchemeFromPath = checkIfSchemeInferredFromPath(pathString); } } @@ -302,7 +302,7 @@ private String getStringForChildPath(Path childPath) { // check getPath() so scheme slashes aren't considered part of the path String separator = uri.getPath().endsWith(Path.SEPARATOR) ? "" : Path.SEPARATOR; - return uri + separator + basename; + return uriToString(uri, inferredSchemeFromPath) + separator + basename; } protected enum PathType { HAS_SCHEME, SCHEMELESS_ABSOLUTE, RELATIVE }; @@ -356,7 +356,7 @@ public static PathData[] expandAsGlob(String pattern, Configuration conf) if (globUri.getAuthority() == null) { matchUri = removeAuthority(matchUri); } - globMatch = matchUri.toString(); + globMatch = uriToString(matchUri, false); break; case SCHEMELESS_ABSOLUTE: // take just the uri's path globMatch = matchUri.getPath(); @@ -438,6 +438,10 @@ private static int findLongestDirPrefix(String cwd, String path, boolean isDir) */ @Override public String toString() { + return uriToString(uri, inferredSchemeFromPath); + } + + private static String uriToString(URI uri, boolean inferredSchemeFromPath) { String scheme = uri.getScheme(); // No interpretation of symbols. Just decode % escaped chars. String decodedRemainder = uri.getSchemeSpecificPart(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java index a250e24627..79df284045 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/util/CommandExecutor.java @@ -24,6 +24,9 @@ import java.io.File; import java.io.PrintStream; import java.util.StringTokenizer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.ArrayList; /** * @@ -32,23 +35,31 @@ public abstract class CommandExecutor { protected String[] getCommandAsArgs(final String cmd, final String masterKey, final String master) { - StringTokenizer tokenizer = new StringTokenizer(cmd, " "); - String[] args = new String[tokenizer.countTokens()]; - - int i = 0; - while (tokenizer.hasMoreTokens()) { - args[i] = tokenizer.nextToken(); + String regex = "\'([^\']*)\'|\"([^\"]*)\"|(\\S+)"; + Matcher matcher = Pattern.compile(regex).matcher(cmd); - args[i] = args[i].replaceAll(masterKey, master); - args[i] = args[i].replaceAll("CLITEST_DATA", - new File(CLITestHelper.TEST_CACHE_DATA_DIR). - toURI().toString().replace(' ', '+')); - args[i] = args[i].replaceAll("USERNAME", System.getProperty("user.name")); + ArrayList args = new ArrayList(); + String arg = null; - i++; - } - - return args; + while (matcher.find()) { + if (matcher.group(1) != null) { + arg = matcher.group(1); + } else if (matcher.group(2) != null) { + arg = matcher.group(2); + } else { + arg = matcher.group(3); + } + + arg = arg.replaceAll(masterKey, master); + arg = arg.replaceAll("CLITEST_DATA", + new File(CLITestHelper.TEST_CACHE_DATA_DIR). + toURI().toString().replace(' ', '+')); + arg = arg.replaceAll("USERNAME", System.getProperty("user.name")); + + args.add(arg); + } + + return args.toArray(new String[0]); } public Result executeCommand(final String cmd) throws Exception { diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 538134f5d8..64ea20b06e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -3435,6 +3435,9 @@ Release 0.23.10 - UNRELEASED HDFS-4998. TestUnderReplicatedBlocks fails intermittently (kihwal) + HDFS-4329. DFSShell issues with directories with spaces in name (Cristina + L. Abad via jeagles) + Release 0.23.9 - 2013-07-08 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 44d2b32f33..563d51a841 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -443,6 +443,153 @@ + + ls: whitespaces in an absolute path to a file + + -fs NAMENODE -mkdir -p "/a path with/whitespaces in directories" + -fs NAMENODE -touchz "/a path with/whitespaces in directories/and file names" + -fs NAMENODE -ls "/a path with/whitespaces in directories" + + + -fs NAMENODE -rm -r "/a path with" + + + + TokenComparator + Found 1 items + + + RegexpComparator + ^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/a path with/whitespaces in directories/and file names + + + + + + ls: whitespaces in a relative path to a file + + -fs NAMENODE -mkdir -p "a path with/whitespaces in directories" + -fs NAMENODE -touchz "a path with/whitespaces in directories/and file names" + -fs NAMENODE -ls "a path with/whitespaces in directories" + + + -fs NAMENODE -rm -r "a path with" + + + + TokenComparator + Found 1 items + + + RegexpComparator + ^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*a path with/whitespaces in directories/and file names + + + + + + ls: whitespaces in a scheme-qualified path to a file + + -fs NAMENODE -mkdir -p "NAMENODE/a path with/whitespaces in directories" + -fs NAMENODE -touchz "NAMENODE/a path with/whitespaces in directories/and file names" + -fs NAMENODE -ls "NAMENODE/a path with/whitespaces in directories" + + + -fs NAMENODE -rm -r "NAMENODE/a path with" + + + + TokenComparator + Found 1 items + + + RegexpComparator + ^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*NAMENODE/a path with/whitespaces in directories/and file names + + + + + + ls: whitespaces in an absolute path to a file, using globbing + + -fs NAMENODE -mkdir -p "/a path with/whitespaces in directories" + -fs NAMENODE -touchz "/a path with/whitespaces in directories/and file names" + -fs NAMENODE -touchz "/a path with/whitespaces in directories/and file names 2" + -fs NAMENODE -ls "/a*/w*" + + + -fs NAMENODE -rm -r "/a path with" + + + + TokenComparator + Found 2 items + + + RegexpComparator + ^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/a path with/whitespaces in directories/and file names + + + RegexpComparator + ^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/a path with/whitespaces in directories/and file names 2 + + + + + + ls: whitespaces in a relative path to a file, using globbing + + -fs NAMENODE -mkdir -p "a path with/whitespaces in directories" + -fs NAMENODE -touchz "a path with/whitespaces in directories/and file names" + -fs NAMENODE -touchz "a path with/whitespaces in directories/and file names 2" + -fs NAMENODE -ls "a*/w*" + + + -fs NAMENODE -rm -r "a path with" + + + + TokenComparator + Found 2 items + + + RegexpComparator + ^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*a path with/whitespaces in directories/and file names + + + RegexpComparator + ^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*a path with/whitespaces in directories/and file names 2 + + + + + + ls: whitespaces in a scheme-qualified path to a file, using globbing + + -fs NAMENODE -mkdir -p "NAMENODE/a path with/whitespaces in directories" + -fs NAMENODE -touchz "NAMENODE/a path with/whitespaces in directories/and file names" + -fs NAMENODE -touchz "NAMENODE/a path with/whitespaces in directories/and file names 2" + -fs NAMENODE -ls "NAMENODE/a*/w*" + + + -fs NAMENODE -rm -r "NAMENODE/a path with" + + + + TokenComparator + Found 2 items + + + RegexpComparator + ^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*NAMENODE/a path with/whitespaces in directories/and file names + + + RegexpComparator + ^-rw-r--r--( )*1( )*[a-z]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*NAMENODE/a path with/whitespaces in directories/and file names 2 + + + + ls: files/directories using absolute path @@ -6503,23 +6650,23 @@ TokenComparator - "data15bytes-15" + data15bytes-15 TokenComparator - "data30bytes-30" + data30bytes-30 TokenComparator - "data60bytes-60" + data60bytes-60 TokenComparator - "data120bytes-120" + data120bytes-120 TokenComparator - "datadir-0" + datadir-0 @@ -6542,23 +6689,23 @@ TokenComparator - "data15bytes-15" + data15bytes-15 TokenComparator - "data30bytes-30" + data30bytes-30 TokenComparator - "data60bytes-60" + data60bytes-60 TokenComparator - "data120bytes-120" + data120bytes-120 TokenComparator - "datadir-0" + datadir-0 @@ -6644,23 +6791,23 @@ TokenComparator - "data15bytes-15" + data15bytes-15 TokenComparator - "data30bytes-30" + data30bytes-30 TokenComparator - "data60bytes-60" + data60bytes-60 TokenComparator - "data120bytes-120" + data120bytes-120 TokenComparator - "datadir-0" + datadir-0 @@ -6731,23 +6878,23 @@ TokenComparator - "data15bytes-15" + data15bytes-15 TokenComparator - "data30bytes-30" + data30bytes-30 TokenComparator - "data60bytes-60" + data60bytes-60 TokenComparator - "data120bytes-120" + data120bytes-120 TokenComparator - "datadir-0" + datadir-0