From 659ea4c540e440004d9f1a7dedefa91c0bec8b04 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Fri, 28 Oct 2011 01:13:49 +0000 Subject: [PATCH] HADOOP-7360. Preserve relative paths that do not contain globs in FsShell. Contributed by Daryn Sharp and Kihwal Lee git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1190109 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/fs/shell/CopyCommands.java | 23 +- .../org/apache/hadoop/fs/shell/Count.java | 2 +- .../java/org/apache/hadoop/fs/shell/Ls.java | 2 +- .../org/apache/hadoop/fs/shell/PathData.java | 349 ++++++++++++++---- .../org/apache/hadoop/cli/CLITestHelper.java | 4 +- .../apache/hadoop/fs/shell/TestPathData.java | 156 ++++++-- 7 files changed, 412 insertions(+), 127 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index cd737733aa..eb8667baee 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -503,6 +503,9 @@ Release 0.23.0 - Unreleased HADOOP-7762. Common side of MR-2736. (eli) + HADOOP-7360. Preserve relative paths that do not contain globs in FsShell. + (Daryn Sharp and Kihwal Lee via szetszwo) + OPTIMIZATIONS HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java index 261a1a3015..7773cf08f7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java @@ -27,10 +27,8 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.ChecksumFileSystem; import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException; import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; import org.apache.hadoop.fs.shell.PathExceptions.PathOperationException; @@ -131,12 +129,10 @@ public static class Get extends CommandWithDestination { private static final String COPYTOLOCAL_PREFIX = "_copyToLocal_"; private boolean copyCrc; private boolean verifyChecksum; - private LocalFileSystem localFs; @Override protected void processOptions(LinkedList args) throws IOException { - localFs = FileSystem.getLocal(getConf()); CommandFormat cf = new CommandFormat( 1, Integer.MAX_VALUE, "crc", "ignoreCrc"); cf.parse(args); @@ -157,16 +153,15 @@ protected void processPath(PathData src, PathData target) copyCrc = false; } - File targetFile = localFs.pathToFile(target.path); if (src.stat.isFile()) { // copy the file and maybe its crc - copyFileToLocal(src, target.path); + copyFileToLocal(src, target); if (copyCrc) { - copyCrcToLocal(src, target.path); + copyFileToLocal(src.getChecksumFile(), target.getChecksumFile()); } } else if (src.stat.isDirectory()) { // create the remote directory structure locally - if (!targetFile.mkdirs()) { + if (!target.toFile().mkdirs()) { throw new PathIOException(target.toString()); } } else { @@ -174,9 +169,9 @@ protected void processPath(PathData src, PathData target) } } - private void copyFileToLocal(PathData src, Path target) + private void copyFileToLocal(PathData src, PathData target) throws IOException { - File targetFile = localFs.pathToFile(target); + File targetFile = target.toFile(); File tmpFile = FileUtil.createLocalTempFile( targetFile, COPYTOLOCAL_PREFIX, true); // too bad we can't tell exactly why it failed... @@ -195,14 +190,6 @@ private void copyFileToLocal(PathData src, Path target) throw e; } } - - private void copyCrcToLocal(PathData src, Path target) - throws IOException { - ChecksumFileSystem srcFs = (ChecksumFileSystem)src.fs; - Path srcPath = srcFs.getChecksumFile(src.path); - src = new PathData(srcFs.getRawFileSystem(), srcPath); - copyFileToLocal(src, localFs.getChecksumFile(target)); - } } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java index 219973f25d..c3b2e7441b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java @@ -81,6 +81,6 @@ protected void processOptions(LinkedList args) { @Override protected void processPath(PathData src) throws IOException { ContentSummary summary = src.fs.getContentSummary(src.path); - out.println(summary.toString(showQuotas) + src.path); + out.println(summary.toString(showQuotas) + src); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java index d6adcf8173..07f3190fe5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java @@ -113,7 +113,7 @@ protected void processPath(PathData item) throws IOException { stat.getGroup(), formatSize(stat.getLen()), dateFormat.format(new Date(stat.getModificationTime())), - item.path.toUri().getPath() + item ); out.println(line); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java index c70e0243ce..e05abf093c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java @@ -21,27 +21,34 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ChecksumFileSystem; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; +import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException; import org.apache.hadoop.fs.shell.PathExceptions.PathIsNotDirectoryException; +import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException; /** * Encapsulates a Path (path), its FileStatus (stat), and its FileSystem (fs). * The stat field will be null if the path does not exist. */ @InterfaceAudience.Private -@InterfaceStability.Evolving +@InterfaceStability.Unstable public class PathData { - protected String string = null; + protected final URI uri; + public final FileSystem fs; public final Path path; public FileStatus stat; - public final FileSystem fs; public boolean exists; /** @@ -53,10 +60,7 @@ public class PathData { * @throws IOException if anything goes wrong... */ public PathData(String pathString, Configuration conf) throws IOException { - this.string = pathString; - this.path = new Path(pathString); - this.fs = path.getFileSystem(conf); - setStat(getStat(fs, path)); + this(FileSystem.get(URI.create(pathString), conf), pathString); } /** @@ -68,87 +72,116 @@ public PathData(String pathString, Configuration conf) throws IOException { * @throws IOException if anything goes wrong... */ public PathData(File localPath, Configuration conf) throws IOException { - this.string = localPath.toString(); - this.path = new Path(this.string); - this.fs = FileSystem.getLocal(conf); - setStat(getStat(fs, path)); + this(FileSystem.getLocal(conf), localPath.toString()); } /** - * Creates an object to wrap the given parameters as fields. - * @param fs the FileSystem - * @param path a Path - * @param stat the FileStatus (may be null if the path doesn't exist) - */ - public PathData(FileSystem fs, Path path, FileStatus stat) { - this.string = path.toString(); - this.path = path; - this.fs = fs; - setStat(stat); - } - - /** - * Convenience ctor that looks up the file status for a path. If the path + * Looks up the file status for a path. If the path * doesn't exist, then the status will be null * @param fs the FileSystem for the path - * @param path the pathname to lookup + * @param pathString a string for a path * @throws IOException if anything goes wrong */ - public PathData(FileSystem fs, Path path) throws IOException { - this(fs, path, getStat(fs, path)); + private PathData(FileSystem fs, String pathString) throws IOException { + this(fs, pathString, lookupStat(fs, pathString, true)); } /** * Creates an object to wrap the given parameters as fields. The string * used to create the path will be recorded since the Path object does not - * return exactly the same string used to initialize it. If the FileStatus - * is not null, then its Path will be used to initialized the path, else - * the string of the path will be used. + * return exactly the same string used to initialize it. * @param fs the FileSystem * @param pathString a String of the path * @param stat the FileStatus (may be null if the path doesn't exist) */ - public PathData(FileSystem fs, String pathString, FileStatus stat) { - this.string = pathString; - this.path = (stat != null) ? stat.getPath() : new Path(pathString); + private PathData(FileSystem fs, String pathString, FileStatus stat) + throws IOException { this.fs = fs; + this.uri = stringToUri(pathString); + this.path = fs.makeQualified(new Path(uri)); setStat(stat); } // need a static method for the ctor above - private static FileStatus getStat(FileSystem fs, Path path) - throws IOException { + /** + * Get the FileStatus info + * @param ignoreFNF if true, stat will be null if the path doesn't exist + * @return FileStatus for the given path + * @throws IOException if anything goes wrong + */ + private static + FileStatus lookupStat(FileSystem fs, String pathString, boolean ignoreFNF) + throws IOException { FileStatus status = null; try { - status = fs.getFileStatus(path); - } catch (FileNotFoundException e) {} // ignore FNF + status = fs.getFileStatus(new Path(pathString)); + } catch (FileNotFoundException e) { + if (!ignoreFNF) throw new PathNotFoundException(pathString); + } + // TODO: should consider wrapping other exceptions into Path*Exceptions return status; } - private void setStat(FileStatus theStat) { - stat = theStat; + private void setStat(FileStatus stat) { + this.stat = stat; exists = (stat != null); } - /** - * Convenience ctor that extracts the path from the given file status - * @param fs the FileSystem for the FileStatus - * @param stat the FileStatus - */ - public PathData(FileSystem fs, FileStatus stat) { - this(fs, stat.getPath(), stat); - } - /** * Updates the paths's file status * @return the updated FileStatus * @throws IOException if anything goes wrong... */ public FileStatus refreshStatus() throws IOException { - setStat(fs.getFileStatus(path)); - return stat; + FileStatus status = null; + try { + status = lookupStat(fs, toString(), false); + } finally { + // always set the status. the caller must get the correct result + // if it catches the exception and later interrogates the status + setStat(status); + } + return status; + } + + protected enum FileTypeRequirement { + SHOULD_NOT_BE_DIRECTORY, SHOULD_BE_DIRECTORY + }; + + /** + * Ensure that the file exists and if it is or is not a directory + * @param typeRequirement Set it to the desired requirement. + * @throws PathIOException if file doesn't exist or the type does not match + * what was specified in typeRequirement. + */ + private void checkIfExists(FileTypeRequirement typeRequirement) + throws PathIOException { + if (!exists) { + throw new PathNotFoundException(toString()); + } + + if ((typeRequirement == FileTypeRequirement.SHOULD_BE_DIRECTORY) + && !stat.isDirectory()) { + throw new PathIsNotDirectoryException(toString()); + } else if ((typeRequirement == FileTypeRequirement.SHOULD_NOT_BE_DIRECTORY) + && stat.isDirectory()) { + throw new PathIsDirectoryException(toString()); + } } + /** + * Return the corresponding crc data for a file. Avoids exposing the fs + * contortions to the caller. + * @return PathData of the crc file + * @throws IOException is anything goes wrong + */ + public PathData getChecksumFile() throws IOException { + checkIfExists(FileTypeRequirement.SHOULD_NOT_BE_DIRECTORY); + ChecksumFileSystem srcFs = (ChecksumFileSystem)fs; + Path srcPath = srcFs.getChecksumFile(path); + return new PathData(srcFs.getRawFileSystem(), srcPath.toString()); + } + /** * Returns a list of PathData objects of the items contained in the given * directory. @@ -156,18 +189,13 @@ public FileStatus refreshStatus() throws IOException { * @throws IOException if anything else goes wrong... */ public PathData[] getDirectoryContents() throws IOException { - if (!stat.isDirectory()) { - throw new PathIsNotDirectoryException(string); - } - + checkIfExists(FileTypeRequirement.SHOULD_BE_DIRECTORY); FileStatus[] stats = fs.listStatus(path); PathData[] items = new PathData[stats.length]; for (int i=0; i < stats.length; i++) { // preserve relative paths - String basename = stats[i].getPath().getName(); - String parent = string; - if (!parent.endsWith(Path.SEPARATOR)) parent += Path.SEPARATOR; - items[i] = new PathData(fs, parent + basename, stats[i]); + String child = getStringForChildPath(stats[i].getPath()); + items[i] = new PathData(fs, child, stats[i]); } return items; } @@ -179,12 +207,30 @@ public PathData[] getDirectoryContents() throws IOException { * @throws IOException if this object does not exist or is not a directory */ public PathData getPathDataForChild(PathData child) throws IOException { - if (!stat.isDirectory()) { - throw new PathIsNotDirectoryException(string); - } - return new PathData(fs, new Path(path, child.path.getName())); + checkIfExists(FileTypeRequirement.SHOULD_BE_DIRECTORY); + return new PathData(fs, getStringForChildPath(child.path)); } + /** + * Given a child of this directory, use the directory's path and the child's + * basename to construct the string to the child. This preserves relative + * paths since Path will fully qualify. + * @param child a path contained within this directory + * @return String of the path relative to this directory + */ + private String getStringForChildPath(Path childPath) { + String basename = childPath.getName(); + if (Path.CUR_DIR.equals(toString())) { + return basename; + } + // check getPath() so scheme slashes aren't considered part of the path + String separator = uri.getPath().endsWith(Path.SEPARATOR) + ? "" : Path.SEPARATOR; + return uri + separator + basename; + } + + protected enum PathType { HAS_SCHEME, SCHEMELESS_ABSOLUTE, RELATIVE }; + /** * Expand the given path as a glob pattern. Non-existent paths do not * throw an exception because creation commands like touch and mkdir need @@ -207,35 +253,184 @@ public static PathData[] expandAsGlob(String pattern, Configuration conf) if (stats == null) { // not a glob & file not found, so add the path with a null stat items = new PathData[]{ new PathData(fs, pattern, null) }; - } else if ( - // this is very ugly, but needed to avoid breaking hdfs tests... - // if a path has no authority, then the FileStatus from globStatus - // will add the "-fs" authority into the path, so we need to sub - // it back out to satisfy the tests - stats.length == 1 - && - stats[0].getPath().equals(fs.makeQualified(globPath))) - { - // if the fq path is identical to the pattern passed, use the pattern - // to initialize the string value - items = new PathData[]{ new PathData(fs, pattern, stats[0]) }; } else { + // figure out what type of glob path was given, will convert globbed + // paths to match the type to preserve relativity + PathType globType; + URI globUri = globPath.toUri(); + if (globUri.getScheme() != null) { + globType = PathType.HAS_SCHEME; + } else if (new File(globUri.getPath()).isAbsolute()) { + globType = PathType.SCHEMELESS_ABSOLUTE; + } else { + globType = PathType.RELATIVE; + } + // convert stats to PathData items = new PathData[stats.length]; int i=0; for (FileStatus stat : stats) { - items[i++] = new PathData(fs, stat); + URI matchUri = stat.getPath().toUri(); + String globMatch = null; + switch (globType) { + case HAS_SCHEME: // use as-is, but remove authority if necessary + if (globUri.getAuthority() == null) { + matchUri = removeAuthority(matchUri); + } + globMatch = matchUri.toString(); + break; + case SCHEMELESS_ABSOLUTE: // take just the uri's path + globMatch = matchUri.getPath(); + break; + case RELATIVE: // make it relative to the current working dir + URI cwdUri = fs.getWorkingDirectory().toUri(); + globMatch = relativize(cwdUri, matchUri, stat.isDirectory()); + break; + } + items[i++] = new PathData(fs, globMatch, stat); } } return items; } + private static URI removeAuthority(URI uri) { + try { + uri = new URI( + uri.getScheme(), "", + uri.getPath(), uri.getQuery(), uri.getFragment() + ); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e.getLocalizedMessage()); + } + return uri; + } + + private static String relativize(URI cwdUri, URI srcUri, boolean isDir) { + String uriPath = srcUri.getPath(); + String cwdPath = cwdUri.getPath(); + if (cwdPath.equals(uriPath)) { + return Path.CUR_DIR; + } + + // find common ancestor + int lastSep = findLongestDirPrefix(cwdPath, uriPath, isDir); + + StringBuilder relPath = new StringBuilder(); + // take the remaining path fragment after the ancestor + if (lastSep < uriPath.length()) { + relPath.append(uriPath.substring(lastSep+1)); + } + + // if cwd has a path fragment after the ancestor, convert them to ".." + if (lastSep < cwdPath.length()) { + while (lastSep != -1) { + if (relPath.length() != 0) relPath.insert(0, Path.SEPARATOR); + relPath.insert(0, ".."); + lastSep = cwdPath.indexOf(Path.SEPARATOR, lastSep+1); + } + } + return relPath.toString(); + } + + private static int findLongestDirPrefix(String cwd, String path, boolean isDir) { + // add the path separator to dirs to simplify finding the longest match + if (!cwd.endsWith(Path.SEPARATOR)) { + cwd += Path.SEPARATOR; + } + if (isDir && !path.endsWith(Path.SEPARATOR)) { + path += Path.SEPARATOR; + } + + // find longest directory prefix + int len = Math.min(cwd.length(), path.length()); + int lastSep = -1; + for (int i=0; i < len; i++) { + if (cwd.charAt(i) != path.charAt(i)) break; + if (cwd.charAt(i) == Path.SEPARATOR_CHAR) lastSep = i; + } + return lastSep; + } + /** * Returns the printable version of the path that is either the path * as given on the commandline, or the full path * @return String of the path */ public String toString() { - return (string != null) ? string : path.toString(); + String scheme = uri.getScheme(); + // No interpretation of symbols. Just decode % escaped chars. + String decodedRemainder = uri.getSchemeSpecificPart(); + + if (scheme == null) { + return decodedRemainder; + } else { + StringBuilder buffer = new StringBuilder(); + buffer.append(scheme); + buffer.append(":"); + buffer.append(decodedRemainder); + return buffer.toString(); + } } + + /** + * Get the path to a local file + * @return File representing the local path + * @throws IllegalArgumentException if this.fs is not the LocalFileSystem + */ + public File toFile() { + if (!(fs instanceof LocalFileSystem)) { + throw new IllegalArgumentException("Not a local path: " + path); + } + return ((LocalFileSystem)fs).pathToFile(path); + } + + /** Construct a URI from a String with unescaped special characters + * that have non-standard sematics. e.g. /, ?, #. A custom parsing + * is needed to prevent misbihaviors. + * @param pathString The input path in string form + * @return URI + */ + private static URI stringToUri(String pathString) { + // We can't use 'new URI(String)' directly. Since it doesn't do quoting + // internally, the internal parser may fail or break the string at wrong + // places. Use of multi-argument ctors will quote those chars for us, + // but we need to do our own parsing and assembly. + + // parse uri components + String scheme = null; + String authority = null; + + int start = 0; + + // parse uri scheme, if any + int colon = pathString.indexOf(':'); + int slash = pathString.indexOf('/'); + if (colon > 0 && (slash == colon +1)) { + // has a non zero-length scheme + scheme = pathString.substring(0, colon); + start = colon + 1; + } + + // parse uri authority, if any + if (pathString.startsWith("//", start) && + (pathString.length()-start > 2)) { + start += 2; + int nextSlash = pathString.indexOf('/', start); + int authEnd = nextSlash > 0 ? nextSlash : pathString.length(); + authority = pathString.substring(start, authEnd); + start = authEnd; + } + + // uri path is the rest of the string. ? or # are not interpreated, + // but any occurrence of them will be quoted by the URI ctor. + String path = pathString.substring(start, pathString.length()); + + // Construct the URI + try { + return new URI(scheme, authority, path, null, null); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e); + } + } + } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java index cdeba83016..3fba5d98a5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/cli/CLITestHelper.java @@ -180,7 +180,7 @@ private void displayResults() { LOG.info(" Comparision result: [" + (resultBoolean ? "pass" : "fail") + "]"); LOG.info(" Expected output: [" + - cd.getExpectedOutput() + "]"); + expandCommand(cd.getExpectedOutput()) + "]"); LOG.info(" Actual output: [" + cd.getActualOutput() + "]"); } @@ -290,7 +290,7 @@ private boolean compareTestOutput(ComparatorData compdata, Result cmdResult) { comparatorType); ComparatorBase comp = (ComparatorBase) comparatorClass.newInstance(); compareOutput = comp.compare(cmdResult.getCommandOutput(), - compdata.getExpectedOutput()); + expandCommand(compdata.getExpectedOutput())); } catch (Exception e) { LOG.info("Error in instantiating the comparator" + e); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java index 170d16cc1a..be72f39694 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java @@ -17,80 +17,180 @@ */ package org.apache.hadoop.fs.shell; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.util.Arrays; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.*; public class TestPathData { protected static Configuration conf; protected static FileSystem fs; protected static String dirString; - protected static Path dir; + protected static Path testDir; protected static PathData item; - + + protected static String[] d1Paths = + new String[] { "d1/f1", "d1/f1.1", "d1/f2" }; + protected static String[] d2Paths = + new String[] { "d2/f3" }; + @BeforeClass public static void initialize() throws Exception { conf = new Configuration(); - fs = FileSystem.getLocal(conf); + fs = FileSystem.getLocal(conf); + testDir = new Path( + System.getProperty("test.build.data", "build/test/data") + "/testPD" + ); + // don't want scheme on the path, just an absolute path + testDir = new Path(fs.makeQualified(testDir).toUri().getPath()); + FileSystem.setDefaultUri(conf, fs.getUri()); + fs.setWorkingDirectory(testDir); + fs.mkdirs(new Path("d1")); + fs.createNewFile(new Path("d1", "f1")); + fs.createNewFile(new Path("d1", "f1.1")); + fs.createNewFile(new Path("d1", "f2")); + fs.mkdirs(new Path("d2")); + fs.create(new Path("d2","f3")); } @Test - public void testWithFsAndPath() throws Exception { - dirString = "/tmp"; - dir = new Path(dirString); - item = new PathData(fs, dir); + public void testWithDirStringAndConf() throws Exception { + dirString = "d1"; + item = new PathData(dirString, conf); checkPathData(); - } - @Test - public void testWithStringAndConf() throws Exception { - dirString = "/tmp"; - dir = new Path(dirString); + // properly implementing symlink support in various commands will require + // trailing slashes to be retained + dirString = "d1/"; item = new PathData(dirString, conf); checkPathData(); } @Test public void testUnqualifiedUriContents() throws Exception { - dirString = "/tmp"; + dirString = "d1"; item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); - for (PathData item : items) { - assertTrue(item.toString().startsWith(dirString)); - } + assertEquals( + sortedString("d1/f1", "d1/f1.1", "d1/f2"), + sortedString(items) + ); } @Test public void testQualifiedUriContents() throws Exception { - dirString = "file:/tmp"; + dirString = fs.makeQualified(new Path("d1")).toString(); item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); - for (PathData item : items) { - assertTrue(item.toString().startsWith(dirString)); - } + assertEquals( + sortedString(dirString+"/f1", dirString+"/f1.1", dirString+"/f2"), + sortedString(items) + ); + } + + @Test + public void testCwdContents() throws Exception { + dirString = Path.CUR_DIR; + item = new PathData(dirString, conf); + PathData[] items = item.getDirectoryContents(); + assertEquals( + sortedString("d1", "d2"), + sortedString(items) + ); + } + + + @Test + public void testToFile() throws Exception { + item = new PathData(".", conf); + assertEquals(new File(testDir.toString()), item.toFile()); + item = new PathData("d1/f1", conf); + assertEquals(new File(testDir+"/d1/f1"), item.toFile()); + item = new PathData(testDir+"/d1/f1", conf); + assertEquals(new File(testDir+"/d1/f1"), item.toFile()); + } + + @Test + public void testAbsoluteGlob() throws Exception { + PathData[] items = PathData.expandAsGlob(testDir+"/d1/f1*", conf); + assertEquals( + sortedString(testDir+"/d1/f1", testDir+"/d1/f1.1"), + sortedString(items) + ); + } + + @Test + public void testRelativeGlob() throws Exception { + PathData[] items = PathData.expandAsGlob("d1/f1*", conf); + assertEquals( + sortedString("d1/f1", "d1/f1.1"), + sortedString(items) + ); + } + + @Test + public void testRelativeGlobBack() throws Exception { + fs.setWorkingDirectory(new Path("d1")); + PathData[] items = PathData.expandAsGlob("../d2/*", conf); + assertEquals( + sortedString("../d2/f3"), + sortedString(items) + ); } @Test public void testWithStringAndConfForBuggyPath() throws Exception { dirString = "file:///tmp"; - dir = new Path(dirString); + testDir = new Path(dirString); item = new PathData(dirString, conf); // this may fail some day if Path is fixed to not crunch the uri // if the authority is null, however we need to test that the PathData // toString() returns the given string, while Path toString() does // the crunching - assertEquals("file:/tmp", dir.toString()); + assertEquals("file:/tmp", testDir.toString()); checkPathData(); } public void checkPathData() throws Exception { - assertEquals(fs, item.fs); - assertEquals(dirString, item.toString()); - assertEquals(dir, item.path); - assertTrue(item.stat != null); - assertTrue(item.stat.isDirectory()); + assertEquals("checking fs", fs, item.fs); + assertEquals("checking string", dirString, item.toString()); + assertEquals("checking path", + fs.makeQualified(new Path(item.toString())), item.path + ); + assertTrue("checking exist", item.stat != null); + assertTrue("checking isDir", item.stat.isDirectory()); + } + + /* junit does a lousy job of comparing arrays + * if the array lengths differ, it just says that w/o showing contents + * this sorts the paths, and builds a string of "i:, ..." suitable + * for a string compare + */ + private static String sortedString(Object ... list) { + String[] strings = new String[list.length]; + for (int i=0; i < list.length; i++) { + strings[i] = String.valueOf(list[i]); + } + Arrays.sort(strings); + + StringBuilder result = new StringBuilder(); + for (int i=0; i < strings.length; i++) { + if (result.length() > 0) { + result.append(", "); + } + result.append(i+":<"+strings[i]+">"); + } + return result.toString(); + } + + private static String sortedString(PathData ... items) { + return sortedString((Object[])items); } }