HADOOP-8906. paths with multiple globs are unreliable. Contributed by Daryn Sharp.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1397704 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jason Darrell Lowe
parent
611ff4ec89
commit
f8bdac0a1b
@ -1054,6 +1054,9 @@ Release 0.23.5 - UNRELEASED
|
||||
|
||||
BUG FIXES
|
||||
|
||||
HADOOP-8906. paths with multiple globs are unreliable. (Daryn Sharp via
|
||||
jlowe)
|
||||
|
||||
Release 0.23.4 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
||||
@ -24,6 +24,7 @@
|
||||
import java.security.PrivilegedExceptionAction;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.EnumSet;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
@ -1574,120 +1575,113 @@ public FileStatus[] globStatus(Path pathPattern) throws IOException {
|
||||
public FileStatus[] globStatus(Path pathPattern, PathFilter filter)
|
||||
throws IOException {
|
||||
String filename = pathPattern.toUri().getPath();
|
||||
List<String> filePatterns = GlobExpander.expand(filename);
|
||||
if (filePatterns.size() == 1) {
|
||||
return globStatusInternal(pathPattern, filter);
|
||||
} else {
|
||||
List<FileStatus> results = new ArrayList<FileStatus>();
|
||||
for (String filePattern : filePatterns) {
|
||||
FileStatus[] files = globStatusInternal(new Path(filePattern), filter);
|
||||
for (FileStatus file : files) {
|
||||
results.add(file);
|
||||
}
|
||||
}
|
||||
return results.toArray(new FileStatus[results.size()]);
|
||||
}
|
||||
}
|
||||
|
||||
private FileStatus[] globStatusInternal(Path pathPattern, PathFilter filter)
|
||||
throws IOException {
|
||||
Path[] parents = new Path[1];
|
||||
int level = 0;
|
||||
String filename = pathPattern.toUri().getPath();
|
||||
List<FileStatus> allMatches = null;
|
||||
|
||||
// path has only zero component
|
||||
if (filename.isEmpty() || Path.SEPARATOR.equals(filename)) {
|
||||
return getFileStatus(new Path[]{pathPattern});
|
||||
}
|
||||
|
||||
// path has at least one component
|
||||
String[] components = filename.split(Path.SEPARATOR);
|
||||
// get the first component
|
||||
if (pathPattern.isAbsolute()) {
|
||||
parents[0] = new Path(Path.SEPARATOR);
|
||||
level = 1;
|
||||
} else {
|
||||
parents[0] = new Path(Path.CUR_DIR);
|
||||
}
|
||||
|
||||
// glob the paths that match the parent path, i.e., [0, components.length-1]
|
||||
boolean[] hasGlob = new boolean[]{false};
|
||||
Path[] parentPaths = globPathsLevel(parents, components, level, hasGlob);
|
||||
FileStatus[] results;
|
||||
if (parentPaths == null || parentPaths.length == 0) {
|
||||
results = null;
|
||||
} else {
|
||||
// Now work on the last component of the path
|
||||
GlobFilter fp = new GlobFilter(components[components.length - 1], filter);
|
||||
if (fp.hasPattern()) { // last component has a pattern
|
||||
// list parent directories and then glob the results
|
||||
try {
|
||||
results = listStatus(parentPaths, fp);
|
||||
} catch (FileNotFoundException e) {
|
||||
results = null;
|
||||
List<String> filePatterns = GlobExpander.expand(filename);
|
||||
for (String filePattern : filePatterns) {
|
||||
Path path = new Path(filePattern.isEmpty() ? Path.CUR_DIR : filePattern);
|
||||
List<FileStatus> matches = globStatusInternal(path, filter);
|
||||
if (matches != null) {
|
||||
if (allMatches == null) {
|
||||
allMatches = matches;
|
||||
} else {
|
||||
allMatches.addAll(matches);
|
||||
}
|
||||
hasGlob[0] = true;
|
||||
} else { // last component does not have a pattern
|
||||
// remove the quoting of metachars in a non-regexp expansion
|
||||
String name = unquotePathComponent(components[components.length - 1]);
|
||||
// get all the path names
|
||||
ArrayList<Path> filteredPaths = new ArrayList<Path>(parentPaths.length);
|
||||
for (int i = 0; i < parentPaths.length; i++) {
|
||||
parentPaths[i] = new Path(parentPaths[i], name);
|
||||
if (fp.accept(parentPaths[i])) {
|
||||
filteredPaths.add(parentPaths[i]);
|
||||
}
|
||||
}
|
||||
// get all their statuses
|
||||
results = getFileStatus(
|
||||
filteredPaths.toArray(new Path[filteredPaths.size()]));
|
||||
}
|
||||
}
|
||||
|
||||
// Decide if the pathPattern contains a glob or not
|
||||
if (results == null) {
|
||||
if (hasGlob[0]) {
|
||||
results = new FileStatus[0];
|
||||
}
|
||||
} else {
|
||||
if (results.length == 0 ) {
|
||||
if (!hasGlob[0]) {
|
||||
results = null;
|
||||
}
|
||||
} else {
|
||||
Arrays.sort(results);
|
||||
}
|
||||
|
||||
FileStatus[] results = null;
|
||||
if (allMatches != null) {
|
||||
results = allMatches.toArray(new FileStatus[allMatches.size()]);
|
||||
} else if (filePatterns.size() > 1) {
|
||||
// no matches with multiple expansions is a non-matching glob
|
||||
results = new FileStatus[0];
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/*
|
||||
* For a path of N components, return a list of paths that match the
|
||||
* components [<code>level</code>, <code>N-1</code>].
|
||||
*/
|
||||
private Path[] globPathsLevel(Path[] parents, String[] filePattern,
|
||||
int level, boolean[] hasGlob) throws IOException {
|
||||
if (level == filePattern.length - 1)
|
||||
return parents;
|
||||
if (parents == null || parents.length == 0) {
|
||||
return null;
|
||||
// sort gripes because FileStatus Comparable isn't parameterized...
|
||||
@SuppressWarnings("unchecked")
|
||||
private List<FileStatus> globStatusInternal(Path pathPattern,
|
||||
PathFilter filter) throws IOException {
|
||||
boolean patternHasGlob = false; // pathPattern has any globs
|
||||
List<FileStatus> matches = new ArrayList<FileStatus>();
|
||||
|
||||
// determine starting point
|
||||
int level = 0;
|
||||
String baseDir = Path.CUR_DIR;
|
||||
if (pathPattern.isAbsolute()) {
|
||||
level = 1; // need to skip empty item at beginning of split list
|
||||
baseDir = Path.SEPARATOR;
|
||||
}
|
||||
GlobFilter fp = new GlobFilter(filePattern[level]);
|
||||
if (fp.hasPattern()) {
|
||||
try {
|
||||
parents = FileUtil.stat2Paths(listStatus(parents, fp));
|
||||
} catch (FileNotFoundException e) {
|
||||
parents = null;
|
||||
|
||||
// parse components and determine if it's a glob
|
||||
String[] components = null;
|
||||
GlobFilter[] filters = null;
|
||||
String filename = pathPattern.toUri().getPath();
|
||||
if (!filename.isEmpty() && !Path.SEPARATOR.equals(filename)) {
|
||||
components = filename.split(Path.SEPARATOR);
|
||||
filters = new GlobFilter[components.length];
|
||||
for (int i=level; i < components.length; i++) {
|
||||
filters[i] = new GlobFilter(components[i]);
|
||||
patternHasGlob |= filters[i].hasPattern();
|
||||
}
|
||||
hasGlob[0] = true;
|
||||
} else { // the component does not have a pattern
|
||||
// remove the quoting of metachars in a non-regexp expansion
|
||||
String name = unquotePathComponent(filePattern[level]);
|
||||
for (int i = 0; i < parents.length; i++) {
|
||||
parents[i] = new Path(parents[i], name);
|
||||
if (!patternHasGlob) {
|
||||
baseDir = unquotePathComponent(filename);
|
||||
components = null; // short through to filter check
|
||||
}
|
||||
}
|
||||
return globPathsLevel(parents, filePattern, level + 1, hasGlob);
|
||||
|
||||
// seed the parent directory path, return if it doesn't exist
|
||||
try {
|
||||
matches.add(getFileStatus(new Path(baseDir)));
|
||||
} catch (FileNotFoundException e) {
|
||||
return patternHasGlob ? matches : null;
|
||||
}
|
||||
|
||||
// skip if there are no components other than the basedir
|
||||
if (components != null) {
|
||||
// iterate through each path component
|
||||
for (int i=level; (i < components.length) && !matches.isEmpty(); i++) {
|
||||
List<FileStatus> children = new ArrayList<FileStatus>();
|
||||
for (FileStatus match : matches) {
|
||||
// don't look for children in a file matched by a glob
|
||||
if (!match.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
if (filters[i].hasPattern()) {
|
||||
// get all children matching the filter
|
||||
FileStatus[] statuses = listStatus(match.getPath(), filters[i]);
|
||||
children.addAll(Arrays.asList(statuses));
|
||||
} else {
|
||||
// the component does not have a pattern
|
||||
String component = unquotePathComponent(components[i]);
|
||||
Path child = new Path(match.getPath(), component);
|
||||
children.add(getFileStatus(child));
|
||||
}
|
||||
} catch (FileNotFoundException e) {
|
||||
// don't care
|
||||
}
|
||||
}
|
||||
matches = children;
|
||||
}
|
||||
}
|
||||
// remove anything that didn't match the filter
|
||||
if (!matches.isEmpty()) {
|
||||
Iterator<FileStatus> iter = matches.iterator();
|
||||
while (iter.hasNext()) {
|
||||
if (!filter.accept(iter.next().getPath())) {
|
||||
iter.remove();
|
||||
}
|
||||
}
|
||||
}
|
||||
// no final paths, if there were any globs return empty list
|
||||
if (matches.isEmpty()) {
|
||||
return patternHasGlob ? matches : null;
|
||||
}
|
||||
Collections.sort(matches);
|
||||
return matches;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2163,30 +2157,6 @@ public void setWriteChecksum(boolean writeChecksum) {
|
||||
//doesn't do anything
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a list of file status objects that corresponds to the list of paths
|
||||
* excluding those non-existent paths.
|
||||
*
|
||||
* @param paths
|
||||
* the list of paths we want information from
|
||||
* @return a list of FileStatus objects
|
||||
* @throws IOException
|
||||
* see specific implementation
|
||||
*/
|
||||
private FileStatus[] getFileStatus(Path[] paths) throws IOException {
|
||||
if (paths == null) {
|
||||
return null;
|
||||
}
|
||||
ArrayList<FileStatus> results = new ArrayList<FileStatus>(paths.length);
|
||||
for (int i = 0; i < paths.length; i++) {
|
||||
try {
|
||||
results.add(getFileStatus(paths[i]));
|
||||
} catch (FileNotFoundException e) { // do nothing
|
||||
}
|
||||
}
|
||||
return results.toArray(new FileStatus[results.size()]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a status object describing the use and capacity of the
|
||||
* file system. If the file system has multiple partitions, the
|
||||
|
||||
@ -17,18 +17,16 @@
|
||||
*/
|
||||
package org.apache.hadoop.fs;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.*;
|
||||
|
||||
public class TestGlobPaths {
|
||||
|
||||
@ -49,26 +47,377 @@ public boolean accept(Path path) {
|
||||
static private MiniDFSCluster dfsCluster;
|
||||
static private FileSystem fs;
|
||||
static final private int NUM_OF_PATHS = 4;
|
||||
static final String USER_DIR = "/user/"+System.getProperty("user.name");
|
||||
static private String USER_DIR;
|
||||
private Path[] path = new Path[NUM_OF_PATHS];
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
try {
|
||||
Configuration conf = new HdfsConfiguration();
|
||||
dfsCluster = new MiniDFSCluster.Builder(conf).build();
|
||||
fs = FileSystem.get(conf);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
Configuration conf = new HdfsConfiguration();
|
||||
dfsCluster = new MiniDFSCluster.Builder(conf).build();
|
||||
fs = FileSystem.get(conf);
|
||||
USER_DIR = fs.getHomeDirectory().toUri().getPath().toString();
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
@AfterClass
|
||||
public static void tearDown() throws Exception {
|
||||
if(dfsCluster!=null) {
|
||||
dfsCluster.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMultiGlob() throws IOException {
|
||||
FileStatus[] status;
|
||||
/*
|
||||
* /dir1/subdir1
|
||||
* /dir1/subdir1/f1
|
||||
* /dir1/subdir1/f2
|
||||
* /dir1/subdir2/f1
|
||||
* /dir2/subdir1
|
||||
* /dir2/subdir2
|
||||
* /dir2/subdir2/f1
|
||||
* /dir3/f1
|
||||
* /dir3/f1
|
||||
* /dir3/f2(dir)
|
||||
* /dir3/subdir2(file)
|
||||
* /dir3/subdir3
|
||||
* /dir3/subdir3/f1
|
||||
* /dir3/subdir3/f1/f1
|
||||
* /dir3/subdir3/f3
|
||||
* /dir4
|
||||
*/
|
||||
|
||||
Path d1 = new Path(USER_DIR, "dir1");
|
||||
Path d11 = new Path(d1, "subdir1");
|
||||
Path d12 = new Path(d1, "subdir2");
|
||||
|
||||
Path f111 = new Path(d11, "f1");
|
||||
fs.createNewFile(f111);
|
||||
Path f112 = new Path(d11, "f2");
|
||||
fs.createNewFile(f112);
|
||||
Path f121 = new Path(d12, "f1");
|
||||
fs.createNewFile(f121);
|
||||
|
||||
Path d2 = new Path(USER_DIR, "dir2");
|
||||
Path d21 = new Path(d2, "subdir1");
|
||||
fs.mkdirs(d21);
|
||||
Path d22 = new Path(d2, "subdir2");
|
||||
Path f221 = new Path(d22, "f1");
|
||||
fs.createNewFile(f221);
|
||||
|
||||
Path d3 = new Path(USER_DIR, "dir3");
|
||||
Path f31 = new Path(d3, "f1");
|
||||
fs.createNewFile(f31);
|
||||
Path d32 = new Path(d3, "f2");
|
||||
fs.mkdirs(d32);
|
||||
Path f32 = new Path(d3, "subdir2"); // fake as a subdir!
|
||||
fs.createNewFile(f32);
|
||||
Path d33 = new Path(d3, "subdir3");
|
||||
Path f333 = new Path(d33, "f3");
|
||||
fs.createNewFile(f333);
|
||||
Path d331 = new Path(d33, "f1");
|
||||
Path f3311 = new Path(d331, "f1");
|
||||
fs.createNewFile(f3311);
|
||||
Path d4 = new Path(USER_DIR, "dir4");
|
||||
fs.mkdirs(d4);
|
||||
|
||||
/*
|
||||
* basic
|
||||
*/
|
||||
Path root = new Path(USER_DIR);
|
||||
status = fs.globStatus(root);
|
||||
checkStatus(status, root);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "x"));
|
||||
assertNull(status);
|
||||
|
||||
status = fs.globStatus(new Path("x"));
|
||||
assertNull(status);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "x/x"));
|
||||
assertNull(status);
|
||||
|
||||
status = fs.globStatus(new Path("x/x"));
|
||||
assertNull(status);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "*"));
|
||||
checkStatus(status, d1, d2, d3, d4);
|
||||
|
||||
status = fs.globStatus(new Path("*"));
|
||||
checkStatus(status, d1, d2, d3, d4);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "*/x"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path("*/x"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "x/*"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path("x/*"));
|
||||
checkStatus(status);
|
||||
|
||||
// make sure full pattern is scanned instead of bailing early with undef
|
||||
status = fs.globStatus(new Path(USER_DIR, "x/x/x/*"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path("x/x/x/*"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "*/*"));
|
||||
checkStatus(status, d11, d12, d21, d22, f31, d32, f32, d33);
|
||||
|
||||
status = fs.globStatus(new Path("*/*"));
|
||||
checkStatus(status, d11, d12, d21, d22, f31, d32, f32, d33);
|
||||
|
||||
/*
|
||||
* one level deep
|
||||
*/
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/*"));
|
||||
checkStatus(status, d11, d12, d21, d22, f31, d32, f32, d33);
|
||||
|
||||
status = fs.globStatus(new Path("dir*/*"));
|
||||
checkStatus(status, d11, d12, d21, d22, f31, d32, f32, d33);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*"));
|
||||
checkStatus(status, d11, d12, d21, d22, f32, d33);
|
||||
|
||||
status = fs.globStatus(new Path("dir*/subdir*"));
|
||||
checkStatus(status, d11, d12, d21, d22, f32, d33);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/f*"));
|
||||
checkStatus(status, f31, d32);
|
||||
|
||||
status = fs.globStatus(new Path("dir*/f*"));
|
||||
checkStatus(status, f31, d32);
|
||||
|
||||
/*
|
||||
* subdir1 globs
|
||||
*/
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1"));
|
||||
checkStatus(status, d11, d21);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/*"));
|
||||
checkStatus(status, f111, f112);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/*/*"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/x"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/x*"));
|
||||
checkStatus(status);
|
||||
|
||||
/*
|
||||
* subdir2 globs
|
||||
*/
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir2"));
|
||||
checkStatus(status, d12, d22, f32);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir2/*"));
|
||||
checkStatus(status, f121, f221);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir2/*/*"));
|
||||
checkStatus(status);
|
||||
|
||||
/*
|
||||
* subdir3 globs
|
||||
*/
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir3"));
|
||||
checkStatus(status, d33);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir3/*"));
|
||||
checkStatus(status, d331, f333);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir3/*/*"));
|
||||
checkStatus(status, f3311);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir3/*/*/*"));
|
||||
checkStatus(status);
|
||||
|
||||
/*
|
||||
* file1 single dir globs
|
||||
*/
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/f1"));
|
||||
checkStatus(status, f111);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/f1*"));
|
||||
checkStatus(status, f111);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/f1/*"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/f1*/*"));
|
||||
checkStatus(status);
|
||||
|
||||
/*
|
||||
* file1 multi-dir globs
|
||||
*/
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1"));
|
||||
checkStatus(status, f111, f121, f221, d331);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1*"));
|
||||
checkStatus(status, f111, f121, f221, d331);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1/*"));
|
||||
checkStatus(status, f3311);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1*/*"));
|
||||
checkStatus(status, f3311);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1*/*"));
|
||||
checkStatus(status, f3311);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1*/x"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f1*/*/*"));
|
||||
checkStatus(status);
|
||||
|
||||
/*
|
||||
* file glob multiple files
|
||||
*/
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*"));
|
||||
checkStatus(status, d11, d12, d21, d22, f32, d33);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/*"));
|
||||
checkStatus(status, f111, f112, f121, f221, d331, f333);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f*"));
|
||||
checkStatus(status, f111, f112, f121, f221, d331, f333);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/f*/*"));
|
||||
checkStatus(status, f3311);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/*/f1"));
|
||||
checkStatus(status, f3311);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir*/*/*"));
|
||||
checkStatus(status, f3311);
|
||||
|
||||
|
||||
// doesn't exist
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/f3"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "dir*/subdir1/f3*"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path("{x}"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path("{x,y}"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path("dir*/{x,y}"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path("dir*/{f1,y}"));
|
||||
checkStatus(status, f31);
|
||||
|
||||
status = fs.globStatus(new Path("{x,y}"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path("/{x/x,y/y}"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path("{x/x,y/y}"));
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path(Path.CUR_DIR));
|
||||
checkStatus(status, new Path(USER_DIR));
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR+"{/dir1}"));
|
||||
checkStatus(status, d1);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR+"{/dir*}"));
|
||||
checkStatus(status, d1, d2, d3, d4);
|
||||
|
||||
/*
|
||||
* true filter
|
||||
*/
|
||||
|
||||
PathFilter trueFilter = new PathFilter() {
|
||||
@Override
|
||||
public boolean accept(Path path) {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
status = fs.globStatus(new Path(Path.SEPARATOR), trueFilter);
|
||||
checkStatus(status, new Path(Path.SEPARATOR));
|
||||
|
||||
status = fs.globStatus(new Path(Path.CUR_DIR), trueFilter);
|
||||
checkStatus(status, new Path(USER_DIR));
|
||||
|
||||
status = fs.globStatus(d1, trueFilter);
|
||||
checkStatus(status, d1);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR), trueFilter);
|
||||
checkStatus(status, new Path(USER_DIR));
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "*"), trueFilter);
|
||||
checkStatus(status, d1, d2, d3, d4);
|
||||
|
||||
status = fs.globStatus(new Path("/x/*"), trueFilter);
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path("/x"), trueFilter);
|
||||
assertNull(status);
|
||||
|
||||
status = fs.globStatus(new Path("/x/x"), trueFilter);
|
||||
assertNull(status);
|
||||
|
||||
/*
|
||||
* false filter
|
||||
*/
|
||||
PathFilter falseFilter = new PathFilter() {
|
||||
@Override
|
||||
public boolean accept(Path path) {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
status = fs.globStatus(new Path(Path.SEPARATOR), falseFilter);
|
||||
assertNull(status);
|
||||
|
||||
status = fs.globStatus(new Path(Path.CUR_DIR), falseFilter);
|
||||
assertNull(status);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR), falseFilter);
|
||||
assertNull(status);
|
||||
|
||||
status = fs.globStatus(new Path(USER_DIR, "*"), falseFilter);
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path("/x/*"), falseFilter);
|
||||
checkStatus(status);
|
||||
|
||||
status = fs.globStatus(new Path("/x"), falseFilter);
|
||||
assertNull(status);
|
||||
|
||||
status = fs.globStatus(new Path("/x/x"), falseFilter);
|
||||
assertNull(status);
|
||||
}
|
||||
|
||||
private void checkStatus(FileStatus[] status, Path ... expectedMatches) {
|
||||
assertNotNull(status);
|
||||
String[] paths = new String[status.length];
|
||||
for (int i=0; i < status.length; i++) {
|
||||
paths[i] = getPathFromStatus(status[i]);
|
||||
}
|
||||
String got = StringUtils.join(paths, "\n");
|
||||
String expected = StringUtils.join(expectedMatches, "\n");
|
||||
assertEquals(expected, got);
|
||||
}
|
||||
|
||||
private String getPathFromStatus(FileStatus status) {
|
||||
return status.getPath().toUri().getPath();
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testPathFilter() throws IOException {
|
||||
@ -98,21 +447,7 @@ public void testPathFilterWithFixedLastComponent() throws IOException {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGlob() throws Exception {
|
||||
//pTestEscape(); // need to wait until HADOOP-1995 is fixed
|
||||
pTestJavaRegexSpecialChars();
|
||||
pTestCurlyBracket();
|
||||
pTestLiteral();
|
||||
pTestAny();
|
||||
pTestClosure();
|
||||
pTestSet();
|
||||
pTestRange();
|
||||
pTestSetExcl();
|
||||
pTestCombination();
|
||||
pTestRelativePath();
|
||||
}
|
||||
|
||||
private void pTestLiteral() throws IOException {
|
||||
public void pTestLiteral() throws IOException {
|
||||
try {
|
||||
String [] files = new String[] {USER_DIR+"/a2c", USER_DIR+"/abc.d"};
|
||||
Path[] matchedPath = prepareTesting(USER_DIR+"/abc.d", files);
|
||||
@ -123,7 +458,8 @@ private void pTestLiteral() throws IOException {
|
||||
}
|
||||
}
|
||||
|
||||
private void pTestEscape() throws IOException {
|
||||
@Test
|
||||
public void pTestEscape() throws IOException {
|
||||
try {
|
||||
String [] files = new String[] {USER_DIR+"/ab\\[c.d"};
|
||||
Path[] matchedPath = prepareTesting(USER_DIR+"/ab\\[c.d", files);
|
||||
@ -134,7 +470,8 @@ private void pTestEscape() throws IOException {
|
||||
}
|
||||
}
|
||||
|
||||
private void pTestAny() throws IOException {
|
||||
@Test
|
||||
public void pTestAny() throws IOException {
|
||||
try {
|
||||
String [] files = new String[] { USER_DIR+"/abc", USER_DIR+"/a2c",
|
||||
USER_DIR+"/a.c", USER_DIR+"/abcd"};
|
||||
@ -148,15 +485,8 @@ private void pTestAny() throws IOException {
|
||||
}
|
||||
}
|
||||
|
||||
private void pTestClosure() throws IOException {
|
||||
pTestClosure1();
|
||||
pTestClosure2();
|
||||
pTestClosure3();
|
||||
pTestClosure4();
|
||||
pTestClosure5();
|
||||
}
|
||||
|
||||
private void pTestClosure1() throws IOException {
|
||||
@Test
|
||||
public void pTestClosure1() throws IOException {
|
||||
try {
|
||||
String [] files = new String[] {USER_DIR+"/a", USER_DIR+"/abc",
|
||||
USER_DIR+"/abc.p", USER_DIR+"/bacd"};
|
||||
@ -170,7 +500,8 @@ private void pTestClosure1() throws IOException {
|
||||
}
|
||||
}
|
||||
|
||||
private void pTestClosure2() throws IOException {
|
||||
@Test
|
||||
public void pTestClosure2() throws IOException {
|
||||
try {
|
||||
String [] files = new String[] {USER_DIR+"/a.", USER_DIR+"/a.txt",
|
||||
USER_DIR+"/a.old.java", USER_DIR+"/.java"};
|
||||
@ -184,7 +515,8 @@ private void pTestClosure2() throws IOException {
|
||||
}
|
||||
}
|
||||
|
||||
private void pTestClosure3() throws IOException {
|
||||
@Test
|
||||
public void pTestClosure3() throws IOException {
|
||||
try {
|
||||
String [] files = new String[] {USER_DIR+"/a.txt.x", USER_DIR+"/ax",
|
||||
USER_DIR+"/ab37x", USER_DIR+"/bacd"};
|
||||
@ -198,7 +530,8 @@ private void pTestClosure3() throws IOException {
|
||||
}
|
||||
}
|
||||
|
||||
private void pTestClosure4() throws IOException {
|
||||
@Test
|
||||
public void pTestClosure4() throws IOException {
|
||||
try {
|
||||
String [] files = new String[] {USER_DIR+"/dir1/file1",
|
||||
USER_DIR+"/dir2/file2",
|
||||
@ -212,7 +545,8 @@ private void pTestClosure4() throws IOException {
|
||||
}
|
||||
}
|
||||
|
||||
private void pTestClosure5() throws IOException {
|
||||
@Test
|
||||
public void pTestClosure5() throws IOException {
|
||||
try {
|
||||
String [] files = new String[] {USER_DIR+"/dir1/file1",
|
||||
USER_DIR+"/file1"};
|
||||
@ -224,7 +558,8 @@ private void pTestClosure5() throws IOException {
|
||||
}
|
||||
}
|
||||
|
||||
private void pTestSet() throws IOException {
|
||||
@Test
|
||||
public void pTestSet() throws IOException {
|
||||
try {
|
||||
String [] files = new String[] {USER_DIR+"/a.c", USER_DIR+"/a.cpp",
|
||||
USER_DIR+"/a.hlp", USER_DIR+"/a.hxy"};
|
||||
@ -238,7 +573,8 @@ private void pTestSet() throws IOException {
|
||||
}
|
||||
}
|
||||
|
||||
private void pTestRange() throws IOException {
|
||||
@Test
|
||||
public void pTestRange() throws IOException {
|
||||
try {
|
||||
String [] files = new String[] {USER_DIR+"/a.d", USER_DIR+"/a.e",
|
||||
USER_DIR+"/a.f", USER_DIR+"/a.h"};
|
||||
@ -252,7 +588,8 @@ private void pTestRange() throws IOException {
|
||||
}
|
||||
}
|
||||
|
||||
private void pTestSetExcl() throws IOException {
|
||||
@Test
|
||||
public void pTestSetExcl() throws IOException {
|
||||
try {
|
||||
String [] files = new String[] {USER_DIR+"/a.d", USER_DIR+"/a.e",
|
||||
USER_DIR+"/a.0", USER_DIR+"/a.h"};
|
||||
@ -265,7 +602,8 @@ private void pTestSetExcl() throws IOException {
|
||||
}
|
||||
}
|
||||
|
||||
private void pTestCombination() throws IOException {
|
||||
@Test
|
||||
public void pTestCombination() throws IOException {
|
||||
try {
|
||||
String [] files = new String[] {"/user/aa/a.c", "/user/bb/a.cpp",
|
||||
"/user1/cc/b.hlp", "/user/dd/a.hxy"};
|
||||
@ -277,7 +615,8 @@ private void pTestCombination() throws IOException {
|
||||
}
|
||||
}
|
||||
|
||||
private void pTestRelativePath() throws IOException {
|
||||
@Test
|
||||
public void pTestRelativePath() throws IOException {
|
||||
try {
|
||||
String [] files = new String[] {"a", "abc", "abc.p", "bacd"};
|
||||
Path[] matchedPath = prepareTesting("a*", files);
|
||||
@ -291,7 +630,8 @@ private void pTestRelativePath() throws IOException {
|
||||
}
|
||||
|
||||
/* Test {xx,yy} */
|
||||
private void pTestCurlyBracket() throws IOException {
|
||||
@Test
|
||||
public void pTestCurlyBracket() throws IOException {
|
||||
Path[] matchedPath;
|
||||
String [] files;
|
||||
try {
|
||||
@ -390,7 +730,8 @@ private void pTestCurlyBracket() throws IOException {
|
||||
}
|
||||
|
||||
/* test that a path name can contain Java regex special characters */
|
||||
private void pTestJavaRegexSpecialChars() throws IOException {
|
||||
@Test
|
||||
public void pTestJavaRegexSpecialChars() throws IOException {
|
||||
try {
|
||||
String[] files = new String[] {USER_DIR+"/($.|+)bc", USER_DIR+"/abc"};
|
||||
Path[] matchedPath = prepareTesting(USER_DIR+"/($.|+)*", files);
|
||||
@ -401,6 +742,7 @@ private void pTestJavaRegexSpecialChars() throws IOException {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private Path[] prepareTesting(String pattern, String[] files)
|
||||
throws IOException {
|
||||
for(int i=0; i<Math.min(NUM_OF_PATHS, files.length); i++) {
|
||||
@ -437,8 +779,9 @@ private Path[] prepareTesting(String pattern, String[] files,
|
||||
return globResults;
|
||||
}
|
||||
|
||||
private void cleanupDFS() throws IOException {
|
||||
fs.delete(new Path("/user"), true);
|
||||
@After
|
||||
public void cleanupDFS() throws IOException {
|
||||
fs.delete(new Path(USER_DIR), true);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user