HADOOP-10392. Use FileSystem#makeQualified(Path) instead of Path#makeQualified(FileSystem) (ajisakaa via aw)
This commit is contained in:
parent
d964062f66
commit
4222c97108
@ -295,8 +295,8 @@ private static void checkDependencies(FileSystem srcFS,
|
|||||||
Path dst)
|
Path dst)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
if (srcFS == dstFS) {
|
if (srcFS == dstFS) {
|
||||||
String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR;
|
String srcq = srcFS.makeQualified(src).toString() + Path.SEPARATOR;
|
||||||
String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR;
|
String dstq = dstFS.makeQualified(dst).toString() + Path.SEPARATOR;
|
||||||
if (dstq.startsWith(srcq)) {
|
if (dstq.startsWith(srcq)) {
|
||||||
if (srcq.length() == dstq.length()) {
|
if (srcq.length() == dstq.length()) {
|
||||||
throw new IOException("Cannot copy " + src + " to itself.");
|
throw new IOException("Cannot copy " + src + " to itself.");
|
||||||
|
@ -505,7 +505,7 @@ private FileStatus getFileStatus(FTPClient client, Path file)
|
|||||||
long modTime = -1; // Modification time of root dir not known.
|
long modTime = -1; // Modification time of root dir not known.
|
||||||
Path root = new Path("/");
|
Path root = new Path("/");
|
||||||
return new FileStatus(length, isDir, blockReplication, blockSize,
|
return new FileStatus(length, isDir, blockReplication, blockSize,
|
||||||
modTime, root.makeQualified(this));
|
modTime, this.makeQualified(root));
|
||||||
}
|
}
|
||||||
String pathName = parentPath.toUri().getPath();
|
String pathName = parentPath.toUri().getPath();
|
||||||
FTPFile[] ftpFiles = client.listFiles(pathName);
|
FTPFile[] ftpFiles = client.listFiles(pathName);
|
||||||
@ -546,7 +546,7 @@ private FileStatus getFileStatus(FTPFile ftpFile, Path parentPath) {
|
|||||||
String group = ftpFile.getGroup();
|
String group = ftpFile.getGroup();
|
||||||
Path filePath = new Path(parentPath, ftpFile.getName());
|
Path filePath = new Path(parentPath, ftpFile.getName());
|
||||||
return new FileStatus(length, isDir, blockReplication, blockSize, modTime,
|
return new FileStatus(length, isDir, blockReplication, blockSize, modTime,
|
||||||
accessTime, permission, user, group, filePath.makeQualified(this));
|
accessTime, permission, user, group, this.makeQualified(filePath));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -1883,7 +1883,7 @@ public Reader(Configuration conf, Option... opts) throws IOException {
|
|||||||
@Deprecated
|
@Deprecated
|
||||||
public Reader(FileSystem fs, Path file,
|
public Reader(FileSystem fs, Path file,
|
||||||
Configuration conf) throws IOException {
|
Configuration conf) throws IOException {
|
||||||
this(conf, file(file.makeQualified(fs)));
|
this(conf, file(fs.makeQualified(file)));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -218,8 +218,8 @@ public void testCopy() throws IOException {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testHomeDirectory() throws IOException {
|
public void testHomeDirectory() throws IOException {
|
||||||
Path home = new Path(System.getProperty("user.home"))
|
Path home = fileSys.makeQualified(
|
||||||
.makeQualified(fileSys);
|
new Path(System.getProperty("user.home")));
|
||||||
Path fsHome = fileSys.getHomeDirectory();
|
Path fsHome = fileSys.getHomeDirectory();
|
||||||
assertEquals(home, fsHome);
|
assertEquals(home, fsHome);
|
||||||
}
|
}
|
||||||
@ -229,7 +229,7 @@ public void testPathEscapes() throws IOException {
|
|||||||
Path path = new Path(TEST_ROOT_DIR, "foo%bar");
|
Path path = new Path(TEST_ROOT_DIR, "foo%bar");
|
||||||
writeFile(fileSys, path, 1);
|
writeFile(fileSys, path, 1);
|
||||||
FileStatus status = fileSys.getFileStatus(path);
|
FileStatus status = fileSys.getFileStatus(path);
|
||||||
assertEquals(path.makeQualified(fileSys), status.getPath());
|
assertEquals(fileSys.makeQualified(path), status.getPath());
|
||||||
cleanupFile(fileSys, path);
|
cleanupFile(fileSys, path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -170,7 +170,7 @@ public int run(String[] argv) throws IOException {
|
|||||||
for(int i = 0; i < argv.length; ++i) {
|
for(int i = 0; i < argv.length; ++i) {
|
||||||
try {
|
try {
|
||||||
if ("-dir".equals(argv[i])) {
|
if ("-dir".equals(argv[i])) {
|
||||||
root = new Path(argv[++i]).makeQualified(fs);
|
root = fs.makeQualified(new Path(argv[++i]));
|
||||||
System.out.println("DIR: " + root.toString());
|
System.out.println("DIR: " + root.toString());
|
||||||
} else if ("-seed".equals(argv[i])) {
|
} else if ("-seed".equals(argv[i])) {
|
||||||
job.setLong("filebench.seed", Long.valueOf(argv[++i]));
|
job.setLong("filebench.seed", Long.valueOf(argv[++i]));
|
||||||
|
@ -50,8 +50,8 @@ public static MiniMRClientCluster create(Class<?> caller, String identifier,
|
|||||||
|
|
||||||
FileSystem fs = FileSystem.get(conf);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
|
|
||||||
Path testRootDir = new Path("target", identifier + "-tmpDir")
|
Path testRootDir = fs.makeQualified(
|
||||||
.makeQualified(fs);
|
new Path("target", identifier + "-tmpDir"));
|
||||||
Path appJar = new Path(testRootDir, "MRAppJar.jar");
|
Path appJar = new Path(testRootDir, "MRAppJar.jar");
|
||||||
|
|
||||||
// Copy MRAppJar and make it private.
|
// Copy MRAppJar and make it private.
|
||||||
|
@ -47,9 +47,9 @@ public class TestCombineFileInputFormat {
|
|||||||
throw new RuntimeException("init failure", e);
|
throw new RuntimeException("init failure", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
private static Path workDir =
|
private static Path workDir = localFs.makeQualified(new Path(
|
||||||
new Path(new Path(System.getProperty("test.build.data", "/tmp")),
|
System.getProperty("test.build.data", "/tmp"),
|
||||||
"TestCombineFileInputFormat").makeQualified(localFs);
|
"TestCombineFileInputFormat"));
|
||||||
|
|
||||||
private static void writeFile(FileSystem fs, Path name,
|
private static void writeFile(FileSystem fs, Path name,
|
||||||
String contents) throws IOException {
|
String contents) throws IOException {
|
||||||
|
@ -53,10 +53,9 @@ public class TestCombineSequenceFileInputFormat {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("deprecation")
|
private static Path workDir = localFs.makeQualified(new Path(
|
||||||
private static Path workDir =
|
System.getProperty("test.build.data", "/tmp"),
|
||||||
new Path(new Path(System.getProperty("test.build.data", "/tmp")),
|
"TestCombineSequenceFileInputFormat"));
|
||||||
"TestCombineSequenceFileInputFormat").makeQualified(localFs);
|
|
||||||
|
|
||||||
@Test(timeout=10000)
|
@Test(timeout=10000)
|
||||||
public void testFormat() throws Exception {
|
public void testFormat() throws Exception {
|
||||||
|
@ -60,10 +60,9 @@ public class TestCombineTextInputFormat {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("deprecation")
|
private static Path workDir = localFs.makeQualified(new Path(
|
||||||
private static Path workDir =
|
System.getProperty("test.build.data", "/tmp"),
|
||||||
new Path(new Path(System.getProperty("test.build.data", "/tmp")),
|
"TestCombineTextInputFormat"));
|
||||||
"TestCombineTextInputFormat").makeQualified(localFs);
|
|
||||||
|
|
||||||
// A reporter that does nothing
|
// A reporter that does nothing
|
||||||
private static final Reporter voidReporter = Reporter.NULL;
|
private static final Reporter voidReporter = Reporter.NULL;
|
||||||
|
@ -84,9 +84,9 @@ public class TestConcatenatedCompressedInput {
|
|||||||
public void after() {
|
public void after() {
|
||||||
ZlibFactory.loadNativeZLib();
|
ZlibFactory.loadNativeZLib();
|
||||||
}
|
}
|
||||||
private static Path workDir =
|
private static Path workDir = localFs.makeQualified(new Path(
|
||||||
new Path(new Path(System.getProperty("test.build.data", "/tmp")),
|
System.getProperty("test.build.data", "/tmp"),
|
||||||
"TestConcatenatedCompressedInput").makeQualified(localFs);
|
"TestConcatenatedCompressedInput"));
|
||||||
|
|
||||||
private static LineReader makeStream(String str) throws IOException {
|
private static LineReader makeStream(String str) throws IOException {
|
||||||
return new LineReader(new ByteArrayInputStream(str.getBytes("UTF-8")),
|
return new LineReader(new ByteArrayInputStream(str.getBytes("UTF-8")),
|
||||||
|
@ -342,8 +342,8 @@ public void testNullKeys() throws Exception {
|
|||||||
values.add(m);
|
values.add(m);
|
||||||
m = m.replace((char)('A' + i - 1), (char)('A' + i));
|
m = m.replace((char)('A' + i - 1), (char)('A' + i));
|
||||||
}
|
}
|
||||||
Path testdir = new Path(
|
Path testdir = fs.makeQualified(new Path(
|
||||||
System.getProperty("test.build.data","/tmp")).makeQualified(fs);
|
System.getProperty("test.build.data","/tmp")));
|
||||||
fs.delete(testdir, true);
|
fs.delete(testdir, true);
|
||||||
Path inFile = new Path(testdir, "nullin/blah");
|
Path inFile = new Path(testdir, "nullin/blah");
|
||||||
SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, inFile,
|
SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, inFile,
|
||||||
|
@ -75,8 +75,8 @@ public class TestMiniMRChildTask {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Path TEST_ROOT_DIR = new Path("target",
|
private static Path TEST_ROOT_DIR = localFs.makeQualified(
|
||||||
TestMiniMRChildTask.class.getName() + "-tmpDir").makeQualified(localFs);
|
new Path("target", TestMiniMRChildTask.class.getName() + "-tmpDir"));
|
||||||
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
|
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -61,10 +61,10 @@ public class TestTextInputFormat {
|
|||||||
throw new RuntimeException("init failure", e);
|
throw new RuntimeException("init failure", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@SuppressWarnings("deprecation")
|
|
||||||
private static Path workDir =
|
private static Path workDir = localFs.makeQualified(new Path(
|
||||||
new Path(new Path(System.getProperty("test.build.data", "/tmp")),
|
System.getProperty("test.build.data", "/tmp"),
|
||||||
"TestTextInputFormat").makeQualified(localFs);
|
"TestTextInputFormat"));
|
||||||
|
|
||||||
@Test (timeout=500000)
|
@Test (timeout=500000)
|
||||||
public void testFormat() throws Exception {
|
public void testFormat() throws Exception {
|
||||||
|
@ -50,8 +50,8 @@ public void testClassLoader() throws Exception {
|
|||||||
assertTrue(job.getClassLoader() instanceof Fake_ClassLoader);
|
assertTrue(job.getClassLoader() instanceof Fake_ClassLoader);
|
||||||
|
|
||||||
FileSystem fs = FileSystem.get(job);
|
FileSystem fs = FileSystem.get(job);
|
||||||
Path testdir = new Path(System.getProperty("test.build.data", "/tmp"))
|
Path testdir = fs.makeQualified(new Path(
|
||||||
.makeQualified(fs);
|
System.getProperty("test.build.data", "/tmp")));
|
||||||
|
|
||||||
Path base = new Path(testdir, "/empty");
|
Path base = new Path(testdir, "/empty");
|
||||||
Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") };
|
Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") };
|
||||||
|
@ -50,8 +50,8 @@ public void testClassLoader() throws Exception {
|
|||||||
assertTrue(conf.getClassLoader() instanceof Fake_ClassLoader);
|
assertTrue(conf.getClassLoader() instanceof Fake_ClassLoader);
|
||||||
|
|
||||||
FileSystem fs = FileSystem.get(conf);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
Path testdir = new Path(System.getProperty("test.build.data", "/tmp"))
|
Path testdir = fs.makeQualified(new Path(
|
||||||
.makeQualified(fs);
|
System.getProperty("test.build.data", "/tmp")));
|
||||||
|
|
||||||
Path base = new Path(testdir, "/empty");
|
Path base = new Path(testdir, "/empty");
|
||||||
Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") };
|
Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") };
|
||||||
|
@ -330,7 +330,7 @@ public void cleanupAllVolumes() throws IOException {
|
|||||||
* Returns the normalized path of a path.
|
* Returns the normalized path of a path.
|
||||||
*/
|
*/
|
||||||
private String normalizePath(String path) {
|
private String normalizePath(String path) {
|
||||||
return (new Path(path)).makeQualified(this.localFileSystem)
|
return this.localFileSystem.makeQualified(new Path(path))
|
||||||
.toUri().getPath();
|
.toUri().getPath();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -128,8 +128,8 @@ public class TestMRJobs {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Path TEST_ROOT_DIR = new Path("target",
|
private static Path TEST_ROOT_DIR = localFs.makeQualified(
|
||||||
TestMRJobs.class.getName() + "-tmpDir").makeQualified(localFs);
|
new Path("target", TestMRJobs.class.getName() + "-tmpDir"));
|
||||||
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
|
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
|
||||||
private static final String OUTPUT_ROOT_DIR = "/tmp/" +
|
private static final String OUTPUT_ROOT_DIR = "/tmp/" +
|
||||||
TestMRJobs.class.getSimpleName();
|
TestMRJobs.class.getSimpleName();
|
||||||
|
@ -73,8 +73,8 @@ public class TestMRJobsWithHistoryService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Path TEST_ROOT_DIR = new Path("target",
|
private static Path TEST_ROOT_DIR = localFs.makeQualified(
|
||||||
TestMRJobs.class.getName() + "-tmpDir").makeQualified(localFs);
|
new Path("target", TestMRJobs.class.getName() + "-tmpDir"));
|
||||||
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
|
static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar");
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
|
@ -473,7 +473,7 @@ void archive(Path parentPath, List<Path> srcPaths,
|
|||||||
conf.setLong(HAR_BLOCKSIZE_LABEL, blockSize);
|
conf.setLong(HAR_BLOCKSIZE_LABEL, blockSize);
|
||||||
conf.setLong(HAR_PARTSIZE_LABEL, partSize);
|
conf.setLong(HAR_PARTSIZE_LABEL, partSize);
|
||||||
conf.set(DST_HAR_LABEL, archiveName);
|
conf.set(DST_HAR_LABEL, archiveName);
|
||||||
conf.set(SRC_PARENT_LABEL, parentPath.makeQualified(fs).toString());
|
conf.set(SRC_PARENT_LABEL, fs.makeQualified(parentPath).toString());
|
||||||
conf.setInt(HAR_REPLICATION_LABEL, repl);
|
conf.setInt(HAR_REPLICATION_LABEL, repl);
|
||||||
Path outputPath = new Path(dest, archiveName);
|
Path outputPath = new Path(dest, archiveName);
|
||||||
FileOutputFormat.setOutputPath(conf, outputPath);
|
FileOutputFormat.setOutputPath(conf, outputPath);
|
||||||
|
@ -447,7 +447,7 @@ private int runJob(Configuration conf, String[] argv)
|
|||||||
|
|
||||||
// Create <ioPath> with 777 permissions
|
// Create <ioPath> with 777 permissions
|
||||||
final FileSystem inputFs = ioPath.getFileSystem(conf);
|
final FileSystem inputFs = ioPath.getFileSystem(conf);
|
||||||
ioPath = ioPath.makeQualified(inputFs);
|
ioPath = inputFs.makeQualified(ioPath);
|
||||||
boolean succeeded = false;
|
boolean succeeded = false;
|
||||||
try {
|
try {
|
||||||
succeeded = FileSystem.mkdirs(inputFs, ioPath,
|
succeeded = FileSystem.mkdirs(inputFs, ioPath,
|
||||||
|
@ -116,7 +116,7 @@ public FSDataOutputStream create(Path path) throws IOException {
|
|||||||
* @throws FileNotFoundException
|
* @throws FileNotFoundException
|
||||||
*/
|
*/
|
||||||
long validateFileNameFormat(Path path) throws FileNotFoundException {
|
long validateFileNameFormat(Path path) throws FileNotFoundException {
|
||||||
path = path.makeQualified(this);
|
path = this.makeQualified(path);
|
||||||
boolean valid = true;
|
boolean valid = true;
|
||||||
long fileSize = 0;
|
long fileSize = 0;
|
||||||
if (!path.toUri().getScheme().equals(getUri().getScheme())) {
|
if (!path.toUri().getScheme().equals(getUri().getScheme())) {
|
||||||
@ -329,4 +329,10 @@ public void setWorkingDirectory(Path newDir) {
|
|||||||
throw new UnsupportedOperationException("SetWorkingDirectory "
|
throw new UnsupportedOperationException("SetWorkingDirectory "
|
||||||
+ "is not supported in pseudo local file system.");
|
+ "is not supported in pseudo local file system.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Path makeQualified(Path path) {
|
||||||
|
// skip FileSystem#checkPath() to validate some other Filesystems
|
||||||
|
return path.makeQualified(this.getUri(), this.getWorkingDirectory());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -48,8 +48,8 @@ static Path getBaseDir() {
|
|||||||
try {
|
try {
|
||||||
final Configuration conf = new Configuration();
|
final Configuration conf = new Configuration();
|
||||||
final FileSystem fs = FileSystem.getLocal(conf).getRaw();
|
final FileSystem fs = FileSystem.getLocal(conf).getRaw();
|
||||||
return new Path(System.getProperty("test.build.data", "/tmp"),
|
return fs.makeQualified(new Path(
|
||||||
"testFilePool").makeQualified(fs);
|
System.getProperty("test.build.data", "/tmp"), "testFilePool"));
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
fail();
|
fail();
|
||||||
}
|
}
|
||||||
|
@ -48,8 +48,8 @@ public class TestFileQueue {
|
|||||||
public static void setup() throws IOException {
|
public static void setup() throws IOException {
|
||||||
final Configuration conf = new Configuration();
|
final Configuration conf = new Configuration();
|
||||||
final FileSystem fs = FileSystem.getLocal(conf).getRaw();
|
final FileSystem fs = FileSystem.getLocal(conf).getRaw();
|
||||||
final Path p = new Path(System.getProperty("test.build.data", "/tmp"),
|
final Path p = fs.makeQualified(new Path(
|
||||||
"testFileQueue").makeQualified(fs);
|
System.getProperty("test.build.data", "/tmp"), "testFileQueue"));
|
||||||
fs.delete(p, true);
|
fs.delete(p, true);
|
||||||
final byte[] b = new byte[BLOCK];
|
final byte[] b = new byte[BLOCK];
|
||||||
for (int i = 0; i < NFILES; ++i) {
|
for (int i = 0; i < NFILES; ++i) {
|
||||||
@ -71,8 +71,8 @@ public static void setup() throws IOException {
|
|||||||
public static void cleanup() throws IOException {
|
public static void cleanup() throws IOException {
|
||||||
final Configuration conf = new Configuration();
|
final Configuration conf = new Configuration();
|
||||||
final FileSystem fs = FileSystem.getLocal(conf).getRaw();
|
final FileSystem fs = FileSystem.getLocal(conf).getRaw();
|
||||||
final Path p = new Path(System.getProperty("test.build.data", "/tmp"),
|
final Path p = fs.makeQualified(new Path(
|
||||||
"testFileQueue").makeQualified(fs);
|
System.getProperty("test.build.data", "/tmp"), "testFileQueue"));
|
||||||
fs.delete(p, true);
|
fs.delete(p, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -224,7 +224,7 @@ public void testPseudoLocalFsFileNames() throws IOException {
|
|||||||
|
|
||||||
// Validate operations on valid qualified path
|
// Validate operations on valid qualified path
|
||||||
path = new Path("myPsedoFile.1237");
|
path = new Path("myPsedoFile.1237");
|
||||||
path = path.makeQualified(pfs);
|
path = pfs.makeQualified(path);
|
||||||
validateGetFileStatus(pfs, path, true);
|
validateGetFileStatus(pfs, path, true);
|
||||||
validateCreate(pfs, path, true);
|
validateCreate(pfs, path, true);
|
||||||
validateOpen(pfs, path, true);
|
validateOpen(pfs, path, true);
|
||||||
|
@ -40,8 +40,8 @@ public class TestUserResolve {
|
|||||||
public static void createRootDir() throws IOException {
|
public static void createRootDir() throws IOException {
|
||||||
conf = new Configuration();
|
conf = new Configuration();
|
||||||
fs = FileSystem.getLocal(conf);
|
fs = FileSystem.getLocal(conf);
|
||||||
rootDir = new Path(new Path(System.getProperty("test.build.data", "/tmp"))
|
rootDir = new Path(fs.makeQualified(new Path(
|
||||||
.makeQualified(fs), "gridmixUserResolve");
|
System.getProperty("test.build.data", "/tmp"))), "gridmixUserResolve");
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -278,7 +278,7 @@ public static void cleanup(String action,
|
|||||||
noteAction(action);
|
noteAction(action);
|
||||||
try {
|
try {
|
||||||
if (fileSystem != null) {
|
if (fileSystem != null) {
|
||||||
fileSystem.delete(new Path(cleanupPath).makeQualified(fileSystem),
|
fileSystem.delete(fileSystem.makeQualified(new Path(cleanupPath)),
|
||||||
true);
|
true);
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
@ -159,7 +159,7 @@ protected void assumeRenameSupported() {
|
|||||||
* @return a qualified path instance
|
* @return a qualified path instance
|
||||||
*/
|
*/
|
||||||
protected Path path(String pathString) {
|
protected Path path(String pathString) {
|
||||||
return new Path(pathString).makeQualified(fs);
|
return fs.makeQualified(new Path(pathString));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -126,7 +126,7 @@ public void testFilePartUpload() throws Throwable {
|
|||||||
SwiftTestUtils.compareByteArrays(src, dest, len);
|
SwiftTestUtils.compareByteArrays(src, dest, len);
|
||||||
FileStatus status;
|
FileStatus status;
|
||||||
|
|
||||||
final Path qualifiedPath = path.makeQualified(fs);
|
final Path qualifiedPath = fs.makeQualified(path);
|
||||||
status = fs.getFileStatus(qualifiedPath);
|
status = fs.getFileStatus(qualifiedPath);
|
||||||
//now see what block location info comes back.
|
//now see what block location info comes back.
|
||||||
//This will vary depending on the Swift version, so the results
|
//This will vary depending on the Swift version, so the results
|
||||||
@ -216,7 +216,7 @@ public void testFilePartUploadNoLengthCheck() throws IOException, URISyntaxExcep
|
|||||||
|
|
||||||
private FileStatus validatePathLen(Path path, int len) throws IOException {
|
private FileStatus validatePathLen(Path path, int len) throws IOException {
|
||||||
//verify that the length is what was written in a direct status check
|
//verify that the length is what was written in a direct status check
|
||||||
final Path qualifiedPath = path.makeQualified(fs);
|
final Path qualifiedPath = fs.makeQualified(path);
|
||||||
FileStatus[] parentDirListing = fs.listStatus(qualifiedPath.getParent());
|
FileStatus[] parentDirListing = fs.listStatus(qualifiedPath.getParent());
|
||||||
StringBuilder listing = lsToString(parentDirListing);
|
StringBuilder listing = lsToString(parentDirListing);
|
||||||
String parentDirLS = listing.toString();
|
String parentDirLS = listing.toString();
|
||||||
|
@ -57,8 +57,8 @@ public class TestHistograms {
|
|||||||
public void testHistograms() throws IOException {
|
public void testHistograms() throws IOException {
|
||||||
final Configuration conf = new Configuration();
|
final Configuration conf = new Configuration();
|
||||||
final FileSystem lfs = FileSystem.getLocal(conf);
|
final FileSystem lfs = FileSystem.getLocal(conf);
|
||||||
final Path rootInputDir = new Path(
|
final Path rootInputDir = lfs.makeQualified(new Path(
|
||||||
System.getProperty("test.tools.input.dir", "")).makeQualified(lfs);
|
System.getProperty("test.tools.input.dir", "target/input")));
|
||||||
final Path rootInputFile = new Path(rootInputDir, "rumen/histogram-tests");
|
final Path rootInputFile = new Path(rootInputDir, "rumen/histogram-tests");
|
||||||
|
|
||||||
|
|
||||||
@ -132,7 +132,7 @@ public static void main(String[] args) throws IOException {
|
|||||||
final FileSystem lfs = FileSystem.getLocal(conf);
|
final FileSystem lfs = FileSystem.getLocal(conf);
|
||||||
|
|
||||||
for (String arg : args) {
|
for (String arg : args) {
|
||||||
Path filePath = new Path(arg).makeQualified(lfs);
|
Path filePath = lfs.makeQualified(new Path(arg));
|
||||||
String fileName = filePath.getName();
|
String fileName = filePath.getName();
|
||||||
if (fileName.startsWith("input")) {
|
if (fileName.startsWith("input")) {
|
||||||
LoggedDiscreteCDF newResult = histogramFileToCDF(filePath, lfs);
|
LoggedDiscreteCDF newResult = histogramFileToCDF(filePath, lfs);
|
||||||
|
@ -22,13 +22,11 @@
|
|||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
|
||||||
import java.net.URLEncoder;
|
import java.net.URLEncoder;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.regex.Pattern;
|
|
||||||
import java.util.TreeMap;
|
import java.util.TreeMap;
|
||||||
import java.util.TreeSet;
|
import java.util.TreeSet;
|
||||||
|
|
||||||
@ -41,12 +39,12 @@
|
|||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.permission.FsAction;
|
||||||
import org.apache.hadoop.mapreduce.MRConfig;
|
import org.apache.hadoop.mapreduce.MRConfig;
|
||||||
import org.apache.hadoop.mapreduce.MRJobConfig;
|
import org.apache.hadoop.mapreduce.MRJobConfig;
|
||||||
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
|
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
|
||||||
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
|
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.FileUtil;
|
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
||||||
import org.apache.hadoop.mapred.FileInputFormat;
|
import org.apache.hadoop.mapred.FileInputFormat;
|
||||||
@ -56,7 +54,6 @@
|
|||||||
import org.apache.hadoop.mapred.JobConf;
|
import org.apache.hadoop.mapred.JobConf;
|
||||||
import org.apache.hadoop.mapred.JobID;
|
import org.apache.hadoop.mapred.JobID;
|
||||||
import org.apache.hadoop.mapred.KeyValueTextInputFormat;
|
import org.apache.hadoop.mapred.KeyValueTextInputFormat;
|
||||||
import org.apache.hadoop.mapred.OutputFormat;
|
|
||||||
import org.apache.hadoop.mapred.RunningJob;
|
import org.apache.hadoop.mapred.RunningJob;
|
||||||
import org.apache.hadoop.mapred.SequenceFileAsTextInputFormat;
|
import org.apache.hadoop.mapred.SequenceFileAsTextInputFormat;
|
||||||
import org.apache.hadoop.mapred.SequenceFileInputFormat;
|
import org.apache.hadoop.mapred.SequenceFileInputFormat;
|
||||||
@ -65,6 +62,7 @@
|
|||||||
import org.apache.hadoop.mapred.lib.LazyOutputFormat;
|
import org.apache.hadoop.mapred.lib.LazyOutputFormat;
|
||||||
import org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorCombiner;
|
import org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorCombiner;
|
||||||
import org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorReducer;
|
import org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorReducer;
|
||||||
|
import org.apache.hadoop.security.AccessControlException;
|
||||||
import org.apache.hadoop.streaming.io.IdentifierResolver;
|
import org.apache.hadoop.streaming.io.IdentifierResolver;
|
||||||
import org.apache.hadoop.streaming.io.InputWriter;
|
import org.apache.hadoop.streaming.io.InputWriter;
|
||||||
import org.apache.hadoop.streaming.io.OutputReader;
|
import org.apache.hadoop.streaming.io.OutputReader;
|
||||||
@ -297,7 +295,10 @@ void parseArgv() {
|
|||||||
try {
|
try {
|
||||||
Path path = new Path(file);
|
Path path = new Path(file);
|
||||||
FileSystem localFs = FileSystem.getLocal(config_);
|
FileSystem localFs = FileSystem.getLocal(config_);
|
||||||
String finalPath = path.makeQualified(localFs).toString();
|
Path qualifiedPath = path.makeQualified(
|
||||||
|
localFs.getUri(), localFs.getWorkingDirectory());
|
||||||
|
validate(qualifiedPath);
|
||||||
|
String finalPath = qualifiedPath.toString();
|
||||||
if(fileList.length() > 0) {
|
if(fileList.length() > 0) {
|
||||||
fileList.append(',');
|
fileList.append(',');
|
||||||
}
|
}
|
||||||
@ -313,7 +314,6 @@ void parseArgv() {
|
|||||||
tmpFiles = tmpFiles + "," + fileList;
|
tmpFiles = tmpFiles + "," + fileList;
|
||||||
}
|
}
|
||||||
config_.set("tmpfiles", tmpFiles);
|
config_.set("tmpfiles", tmpFiles);
|
||||||
validate(packageFiles_);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
String fsName = cmdLine.getOptionValue("dfs");
|
String fsName = cmdLine.getOptionValue("dfs");
|
||||||
@ -391,14 +391,13 @@ private Option createBoolOption(String name, String desc){
|
|||||||
return OptionBuilder.withDescription(desc).create(name);
|
return OptionBuilder.withDescription(desc).create(name);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void validate(final List<String> values)
|
private void validate(final Path path) throws IOException {
|
||||||
throws IllegalArgumentException {
|
try {
|
||||||
for (String file : values) {
|
path.getFileSystem(config_).access(path, FsAction.READ);
|
||||||
File f = new File(file);
|
} catch (FileNotFoundException e) {
|
||||||
if (!FileUtil.canRead(f)) {
|
fail("File: " + path + " does not exist.");
|
||||||
fail("File: " + f.getAbsolutePath()
|
} catch (AccessControlException e) {
|
||||||
+ " does not exist, or is not readable.");
|
fail("File: " + path + " is not readable.");
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user