From 4222c971080f2b150713727092c7197df58c88e5 Mon Sep 17 00:00:00 2001 From: Allen Wittenauer Date: Fri, 11 Aug 2017 09:25:56 -0700 Subject: [PATCH] HADOOP-10392. Use FileSystem#makeQualified(Path) instead of Path#makeQualified(FileSystem) (ajisakaa via aw) --- .../java/org/apache/hadoop/fs/FileUtil.java | 4 +-- .../apache/hadoop/fs/ftp/FTPFileSystem.java | 4 +-- .../org/apache/hadoop/io/SequenceFile.java | 2 +- .../apache/hadoop/fs/TestLocalFileSystem.java | 6 ++--- .../java/org/apache/hadoop/io/FileBench.java | 2 +- .../mapred/MiniMRClientClusterFactory.java | 4 +-- .../mapred/TestCombineFileInputFormat.java | 6 ++--- .../TestCombineSequenceFileInputFormat.java | 7 +++-- .../mapred/TestCombineTextInputFormat.java | 7 +++-- .../TestConcatenatedCompressedInput.java | 6 ++--- .../org/apache/hadoop/mapred/TestMapRed.java | 4 +-- .../hadoop/mapred/TestMiniMRChildTask.java | 4 +-- .../hadoop/mapred/TestTextInputFormat.java | 8 +++--- .../TestWrappedRecordReaderClassloader.java | 4 +-- .../lib/join/TestWrappedRRClassloader.java | 4 +-- .../mapreduce/util/MRAsyncDiskService.java | 2 +- .../hadoop/mapreduce/v2/TestMRJobs.java | 4 +-- .../v2/TestMRJobsWithHistoryService.java | 4 +-- .../apache/hadoop/tools/HadoopArchives.java | 2 +- .../apache/hadoop/mapred/gridmix/Gridmix.java | 2 +- .../hadoop/mapred/gridmix/PseudoLocalFs.java | 8 +++++- .../hadoop/mapred/gridmix/TestFilePool.java | 4 +-- .../hadoop/mapred/gridmix/TestFileQueue.java | 8 +++--- .../mapred/gridmix/TestPseudoLocalFs.java | 2 +- .../mapred/gridmix/TestUserResolve.java | 4 +-- .../hadoop/fs/swift/util/SwiftTestUtils.java | 2 +- .../fs/swift/SwiftFileSystemBaseTest.java | 2 +- ...TestSwiftFileSystemPartitionedUploads.java | 4 +-- .../hadoop/tools/rumen/TestHistograms.java | 6 ++--- .../apache/hadoop/streaming/StreamJob.java | 27 +++++++++---------- 30 files changed, 78 insertions(+), 75 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index eb8a5c3c1c..72b9615832 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -295,8 +295,8 @@ private static void checkDependencies(FileSystem srcFS, Path dst) throws IOException { if (srcFS == dstFS) { - String srcq = src.makeQualified(srcFS).toString() + Path.SEPARATOR; - String dstq = dst.makeQualified(dstFS).toString() + Path.SEPARATOR; + String srcq = srcFS.makeQualified(src).toString() + Path.SEPARATOR; + String dstq = dstFS.makeQualified(dst).toString() + Path.SEPARATOR; if (dstq.startsWith(srcq)) { if (srcq.length() == dstq.length()) { throw new IOException("Cannot copy " + src + " to itself."); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java index 4c1236baa4..644cf4e50e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java @@ -505,7 +505,7 @@ private FileStatus getFileStatus(FTPClient client, Path file) long modTime = -1; // Modification time of root dir not known. Path root = new Path("/"); return new FileStatus(length, isDir, blockReplication, blockSize, - modTime, root.makeQualified(this)); + modTime, this.makeQualified(root)); } String pathName = parentPath.toUri().getPath(); FTPFile[] ftpFiles = client.listFiles(pathName); @@ -546,7 +546,7 @@ private FileStatus getFileStatus(FTPFile ftpFile, Path parentPath) { String group = ftpFile.getGroup(); Path filePath = new Path(parentPath, ftpFile.getName()); return new FileStatus(length, isDir, blockReplication, blockSize, modTime, - accessTime, permission, user, group, filePath.makeQualified(this)); + accessTime, permission, user, group, this.makeQualified(filePath)); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java index 2cc0e40aac..f42848b00c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java @@ -1883,7 +1883,7 @@ public Reader(Configuration conf, Option... opts) throws IOException { @Deprecated public Reader(FileSystem fs, Path file, Configuration conf) throws IOException { - this(conf, file(file.makeQualified(fs))); + this(conf, file(fs.makeQualified(file))); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index 357c683227..90eaa2a65f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -218,8 +218,8 @@ public void testCopy() throws IOException { @Test public void testHomeDirectory() throws IOException { - Path home = new Path(System.getProperty("user.home")) - .makeQualified(fileSys); + Path home = fileSys.makeQualified( + new Path(System.getProperty("user.home"))); Path fsHome = fileSys.getHomeDirectory(); assertEquals(home, fsHome); } @@ -229,7 +229,7 @@ public void testPathEscapes() throws IOException { Path path = new Path(TEST_ROOT_DIR, "foo%bar"); writeFile(fileSys, path, 1); FileStatus status = fileSys.getFileStatus(path); - assertEquals(path.makeQualified(fileSys), status.getPath()); + assertEquals(fileSys.makeQualified(path), status.getPath()); cleanupFile(fileSys, path); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/FileBench.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/FileBench.java index 0a9d0e9391..ef68cdff51 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/FileBench.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/io/FileBench.java @@ -170,7 +170,7 @@ public int run(String[] argv) throws IOException { for(int i = 0; i < argv.length; ++i) { try { if ("-dir".equals(argv[i])) { - root = new Path(argv[++i]).makeQualified(fs); + root = fs.makeQualified(new Path(argv[++i])); System.out.println("DIR: " + root.toString()); } else if ("-seed".equals(argv[i])) { job.setLong("filebench.seed", Long.valueOf(argv[++i])); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java index 023da48082..85c534bfb8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java @@ -50,8 +50,8 @@ public static MiniMRClientCluster create(Class caller, String identifier, FileSystem fs = FileSystem.get(conf); - Path testRootDir = new Path("target", identifier + "-tmpDir") - .makeQualified(fs); + Path testRootDir = fs.makeQualified( + new Path("target", identifier + "-tmpDir")); Path appJar = new Path(testRootDir, "MRAppJar.jar"); // Copy MRAppJar and make it private. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java index ca3c2dfe5b..de7880dc65 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineFileInputFormat.java @@ -47,9 +47,9 @@ public class TestCombineFileInputFormat { throw new RuntimeException("init failure", e); } } - private static Path workDir = - new Path(new Path(System.getProperty("test.build.data", "/tmp")), - "TestCombineFileInputFormat").makeQualified(localFs); + private static Path workDir = localFs.makeQualified(new Path( + System.getProperty("test.build.data", "/tmp"), + "TestCombineFileInputFormat")); private static void writeFile(FileSystem fs, Path name, String contents) throws IOException { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java index 8d0203e925..8cdaa8024c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineSequenceFileInputFormat.java @@ -53,10 +53,9 @@ public class TestCombineSequenceFileInputFormat { } } - @SuppressWarnings("deprecation") - private static Path workDir = - new Path(new Path(System.getProperty("test.build.data", "/tmp")), - "TestCombineSequenceFileInputFormat").makeQualified(localFs); + private static Path workDir = localFs.makeQualified(new Path( + System.getProperty("test.build.data", "/tmp"), + "TestCombineSequenceFileInputFormat")); @Test(timeout=10000) public void testFormat() throws Exception { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java index ca86dd55f0..581e62b639 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestCombineTextInputFormat.java @@ -60,10 +60,9 @@ public class TestCombineTextInputFormat { } } - @SuppressWarnings("deprecation") - private static Path workDir = - new Path(new Path(System.getProperty("test.build.data", "/tmp")), - "TestCombineTextInputFormat").makeQualified(localFs); + private static Path workDir = localFs.makeQualified(new Path( + System.getProperty("test.build.data", "/tmp"), + "TestCombineTextInputFormat")); // A reporter that does nothing private static final Reporter voidReporter = Reporter.NULL; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java index 22a05c5b41..15d651d18f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestConcatenatedCompressedInput.java @@ -84,9 +84,9 @@ public class TestConcatenatedCompressedInput { public void after() { ZlibFactory.loadNativeZLib(); } - private static Path workDir = - new Path(new Path(System.getProperty("test.build.data", "/tmp")), - "TestConcatenatedCompressedInput").makeQualified(localFs); + private static Path workDir = localFs.makeQualified(new Path( + System.getProperty("test.build.data", "/tmp"), + "TestConcatenatedCompressedInput")); private static LineReader makeStream(String str) throws IOException { return new LineReader(new ByteArrayInputStream(str.getBytes("UTF-8")), diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java index d60905ed05..af09e09535 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapRed.java @@ -342,8 +342,8 @@ public void testNullKeys() throws Exception { values.add(m); m = m.replace((char)('A' + i - 1), (char)('A' + i)); } - Path testdir = new Path( - System.getProperty("test.build.data","/tmp")).makeQualified(fs); + Path testdir = fs.makeQualified(new Path( + System.getProperty("test.build.data","/tmp"))); fs.delete(testdir, true); Path inFile = new Path(testdir, "nullin/blah"); SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, inFile, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java index f690118c80..51f0120d73 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java @@ -75,8 +75,8 @@ public class TestMiniMRChildTask { } } - private static Path TEST_ROOT_DIR = new Path("target", - TestMiniMRChildTask.class.getName() + "-tmpDir").makeQualified(localFs); + private static Path TEST_ROOT_DIR = localFs.makeQualified( + new Path("target", TestMiniMRChildTask.class.getName() + "-tmpDir")); static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar"); /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java index 5106c3843f..67bd497763 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java @@ -61,10 +61,10 @@ public class TestTextInputFormat { throw new RuntimeException("init failure", e); } } - @SuppressWarnings("deprecation") - private static Path workDir = - new Path(new Path(System.getProperty("test.build.data", "/tmp")), - "TestTextInputFormat").makeQualified(localFs); + + private static Path workDir = localFs.makeQualified(new Path( + System.getProperty("test.build.data", "/tmp"), + "TestTextInputFormat")); @Test (timeout=500000) public void testFormat() throws Exception { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestWrappedRecordReaderClassloader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestWrappedRecordReaderClassloader.java index ae5572f5dc..785898d33e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestWrappedRecordReaderClassloader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/join/TestWrappedRecordReaderClassloader.java @@ -50,8 +50,8 @@ public void testClassLoader() throws Exception { assertTrue(job.getClassLoader() instanceof Fake_ClassLoader); FileSystem fs = FileSystem.get(job); - Path testdir = new Path(System.getProperty("test.build.data", "/tmp")) - .makeQualified(fs); + Path testdir = fs.makeQualified(new Path( + System.getProperty("test.build.data", "/tmp"))); Path base = new Path(testdir, "/empty"); Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") }; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestWrappedRRClassloader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestWrappedRRClassloader.java index 680e246b4e..e3d7fa05f1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestWrappedRRClassloader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/join/TestWrappedRRClassloader.java @@ -50,8 +50,8 @@ public void testClassLoader() throws Exception { assertTrue(conf.getClassLoader() instanceof Fake_ClassLoader); FileSystem fs = FileSystem.get(conf); - Path testdir = new Path(System.getProperty("test.build.data", "/tmp")) - .makeQualified(fs); + Path testdir = fs.makeQualified(new Path( + System.getProperty("test.build.data", "/tmp"))); Path base = new Path(testdir, "/empty"); Path[] src = { new Path(base, "i0"), new Path("i1"), new Path("i2") }; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java index 4446756432..be4638579e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MRAsyncDiskService.java @@ -330,7 +330,7 @@ public void cleanupAllVolumes() throws IOException { * Returns the normalized path of a path. */ private String normalizePath(String path) { - return (new Path(path)).makeQualified(this.localFileSystem) + return this.localFileSystem.makeQualified(new Path(path)) .toUri().getPath(); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java index c6d2168261..274f405529 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java @@ -128,8 +128,8 @@ public class TestMRJobs { } } - private static Path TEST_ROOT_DIR = new Path("target", - TestMRJobs.class.getName() + "-tmpDir").makeQualified(localFs); + private static Path TEST_ROOT_DIR = localFs.makeQualified( + new Path("target", TestMRJobs.class.getName() + "-tmpDir")); static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar"); private static final String OUTPUT_ROOT_DIR = "/tmp/" + TestMRJobs.class.getSimpleName(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java index f9236a926a..98a6de2ce0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java @@ -73,8 +73,8 @@ public class TestMRJobsWithHistoryService { } } - private static Path TEST_ROOT_DIR = new Path("target", - TestMRJobs.class.getName() + "-tmpDir").makeQualified(localFs); + private static Path TEST_ROOT_DIR = localFs.makeQualified( + new Path("target", TestMRJobs.class.getName() + "-tmpDir")); static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar"); @Before diff --git a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java index c2097dc673..8ad860036c 100644 --- a/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java +++ b/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java @@ -473,7 +473,7 @@ void archive(Path parentPath, List srcPaths, conf.setLong(HAR_BLOCKSIZE_LABEL, blockSize); conf.setLong(HAR_PARTSIZE_LABEL, partSize); conf.set(DST_HAR_LABEL, archiveName); - conf.set(SRC_PARENT_LABEL, parentPath.makeQualified(fs).toString()); + conf.set(SRC_PARENT_LABEL, fs.makeQualified(parentPath).toString()); conf.setInt(HAR_REPLICATION_LABEL, repl); Path outputPath = new Path(dest, archiveName); FileOutputFormat.setOutputPath(conf, outputPath); diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java index 4386bc1bb2..3507b7f46b 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java @@ -447,7 +447,7 @@ private int runJob(Configuration conf, String[] argv) // Create with 777 permissions final FileSystem inputFs = ioPath.getFileSystem(conf); - ioPath = ioPath.makeQualified(inputFs); + ioPath = inputFs.makeQualified(ioPath); boolean succeeded = false; try { succeeded = FileSystem.mkdirs(inputFs, ioPath, diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java index d7ef563c95..15fc68e2d1 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java @@ -116,7 +116,7 @@ public FSDataOutputStream create(Path path) throws IOException { * @throws FileNotFoundException */ long validateFileNameFormat(Path path) throws FileNotFoundException { - path = path.makeQualified(this); + path = this.makeQualified(path); boolean valid = true; long fileSize = 0; if (!path.toUri().getScheme().equals(getUri().getScheme())) { @@ -329,4 +329,10 @@ public void setWorkingDirectory(Path newDir) { throw new UnsupportedOperationException("SetWorkingDirectory " + "is not supported in pseudo local file system."); } + + @Override + public Path makeQualified(Path path) { + // skip FileSystem#checkPath() to validate some other Filesystems + return path.makeQualified(this.getUri(), this.getWorkingDirectory()); + } } diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFilePool.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFilePool.java index 4be90c6aeb..a75414accc 100644 --- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFilePool.java +++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFilePool.java @@ -48,8 +48,8 @@ static Path getBaseDir() { try { final Configuration conf = new Configuration(); final FileSystem fs = FileSystem.getLocal(conf).getRaw(); - return new Path(System.getProperty("test.build.data", "/tmp"), - "testFilePool").makeQualified(fs); + return fs.makeQualified(new Path( + System.getProperty("test.build.data", "/tmp"), "testFilePool")); } catch (IOException e) { fail(); } diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFileQueue.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFileQueue.java index a4668ee147..e68e83f6c6 100644 --- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFileQueue.java +++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestFileQueue.java @@ -48,8 +48,8 @@ public class TestFileQueue { public static void setup() throws IOException { final Configuration conf = new Configuration(); final FileSystem fs = FileSystem.getLocal(conf).getRaw(); - final Path p = new Path(System.getProperty("test.build.data", "/tmp"), - "testFileQueue").makeQualified(fs); + final Path p = fs.makeQualified(new Path( + System.getProperty("test.build.data", "/tmp"), "testFileQueue")); fs.delete(p, true); final byte[] b = new byte[BLOCK]; for (int i = 0; i < NFILES; ++i) { @@ -71,8 +71,8 @@ public static void setup() throws IOException { public static void cleanup() throws IOException { final Configuration conf = new Configuration(); final FileSystem fs = FileSystem.getLocal(conf).getRaw(); - final Path p = new Path(System.getProperty("test.build.data", "/tmp"), - "testFileQueue").makeQualified(fs); + final Path p = fs.makeQualified(new Path( + System.getProperty("test.build.data", "/tmp"), "testFileQueue")); fs.delete(p, true); } diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java index a607ece129..7179c5d0dd 100644 --- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java +++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestPseudoLocalFs.java @@ -224,7 +224,7 @@ public void testPseudoLocalFsFileNames() throws IOException { // Validate operations on valid qualified path path = new Path("myPsedoFile.1237"); - path = path.makeQualified(pfs); + path = pfs.makeQualified(path); validateGetFileStatus(pfs, path, true); validateCreate(pfs, path, true); validateOpen(pfs, path, true); diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestUserResolve.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestUserResolve.java index 8050f33b79..44075157f5 100644 --- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestUserResolve.java +++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestUserResolve.java @@ -40,8 +40,8 @@ public class TestUserResolve { public static void createRootDir() throws IOException { conf = new Configuration(); fs = FileSystem.getLocal(conf); - rootDir = new Path(new Path(System.getProperty("test.build.data", "/tmp")) - .makeQualified(fs), "gridmixUserResolve"); + rootDir = new Path(fs.makeQualified(new Path( + System.getProperty("test.build.data", "/tmp"))), "gridmixUserResolve"); } /** diff --git a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java index f91ba3013c..726045ee9e 100644 --- a/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java +++ b/hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java @@ -278,7 +278,7 @@ public static void cleanup(String action, noteAction(action); try { if (fileSystem != null) { - fileSystem.delete(new Path(cleanupPath).makeQualified(fileSystem), + fileSystem.delete(fileSystem.makeQualified(new Path(cleanupPath)), true); } } catch (Exception e) { diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java index 12f58e6433..99e03c7a71 100644 --- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java @@ -159,7 +159,7 @@ protected void assumeRenameSupported() { * @return a qualified path instance */ protected Path path(String pathString) { - return new Path(pathString).makeQualified(fs); + return fs.makeQualified(new Path(pathString)); } /** diff --git a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java index f344093158..b42abcddf5 100644 --- a/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java +++ b/hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java @@ -126,7 +126,7 @@ public void testFilePartUpload() throws Throwable { SwiftTestUtils.compareByteArrays(src, dest, len); FileStatus status; - final Path qualifiedPath = path.makeQualified(fs); + final Path qualifiedPath = fs.makeQualified(path); status = fs.getFileStatus(qualifiedPath); //now see what block location info comes back. //This will vary depending on the Swift version, so the results @@ -216,7 +216,7 @@ public void testFilePartUploadNoLengthCheck() throws IOException, URISyntaxExcep private FileStatus validatePathLen(Path path, int len) throws IOException { //verify that the length is what was written in a direct status check - final Path qualifiedPath = path.makeQualified(fs); + final Path qualifiedPath = fs.makeQualified(path); FileStatus[] parentDirListing = fs.listStatus(qualifiedPath.getParent()); StringBuilder listing = lsToString(parentDirListing); String parentDirLS = listing.toString(); diff --git a/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java b/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java index 206095a257..52caaf5575 100644 --- a/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java +++ b/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java @@ -57,8 +57,8 @@ public class TestHistograms { public void testHistograms() throws IOException { final Configuration conf = new Configuration(); final FileSystem lfs = FileSystem.getLocal(conf); - final Path rootInputDir = new Path( - System.getProperty("test.tools.input.dir", "")).makeQualified(lfs); + final Path rootInputDir = lfs.makeQualified(new Path( + System.getProperty("test.tools.input.dir", "target/input"))); final Path rootInputFile = new Path(rootInputDir, "rumen/histogram-tests"); @@ -132,7 +132,7 @@ public static void main(String[] args) throws IOException { final FileSystem lfs = FileSystem.getLocal(conf); for (String arg : args) { - Path filePath = new Path(arg).makeQualified(lfs); + Path filePath = lfs.makeQualified(new Path(arg)); String fileName = filePath.getName(); if (fileName.startsWith("input")) { LoggedDiscreteCDF newResult = histogramFileToCDF(filePath, lfs); diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java index 9f5b293b36..0b239d0ea4 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java @@ -22,13 +22,11 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.net.URLEncoder; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.regex.Pattern; import java.util.TreeMap; import java.util.TreeSet; @@ -41,12 +39,12 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.mapred.FileInputFormat; @@ -56,7 +54,6 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.KeyValueTextInputFormat; -import org.apache.hadoop.mapred.OutputFormat; import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapred.SequenceFileAsTextInputFormat; import org.apache.hadoop.mapred.SequenceFileInputFormat; @@ -65,6 +62,7 @@ import org.apache.hadoop.mapred.lib.LazyOutputFormat; import org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorCombiner; import org.apache.hadoop.mapred.lib.aggregate.ValueAggregatorReducer; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.streaming.io.IdentifierResolver; import org.apache.hadoop.streaming.io.InputWriter; import org.apache.hadoop.streaming.io.OutputReader; @@ -297,7 +295,10 @@ void parseArgv() { try { Path path = new Path(file); FileSystem localFs = FileSystem.getLocal(config_); - String finalPath = path.makeQualified(localFs).toString(); + Path qualifiedPath = path.makeQualified( + localFs.getUri(), localFs.getWorkingDirectory()); + validate(qualifiedPath); + String finalPath = qualifiedPath.toString(); if(fileList.length() > 0) { fileList.append(','); } @@ -313,7 +314,6 @@ void parseArgv() { tmpFiles = tmpFiles + "," + fileList; } config_.set("tmpfiles", tmpFiles); - validate(packageFiles_); } String fsName = cmdLine.getOptionValue("dfs"); @@ -391,14 +391,13 @@ private Option createBoolOption(String name, String desc){ return OptionBuilder.withDescription(desc).create(name); } - private void validate(final List values) - throws IllegalArgumentException { - for (String file : values) { - File f = new File(file); - if (!FileUtil.canRead(f)) { - fail("File: " + f.getAbsolutePath() - + " does not exist, or is not readable."); - } + private void validate(final Path path) throws IOException { + try { + path.getFileSystem(config_).access(path, FsAction.READ); + } catch (FileNotFoundException e) { + fail("File: " + path + " does not exist."); + } catch (AccessControlException e) { + fail("File: " + path + " is not readable."); } }