From 22cb0ec82a663137dfbaef7a088fb1ac76df6568 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Fri, 25 May 2012 04:10:15 +0000 Subject: [PATCH] HADOOP-8422. Deprecate FileSystem#getDefault* and getServerDefault methods that don't take a Path argument. Contributed by Eli Collins git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1342495 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/fs/FileSystem.java | 12 ++++++++++-- .../org/apache/hadoop/fs/RawLocalFileSystem.java | 4 ++-- .../org/apache/hadoop/fs/FileSystemTestHelper.java | 2 +- .../hadoop/fs/s3/S3FileSystemContractBaseTest.java | 6 ++---- .../s3native/NativeS3FileSystemContractBaseTest.java | 4 ++-- .../apache/hadoop/fs/http/server/FSOperations.java | 4 ++-- .../org/apache/hadoop/hdfs/TestDFSPermission.java | 2 +- .../test/java/org/apache/hadoop/hdfs/TestQuota.java | 6 +++--- .../org/apache/hadoop/security/TestPermission.java | 5 +++-- 10 files changed, 29 insertions(+), 19 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 057f97fa6f..e004829ad4 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -173,6 +173,9 @@ Release 2.0.1-alpha - UNRELEASED HADOOP-8398. Cleanup BlockLocation. (eli) + HADOOP-8422. Deprecate FileSystem#getDefault* and getServerDefault + methods that don't take a Path argument. (eli) + BUG FIXES HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index b8879a29d5..fe908beddd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -615,7 +615,9 @@ public BlockLocation[] getFileBlockLocations(Path p, * Return a set of server default configuration values * @return server default configuration values * @throws IOException + * @deprecated use {@link #getServerDefaults(Path)} instead */ + @Deprecated public FsServerDefaults getServerDefaults() throws IOException { Configuration conf = getConf(); return new FsServerDefaults(getDefaultBlockSize(), @@ -1939,8 +1941,12 @@ public long getBlockSize(Path f) throws IOException { return getFileStatus(f).getBlockSize(); } - /** Return the number of bytes that large input files should be optimally - * be split into to minimize i/o time. */ + /** + * Return the number of bytes that large input files should be optimally + * be split into to minimize i/o time. + * @deprecated use {@link #getDefaultBlockSize(Path)} instead + */ + @Deprecated public long getDefaultBlockSize() { // default to 32MB: large enough to minimize the impact of seeks return getConf().getLong("fs.local.block.size", 32 * 1024 * 1024); @@ -1958,7 +1964,9 @@ public long getDefaultBlockSize(Path f) { /** * Get the default replication. + * @deprecated use {@link #getDefaultReplication(Path)} instead */ + @Deprecated public short getDefaultReplication() { return 1; } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index 502847960d..85288df320 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -316,7 +316,7 @@ public FileStatus[] listStatus(Path f) throws IOException { } if (localf.isFile()) { return new FileStatus[] { - new RawLocalFileStatus(localf, getDefaultBlockSize(), this) }; + new RawLocalFileStatus(localf, getDefaultBlockSize(f), this) }; } String[] names = localf.list(); @@ -444,7 +444,7 @@ public String toString() { public FileStatus getFileStatus(Path f) throws IOException { File path = pathToFile(f); if (path.exists()) { - return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(), this); + return new RawLocalFileStatus(pathToFile(f), getDefaultBlockSize(f), this); } else { throw new FileNotFoundException("File " + f + " does not exist"); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java index 035a0165fb..c175d53ecc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java @@ -113,7 +113,7 @@ public static long createFile(FileSystem fSys, Path path, int numBlocks, public static long createFile(FileSystem fSys, Path path, int numBlocks, int blockSize, boolean createParent) throws IOException { - return createFile(fSys, path, numBlocks, blockSize, fSys.getDefaultReplication(), true); + return createFile(fSys, path, numBlocks, blockSize, fSys.getDefaultReplication(path), true); } public static long createFile(FileSystem fSys, Path path, int numBlocks, diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java index d245804d87..d1770d3b88 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/S3FileSystemContractBaseTest.java @@ -47,11 +47,9 @@ protected void tearDown() throws Exception { } public void testBlockSize() throws Exception { - - long newBlockSize = fs.getDefaultBlockSize() * 2; - fs.getConf().setLong("fs.s3.block.size", newBlockSize); - Path file = path("/test/hadoop/file"); + long newBlockSize = fs.getDefaultBlockSize(file) * 2; + fs.getConf().setLong("fs.s3.block.size", newBlockSize); createFile(file); assertEquals("Double default block size", newBlockSize, fs.getFileStatus(file).getBlockSize()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java index d98021b796..60010e4364 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java @@ -141,11 +141,11 @@ public void testEmptyFile() throws Exception { public void testBlockSize() throws Exception { Path file = path("/test/hadoop/file"); createFile(file); - assertEquals("Default block size", fs.getDefaultBlockSize(), + assertEquals("Default block size", fs.getDefaultBlockSize(file), fs.getFileStatus(file).getBlockSize()); // Block size is determined at read time - long newBlockSize = fs.getDefaultBlockSize() * 2; + long newBlockSize = fs.getDefaultBlockSize(file) * 2; fs.getConf().setLong("fs.s3n.block.size", newBlockSize); assertEquals("Double default block size", newBlockSize, fs.getFileStatus(file).getBlockSize()); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java index 4b3b587a24..9e51ce8a77 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java @@ -288,10 +288,10 @@ public FSCreate(InputStream is, String path, String perm, boolean override, shor @Override public Void execute(FileSystem fs) throws IOException { if (replication == -1) { - replication = fs.getDefaultReplication(); + replication = fs.getDefaultReplication(path); } if (blockSize == -1) { - blockSize = fs.getDefaultBlockSize(); + blockSize = fs.getDefaultBlockSize(path); } FsPermission fsPermission = getPermission(permission); int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java index 38a837247a..2474242553 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSPermission.java @@ -202,7 +202,7 @@ private void create(OpType op, Path name, short umask, case CREATE: FSDataOutputStream out = fs.create(name, permission, true, conf.getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), - fs.getDefaultReplication(), fs.getDefaultBlockSize(), null); + fs.getDefaultReplication(name), fs.getDefaultBlockSize(name), null); out.close(); break; case MKDIRS: diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java index b1d31da930..a747a333fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestQuota.java @@ -875,8 +875,8 @@ public void testMultipleFilesSmallerThanOneBlock() throws Exception { // 6kb block // 192kb quota final int FILE_SIZE = 1024; - final int QUOTA_SIZE = 32 * (int) fs.getDefaultBlockSize(); - assertEquals(6 * 1024, fs.getDefaultBlockSize()); + final int QUOTA_SIZE = 32 * (int) fs.getDefaultBlockSize(dir); + assertEquals(6 * 1024, fs.getDefaultBlockSize(dir)); assertEquals(192 * 1024, QUOTA_SIZE); // Create the dir and set the quota. We need to enable the quota before @@ -903,7 +903,7 @@ public void testMultipleFilesSmallerThanOneBlock() throws Exception { assertEquals("Invalid space consumed", 59 * FILE_SIZE * 3, c.getSpaceConsumed()); assertEquals("Invalid space consumed", QUOTA_SIZE - (59 * FILE_SIZE * 3), - 3 * (fs.getDefaultBlockSize() - FILE_SIZE)); + 3 * (fs.getDefaultBlockSize(dir) - FILE_SIZE)); // Now check that trying to create another file violates the quota try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java index 3823822d0c..843f422448 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermission.java @@ -123,9 +123,10 @@ public void testCreate() throws Exception { checkPermission(fs, "/aa/1/aa/2/aa/3", dirPerm); FsPermission filePerm = new FsPermission((short)0444); - FSDataOutputStream out = fs.create(new Path("/b1/b2/b3.txt"), filePerm, + Path p = new Path("/b1/b2/b3.txt"); + FSDataOutputStream out = fs.create(p, filePerm, true, conf.getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), - fs.getDefaultReplication(), fs.getDefaultBlockSize(), null); + fs.getDefaultReplication(p), fs.getDefaultBlockSize(p), null); out.write(123); out.close(); checkPermission(fs, "/b1", inheritPerm);