From b91305119b434d23b99ae7e755aea6639f48b6ab Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 3 Oct 2017 11:23:40 -0700 Subject: [PATCH] HDFS-11968. ViewFS: StoragePolicies commands fail with HDFS federation. Contributed by Mukul Kumar Singh. --- .../apache/hadoop/fs/viewfs/InodeTree.java | 3 +- .../hadoop/hdfs/tools/StoragePolicyAdmin.java | 61 ++++++++------ .../hdfs/tools/TestStoragePolicyCommands.java | 11 +-- .../TestViewFSStoragePolicyCommands.java | 80 +++++++++++++++++++ .../TestWebHDFSStoragePolicyCommands.java | 42 ++++++++++ 5 files changed, 168 insertions(+), 29 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestViewFSStoragePolicyCommands.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestWebHDFSStoragePolicyCommands.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java index 665c9c9acb..7274acd71a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java @@ -407,7 +407,8 @@ ResolveResult resolve(final String p, final boolean resolveLastComponent) for (int j = 1; j <= i; ++j) { failedAt.append('/').append(path[j]); } - throw (new FileNotFoundException(failedAt.toString())); + throw (new FileNotFoundException("File/Directory does not exist: " + + failedAt.toString())); } if (nextInode instanceof INodeLink) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java index 9c7d048701..d5e5b4d7b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java @@ -19,8 +19,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.BlockStoragePolicySpi; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -29,6 +31,7 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.Arrays; import java.util.Collection; @@ -100,11 +103,12 @@ public String getLongUsage() { @Override public int run(Configuration conf, List args) throws IOException { - final DistributedFileSystem dfs = AdminHelper.getDFS(conf); + final FileSystem fs = FileSystem.get(conf); try { - Collection policies = dfs.getAllStoragePolicies(); + Collection policies = + fs.getAllStoragePolicies(); System.out.println("Block Storage Policies:"); - for (BlockStoragePolicy policy : policies) { + for (BlockStoragePolicySpi policy : policies) { if (policy != null) { System.out.println("\t" + policy); } @@ -149,32 +153,43 @@ public int run(Configuration conf, List args) throws IOException { } Path p = new Path(path); - final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf); + final FileSystem fs = FileSystem.get(conf); try { - HdfsFileStatus status = dfs.getClient().getFileInfo( - Path.getPathWithoutSchemeAndAuthority(p).toString()); - if (status == null) { + FileStatus status; + try { + status = fs.getFileStatus(p); + } catch (FileNotFoundException e) { System.err.println("File/Directory does not exist: " + path); return 2; } - byte storagePolicyId = status.getStoragePolicy(); - if (storagePolicyId == HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) { - System.out.println("The storage policy of " + path + " is unspecified"); - return 0; - } - Collection policies = dfs.getAllStoragePolicies(); - for (BlockStoragePolicy policy : policies) { - if (policy.getId() == storagePolicyId) { - System.out.println("The storage policy of " + path + ":\n" + policy); + + if (status instanceof HdfsFileStatus) { + byte storagePolicyId = ((HdfsFileStatus)status).getStoragePolicy(); + if (storagePolicyId == + HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) { + System.out.println("The storage policy of " + path + + " is unspecified"); return 0; } + Collection policies = + fs.getAllStoragePolicies(); + for (BlockStoragePolicySpi policy : policies) { + if (policy instanceof BlockStoragePolicy) { + if (((BlockStoragePolicy)policy).getId() == storagePolicyId) { + System.out.println("The storage policy of " + path + + ":\n" + policy); + return 0; + } + } + } } + System.err.println(getName() + " is not supported for filesystem " + + fs.getScheme() + " on path " + path); + return 2; } catch (Exception e) { System.err.println(AdminHelper.prettifyException(e)); return 2; } - System.err.println("Cannot identify the storage policy for " + path); - return 2; } } @@ -218,9 +233,9 @@ public int run(Configuration conf, List args) throws IOException { return 1; } Path p = new Path(path); - final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf); + final FileSystem fs = FileSystem.get(conf); try { - dfs.setStoragePolicy(p, policyName); + fs.setStoragePolicy(p, policyName); System.out.println("Set storage policy " + policyName + " on " + path); } catch (Exception e) { System.err.println(AdminHelper.prettifyException(e)); @@ -264,9 +279,9 @@ public int run(Configuration conf, List args) throws IOException { } Path p = new Path(path); - final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf); + final FileSystem fs = FileSystem.get(conf); try { - dfs.unsetStoragePolicy(p); + fs.unsetStoragePolicy(p); System.out.println("Unset storage policy from " + path); } catch (Exception e) { System.err.println(AdminHelper.prettifyException(e)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java index 149dabb06f..f31c739950 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestStoragePolicyCommands.java @@ -18,11 +18,12 @@ package org.apache.hadoop.hdfs.tools; import java.io.IOException; +import java.net.URISyntaxException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; @@ -38,12 +39,12 @@ public class TestStoragePolicyCommands { private static final short REPL = 1; private static final int SIZE = 128; - private static Configuration conf; - private static MiniDFSCluster cluster; - private static DistributedFileSystem fs; + protected static Configuration conf; + protected static MiniDFSCluster cluster; + protected static FileSystem fs; @Before - public void clusterSetUp() throws IOException { + public void clusterSetUp() throws IOException, URISyntaxException { conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPL).build(); cluster.waitActive(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestViewFSStoragePolicyCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestViewFSStoragePolicyCommands.java new file mode 100644 index 0000000000..b3bb3c4373 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestViewFSStoragePolicyCommands.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FsConstants; + +import org.apache.hadoop.fs.viewfs.ConfigUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.MiniDFSNNTopology; + +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +/** + * Test StoragePolicyAdmin commands with ViewFileSystem. + */ +public class TestViewFSStoragePolicyCommands extends TestStoragePolicyCommands { + + @Before + public void clusterSetUp() throws IOException { + conf = new HdfsConfiguration(); + String clusterName = "cluster"; + cluster = + new MiniDFSCluster.Builder(conf).nnTopology( + MiniDFSNNTopology.simpleFederatedTopology(2)) + .numDataNodes(2) + .build(); + cluster.waitActive(); + DistributedFileSystem hdfs1 = cluster.getFileSystem(0); + DistributedFileSystem hdfs2 = cluster.getFileSystem(1); + + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, + FsConstants.VIEWFS_SCHEME +"://" + clusterName); + + Path base1 = new Path("/user1"); + Path base2 = new Path("/user2"); + hdfs1.delete(base1, true); + hdfs2.delete(base2, true); + hdfs1.mkdirs(base1); + hdfs2.mkdirs(base2); + ConfigUtil.addLink(conf, clusterName, "/foo", + hdfs1.makeQualified(base1).toUri()); + ConfigUtil.addLink(conf, clusterName, "/hdfs2", + hdfs2.makeQualified(base2).toUri()); + fs = FileSystem.get(conf); + } + + /** + * Storage policy operation on the viewfs root should fail. + */ + @Test + public void testStoragePolicyRoot() throws Exception { + final StoragePolicyAdmin admin = new StoragePolicyAdmin(conf); + DFSTestUtil.toolRun(admin, "-getStoragePolicy -path /", 2, + "is not supported for filesystem viewfs on path /"); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestWebHDFSStoragePolicyCommands.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestWebHDFSStoragePolicyCommands.java new file mode 100644 index 0000000000..f10205cfc9 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestWebHDFSStoragePolicyCommands.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools; + +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.hdfs.web.WebHdfsConstants; +import org.apache.hadoop.hdfs.web.WebHdfsTestUtil; +import org.junit.Before; + +import java.io.IOException; +import java.net.URISyntaxException; + +/** + * Test StoragePolicyAdmin commands with WebHDFS. + */ +public class TestWebHDFSStoragePolicyCommands + extends TestStoragePolicyCommands { + + @Before + public void clusterSetUp() throws IOException, URISyntaxException { + super.clusterSetUp(); + fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, + WebHdfsConstants.WEBHDFS_SCHEME); + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, + fs.getUri().toString()); + } +}