From 30eceec3420fc6be00d3878ba787bd9518d3ca0e Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Mon, 10 Sep 2018 09:10:51 +0530 Subject: [PATCH] HDFS-13806. EC: No error message for unsetting EC policy of the directory inherits the erasure coding policy from an ancestor directory. Contributed by Ayush Saxena. --- .../org/apache/hadoop/hdfs/DFSClient.java | 3 +- .../hdfs/protocol/NoECPolicySetException.java | 37 +++++++++++++++++++ .../server/namenode/FSDirErasureCodingOp.java | 4 ++ .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 7 ++++ .../TestUnsetAndChangeDirectoryEcPolicy.java | 23 +++++++++--- .../test/resources/testErasureCodingConf.xml | 24 ++++++++++++ 6 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/NoECPolicySetException.java diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 61568d56ab..f4d11b9fd6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -131,6 +131,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; +import org.apache.hadoop.hdfs.protocol.NoECPolicySetException; import org.apache.hadoop.hdfs.protocol.OpenFileEntry; import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType; @@ -2750,7 +2751,7 @@ public void unsetErasureCodingPolicy(String src) throws IOException { throw re.unwrapRemoteException(AccessControlException.class, SafeModeException.class, UnresolvedPathException.class, - FileNotFoundException.class); + FileNotFoundException.class, NoECPolicySetException.class); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/NoECPolicySetException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/NoECPolicySetException.java new file mode 100644 index 0000000000..de3054a4d6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/NoECPolicySetException.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.protocol; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + *Thrown when no EC policy is set explicitly on the directory. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class NoECPolicySetException extends IOException { + private static final long serialVersionUID = 1L; + + public NoECPolicySetException(String msg) { + super(msg); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 2ba840e8ab..b0bc5e40eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.protocol.NoECPolicySetException; import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.WritableUtils; @@ -206,6 +207,9 @@ static FileStatus unsetErasureCodingPolicy(final FSNamesystem fsn, } if (xAttrs != null) { fsn.getEditLog().logRemoveXAttrs(src, xAttrs, logRetryCache); + } else { + throw new NoECPolicySetException( + "No erasure coding policy explicitly set on " + src); } return fsd.getAuditFileInfo(iip); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index 56d453bb68..903a1e2ef9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; +import org.apache.hadoop.hdfs.protocol.NoECPolicySetException; import org.apache.hadoop.hdfs.util.ECPolicyLoader; import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import org.apache.hadoop.tools.TableListing; @@ -424,6 +425,12 @@ public int run(Configuration conf, List args) throws IOException { "non-empty directory will not automatically convert existing" + " files to replicated data."); } + } catch (NoECPolicySetException e) { + System.err.println(AdminHelper.prettifyException(e)); + System.err.println("Use '-setPolicy -path -replicate' to enforce" + + " default replication policy irrespective of EC policy" + + " defined on parent."); + return 2; } catch (Exception e) { System.err.println(AdminHelper.prettifyException(e)); return 2; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java index aa146945d2..4f33ce7596 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.NoECPolicySetException; import org.apache.hadoop.io.erasurecode.CodecUtil; import org.apache.hadoop.io.erasurecode.ErasureCodeNative; import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory; @@ -98,7 +99,11 @@ public void testUnsetEcPolicy() throws Exception { fs.mkdirs(dirPath); // Test unset a directory which has no EC policy - fs.unsetErasureCodingPolicy(dirPath); + try { + fs.unsetErasureCodingPolicy(dirPath); + fail(); + } catch (NoECPolicySetException e) { + } // Set EC policy on directory fs.setErasureCodingPolicy(dirPath, ecPolicy.getName()); @@ -126,8 +131,8 @@ public void testUnsetEcPolicy() throws Exception { } /* - * Test nested directory with different EC policy. - */ + * Test nested directory with different EC policy. + */ @Test public void testNestedEcPolicy() throws Exception { final int numBlocks = 1; @@ -199,7 +204,11 @@ public void testUnsetRootDirEcPolicy() throws Exception { final Path replicateFilePath = new Path(rootPath, "rep_file"); // Test unset root path which has no EC policy - fs.unsetErasureCodingPolicy(rootPath); + try { + fs.unsetErasureCodingPolicy(rootPath); + fail(); + } catch (NoECPolicySetException e) { + } // Set EC policy on root path fs.setErasureCodingPolicy(rootPath, ecPolicy.getName()); DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L); @@ -238,7 +247,11 @@ public void testChangeRootDirEcPolicy() throws Exception { final ErasureCodingPolicy ec32Policy = SystemErasureCodingPolicies .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID); - fs.unsetErasureCodingPolicy(rootPath); + try { + fs.unsetErasureCodingPolicy(rootPath); + fail(); + } catch (NoECPolicySetException e) { + } fs.setErasureCodingPolicy(rootPath, ecPolicy.getName()); // Create RS(6,3) EC policy file DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml index b47d50f37f..34f5176439 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -359,6 +359,30 @@ + + unsetPolicy : unset on non EC directory + + -fs NAMENODE -mkdir /ecdir + -fs NAMENODE -mkdir /ecdir/child + -fs NAMENODE -unsetPolicy -path /ecdir/child + + + -fs NAMENODE -rm /ecdir/child/ecfile + -fs NAMENODE -rmdir /ecdir/child + -fs NAMENODE -rmdir /ecdir + + + + SubstringComparator + NoECPolicySetException: No erasure coding policy explicitly set on /ecdir/child + + + SubstringComparator + Use '-setPolicy -path <PATH> -replicate' to enforce default replication policy irrespective of EC policy defined on parent. + + + + unsetPolicy : unset policy on non-empty directory