diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index d114f0f055..1f6022ca35 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -2783,6 +2783,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, namenode.removeErasureCodingPolicy(ecPolicyName); } + public void enableErasureCodingPolicy(String ecPolicyName) + throws IOException { + checkOpen(); + namenode.enableErasureCodingPolicy(ecPolicyName); + } + + public void disableErasureCodingPolicy(String ecPolicyName) + throws IOException { + checkOpen(); + namenode.disableErasureCodingPolicy(ecPolicyName); + } + public DFSInotifyEventInputStream getInotifyEventStream() throws IOException { checkOpen(); return new DFSInotifyEventInputStream(namenode, tracer); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index f8af4abff4..34c631a66b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -2618,6 +2618,28 @@ public class DistributedFileSystem extends FileSystem { dfs.removeErasureCodingPolicy(ecPolicyName); } + /** + * Enable erasure coding policy. + * + * @param ecPolicyName The name of the policy to be enabled. + * @throws IOException + */ + public void enableErasureCodingPolicy(String ecPolicyName) + throws IOException { + dfs.enableErasureCodingPolicy(ecPolicyName); + } + + /** + * Disable erasure coding policy. + * + * @param ecPolicyName The name of the policy to be disabled. + * @throws IOException + */ + public void disableErasureCodingPolicy(String ecPolicyName) + throws IOException { + dfs.disableErasureCodingPolicy(ecPolicyName); + } + /** * Unset the erasure coding policy from the source path. * diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java index 701bf0fa5b..abf341ef3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java @@ -548,6 +548,28 @@ public class HdfsAdmin { dfs.removeErasureCodingPolicy(ecPolicyName); } + /** + * Enable erasure coding policy. + * + * @param ecPolicyName The name of the policy to be enabled. + * @throws IOException + */ + public void enableErasureCodingPolicy(String ecPolicyName) + throws IOException { + dfs.enableErasureCodingPolicy(ecPolicyName); + } + + /** + * Disable erasure coding policy. + * + * @param ecPolicyName The name of the policy to be disabled. + * @throws IOException + */ + public void disableErasureCodingPolicy(String ecPolicyName) + throws IOException { + dfs.disableErasureCodingPolicy(ecPolicyName); + } + private void provisionEZTrash(Path path) throws IOException { // make sure the path is an EZ EncryptionZone ez = dfs.getEZForPath(path); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 313f322794..45c6b3269b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -1570,6 +1570,23 @@ public interface ClientProtocol { @AtMostOnce void removeErasureCodingPolicy(String ecPolicyName) throws IOException; + /** + * Enable erasure coding policy. + * @param ecPolicyName The name of the policy to be enabled. + * @throws IOException + */ + @AtMostOnce + void enableErasureCodingPolicy(String ecPolicyName) throws IOException; + + /** + * Disable erasure coding policy. + * @param ecPolicyName The name of the policy to be disabled. + * @throws IOException + */ + @AtMostOnce + void disableErasureCodingPolicy(String ecPolicyName) throws IOException; + + /** * Get the erasure coding policies loaded in Namenode. * diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index d913f81602..388788c89b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -186,6 +186,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodin import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.RemoveErasureCodingPolicyRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.EnableErasureCodingPolicyRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.DisableErasureCodingPolicyRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingCodecsRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingCodecsResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyRequestProto; @@ -1708,6 +1710,34 @@ public class ClientNamenodeProtocolTranslatorPB implements } } + @Override + public void enableErasureCodingPolicy(String ecPolicyName) + throws IOException { + EnableErasureCodingPolicyRequestProto.Builder builder = + EnableErasureCodingPolicyRequestProto.newBuilder(); + builder.setEcPolicyName(ecPolicyName); + EnableErasureCodingPolicyRequestProto req = builder.build(); + try { + rpcProxy.enableErasureCodingPolicy(null, req); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + + @Override + public void disableErasureCodingPolicy(String ecPolicyName) + throws IOException { + DisableErasureCodingPolicyRequestProto.Builder builder = + DisableErasureCodingPolicyRequestProto.newBuilder(); + builder.setEcPolicyName(ecPolicyName); + DisableErasureCodingPolicyRequestProto req = builder.build(); + try { + rpcProxy.disableErasureCodingPolicy(null, req); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + @Override public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException { try { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto index d3e15d054a..4f44c5e7dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto @@ -957,6 +957,10 @@ service ClientNamenodeProtocol { returns(AddErasureCodingPoliciesResponseProto); rpc removeErasureCodingPolicy(RemoveErasureCodingPolicyRequestProto) returns(RemoveErasureCodingPolicyResponseProto); + rpc enableErasureCodingPolicy(EnableErasureCodingPolicyRequestProto) + returns(EnableErasureCodingPolicyResponseProto); + rpc disableErasureCodingPolicy(DisableErasureCodingPolicyRequestProto) + returns(DisableErasureCodingPolicyResponseProto); rpc getErasureCodingPolicy(GetErasureCodingPolicyRequestProto) returns(GetErasureCodingPolicyResponseProto); rpc getErasureCodingCodecs(GetErasureCodingCodecsRequestProto) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto index ed0fd8139e..65baab65a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto @@ -68,6 +68,20 @@ message RemoveErasureCodingPolicyRequestProto { message RemoveErasureCodingPolicyResponseProto { } +message EnableErasureCodingPolicyRequestProto { + required string ecPolicyName = 1; +} + +message EnableErasureCodingPolicyResponseProto { +} + +message DisableErasureCodingPolicyRequestProto { + required string ecPolicyName = 1; +} + +message DisableErasureCodingPolicyResponseProto { +} + message UnsetErasureCodingPolicyRequestProto { required string src = 1; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 16adc709e0..4ac49fe12f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -230,6 +230,10 @@ import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodin import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.RemoveErasureCodingPolicyRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.RemoveErasureCodingPolicyResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.EnableErasureCodingPolicyRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.EnableErasureCodingPolicyResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.DisableErasureCodingPolicyRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.DisableErasureCodingPolicyResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.UnsetErasureCodingPolicyRequestProto; @@ -1707,6 +1711,30 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements } } + @Override + public EnableErasureCodingPolicyResponseProto enableErasureCodingPolicy( + RpcController controller, EnableErasureCodingPolicyRequestProto request) + throws ServiceException { + try { + server.enableErasureCodingPolicy(request.getEcPolicyName()); + return EnableErasureCodingPolicyResponseProto.newBuilder().build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public DisableErasureCodingPolicyResponseProto disableErasureCodingPolicy( + RpcController controller, DisableErasureCodingPolicyRequestProto request) + throws ServiceException { + try { + server.disableErasureCodingPolicy(request.getEcPolicyName()); + return DisableErasureCodingPolicyResponseProto.newBuilder().build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + @Override public GetErasureCodingPolicyResponseProto getErasureCodingPolicy(RpcController controller, GetErasureCodingPolicyRequestProto request) throws ServiceException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index d22e47e5a0..266d45cdc0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -212,6 +212,10 @@ public final class ErasureCodingPolicyManager { // This is a placeholder for HDFS-7337. } + /** + * Add an erasure coding policy. + * @return the added policy + */ public synchronized ErasureCodingPolicy addPolicy(ErasureCodingPolicy policy) throws IllegalECPolicyException { if (!CodecUtil.hasCodec(policy.getCodecName())) { @@ -251,6 +255,9 @@ public final class ErasureCodingPolicyManager { return (byte) (currentId + 1); } + /** + * Remove an User erasure coding policy by policyName. + */ public synchronized void removePolicy(String name) { if (SystemErasureCodingPolicies.getByName(name) != null) { throw new IllegalArgumentException("System erasure coding policy " + @@ -268,4 +275,53 @@ public final class ErasureCodingPolicyManager { public List getRemovedPolicies() { return removedPoliciesByName.values().stream().collect(Collectors.toList()); } + + /** + * Disable an erasure coding policy by policyName. + */ + public synchronized void disablePolicy(String name) { + ErasureCodingPolicy sysEcPolicy = SystemErasureCodingPolicies + .getByName(name); + ErasureCodingPolicy userEcPolicy = userPoliciesByName.get(name); + LOG.info("Disable the erasure coding policy " + name); + if (sysEcPolicy == null && + userEcPolicy == null) { + throw new IllegalArgumentException("The policy name " + + name + " does not exists"); + } + + if(sysEcPolicy != null){ + enabledPoliciesByName.remove(name); + removedPoliciesByName.put(name, sysEcPolicy); + } + if(userEcPolicy != null){ + enabledPoliciesByName.remove(name); + removedPoliciesByName.put(name, userEcPolicy); + } + } + + /** + * Enable an erasure coding policy by policyName. + */ + public synchronized void enablePolicy(String name) { + ErasureCodingPolicy sysEcPolicy = SystemErasureCodingPolicies + .getByName(name); + ErasureCodingPolicy userEcPolicy = userPoliciesByName.get(name); + LOG.info("Enable the erasure coding policy " + name); + if (sysEcPolicy == null && + userEcPolicy == null) { + throw new IllegalArgumentException("The policy name " + + name + " does not exists"); + } + + if(sysEcPolicy != null){ + enabledPoliciesByName.put(name, sysEcPolicy); + removedPoliciesByName.remove(name); + } + if(userEcPolicy != null) { + enabledPoliciesByName.put(name, userEcPolicy); + removedPoliciesByName.remove(name); + } + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index a0402628e4..681f217d40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -231,6 +231,18 @@ final class FSDirErasureCodingOp { fsn.getErasureCodingPolicyManager().removePolicy(ecPolicyName); } + static void enableErasureCodePolicy(final FSNamesystem fsn, + String ecPolicyName) throws IOException { + Preconditions.checkNotNull(ecPolicyName); + fsn.getErasureCodingPolicyManager().enablePolicy(ecPolicyName); + } + + static void disableErasureCodePolicy(final FSNamesystem fsn, + String ecPolicyName) throws IOException { + Preconditions.checkNotNull(ecPolicyName); + fsn.getErasureCodingPolicyManager().disablePolicy(ecPolicyName); + } + private static List removeErasureCodingPolicyXAttr( final FSNamesystem fsn, final INodesInPath srcIIP) throws IOException { FSDirectory fsd = fsn.getFSDirectory(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index f7acb553f2..9872cd7720 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -7060,6 +7060,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, AddECPolicyResponse[] addECPolicies(ErasureCodingPolicy[] policies) throws IOException { final String operationName = "addECPolicies"; + String addECPolicyName = ""; checkOperation(OperationCategory.WRITE); List responses = new ArrayList<>(); boolean success = false; @@ -7070,6 +7071,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, try { ErasureCodingPolicy newPolicy = FSDirErasureCodingOp.addErasureCodePolicy(this, policy); + addECPolicyName = newPolicy.getName(); responses.add(new AddECPolicyResponse(newPolicy)); } catch (IllegalECPolicyException e) { responses.add(new AddECPolicyResponse(policy, e)); @@ -7082,7 +7084,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, if (success) { getEditLog().logSync(); } - logAuditEvent(success, operationName, null, null, null); + logAuditEvent(success, operationName, addECPolicyName, null, null); } } @@ -7108,6 +7110,58 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, } } + /** + * Enable an erasure coding policy. + * @param ecPolicyName the name of the policy to be enabled + * @throws IOException + */ + void enableErasureCodingPolicy(String ecPolicyName) throws IOException { + final String operationName = "enableErasureCodingPolicy"; + checkOperation(OperationCategory.WRITE); + boolean success = false; + LOG.info("Enable the erasure coding policy " + ecPolicyName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); + checkNameNodeSafeMode("Cannot enable erasure coding policy " + + ecPolicyName); + FSDirErasureCodingOp.enableErasureCodePolicy(this, ecPolicyName); + success = true; + } finally { + writeUnlock(operationName); + if (success) { + getEditLog().logSync(); + } + logAuditEvent(success, operationName, ecPolicyName, null, null); + } + } + + /** + * Disable an erasure coding policy. + * @param ecPolicyName the name of the policy to be disabled + * @throws IOException + */ + void disableErasureCodingPolicy(String ecPolicyName) throws IOException { + final String operationName = "disableErasureCodingPolicy"; + checkOperation(OperationCategory.WRITE); + boolean success = false; + LOG.info("Disable the erasure coding policy " + ecPolicyName); + writeLock(); + try { + checkOperation(OperationCategory.WRITE); + checkNameNodeSafeMode("Cannot disable erasure coding policy " + + ecPolicyName); + FSDirErasureCodingOp.disableErasureCodePolicy(this, ecPolicyName); + success = true; + } finally { + writeUnlock(operationName); + if (success) { + getEditLog().logSync(); + } + logAuditEvent(success, operationName, ecPolicyName, null, null); + } + } + /** * Unset an erasure coding policy from the given path. * @param srcArg The path of the target directory. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index fd5a05be46..39d93dff96 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -2307,6 +2307,22 @@ public class NameNodeRpcServer implements NamenodeProtocols { namesystem.removeErasureCodingPolicy(ecPolicyName); } + @Override // ClientProtocol + public void enableErasureCodingPolicy(String ecPolicyName) + throws IOException { + checkNNStartup(); + namesystem.checkSuperuserPrivilege(); + namesystem.enableErasureCodingPolicy(ecPolicyName); + } + + @Override // ClientProtocol + public void disableErasureCodingPolicy(String ecPolicyName) + throws IOException { + checkNNStartup(); + namesystem.checkSuperuserPrivilege(); + namesystem.disableErasureCodingPolicy(ecPolicyName); + } + @Override // ReconfigurationProtocol public void startReconfiguration() throws IOException { checkNNStartup(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index a983056216..5006b5a20e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -463,6 +463,101 @@ public class ECAdmin extends Configured implements Tool { } } + /** Command to enable an existing erasure coding policy. */ + private static class EnableECPolicyCommand implements AdminHelper.Command { + @Override + public String getName() { + return "-enablePolicy"; + } + + @Override + public String getShortUsage() { + return "[" + getName() + " -policy ]\n"; + } + + @Override + public String getLongUsage() { + TableListing listing = AdminHelper.getOptionDescriptionListing(); + listing.addRow("", "The name of the erasure coding policy"); + return getShortUsage() + "\n" + + "Enable the erasure coding policy.\n\n" + + listing.toString(); + } + + @Override + public int run(Configuration conf, List args) throws IOException { + final String ecPolicyName = StringUtils.popOptionWithArgument("-policy", + args); + if (ecPolicyName == null) { + System.err.println("Please specify the policy name.\nUsage: " + + getLongUsage()); + return 1; + } + if (args.size() > 0) { + System.err.println(getName() + ": Too many arguments"); + return 1; + } + + final DistributedFileSystem dfs = AdminHelper.getDFS(conf); + try { + dfs.enableErasureCodingPolicy(ecPolicyName); + System.out.println("Erasure coding policy " + ecPolicyName + + " is enabled"); + } catch (IOException e) { + System.err.println(AdminHelper.prettifyException(e)); + return 2; + } + return 0; + } + } + + /** Command to disable an existing erasure coding policy. */ + private static class DisableECPolicyCommand implements AdminHelper.Command { + @Override + public String getName() { + return "-disablePolicy"; + } + + @Override + public String getShortUsage() { + return "[" + getName() + " -policy ]\n"; + } + + @Override + public String getLongUsage() { + TableListing listing = AdminHelper.getOptionDescriptionListing(); + listing.addRow("", "The name of the erasure coding policy"); + return getShortUsage() + "\n" + + "Disable the erasure coding policy.\n\n" + + listing.toString(); + } + + @Override + public int run(Configuration conf, List args) throws IOException { + final String ecPolicyName = StringUtils.popOptionWithArgument("-policy", + args); + if (ecPolicyName == null) { + System.err.println("Please specify the policy name.\nUsage: " + + getLongUsage()); + return 1; + } + if (args.size() > 0) { + System.err.println(getName() + ": Too many arguments"); + return 1; + } + + final DistributedFileSystem dfs = AdminHelper.getDFS(conf); + try { + dfs.disableErasureCodingPolicy(ecPolicyName); + System.out.println("Erasure coding policy " + ecPolicyName + + " is disabled"); + } catch (IOException e) { + System.err.println(AdminHelper.prettifyException(e)); + return 2; + } + return 0; + } + } private static final AdminHelper.Command[] COMMANDS = { new ListECPoliciesCommand(), @@ -471,6 +566,8 @@ public class ECAdmin extends Configured implements Tool { new RemoveECPolicyCommand(), new SetECPolicyCommand(), new UnsetECPolicyCommand(), - new ListECCodecsCommand() + new ListECCodecsCommand(), + new EnableECPolicyCommand(), + new DisableECPolicyCommand() }; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index 38bdc40fab..5903a36e84 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -446,6 +446,8 @@ Usage: [-listPolicies] [-addPolicies -policyFile ] [-listCodecs] + [-enablePolicy -policy ] + [-disablePolicy -policy ] [-help [cmd ...]] | COMMAND\_OPTION | Description | @@ -456,6 +458,8 @@ Usage: |-listPolicies| Lists all supported ErasureCoding policies| |-addPolicies| Add a list of erasure coding policies| |-listCodecs| Get the list of supported erasure coding codecs and coders in system| +|-enablePolicy| Enable an ErasureCoding policy in system| +|-disablePolicy| Disable an ErasureCoding policy in system| Runs the ErasureCoding CLI. See [HDFS ErasureCoding](./HDFSErasureCoding.html#Administrative_commands) for more information on this command. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 4d0b2e6560..69e8ef280e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -156,6 +156,8 @@ Deployment [-listPolicies] [-addPolicies -policyFile ] [-listCodecs] + [-enablePolicy -policy ] + [-disablePolicy -policy ] [-help [cmd ...]] Below are the details about each command. @@ -190,4 +192,12 @@ Below are the details about each command. * `[-removePolicy -policy ]` - Remove an erasure coding policy. \ No newline at end of file + Remove an erasure coding policy. + +* `[-enablePolicy -policy ]` + + Enable an erasure coding policy. + +* `[-disablePolicy -policy ]` + + Disable an erasure coding policy. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 1d24f5242b..b35d374786 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -1567,4 +1567,52 @@ public class TestDistributedFileSystem { } } } + + @Test + public void testEnableAndDisableErasureCodingPolicy() throws Exception { + Configuration conf = getTestConfiguration(); + MiniDFSCluster cluster = null; + + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + DistributedFileSystem fs = cluster.getFileSystem(); + ECSchema toAddSchema = new ECSchema("rs", 3, 2); + ErasureCodingPolicy toAddPolicy = + new ErasureCodingPolicy(toAddSchema, 128 * 1024, (byte) 254); + String policyName = toAddPolicy.getName(); + ErasureCodingPolicy[] policies = + new ErasureCodingPolicy[]{toAddPolicy}; + fs.addErasureCodingPolicies(policies); + assertEquals(policyName, ErasureCodingPolicyManager.getInstance(). + getByName(policyName).getName()); + fs.disableErasureCodingPolicy(policyName); + assertEquals(policyName, ErasureCodingPolicyManager.getInstance(). + getRemovedPolicies().get(0).getName()); + fs.enableErasureCodingPolicy(policyName); + assertEquals(policyName, ErasureCodingPolicyManager.getInstance(). + getByName(policyName).getName()); + + //test enable a policy that doesn't exist + try { + fs.enableErasureCodingPolicy("notExistECName"); + Assert.fail("enable the policy that doesn't exist should fail"); + } catch (Exception e) { + GenericTestUtils.assertExceptionContains("does not exists", e); + // pass + } + + //test disable a policy that doesn't exist + try { + fs.disableErasureCodingPolicy("notExistECName"); + Assert.fail("disable the policy that doesn't exist should fail"); + } catch (Exception e) { + GenericTestUtils.assertExceptionContains("does not exists", e); + // pass + } + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml index 26acc1f5f7..791e685cea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -163,6 +163,44 @@ + + help: enablePolicy command + + -fs NAMENODE -help enablePolicy + + + + + + SubstringComparator + Enable the erasure coding policy + + + SubstringComparator + [-enablePolicy -policy <policy>] + + + + + + help: disablePolicy command + + -fs NAMENODE -help disablePolicy + + + + + + SubstringComparator + Disable the erasure coding policy + + + SubstringComparator + [-disablePolicy -policy <policy>] + + + + setPolicy : set erasure coding policy on a directory to encode files @@ -349,6 +387,70 @@ + + enablePolicy : enable the erasure coding policy + + -fs NAMENODE -enablePolicy -policy RS-6-3-64k + + + + + + SubstringComparator + Erasure coding policy RS-6-3-64k is enabled + + + + + + enablePolicy : enable the erasure coding policy twice + + -fs NAMENODE -enablePolicy -policy RS-6-3-64k + -fs NAMENODE -enablePolicy -policy RS-6-3-64k + + + + + + SubstringComparator + Erasure coding policy RS-6-3-64k is enabled + + + + + + disablePolicy : disable the erasure coding policy + + -fs NAMENODE -disablePolicy -policy RS-6-3-64k + + + -fs NAMENODE -enablePolicy -policy RS-6-3-64k + + + + SubstringComparator + Erasure coding policy RS-6-3-64k is disabled + + + + + + disablePolicy : disable the erasure coding policy twice + + -fs NAMENODE -disablePolicy -policy RS-6-3-64k + -fs NAMENODE -disablePolicy -policy RS-6-3-64k + + + -fs NAMENODE -enablePolicy -policy RS-6-3-64k + + + + SubstringComparator + Erasure coding policy RS-6-3-64k is disabled + + + + setPolicy : illegal parameters - path is missing @@ -541,6 +643,66 @@ + + enablePolicy : illegal parameters - policy is missing + + -fs NAMENODE -enablePolicy RS-6-3-64k + + + + + + SubstringComparator + Please specify the policy name + + + + + + enablePolicy : illegal parameters - too many parameters + + -fs NAMENODE -enablePolicy -policy RS-6-3-64k RS-3-2-64k + + + + + + SubstringComparator + -enablePolicy: Too many arguments + + + + + + disablePolicy : illegal parameters - policy is missing + + -fs NAMENODE -disablePolicy RS-6-3-64k + + + + + + SubstringComparator + Please specify the policy name + + + + + + disablePolicy : illegal parameters - too many parameters + + -fs NAMENODE -disablePolicy -policy RS-6-3-64k RS-3-2-64k + + + + + + SubstringComparator + -disablePolicy: Too many arguments + + + + listCodecs : illegal parameters - too many parameters