big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11606. Add CLI cmd to remove an erasure code policy. Contributed by Tim Yao

Browse Source
This commit is contained in:
Kai Zheng 2017-06-21 14:01:55 +08:00
parent a010b330e7
commit 5db3f98468
15 changed files with 242 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
View File

@ -2777,6 +2777,12 @@ public AddECPolicyResponse[] addErasureCodingPolicies(
return namenode.addErasureCodingPolicies(policies);
}
public void removeErasureCodingPolicy(String ecPolicyName)
throws IOException {
checkOpen();
namenode.removeErasureCodingPolicy(ecPolicyName);
}
public DFSInotifyEventInputStream getInotifyEventStream() throws IOException {
checkOpen();
return new DFSInotifyEventInputStream(namenode, tracer);

11
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -2607,6 +2607,17 @@ public AddECPolicyResponse[] addErasureCodingPolicies(
return dfs.addErasureCodingPolicies(policies);
}
/**
* Remove erasure coding policy.
*
* @param ecPolicyName The name of the policy to be removed.
* @throws IOException
*/
public void removeErasureCodingPolicy(String ecPolicyName)
throws IOException {
dfs.removeErasureCodingPolicy(ecPolicyName);
}
/**
* Unset the erasure coding policy from the source path.
*

29
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
View File

@ -41,15 +41,16 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSInotifyEventInputStream;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
/**
* The public API for performing administrative functions on HDFS. Those writing
@ -521,6 +522,32 @@ public void unsetErasureCodingPolicy(final Path path) throws IOException {
dfs.unsetErasureCodingPolicy(path);
}
/**
* Add Erasure coding policies to HDFS. For each policy input, schema and
* cellSize are musts, name and id are ignored. They will be automatically
* created and assigned by Namenode once the policy is successfully added, and
* will be returned in the response.
*
* @param policies The user defined ec policy list to add.
* @return Return the response list of adding operations.
* @throws IOException
*/
public AddECPolicyResponse[] addErasureCodingPolicies(
ErasureCodingPolicy[] policies) throws IOException {
return dfs.addErasureCodingPolicies(policies);
}
/**
* Remove erasure coding policy.
*
* @param ecPolicyName The name of the policy to be removed.
* @throws IOException
*/
public void removeErasureCodingPolicy(String ecPolicyName)
throws IOException {
dfs.removeErasureCodingPolicy(ecPolicyName);
}
private void provisionEZTrash(Path path) throws IOException {
// make sure the path is an EZ
EncryptionZone ez = dfs.getEZForPath(path);

8
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
View File

@ -1562,6 +1562,14 @@ void setErasureCodingPolicy(String src, String ecPolicyName)
AddECPolicyResponse[] addErasureCodingPolicies(
ErasureCodingPolicy[] policies) throws IOException;
/**
* Remove erasure coding policy.
* @param ecPolicyName The name of the policy to be removed.
* @throws IOException
*/
@AtMostOnce
void removeErasureCodingPolicy(String ecPolicyName) throws IOException;
/**
* Get the erasure coding policies loaded in Namenode.
*

15
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
View File

@ -185,6 +185,7 @@
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPoliciesResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.RemoveErasureCodingPolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingCodecsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingCodecsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyRequestProto;
@ -1693,6 +1694,20 @@ public AddECPolicyResponse[] addErasureCodingPolicies(
}
}
@Override
public void removeErasureCodingPolicy(String ecPolicyName)
throws IOException {
RemoveErasureCodingPolicyRequestProto.Builder builder =
RemoveErasureCodingPolicyRequestProto.newBuilder();
builder.setEcPolicyName(ecPolicyName);
RemoveErasureCodingPolicyRequestProto req = builder.build();
try {
rpcProxy.removeErasureCodingPolicy(null, req);
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
}
@Override
public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException {
try {

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
View File

@ -955,6 +955,8 @@ service ClientNamenodeProtocol {
returns(GetErasureCodingPoliciesResponseProto);
rpc addErasureCodingPolicies(AddErasureCodingPoliciesRequestProto)
returns(AddErasureCodingPoliciesResponseProto);
rpc removeErasureCodingPolicy(RemoveErasureCodingPolicyRequestProto)
returns(RemoveErasureCodingPolicyResponseProto);
rpc getErasureCodingPolicy(GetErasureCodingPolicyRequestProto)
returns(GetErasureCodingPolicyResponseProto);
rpc getErasureCodingCodecs(GetErasureCodingCodecsRequestProto)

7
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
View File

@ -61,6 +61,13 @@ message AddErasureCodingPoliciesResponseProto {
repeated AddECPolicyResponseProto responses = 1;
}
message RemoveErasureCodingPolicyRequestProto {
required string ecPolicyName = 1;
}
message RemoveErasureCodingPolicyResponseProto {
}
message UnsetErasureCodingPolicyRequestProto {
required string src = 1;
}

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
View File

@ -228,6 +228,8 @@
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPoliciesResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.RemoveErasureCodingPolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.RemoveErasureCodingPolicyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.UnsetErasureCodingPolicyRequestProto;
@ -1693,6 +1695,18 @@ public AddErasureCodingPoliciesResponseProto addErasureCodingPolicies(
}
}
@Override
public RemoveErasureCodingPolicyResponseProto removeErasureCodingPolicy(
RpcController controller, RemoveErasureCodingPolicyRequestProto request)
throws ServiceException {
try {
server.removeErasureCodingPolicy(request.getEcPolicyName());
return RemoveErasureCodingPolicyResponseProto.newBuilder().build();
} catch (IOException e) {
throw new ServiceException(e);
}
}
@Override
public GetErasureCodingPolicyResponseProto getErasureCodingPolicy(RpcController controller,
GetErasureCodingPolicyRequestProto request) throws ServiceException {

25
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
View File

@ -29,6 +29,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.stream.Collectors;
@ -67,6 +68,11 @@ public final class ErasureCodingPolicyManager {
*/
private Map<Byte, ErasureCodingPolicy> userPoliciesByID;
/**
* All removed policies sorted by name.
*/
private Map<String, ErasureCodingPolicy> removedPoliciesByName;
/**
* All enabled policies maintained in NN memory for fast querying,
* identified and sorted by its name.
@ -91,6 +97,7 @@ public void init(Configuration conf) {
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_DEFAULT);
this.userPoliciesByID = new TreeMap<>();
this.userPoliciesByName = new TreeMap<>();
this.removedPoliciesByName = new TreeMap<>();
this.enabledPoliciesByName = new TreeMap<>();
for (String policyName : policyNames) {
if (policyName.trim().isEmpty()) {
@ -243,4 +250,22 @@ private byte getNextAvailablePolicyID() {
.max(Byte::compareTo).orElse(USER_DEFINED_POLICY_START_ID);
return (byte) (currentId + 1);
}
public synchronized void removePolicy(String name) {
if (SystemErasureCodingPolicies.getByName(name) != null) {
throw new IllegalArgumentException("System erasure coding policy " +
name + " cannot be removed");
}
ErasureCodingPolicy policy = userPoliciesByName.get(name);
if (policy == null) {
throw new IllegalArgumentException("The policy name " +
name + " does not exists");
}
enabledPoliciesByName.remove(name);
removedPoliciesByName.put(name, policy);
}
public List<ErasureCodingPolicy> getRemovedPolicies() {
return removedPoliciesByName.values().stream().collect(Collectors.toList());
}
}

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
View File

@ -218,6 +218,19 @@ static ErasureCodingPolicy addErasureCodePolicy(final FSNamesystem fsn,
return fsn.getErasureCodingPolicyManager().addPolicy(policy);
}
/**
* Remove an erasure coding policy.
*
* @param fsn namespace
* @param ecPolicyName the name of the policy to be removed
* @throws IOException
*/
static void removeErasureCodePolicy(final FSNamesystem fsn,
String ecPolicyName) throws IOException {
Preconditions.checkNotNull(ecPolicyName);
fsn.getErasureCodingPolicyManager().removePolicy(ecPolicyName);
}
private static List<XAttr> removeErasureCodingPolicyXAttr(
final FSNamesystem fsn, final INodesInPath srcIIP) throws IOException {
FSDirectory fsd = fsn.getFSDirectory();

22
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -7086,6 +7086,28 @@ AddECPolicyResponse[] addECPolicies(ErasureCodingPolicy[] policies)
}
}
/**
* Remove an erasure coding policy.
* @param ecPolicyName the name of the policy to be removed
* @throws IOException
*/
void removeErasureCodingPolicy(String ecPolicyName) throws IOException {
final String operationName = "removeErasureCodingPolicy";
checkOperation(OperationCategory.WRITE);
boolean success = false;
writeLock();
try {
FSDirErasureCodingOp.removeErasureCodePolicy(this, ecPolicyName);
success = true;
} finally {
writeUnlock(operationName);
if (success) {
getEditLog().logSync();
}
logAuditEvent(success, operationName, null, null, null);
}
}
/**
* Unset an erasure coding policy from the given path.
* @param srcArg The path of the target directory.

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
View File

@ -2301,6 +2301,13 @@ public AddECPolicyResponse[] addErasureCodingPolicies(
return namesystem.addECPolicies(policies);
}
@Override
public void removeErasureCodingPolicy(String ecPolicyName)
throws IOException {
checkNNStartup();
namesystem.removeErasureCodingPolicy(ecPolicyName);
}
@Override // ReconfigurationProtocol
public void startReconfiguration() throws IOException {
checkNNStartup();

54
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
View File

@ -199,7 +199,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
}
}
/** Command to get the erasure coding policy for a file or directory */
/** Command to get the erasure coding policy for a file or directory. */
private static class GetECPolicyCommand implements AdminHelper.Command {
@Override
public String getName() {
@ -254,7 +254,54 @@ public int run(Configuration conf, List<String> args) throws IOException {
}
}
/** Command to set the erasure coding policy to a file/directory */
/** Command to remove an erasure coding policy. */
private static class RemoveECPolicyCommand implements AdminHelper.Command {
@Override
public String getName() {
return "-removePolicy";
}
@Override
public String getShortUsage() {
return "[" + getName() + " -policy <policy>]\n";
}
@Override
public String getLongUsage() {
TableListing listing = AdminHelper.getOptionDescriptionListing();
listing.addRow("<policy>", "The name of the erasure coding policy");
return getShortUsage() + "\n" +
"Remove an erasure coding policy.\n" +
listing.toString();
}
@Override
public int run(Configuration conf, List<String> args) throws IOException {
final String ecPolicyName = StringUtils.popOptionWithArgument(
"-policy", args);
if (ecPolicyName == null) {
System.err.println("Please specify the policy name.\nUsage: " +
getLongUsage());
return 1;
}
if (args.size() > 0) {
System.err.println(getName() + ": Too many arguments");
return 1;
}
final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
try {
dfs.removeErasureCodingPolicy(ecPolicyName);
System.out.println("Erasure coding policy " + ecPolicyName +
"is removed");
} catch (IOException e) {
System.err.println(AdminHelper.prettifyException(e));
return 2;
}
return 0;
}
}
/** Command to set the erasure coding policy to a file/directory. */
private static class SetECPolicyCommand implements AdminHelper.Command {
@Override
public String getName() {
@ -313,7 +360,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
}
}
/** Command to unset the erasure coding policy set for a file/directory */
/** Command to unset the erasure coding policy set for a file/directory. */
private static class UnsetECPolicyCommand
implements AdminHelper.Command {
@ -421,6 +468,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
new ListECPoliciesCommand(),
new AddECPoliciesCommand(),
new GetECPolicyCommand(),
new RemoveECPolicyCommand(),
new SetECPolicyCommand(),
new UnsetECPolicyCommand(),
new ListECCodecsCommand()

4
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
View File

@ -187,3 +187,7 @@ Below are the details about each command.
* `[-listCodecs]`
Get the list of supported erasure coding codecs and coders in system. A coder is an implementation of a codec. A codec can have different implementations, thus different coders. The coders for a codec are listed in a fall back order.
* `[-removePolicy -policy <policyName>]`
Remove an erasure coding policy.

29
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
View File

@ -78,6 +78,7 @@
import org.apache.hadoop.hdfs.client.impl.LeaseRenewer;
import org.apache.hadoop.hdfs.DFSOpsCountStatistics.OpType;
import org.apache.hadoop.hdfs.net.Peer;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
@ -85,7 +86,9 @@
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.net.DNSToSwitchMapping;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.ScriptBasedMapping;
@ -1538,4 +1541,30 @@ public void testDFSDataOutputStreamBuilderForAppend() throws IOException {
assertEquals(16 * 2, status.getLen());
}
}
@Test
public void testRemoveErasureCodingPolicy() throws Exception {
Configuration conf = getTestConfiguration();
MiniDFSCluster cluster = null;
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
DistributedFileSystem fs = cluster.getFileSystem();
ECSchema toAddSchema = new ECSchema("rs", 3, 2);
ErasureCodingPolicy toAddPolicy =
new ErasureCodingPolicy(toAddSchema, 128 * 1024, (byte) 254);
String policyName = toAddPolicy.getName();
ErasureCodingPolicy[] policies = new ErasureCodingPolicy[]{toAddPolicy};
fs.addErasureCodingPolicies(policies);
assertEquals(policyName, ErasureCodingPolicyManager.getInstance().
getByName(policyName).getName());
fs.removeErasureCodingPolicy(policyName);
assertEquals(policyName, ErasureCodingPolicyManager.getInstance().
getRemovedPolicies().get(0).getName());
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
}