diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java index d3c3b6bfc9..73b8f560ad 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java @@ -50,6 +50,7 @@ private ErasureCodeConstants() { public static final ECSchema REPLICATION_1_2_SCHEMA = new ECSchema( REPLICATION_CODEC_NAME, 1, 2); + public static final byte MAX_POLICY_ID = Byte.MAX_VALUE; public static final byte USER_DEFINED_POLICY_START_ID = (byte) 64; public static final byte REPLICATION_POLICY_ID = (byte) 63; public static final String REPLICATION_POLICY_NAME = REPLICATION_CODEC_NAME; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index 90699b43a4..62c7f6045a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -253,6 +253,14 @@ public synchronized ErasureCodingPolicy addPolicy( return p; } } + + if (getCurrentMaxPolicyID() == ErasureCodeConstants.MAX_POLICY_ID) { + throw new HadoopIllegalArgumentException("Adding erasure coding " + + "policy failed because the number of policies stored in the " + + "system already reached the threshold, which is " + + ErasureCodeConstants.MAX_POLICY_ID); + } + policy.setName(assignedNewName); policy.setId(getNextAvailablePolicyID()); this.policiesByName.put(policy.getName(), policy); @@ -261,12 +269,14 @@ public synchronized ErasureCodingPolicy addPolicy( return policy; } + private byte getCurrentMaxPolicyID() { + return policiesByID.keySet().stream().max(Byte::compareTo).orElse((byte)0); + } + private byte getNextAvailablePolicyID() { - byte currentId = this.policiesByID.keySet().stream() - .max(Byte::compareTo) - .filter(id -> id >= ErasureCodeConstants.USER_DEFINED_POLICY_START_ID) - .orElse(ErasureCodeConstants.USER_DEFINED_POLICY_START_ID); - return (byte) (currentId + 1); + byte nextPolicyID = (byte)(getCurrentMaxPolicyID() + 1); + return nextPolicyID > ErasureCodeConstants.USER_DEFINED_POLICY_START_ID ? + nextPolicyID : ErasureCodeConstants.USER_DEFINED_POLICY_START_ID; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index a171665b3f..270201a3c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -193,7 +193,7 @@ Below are the details about each command. * `[-addPolicies -policyFile ]` - Add a list of erasure coding policies. Please refer etc/hadoop/user_ec_policies.xml.template for the example policy file. The maximum cell size is defined in property 'dfs.namenode.ec.policies.max.cellsize' with the default value 4MB. + Add a list of erasure coding policies. Please refer etc/hadoop/user_ec_policies.xml.template for the example policy file. The maximum cell size is defined in property 'dfs.namenode.ec.policies.max.cellsize' with the default value 4MB. Currently HDFS allows the user to add 64 policies in total, and the added policy ID is in range of 64 to 127. Adding policy will fail if there are already 64 policies added. * `[-listCodecs]` diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java index 2c2b05e386..06ac0736f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java @@ -751,6 +751,33 @@ public Object run() throws Exception { }); } + @Test + public void testAddECPoliciesExceeded() throws Exception { + ECSchema toAddSchema = new ECSchema("rs", 3, 2); + int allowNumPolicies = ErasureCodeConstants.MAX_POLICY_ID - + ErasureCodeConstants.USER_DEFINED_POLICY_START_ID + 1; + for (int i = 0; i < allowNumPolicies; i++) { + ErasureCodingPolicy erasureCodingPolicy = new ErasureCodingPolicy( + toAddSchema, 1024 + 1024 * i); + ErasureCodingPolicy[] policyArray = + new ErasureCodingPolicy[]{erasureCodingPolicy}; + AddErasureCodingPolicyResponse[] responses = + fs.addErasureCodingPolicies(policyArray); + assertEquals(1, responses.length); + assertTrue(responses[0].isSucceed()); + assertEquals(responses[0].getPolicy().getId(), + ErasureCodeConstants.USER_DEFINED_POLICY_START_ID + i); + } + ErasureCodingPolicy erasureCodingPolicy = new ErasureCodingPolicy( + toAddSchema, 1024 + 1024 * allowNumPolicies); + ErasureCodingPolicy[] policyArray = + new ErasureCodingPolicy[]{erasureCodingPolicy}; + AddErasureCodingPolicyResponse[] responses = + fs.addErasureCodingPolicies(policyArray); + assertEquals(1, responses.length); + assertFalse(responses[0].isSucceed()); + } + @Test public void testReplicationPolicy() throws Exception { ErasureCodingPolicy replicaPolicy =