From b5d4c7dc76ddb3e0af95d792c2cbc0f99353a42a Mon Sep 17 00:00:00 2001 From: Kai Zheng Date: Wed, 20 Apr 2016 15:47:01 +0800 Subject: [PATCH] HADOOP-12924. Configure raw erasure coders for supported codecs. Contributed by Rui Li. --- .../hadoop/fs/CommonConfigurationKeys.java | 19 ++- .../hadoop/io/erasurecode/CodecUtil.java | 114 +++++++++++------- .../io/erasurecode/ErasureCodeConstants.java | 40 ++++++ .../coder/HHXORErasureEncoder.java | 2 +- .../erasurecode/coder/RSErasureDecoder.java | 1 + .../erasurecode/coder/RSErasureEncoder.java | 3 +- .../src/main/resources/core-default.xml | 8 ++ .../erasurecode/TestCodecRawCoderMapping.java | 85 +++++++++++++ .../hadoop/io/erasurecode/TestCoderBase.java | 2 +- .../coder/TestHHXORErasureCoder.java | 2 +- .../erasurecode/coder/TestRSErasureCoder.java | 2 +- .../hadoop/hdfs/DFSStripedInputStream.java | 2 +- .../hadoop/hdfs/DFSStripedOutputStream.java | 2 +- .../hdfs/protocol/ErasureCodingPolicy.java | 4 + .../hadoop/hdfs/protocol/HdfsConstants.java | 5 +- .../erasurecode/StripedReconstructor.java | 2 +- .../namenode/ErasureCodingPolicyManager.java | 14 ++- .../src/site/markdown/HDFSErasureCoding.md | 6 +- .../hadoop/hdfs/StripedFileTestUtil.java | 3 +- .../hdfs/TestDFSStripedInputStream.java | 2 +- .../test/resources/testErasureCodingConf.xml | 12 +- 21 files changed, 255 insertions(+), 75 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index a708900e63..9f524476db 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -21,6 +21,9 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.http.lib.StaticUserWebFilter; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy; +import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory; /** * This class contains constants for configuration keys used @@ -156,13 +159,23 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { /** Supported erasure codec classes */ public static final String IO_ERASURECODE_CODECS_KEY = "io.erasurecode.codecs"; - /** Raw coder factory for the RS codec. */ - public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY = - "io.erasurecode.codec.rs.rawcoder"; + /** Raw coder factory for the RS default codec. */ + public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY = + "io.erasurecode.codec.rs-default.rawcoder"; + public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT = + RSRawErasureCoderFactory.class.getCanonicalName(); + + /** Raw coder factory for the RS legacy codec. */ + public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY = + "io.erasurecode.codec.rs-legacy.rawcoder"; + public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT = + RSRawErasureCoderFactoryLegacy.class.getCanonicalName(); /** Raw coder factory for the XOR codec. */ public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY = "io.erasurecode.codec.xor.rawcoder"; + public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT = + XORRawErasureCoderFactory.class.getCanonicalName(); /** * Service Authorization diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java index a2354b67c9..fcce071b50 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java @@ -17,17 +17,14 @@ */ package org.apache.hadoop.io.erasurecode; +import com.google.common.base.Preconditions; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder; -import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; -import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder; -import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder; /** * A codec & coder utility to help create raw coders conveniently. @@ -42,39 +39,55 @@ private CodecUtil() { } * @param conf configuration possibly with some items to configure the coder * @param numDataUnits number of data units in a coding group * @param numParityUnits number of parity units in a coding group + * @param codec the codec to use. If null, will use the default codec * @return raw encoder */ public static RawErasureEncoder createRSRawEncoder( - Configuration conf, int numDataUnits, int numParityUnits) { - RawErasureCoder rawCoder = createRawCoder(conf, - CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, - true, numDataUnits, numParityUnits); - if (rawCoder == null) { - rawCoder = new RSRawEncoder(numDataUnits, numParityUnits); + Configuration conf, int numDataUnits, int numParityUnits, String codec) { + Preconditions.checkNotNull(conf); + if (codec == null) { + codec = ErasureCodeConstants.RS_DEFAULT_CODEC_NAME; } - + RawErasureCoder rawCoder = createRawCoder(conf, + getFactNameFromCodec(conf, codec), true, numDataUnits, numParityUnits); return (RawErasureEncoder) rawCoder; } + /** + * Create RS raw encoder using the default codec. + */ + public static RawErasureEncoder createRSRawEncoder( + Configuration conf, int numDataUnits, int numParityUnits) { + return createRSRawEncoder(conf, numDataUnits, numParityUnits, null); + } + /** * Create RS raw decoder according to configuration. * @param conf configuration possibly with some items to configure the coder * @param numDataUnits number of data units in a coding group * @param numParityUnits number of parity units in a coding group + * @param codec the codec to use. If null, will use the default codec * @return raw decoder */ public static RawErasureDecoder createRSRawDecoder( - Configuration conf, int numDataUnits, int numParityUnits) { - RawErasureCoder rawCoder = createRawCoder(conf, - CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, - false, numDataUnits, numParityUnits); - if (rawCoder == null) { - rawCoder = new RSRawDecoder(numDataUnits, numParityUnits); + Configuration conf, int numDataUnits, int numParityUnits, String codec) { + Preconditions.checkNotNull(conf); + if (codec == null) { + codec = ErasureCodeConstants.RS_DEFAULT_CODEC_NAME; } - + RawErasureCoder rawCoder = createRawCoder(conf, + getFactNameFromCodec(conf, codec), false, numDataUnits, numParityUnits); return (RawErasureDecoder) rawCoder; } + /** + * Create RS raw decoder using the default codec. + */ + public static RawErasureDecoder createRSRawDecoder( + Configuration conf, int numDataUnits, int numParityUnits) { + return createRSRawDecoder(conf, numDataUnits, numParityUnits, null); + } + /** * Create XOR raw encoder according to configuration. * @param conf configuration possibly with some items to configure the coder @@ -84,13 +97,10 @@ public static RawErasureDecoder createRSRawDecoder( */ public static RawErasureEncoder createXORRawEncoder( Configuration conf, int numDataUnits, int numParityUnits) { + Preconditions.checkNotNull(conf); RawErasureCoder rawCoder = createRawCoder(conf, - CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY, + getFactNameFromCodec(conf, ErasureCodeConstants.XOR_CODEC_NAME), true, numDataUnits, numParityUnits); - if (rawCoder == null) { - rawCoder = new XORRawEncoder(numDataUnits, numParityUnits); - } - return (RawErasureEncoder) rawCoder; } @@ -103,51 +113,65 @@ public static RawErasureEncoder createXORRawEncoder( */ public static RawErasureDecoder createXORRawDecoder( Configuration conf, int numDataUnits, int numParityUnits) { + Preconditions.checkNotNull(conf); RawErasureCoder rawCoder = createRawCoder(conf, - CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY, + getFactNameFromCodec(conf, ErasureCodeConstants.XOR_CODEC_NAME), false, numDataUnits, numParityUnits); - if (rawCoder == null) { - rawCoder = new XORRawDecoder(numDataUnits, numParityUnits); - } - return (RawErasureDecoder) rawCoder; } /** * Create raw coder using specified conf and raw coder factory key. * @param conf configuration possibly with some items to configure the coder - * @param rawCoderFactoryKey configuration key to find the raw coder factory + * @param rawCoderFactory name of the raw coder factory * @param isEncoder is encoder or not we're going to create * @param numDataUnits number of data units in a coding group * @param numParityUnits number of parity units in a coding group * @return raw coder */ public static RawErasureCoder createRawCoder(Configuration conf, - String rawCoderFactoryKey, boolean isEncoder, int numDataUnits, + String rawCoderFactory, boolean isEncoder, int numDataUnits, int numParityUnits) { - if (conf == null) { - return null; - } - - Class factClass = null; - factClass = conf.getClass(rawCoderFactoryKey, - factClass, RawErasureCoderFactory.class); - - if (factClass == null) { - return null; - } - RawErasureCoderFactory fact; try { + Class factClass = conf.getClassByName( + rawCoderFactory).asSubclass(RawErasureCoderFactory.class); fact = factClass.newInstance(); - } catch (InstantiationException e) { - throw new RuntimeException("Failed to create raw coder", e); - } catch (IllegalAccessException e) { + } catch (ClassNotFoundException | InstantiationException | + IllegalAccessException e) { throw new RuntimeException("Failed to create raw coder", e); } return isEncoder ? fact.createEncoder(numDataUnits, numParityUnits) : fact.createDecoder(numDataUnits, numParityUnits); } + + private static String getFactNameFromCodec(Configuration conf, String codec) { + switch (codec) { + case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME: + return conf.get( + CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CommonConfigurationKeys. + IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT); + case ErasureCodeConstants.RS_LEGACY_CODEC_NAME: + return conf.get( + CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, + CommonConfigurationKeys. + IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT); + case ErasureCodeConstants.XOR_CODEC_NAME: + return conf.get( + CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY, + CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT); + default: + // For custom codec, we throw exception if the factory is not configured + String rawCoderKey = "io.erasurecode.codec." + codec + ".rawcoder"; + String factName = conf.get(rawCoderKey); + if (factName == null) { + throw new IllegalArgumentException("Raw coder factory not configured " + + "for custom codec " + codec); + } + return factName; + } + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java new file mode 100644 index 0000000000..1fb94882b5 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode; + +/** + * Constants related to the erasure code feature. + */ +public final class ErasureCodeConstants { + + private ErasureCodeConstants(){ + } + + public static final String RS_DEFAULT_CODEC_NAME = "rs-default"; + public static final String RS_LEGACY_CODEC_NAME = "rs-legacy"; + public static final String XOR_CODEC_NAME = "xor"; + + public static final ECSchema RS_6_3_SCHEMA = new ECSchema( + RS_DEFAULT_CODEC_NAME, 6, 3); + + public static final ECSchema RS_3_2_SCHEMA = new ECSchema( + RS_DEFAULT_CODEC_NAME, 3, 2); + + public static final ECSchema RS_6_3_LEGACY_SCHEMA = new ECSchema( + RS_LEGACY_CODEC_NAME, 6, 3); +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java index f30572faef..a402469c84 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java @@ -65,7 +65,7 @@ protected ErasureCodingStep prepareEncodingStep( private RawErasureEncoder checkCreateRSRawEncoder() { if (rsRawEncoder == null) { rsRawEncoder = CodecUtil.createRSRawEncoder(getConf(), - getNumDataUnits(), getNumParityUnits()); + getNumDataUnits(), getNumParityUnits()); } return rsRawEncoder; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java index 6728945bfb..47efd297bf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java @@ -54,6 +54,7 @@ protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) { private RawErasureDecoder checkCreateRSRawDecoder() { if (rsRawDecoder == null) { + // TODO: we should create the raw coder according to codec. rsRawDecoder = CodecUtil.createRSRawDecoder(getConf(), getNumDataUnits(), getNumParityUnits()); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java index 7784573c5d..4806d9e811 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java @@ -54,8 +54,9 @@ protected ErasureCodingStep prepareEncodingStep(final ECBlockGroup blockGroup) { private RawErasureEncoder checkCreateRSRawEncoder() { if (rawEncoder == null) { + // TODO: we should create the raw coder according to codec. rawEncoder = CodecUtil.createRSRawEncoder(getConf(), - getNumDataUnits(), getNumParityUnits()); + getNumDataUnits(), getNumParityUnits()); } return rawEncoder; } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 96b108f0f3..ac8bc5a461 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -506,6 +506,14 @@ + + io.erasurecode.codec.rs-default.rawcoder + org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory + + Raw coder implementation for the rs-default codec. + + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java new file mode 100644 index 0000000000..5075966221 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.erasurecode; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoderLegacy; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder; +import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoderLegacy; +import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder; +import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +/** + * Test the codec to raw coder mapping. + */ +public class TestCodecRawCoderMapping { + + private static Configuration conf; + private static final int numDataUnit = 6; + private static final int numParityUnit = 3; + + @Before + public void setup() { + conf = new Configuration(); + } + + @Test + public void testRSDefaultRawCoder() { + // should return default raw coder of rs-default codec + RawErasureEncoder encoder = CodecUtil.createRSRawEncoder( + conf, numDataUnit, numParityUnit); + Assert.assertTrue(encoder instanceof RSRawEncoder); + RawErasureDecoder decoder = CodecUtil.createRSRawDecoder( + conf, numDataUnit, numParityUnit); + Assert.assertTrue(decoder instanceof RSRawDecoder); + + // should return default raw coder of rs-legacy codec + encoder = CodecUtil.createRSRawEncoder(conf, numDataUnit, numParityUnit, + ErasureCodeConstants.RS_LEGACY_CODEC_NAME); + Assert.assertTrue(encoder instanceof RSRawEncoderLegacy); + decoder = CodecUtil.createRSRawDecoder(conf, numDataUnit, numParityUnit, + ErasureCodeConstants.RS_LEGACY_CODEC_NAME); + Assert.assertTrue(decoder instanceof RSRawDecoderLegacy); + } + + @Test + public void testDedicatedRawCoderKey() { + String dummyFactName = "DummyNoneExistingFactory"; + // set the dummy factory to rs-legacy and create a raw coder + // with rs-default, which is OK as the raw coder key is not used + conf.set(CommonConfigurationKeys. + IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName); + RawErasureEncoder encoder = CodecUtil.createRSRawEncoder(conf, numDataUnit, + numParityUnit, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME); + Assert.assertTrue(encoder instanceof RSRawEncoder); + // now create the raw coder with rs-legacy, which should throw exception + try { + CodecUtil.createRSRawEncoder(conf, numDataUnit, numParityUnit, + ErasureCodeConstants.RS_LEGACY_CODEC_NAME); + Assert.fail(); + } catch (Exception e) { + GenericTestUtils.assertExceptionContains("Failed to create raw coder", e); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java index 57213361c0..633e064176 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java @@ -112,7 +112,7 @@ protected void setAllowDump(boolean allowDump) { protected void prepare(Configuration conf, int numDataUnits, int numParityUnits, int[] erasedDataIndexes, int[] erasedParityIndexes, boolean usingFixedData) { - this.conf = conf; + this.conf = conf != null ? conf : new Configuration(); this.numDataUnits = numDataUnits; this.numParityUnits = numParityUnits; this.erasedDataIndexes = erasedDataIndexes != null ? diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java index ad346e08fa..09d1ec2020 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java @@ -50,7 +50,7 @@ public void testCodingDirectBufferWithConf_10x4_erasing_d0() { * This tests if the configuration items work or not. */ Configuration conf = new Configuration(); - conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, + conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, RSRawErasureCoderFactory.class.getCanonicalName()); prepare(conf, 10, 4, new int[]{0}, new int[0]); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java index ee2348e9c8..b0fe8f8530 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java @@ -57,7 +57,7 @@ public void testCodingDirectBufferWithConf_10x4_erasing_d0() { * This tests if the configuration items work or not. */ Configuration conf = new Configuration(); - conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, + conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, RSRawErasureCoderFactory.class.getCanonicalName()); prepare(conf, 10, 4, new int[]{0}, new int[0]); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java index 38236ad382..1944782a15 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java @@ -185,7 +185,7 @@ void skip() { readingService = new ExecutorCompletionService<>(dfsClient.getStripedReadsThreadPool()); decoder = CodecUtil.createRSRawDecoder(dfsClient.getConfiguration(), - dataBlkNum, parityBlkNum); + dataBlkNum, parityBlkNum, ecPolicy.getCodecName()); if (DFSClient.LOG.isDebugEnabled()) { DFSClient.LOG.debug("Creating an striped input stream for file " + src); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 3dfdbee39f..b049286e41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -287,7 +287,7 @@ private void flipDataBuffers() { ExecutorCompletionService<>(flushAllExecutor); encoder = CodecUtil.createRSRawEncoder(dfsClient.getConfiguration(), - numDataBlocks, numParityBlocks); + numDataBlocks, numParityBlocks, ecPolicy.getCodecName()); coordinator = new Coordinator(numAllBlocks); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java index 2c748d23b4..c320fdfd97 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java @@ -72,6 +72,10 @@ public int getNumParityUnits() { return schema.getNumParityUnits(); } + public String getCodecName() { + return schema.getCodecName(); + } + public byte getId() { return id; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index 3c5c44137c..8df2d54964 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -22,7 +22,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.util.StringUtils; @InterfaceAudience.Private @@ -145,11 +144,9 @@ public enum DatanodeReportType { ALL, LIVE, DEAD, DECOMMISSIONING } - public static final ECSchema RS_6_3_SCHEMA = new ECSchema("rs", 6, 3); public static final byte RS_6_3_POLICY_ID = 0; - - public static final ECSchema RS_3_2_SCHEMA = new ECSchema("rs", 3, 2); public static final byte RS_3_2_POLICY_ID = 1; + public static final byte RS_6_3_LEGACY_POLICY_ID = 2; /* Hidden constructor */ protected HdfsConstants() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java index a0a5f836b6..1b59b22ec2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java @@ -216,7 +216,7 @@ void reconstructAndTransfer() throws IOException { private void initDecoderIfNecessary() { if (decoder == null) { decoder = CodecUtil.createRSRawDecoder(conf, ecPolicy.getNumDataUnits(), - ecPolicy.getNumParityUnits()); + ecPolicy.getNumParityUnits(), ecPolicy.getCodecName()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java index eee80d36b2..eaf63f9d86 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java @@ -20,6 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import java.util.Map; import java.util.TreeMap; @@ -39,15 +40,18 @@ public final class ErasureCodingPolicyManager { */ private static final int DEFAULT_CELLSIZE = 64 * 1024; private static final ErasureCodingPolicy SYS_POLICY1 = - new ErasureCodingPolicy(HdfsConstants.RS_6_3_SCHEMA, DEFAULT_CELLSIZE, - HdfsConstants.RS_6_3_POLICY_ID); + new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_SCHEMA, + DEFAULT_CELLSIZE, HdfsConstants.RS_6_3_POLICY_ID); private static final ErasureCodingPolicy SYS_POLICY2 = - new ErasureCodingPolicy(HdfsConstants.RS_3_2_SCHEMA, DEFAULT_CELLSIZE, - HdfsConstants.RS_3_2_POLICY_ID); + new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA, + DEFAULT_CELLSIZE, HdfsConstants.RS_3_2_POLICY_ID); + private static final ErasureCodingPolicy SYS_POLICY3 = + new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_LEGACY_SCHEMA, + DEFAULT_CELLSIZE, HdfsConstants.RS_6_3_LEGACY_POLICY_ID); //We may add more later. private static final ErasureCodingPolicy[] SYS_POLICIES = - new ErasureCodingPolicy[]{SYS_POLICY1, SYS_POLICY2}; + new ErasureCodingPolicy[]{SYS_POLICY1, SYS_POLICY2, SYS_POLICY3}; /** * All active policies maintained in NN memory for fast querying, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 5b3aa34833..9066a1561a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -96,8 +96,10 @@ Deployment ### Configuration keys The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys: - `io.erasurecode.codec.rs.rawcoder` and `io.erasurecode.codec.xor.rawcoder`. - The default implementations for both of these codecs are pure Java. + `io.erasurecode.codec.rs-default.rawcoder` for the default RS codec, + `io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec, + `io.erasurecode.codec.xor.rawcoder` for the XOR codec. + The default implementations for all of these codecs are pure Java. Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters: diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java index 6d0dfa86c8..ccc94a461e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java @@ -491,7 +491,8 @@ static void verifyParityBlocks(Configuration conf, final long size, } } final RawErasureEncoder encoder = - CodecUtil.createRSRawEncoder(conf, dataBytes.length, parityBytes.length); + CodecUtil.createRSRawEncoder(conf, dataBytes.length, parityBytes.length, + TEST_EC_POLICY.getCodecName()); encoder.encode(dataBytes, expectedParityBytes); for (int i = 0; i < parityBytes.length; i++) { if (checkSet.contains(i + dataBytes.length)){ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index 4dcb654ebc..e4f7ac049c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -218,7 +218,7 @@ public void testPreadWithDNFailure() throws Exception { } RawErasureDecoder rawDecoder = CodecUtil.createRSRawDecoder(conf, - DATA_BLK_NUM, PARITY_BLK_NUM); + DATA_BLK_NUM, PARITY_BLK_NUM, ecPolicy.getCodecName()); // Update the expected content for decoded data int[] missingBlkIdx = new int[PARITY_BLK_NUM]; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml index fd7cf9d5e2..1c782c3c27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -109,7 +109,7 @@ setPolicy : set erasure coding policy on a directory to encode files -fs NAMENODE -mkdir /ecdir - -fs NAMENODE -setPolicy -p RS-6-3-64k /ecdir + -fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir -fs NAMENODE -rmdir /ecdir @@ -179,7 +179,7 @@ getPolicy : get EC policy information at specified path, which doesn't have an EC policy -fs NAMENODE -mkdir /ecdir - -fs NAMENODE -setPolicy -p RS-6-3-64k /ecdir + -fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir -fs NAMENODE -getPolicy /ecdir @@ -188,7 +188,7 @@ SubstringComparator - ErasureCodingPolicy=[Name=RS-6-3-64k + ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k @@ -197,7 +197,7 @@ getPolicy : get EC policy information at specified path, which doesn't have an EC policy -fs NAMENODE -mkdir /ecdir - -fs NAMENODE -setPolicy -p RS-6-3-64k /ecdir + -fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir -fs NAMENODE -touchz /ecdir/ecfile -fs NAMENODE -getPolicy /ecdir/ecfile @@ -208,7 +208,7 @@ SubstringComparator - ErasureCodingPolicy=[Name=RS-6-3-64k + ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k @@ -223,7 +223,7 @@ SubstringComparator - RS-6-3 + RS-DEFAULT-6-3