big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-12924. Configure raw erasure coders for supported codecs. Contributed by Rui Li.

Browse Source
This commit is contained in:
Kai Zheng 2016-04-20 15:47:01 +08:00
parent be0bce1b71
commit b5d4c7dc76
21 changed files with 255 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
View File

@ -21,6 +21,9 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.http.lib.StaticUserWebFilter;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy;
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
/**
* This class contains constants for configuration keys used
@ -156,13 +159,23 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
/** Supported erasure codec classes */
public static final String IO_ERASURECODE_CODECS_KEY = "io.erasurecode.codecs";
/** Raw coder factory for the RS codec. */
public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY =
"io.erasurecode.codec.rs.rawcoder";
/** Raw coder factory for the RS default codec. */
public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY =
"io.erasurecode.codec.rs-default.rawcoder";
public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT =
RSRawErasureCoderFactory.class.getCanonicalName();
/** Raw coder factory for the RS legacy codec. */
public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY =
"io.erasurecode.codec.rs-legacy.rawcoder";
public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT =
RSRawErasureCoderFactoryLegacy.class.getCanonicalName();
/** Raw coder factory for the XOR codec. */
public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY =
"io.erasurecode.codec.xor.rawcoder";
public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT =
XORRawErasureCoderFactory.class.getCanonicalName();
/**
* Service Authorization

114
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
View File

@ -17,17 +17,14 @@
*/
package org.apache.hadoop.io.erasurecode;
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder;
/**
* A codec & coder utility to help create raw coders conveniently.
@ -42,39 +39,55 @@ private CodecUtil() { }
* @param conf configuration possibly with some items to configure the coder
* @param numDataUnits number of data units in a coding group
* @param numParityUnits number of parity units in a coding group
* @param codec the codec to use. If null, will use the default codec
* @return raw encoder
*/
public static RawErasureEncoder createRSRawEncoder(
Configuration conf, int numDataUnits, int numParityUnits) {
RawErasureCoder rawCoder = createRawCoder(conf,
CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
true, numDataUnits, numParityUnits);
if (rawCoder == null) {
rawCoder = new RSRawEncoder(numDataUnits, numParityUnits);
Configuration conf, int numDataUnits, int numParityUnits, String codec) {
Preconditions.checkNotNull(conf);
if (codec == null) {
codec = ErasureCodeConstants.RS_DEFAULT_CODEC_NAME;
}
RawErasureCoder rawCoder = createRawCoder(conf,
getFactNameFromCodec(conf, codec), true, numDataUnits, numParityUnits);
return (RawErasureEncoder) rawCoder;
}
/**
* Create RS raw encoder using the default codec.
*/
public static RawErasureEncoder createRSRawEncoder(
Configuration conf, int numDataUnits, int numParityUnits) {
return createRSRawEncoder(conf, numDataUnits, numParityUnits, null);
}
/**
* Create RS raw decoder according to configuration.
* @param conf configuration possibly with some items to configure the coder
* @param numDataUnits number of data units in a coding group
* @param numParityUnits number of parity units in a coding group
* @param codec the codec to use. If null, will use the default codec
* @return raw decoder
*/
public static RawErasureDecoder createRSRawDecoder(
Configuration conf, int numDataUnits, int numParityUnits) {
RawErasureCoder rawCoder = createRawCoder(conf,
CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
false, numDataUnits, numParityUnits);
if (rawCoder == null) {
rawCoder = new RSRawDecoder(numDataUnits, numParityUnits);
Configuration conf, int numDataUnits, int numParityUnits, String codec) {
Preconditions.checkNotNull(conf);
if (codec == null) {
codec = ErasureCodeConstants.RS_DEFAULT_CODEC_NAME;
}
RawErasureCoder rawCoder = createRawCoder(conf,
getFactNameFromCodec(conf, codec), false, numDataUnits, numParityUnits);
return (RawErasureDecoder) rawCoder;
}
/**
* Create RS raw decoder using the default codec.
*/
public static RawErasureDecoder createRSRawDecoder(
Configuration conf, int numDataUnits, int numParityUnits) {
return createRSRawDecoder(conf, numDataUnits, numParityUnits, null);
}
/**
* Create XOR raw encoder according to configuration.
* @param conf configuration possibly with some items to configure the coder
@ -84,13 +97,10 @@ public static RawErasureDecoder createRSRawDecoder(
*/
public static RawErasureEncoder createXORRawEncoder(
Configuration conf, int numDataUnits, int numParityUnits) {
Preconditions.checkNotNull(conf);
RawErasureCoder rawCoder = createRawCoder(conf,
CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
getFactNameFromCodec(conf, ErasureCodeConstants.XOR_CODEC_NAME),
true, numDataUnits, numParityUnits);
if (rawCoder == null) {
rawCoder = new XORRawEncoder(numDataUnits, numParityUnits);
}
return (RawErasureEncoder) rawCoder;
}
@ -103,51 +113,65 @@ public static RawErasureEncoder createXORRawEncoder(
*/
public static RawErasureDecoder createXORRawDecoder(
Configuration conf, int numDataUnits, int numParityUnits) {
Preconditions.checkNotNull(conf);
RawErasureCoder rawCoder = createRawCoder(conf,
CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
getFactNameFromCodec(conf, ErasureCodeConstants.XOR_CODEC_NAME),
false, numDataUnits, numParityUnits);
if (rawCoder == null) {
rawCoder = new XORRawDecoder(numDataUnits, numParityUnits);
}
return (RawErasureDecoder) rawCoder;
}
/**
* Create raw coder using specified conf and raw coder factory key.
* @param conf configuration possibly with some items to configure the coder
* @param rawCoderFactoryKey configuration key to find the raw coder factory
* @param rawCoderFactory name of the raw coder factory
* @param isEncoder is encoder or not we're going to create
* @param numDataUnits number of data units in a coding group
* @param numParityUnits number of parity units in a coding group
* @return raw coder
*/
public static RawErasureCoder createRawCoder(Configuration conf,
String rawCoderFactoryKey, boolean isEncoder, int numDataUnits,
String rawCoderFactory, boolean isEncoder, int numDataUnits,
int numParityUnits) {
if (conf == null) {
return null;
}
Class<? extends RawErasureCoderFactory> factClass = null;
factClass = conf.getClass(rawCoderFactoryKey,
factClass, RawErasureCoderFactory.class);
if (factClass == null) {
return null;
}
RawErasureCoderFactory fact;
try {
Class<? extends RawErasureCoderFactory> factClass = conf.getClassByName(
rawCoderFactory).asSubclass(RawErasureCoderFactory.class);
fact = factClass.newInstance();
} catch (InstantiationException e) {
throw new RuntimeException("Failed to create raw coder", e);
} catch (IllegalAccessException e) {
} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException e) {
throw new RuntimeException("Failed to create raw coder", e);
}
return isEncoder ? fact.createEncoder(numDataUnits, numParityUnits) :
fact.createDecoder(numDataUnits, numParityUnits);
}
private static String getFactNameFromCodec(Configuration conf, String codec) {
switch (codec) {
case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME:
return conf.get(
CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
CommonConfigurationKeys.
IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT);
case ErasureCodeConstants.RS_LEGACY_CODEC_NAME:
return conf.get(
CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY,
CommonConfigurationKeys.
IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_DEFAULT);
case ErasureCodeConstants.XOR_CODEC_NAME:
return conf.get(
CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_DEFAULT);
default:
// For custom codec, we throw exception if the factory is not configured
String rawCoderKey = "io.erasurecode.codec." + codec + ".rawcoder";
String factName = conf.get(rawCoderKey);
if (factName == null) {
throw new IllegalArgumentException("Raw coder factory not configured " +
"for custom codec " + codec);
}
return factName;
}
}
}

40
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java Normal file
View File

@ -0,0 +1,40 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io.erasurecode;
/**
* Constants related to the erasure code feature.
*/
public final class ErasureCodeConstants {
private ErasureCodeConstants(){
}
public static final String RS_DEFAULT_CODEC_NAME = "rs-default";
public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
public static final String XOR_CODEC_NAME = "xor";
public static final ECSchema RS_6_3_SCHEMA = new ECSchema(
RS_DEFAULT_CODEC_NAME, 6, 3);
public static final ECSchema RS_3_2_SCHEMA = new ECSchema(
RS_DEFAULT_CODEC_NAME, 3, 2);
public static final ECSchema RS_6_3_LEGACY_SCHEMA = new ECSchema(
RS_LEGACY_CODEC_NAME, 6, 3);
}

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java
View File

@ -65,7 +65,7 @@ protected ErasureCodingStep prepareEncodingStep(
private RawErasureEncoder checkCreateRSRawEncoder() {
if (rsRawEncoder == null) {
rsRawEncoder = CodecUtil.createRSRawEncoder(getConf(),
getNumDataUnits(), getNumParityUnits());
getNumDataUnits(), getNumParityUnits());
}
return rsRawEncoder;
}

1
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
View File

@ -54,6 +54,7 @@ protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) {
private RawErasureDecoder checkCreateRSRawDecoder() {
if (rsRawDecoder == null) {
// TODO: we should create the raw coder according to codec.
rsRawDecoder = CodecUtil.createRSRawDecoder(getConf(),
getNumDataUnits(), getNumParityUnits());
}

3
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
View File

@ -54,8 +54,9 @@ protected ErasureCodingStep prepareEncodingStep(final ECBlockGroup blockGroup) {
private RawErasureEncoder checkCreateRSRawEncoder() {
if (rawEncoder == null) {
// TODO: we should create the raw coder according to codec.
rawEncoder = CodecUtil.createRSRawEncoder(getConf(),
getNumDataUnits(), getNumParityUnits());
getNumDataUnits(), getNumParityUnits());
}
return rawEncoder;
}

8
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -506,6 +506,14 @@
</description>
</property>
<property>
<name>io.erasurecode.codec.rs-default.rawcoder</name>
<value>org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value>
<description>
Raw coder implementation for the rs-default codec.
</description>
</property>
<!-- file system properties -->
<property>

85
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java Normal file
View File

@ -0,0 +1,85 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io.erasurecode;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoderLegacy;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoderLegacy;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
/**
* Test the codec to raw coder mapping.
*/
public class TestCodecRawCoderMapping {
private static Configuration conf;
private static final int numDataUnit = 6;
private static final int numParityUnit = 3;
@Before
public void setup() {
conf = new Configuration();
}
@Test
public void testRSDefaultRawCoder() {
// should return default raw coder of rs-default codec
RawErasureEncoder encoder = CodecUtil.createRSRawEncoder(
conf, numDataUnit, numParityUnit);
Assert.assertTrue(encoder instanceof RSRawEncoder);
RawErasureDecoder decoder = CodecUtil.createRSRawDecoder(
conf, numDataUnit, numParityUnit);
Assert.assertTrue(decoder instanceof RSRawDecoder);
// should return default raw coder of rs-legacy codec
encoder = CodecUtil.createRSRawEncoder(conf, numDataUnit, numParityUnit,
ErasureCodeConstants.RS_LEGACY_CODEC_NAME);
Assert.assertTrue(encoder instanceof RSRawEncoderLegacy);
decoder = CodecUtil.createRSRawDecoder(conf, numDataUnit, numParityUnit,
ErasureCodeConstants.RS_LEGACY_CODEC_NAME);
Assert.assertTrue(decoder instanceof RSRawDecoderLegacy);
}
@Test
public void testDedicatedRawCoderKey() {
String dummyFactName = "DummyNoneExistingFactory";
// set the dummy factory to rs-legacy and create a raw coder
// with rs-default, which is OK as the raw coder key is not used
conf.set(CommonConfigurationKeys.
IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName);
RawErasureEncoder encoder = CodecUtil.createRSRawEncoder(conf, numDataUnit,
numParityUnit, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME);
Assert.assertTrue(encoder instanceof RSRawEncoder);
// now create the raw coder with rs-legacy, which should throw exception
try {
CodecUtil.createRSRawEncoder(conf, numDataUnit, numParityUnit,
ErasureCodeConstants.RS_LEGACY_CODEC_NAME);
Assert.fail();
} catch (Exception e) {
GenericTestUtils.assertExceptionContains("Failed to create raw coder", e);
}
}
}

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
View File

@ -112,7 +112,7 @@ protected void setAllowDump(boolean allowDump) {
protected void prepare(Configuration conf, int numDataUnits,
int numParityUnits, int[] erasedDataIndexes,
int[] erasedParityIndexes, boolean usingFixedData) {
this.conf = conf;
this.conf = conf != null ? conf : new Configuration();
this.numDataUnits = numDataUnits;
this.numParityUnits = numParityUnits;
this.erasedDataIndexes = erasedDataIndexes != null ?

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
View File

@ -50,7 +50,7 @@ public void testCodingDirectBufferWithConf_10x4_erasing_d0() {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
View File

@ -57,7 +57,7 @@ public void testCodingDirectBufferWithConf_10x4_erasing_d0() {
* This tests if the configuration items work or not.
*/
Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY,
RSRawErasureCoderFactory.class.getCanonicalName());
prepare(conf, 10, 4, new int[]{0}, new int[0]);

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
View File

@ -185,7 +185,7 @@ void skip() {
readingService =
new ExecutorCompletionService<>(dfsClient.getStripedReadsThreadPool());
decoder = CodecUtil.createRSRawDecoder(dfsClient.getConfiguration(),
dataBlkNum, parityBlkNum);
dataBlkNum, parityBlkNum, ecPolicy.getCodecName());
if (DFSClient.LOG.isDebugEnabled()) {
DFSClient.LOG.debug("Creating an striped input stream for file " + src);
}

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
View File

@ -287,7 +287,7 @@ private void flipDataBuffers() {
ExecutorCompletionService<>(flushAllExecutor);
encoder = CodecUtil.createRSRawEncoder(dfsClient.getConfiguration(),
numDataBlocks, numParityBlocks);
numDataBlocks, numParityBlocks, ecPolicy.getCodecName());
coordinator = new Coordinator(numAllBlocks);
try {

4
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
View File

@ -72,6 +72,10 @@ public int getNumParityUnits() {
return schema.getNumParityUnits();
}
public String getCodecName() {
return schema.getCodecName();
}
public byte getId() {
return id;
}

5
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
View File

@ -22,7 +22,6 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.util.StringUtils;
@InterfaceAudience.Private
@ -145,11 +144,9 @@ public enum DatanodeReportType {
ALL, LIVE, DEAD, DECOMMISSIONING
}
public static final ECSchema RS_6_3_SCHEMA = new ECSchema("rs", 6, 3);
public static final byte RS_6_3_POLICY_ID = 0;
public static final ECSchema RS_3_2_SCHEMA = new ECSchema("rs", 3, 2);
public static final byte RS_3_2_POLICY_ID = 1;
public static final byte RS_6_3_LEGACY_POLICY_ID = 2;
/* Hidden constructor */
protected HdfsConstants() {

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
View File

@ -216,7 +216,7 @@ void reconstructAndTransfer() throws IOException {
private void initDecoderIfNecessary() {
if (decoder == null) {
decoder = CodecUtil.createRSRawDecoder(conf, ecPolicy.getNumDataUnits(),
ecPolicy.getNumParityUnits());
ecPolicy.getNumParityUnits(), ecPolicy.getCodecName());
}
}

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
View File

@ -20,6 +20,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import java.util.Map;
import java.util.TreeMap;
@ -39,15 +40,18 @@ public final class ErasureCodingPolicyManager {
*/
private static final int DEFAULT_CELLSIZE = 64 * 1024;
private static final ErasureCodingPolicy SYS_POLICY1 =
new ErasureCodingPolicy(HdfsConstants.RS_6_3_SCHEMA, DEFAULT_CELLSIZE,
HdfsConstants.RS_6_3_POLICY_ID);
new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_SCHEMA,
DEFAULT_CELLSIZE, HdfsConstants.RS_6_3_POLICY_ID);
private static final ErasureCodingPolicy SYS_POLICY2 =
new ErasureCodingPolicy(HdfsConstants.RS_3_2_SCHEMA, DEFAULT_CELLSIZE,
HdfsConstants.RS_3_2_POLICY_ID);
new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA,
DEFAULT_CELLSIZE, HdfsConstants.RS_3_2_POLICY_ID);
private static final ErasureCodingPolicy SYS_POLICY3 =
new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_LEGACY_SCHEMA,
DEFAULT_CELLSIZE, HdfsConstants.RS_6_3_LEGACY_POLICY_ID);
//We may add more later.
private static final ErasureCodingPolicy[] SYS_POLICIES =
new ErasureCodingPolicy[]{SYS_POLICY1, SYS_POLICY2};
new ErasureCodingPolicy[]{SYS_POLICY1, SYS_POLICY2, SYS_POLICY3};
/**
* All active policies maintained in NN memory for fast querying,

6
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
View File

@ -96,8 +96,10 @@ Deployment
### Configuration keys
The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys:
`io.erasurecode.codec.rs.rawcoder` and `io.erasurecode.codec.xor.rawcoder`.
The default implementations for both of these codecs are pure Java.
`io.erasurecode.codec.rs-default.rawcoder` for the default RS codec,
`io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec,
`io.erasurecode.codec.xor.rawcoder` for the XOR codec.
The default implementations for all of these codecs are pure Java.
Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters:

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
View File

@ -491,7 +491,8 @@ static void verifyParityBlocks(Configuration conf, final long size,
}
}
final RawErasureEncoder encoder =
CodecUtil.createRSRawEncoder(conf, dataBytes.length, parityBytes.length);
CodecUtil.createRSRawEncoder(conf, dataBytes.length, parityBytes.length,
TEST_EC_POLICY.getCodecName());
encoder.encode(dataBytes, expectedParityBytes);
for (int i = 0; i < parityBytes.length; i++) {
if (checkSet.contains(i + dataBytes.length)){

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
View File

@ -218,7 +218,7 @@ public void testPreadWithDNFailure() throws Exception {
}
RawErasureDecoder rawDecoder = CodecUtil.createRSRawDecoder(conf,
DATA_BLK_NUM, PARITY_BLK_NUM);
DATA_BLK_NUM, PARITY_BLK_NUM, ecPolicy.getCodecName());
// Update the expected content for decoded data
int[] missingBlkIdx = new int[PARITY_BLK_NUM];

12
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
View File

@ -109,7 +109,7 @@
<description>setPolicy : set erasure coding policy on a directory to encode files</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -p RS-6-3-64k /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command>
@ -179,7 +179,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -p RS-6-3-64k /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
@ -188,7 +188,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>ErasureCodingPolicy=[Name=RS-6-3-64k</expected-output>
<expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@ -197,7 +197,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -p RS-6-3-64k /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -setPolicy -p RS-DEFAULT-6-3-64k /ecdir</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/ecfile</command>
<ec-admin-command>-fs NAMENODE -getPolicy /ecdir/ecfile</ec-admin-command>
</test-commands>
@ -208,7 +208,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>ErasureCodingPolicy=[Name=RS-6-3-64k</expected-output>
<expected-output>ErasureCodingPolicy=[Name=RS-DEFAULT-6-3-64k</expected-output>
</comparator>
</comparators>
</test>
@ -223,7 +223,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>RS-6-3</expected-output>
<expected-output>RS-DEFAULT-6-3</expected-output>
</comparator>
</comparators>
</test>