From c1a52b04d0cc5ad5c86ae93043655f313386f7f9 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 24 Feb 2017 10:58:45 -0800 Subject: [PATCH] HDFS-11427. Rename rs-default to rs. --- .../hadoop/io/erasurecode/CodecUtil.java | 28 +++++++-------- .../io/erasurecode/ErasureCodeConstants.java | 8 ++--- .../coder/HHXORErasureDecoder.java | 2 +- .../coder/HHXORErasureEncoder.java | 2 +- .../erasurecode/coder/RSErasureDecoder.java | 2 +- .../erasurecode/coder/RSErasureEncoder.java | 2 +- .../src/main/resources/core-default.xml | 4 +-- .../erasurecode/TestCodecRawCoderMapping.java | 10 +++--- .../coder/TestHHXORErasureCoder.java | 2 +- .../erasurecode/coder/TestRSErasureCoder.java | 2 +- .../src/site/markdown/HDFSErasureCoding.md | 6 ++-- ...estDFSRSDefault10x4StripedInputStream.java | 2 +- ...stDFSRSDefault10x4StripedOutputStream.java | 2 +- ...ult10x4StripedOutputStreamWithFailure.java | 4 +-- .../hdfs/TestDFSStripedInputStream.java | 2 +- .../hdfs/TestDFSStripedOutputStream.java | 2 +- ...TestDFSStripedOutputStreamWithFailure.java | 2 +- .../hdfs/TestReconstructStripedFile.java | 2 +- .../TestUnsetAndChangeDirectoryEcPolicy.java | 2 +- .../test/resources/testErasureCodingConf.xml | 36 +++++++++---------- 20 files changed, 61 insertions(+), 61 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java index 977bacb43c..861451aa91 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java @@ -55,9 +55,9 @@ public final class CodecUtil { public static final String IO_ERASURECODE_CODEC_XOR = XORErasureCodec.class.getCanonicalName(); /** Erasure coder Reed-Solomon codec. */ - public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_KEY = + public static final String IO_ERASURECODE_CODEC_RS_KEY = "io.erasurecode.codec.rs"; - public static final String IO_ERASURECODE_CODEC_RS_DEFAULT = + public static final String IO_ERASURECODE_CODEC_RS = RSErasureCodec.class.getCanonicalName(); /** Erasure coder hitch hiker XOR codec. */ public static final String IO_ERASURECODE_CODEC_HHXOR_KEY = @@ -67,10 +67,10 @@ public final class CodecUtil { /** Supported erasure codec classes. */ - /** Raw coder factory for the RS default codec. */ - public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY = - "io.erasurecode.codec.rs-default.rawcoder"; - public static final String IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT = + /** Raw coder factory for the RS codec. */ + public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY = + "io.erasurecode.codec.rs.rawcoder"; + public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT = RSRawErasureCoderFactory.class.getCanonicalName(); /** Raw coder factory for the RS legacy codec. */ @@ -183,10 +183,10 @@ public final class CodecUtil { private static String getRawCoderFactNameFromCodec(Configuration conf, String codec) { switch (codec) { - case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME: + case ErasureCodeConstants.RS_CODEC_NAME: return conf.get( - IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, - IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_DEFAULT); + IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, + IO_ERASURECODE_CODEC_RS_RAWCODER_DEFAULT); case ErasureCodeConstants.RS_LEGACY_CODEC_NAME: return conf.get( IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, @@ -233,15 +233,15 @@ public final class CodecUtil { private static String getCodecClassName(Configuration conf, String codec) { switch (codec) { - case ErasureCodeConstants.RS_DEFAULT_CODEC_NAME: + case ErasureCodeConstants.RS_CODEC_NAME: return conf.get( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY, - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT); + CodecUtil.IO_ERASURECODE_CODEC_RS_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS); case ErasureCodeConstants.RS_LEGACY_CODEC_NAME: //TODO:rs-legacy should be handled differently. return conf.get( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_KEY, - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT); + CodecUtil.IO_ERASURECODE_CODEC_RS_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS); case ErasureCodeConstants.XOR_CODEC_NAME: return conf.get( CodecUtil.IO_ERASURECODE_CODEC_XOR_KEY, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java index e168909e7c..c830bb2b6f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java @@ -25,16 +25,16 @@ public final class ErasureCodeConstants { private ErasureCodeConstants() { } - public static final String RS_DEFAULT_CODEC_NAME = "rs-default"; + public static final String RS_CODEC_NAME = "rs"; public static final String RS_LEGACY_CODEC_NAME = "rs-legacy"; public static final String XOR_CODEC_NAME = "xor"; public static final String HHXOR_CODEC_NAME = "hhxor"; public static final ECSchema RS_6_3_SCHEMA = new ECSchema( - RS_DEFAULT_CODEC_NAME, 6, 3); + RS_CODEC_NAME, 6, 3); public static final ECSchema RS_3_2_SCHEMA = new ECSchema( - RS_DEFAULT_CODEC_NAME, 3, 2); + RS_CODEC_NAME, 3, 2); public static final ECSchema RS_6_3_LEGACY_SCHEMA = new ECSchema( RS_LEGACY_CODEC_NAME, 6, 3); @@ -43,5 +43,5 @@ public final class ErasureCodeConstants { XOR_CODEC_NAME, 2, 1); public static final ECSchema RS_10_4_SCHEMA = new ECSchema( - RS_DEFAULT_CODEC_NAME, 10, 4); + RS_CODEC_NAME, 10, 4); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java index 05e9384b04..4747e49c0e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureDecoder.java @@ -67,7 +67,7 @@ public class HHXORErasureDecoder extends ErasureDecoder { private RawErasureDecoder checkCreateRSRawDecoder() { if (rsRawDecoder == null) { rsRawDecoder = CodecUtil.createRawDecoder(getConf(), - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions()); + ErasureCodeConstants.RS_CODEC_NAME, getOptions()); } return rsRawDecoder; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java index 7a15a05036..6b858b6b55 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/HHXORErasureEncoder.java @@ -61,7 +61,7 @@ public class HHXORErasureEncoder extends ErasureEncoder { private RawErasureEncoder checkCreateRSRawEncoder() { if (rsRawEncoder == null) { rsRawEncoder = CodecUtil.createRawEncoder(getConf(), - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions()); + ErasureCodeConstants.RS_CODEC_NAME, getOptions()); } return rsRawEncoder; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java index 6e679c3e8c..ba7f773180 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java @@ -52,7 +52,7 @@ public class RSErasureDecoder extends ErasureDecoder { private RawErasureDecoder checkCreateRSRawDecoder() { if (rsRawDecoder == null) { rsRawDecoder = CodecUtil.createRawDecoder(getConf(), - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions()); + ErasureCodeConstants.RS_CODEC_NAME, getOptions()); } return rsRawDecoder; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java index 7a09b92d4d..193309dc29 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java @@ -53,7 +53,7 @@ public class RSErasureEncoder extends ErasureEncoder { if (rawEncoder == null) { // TODO: we should create the raw coder according to codec. rawEncoder = CodecUtil.createRawEncoder(getConf(), - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, getOptions()); + ErasureCodeConstants.RS_CODEC_NAME, getOptions()); } return rawEncoder; } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index d8136eeacb..08ca05b235 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -657,10 +657,10 @@ - io.erasurecode.codec.rs-default.rawcoder + io.erasurecode.codec.rs.rawcoder org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory - Raw coder implementation for the rs-default codec. The default value is a + Raw coder implementation for the rs codec. The default value is a pure Java implementation. There is also a native implementation. Its value is org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java index 0db001a47e..7f7fcf32ae 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java @@ -47,12 +47,12 @@ public class TestCodecRawCoderMapping { public void testRSDefaultRawCoder() { ErasureCoderOptions coderOptions = new ErasureCoderOptions( numDataUnit, numParityUnit); - // should return default raw coder of rs-default codec + // should return default raw coder of rs codec RawErasureEncoder encoder = CodecUtil.createRawEncoder( - conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); + conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); Assert.assertTrue(encoder instanceof RSRawEncoder); RawErasureDecoder decoder = CodecUtil.createRawDecoder( - conf, ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); + conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions); Assert.assertTrue(decoder instanceof RSRawDecoder); // should return default raw coder of rs-legacy codec @@ -71,11 +71,11 @@ public class TestCodecRawCoderMapping { String dummyFactName = "DummyNoneExistingFactory"; // set the dummy factory to rs-legacy and create a raw coder - // with rs-default, which is OK as the raw coder key is not used + // with rs, which is OK as the raw coder key is not used conf.set(CodecUtil. IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODER_KEY, dummyFactName); RawErasureEncoder encoder = CodecUtil.createRawEncoder(conf, - ErasureCodeConstants.RS_DEFAULT_CODEC_NAME, coderOptions); + ErasureCodeConstants.RS_CODEC_NAME, coderOptions); Assert.assertTrue(encoder instanceof RSRawEncoder); // now create the raw coder with rs-legacy, which should throw exception try { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java index 1eca1e1d1d..91c4ba8c1a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java @@ -50,7 +50,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase { * This tests if the configuration items work or not. */ Configuration conf = new Configuration(); - conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, RSRawErasureCoderFactory.class.getCanonicalName()); prepare(conf, 10, 4, new int[]{0}, new int[0]); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java index 5f36c9996f..3b18347d76 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java @@ -57,7 +57,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase { * This tests if the configuration items work or not. */ Configuration conf = new Configuration(); - conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, RSRawErasureCoderFactory.class.getCanonicalName()); prepare(conf, 10, 4, new int[]{0}, new int[0]); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 0283e2b003..36fb61dfab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -64,7 +64,7 @@ Architecture 2. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work. - There are four policies currently being supported: RS-DEFAULT-3-2-64k, RS-DEFAULT-6-3-64k, RS-DEFAULT-10-4-64k and RS-LEGACY-6-3-64k. All with default cell size of 64KB. The system default policy is RS-DEFAULT-6-3-64k which use the default schema RS_6_3_SCHEMA with a cell size of 64KB. + Five policies are currently supported: RS-3-2-64k, RS-6-3-64k, RS-10-4-64k, RS-LEGACY-6-3-64k, and XOR-2-1-64k. All with default cell size of 64KB. The system default policy is RS-6-3-64k which use the default schema RS_6_3_SCHEMA with a cell size of 64KB. * **Intel ISA-L** Intel ISA-L stands for Intel Intelligent Storage Acceleration Library. ISA-L is a collection of optimized low-level functions used primarily in storage applications. It includes a fast block Reed-Solomon type erasure codes optimized for Intel AVX and AVX2 instruction sets. @@ -91,7 +91,7 @@ Deployment ### Configuration keys The codec implementation for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys: - `io.erasurecode.codec.rs-default.rawcoder` for the default RS codec, + `io.erasurecode.codec.rs.rawcoder` for the default RS codec, `io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec, `io.erasurecode.codec.xor.rawcoder` for the XOR codec. The default implementations for all of these codecs are pure Java. For default RS codec, there is also a native implementation which leverages Intel ISA-L library to improve the performance of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for more detail information. @@ -108,7 +108,7 @@ Deployment HDFS native implementation of default RS codec leverages Intel ISA-L library to improve the encoding and decoding calculation. To enable and use Intel ISA-L, there are three steps. 1. Build ISA-L library. Please refer to the offical site "https://github.com/01org/isa-l/" for detail information. 2. Build Hadoop with ISA-L support. Please refer to "Intel ISA-L build options" section in "Build instructions for Hadoop"(BUILDING.txt) document. Use -Dbundle.isal to copy the contents of the isal.lib directory into the final tar file. Deploy hadoop with the tar file. Make sure ISA-L library is available on both HDFS client and DataNodes. - 3. Configure the `io.erasurecode.codec.rs-default.rawcoder` key with value `org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory` on HDFS client and DataNodes. + 3. Configure the `io.erasurecode.codec.rs.rawcoder` key with value `org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory` on HDFS client and DataNodes. To check ISA-L library enable state, try "Hadoop checknative" command. It will tell you if ISA-L library is enabled or not. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java index fc0ee37558..3e6d1e4052 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedInputStream.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; /** - * This tests read operation of DFS striped file with RS-DEFAULT-10-4-64k + * This tests read operation of DFS striped file with RS-10-4-64k * erasure code policy. */ public class TestDFSRSDefault10x4StripedInputStream extends diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java index 37821c1b84..1ea839a863 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStream.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; /** - * This tests write operation of DFS striped file with RS-DEFAULT-10-4-64k + * This tests write operation of DFS striped file with RS-10-4-64k * erasure code policy. */ public class TestDFSRSDefault10x4StripedOutputStream diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java index 1b2ec42875..340fec52fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRSDefault10x4StripedOutputStreamWithFailure.java @@ -22,7 +22,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; /** - * This tests write operation of DFS striped file with RS-DEFAULT-10-4-64k + * This tests write operation of DFS striped file with RS-10-4-64k * erasure code policy under Datanode failure conditions. */ public class TestDFSRSDefault10x4StripedOutputStreamWithFailure @@ -33,4 +33,4 @@ public class TestDFSRSDefault10x4StripedOutputStreamWithFailure return ErasureCodingPolicyManager.getPolicyByPolicyID( HdfsConstants.RS_10_4_POLICY_ID); } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index 121b9a4138..9b699045b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -96,7 +96,7 @@ public class TestDFSStripedInputStream { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } SimulatedFSDataset.setFactory(conf); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index 5bde16efaf..8834e14c22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -85,7 +85,7 @@ public class TestDFSStripedOutputStream { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index 0baf9cc327..fabd4176da 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -214,7 +214,7 @@ public class TestDFSStripedOutputStreamWithFailure { final int numDNs = dataBlocks + parityBlocks; if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index 4960c5816c..08b53c2327 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -100,7 +100,7 @@ public class TestReconstructStripedFile { false); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java index 1a4086e711..f26aa06a7c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java @@ -69,7 +69,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); if (ErasureCodeNative.isNativeCodeLoaded()) { conf.set( - CodecUtil.IO_ERASURECODE_CODEC_RS_DEFAULT_RAWCODER_KEY, + CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, NativeRSRawErasureCoderFactory.class.getCanonicalName()); } cluster = new MiniDFSCluster.Builder(conf).numDataNodes( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml index 82b71def80..9b9003a541 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -133,7 +133,7 @@ setPolicy : set erasure coding policy on a directory to encode files -fs NAMENODE -mkdir /ecdir - -fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir + -fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir -fs NAMENODE -rmdir /ecdir @@ -141,7 +141,7 @@ SubstringComparator - Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir + Set erasure coding policy RS-6-3-64k on /ecdir @@ -150,8 +150,8 @@ setPolicy : set a policy twice -fs NAMENODE -mkdir /ecdir - -fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir - -fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir + -fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir + -fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir -fs NAMENODE -rmdir /ecdir @@ -159,7 +159,7 @@ SubstringComparator - Set erasure coding policy RS-DEFAULT-6-3-64k on /ecdir + Set erasure coding policy RS-6-3-64k on /ecdir @@ -168,7 +168,7 @@ unsetPolicy : unset policy and get -fs NAMENODE -mkdir /ecdir - -fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir + -fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir -fs NAMENODE -unsetPolicy -path /ecdir -fs NAMENODE -getPolicy -path /ecdir @@ -187,8 +187,8 @@ setPolicy : change different policy and get -fs NAMENODE -mkdir /ecdir - -fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir - -fs NAMENODE -setPolicy -policy RS-DEFAULT-3-2-64k -path /ecdir + -fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir + -fs NAMENODE -setPolicy -policy RS-3-2-64k -path /ecdir -fs NAMENODE -getPolicy -path /ecdir @@ -197,7 +197,7 @@ SubstringComparator - RS-DEFAULT-3-2-64k + RS-3-2-64k @@ -207,7 +207,7 @@ -fs NAMENODE -mkdir /ecdir -fs NAMENODE -mkdir /ecdir/child - -fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir + -fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir -fs NAMENODE -unsetPolicy -path /ecdir/child -fs NAMENODE -touchz /ecdir/child/ecfile -fs NAMENODE -getPolicy -path /ecdir/child/ecfile @@ -220,7 +220,7 @@ SubstringComparator - RS-DEFAULT-6-3-64k + RS-6-3-64k @@ -246,7 +246,7 @@ getPolicy : get EC policy information at specified path, which doesn't have an EC policy -fs NAMENODE -mkdir /ecdir - -fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir + -fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir -fs NAMENODE -getPolicy -path /ecdir @@ -255,7 +255,7 @@ SubstringComparator - RS-DEFAULT-6-3-64k + RS-6-3-64k @@ -264,7 +264,7 @@ getPolicy : get EC policy information at specified path, which doesn't have an EC policy -fs NAMENODE -mkdir /ecdir - -fs NAMENODE -setPolicy -policy RS-DEFAULT-6-3-64k -path /ecdir + -fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir -fs NAMENODE -touchz /ecdir/ecfile -fs NAMENODE -getPolicy -path /ecdir/ecfile @@ -275,7 +275,7 @@ SubstringComparator - RS-DEFAULT-6-3-64k + RS-6-3-64k @@ -290,7 +290,7 @@ SubstringComparator - RS-DEFAULT-6-3 + RS-6-3 @@ -334,7 +334,7 @@ setPolicy : illegal parameters - too many arguments -fs NAMENODE -mkdir /ecdir - -fs NAMENODE -setPolicy -path /ecdir1 -policy RS-DEFAULT-3-2-64k /ecdir2 + -fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k /ecdir2 -fs NAMENODE -rmdir /ecdir @@ -367,7 +367,7 @@ setPolicy : illegal parameters - no such file - -fs NAMENODE -setPolicy -path /ecdir -policy RS-DEFAULT-3-2-64k + -fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-64k