big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14261. Some refactoring work for erasure coding raw coder. Contributed by Lin Zeng.

Browse Source
This commit is contained in:
Andrew Wang 2017-04-21 11:35:30 -07:00
parent b0803388fc
commit a22fe02fba
18 changed files with 44 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
View File

@ -33,7 +33,7 @@
import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy;
import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
@ -80,7 +80,7 @@ public final class CodecUtil {
public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY =
IO_ERASURECODE_CODEC + "rs-legacy.rawcoders";
public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT =
RSRawErasureCoderFactoryLegacy.class.getCanonicalName();
RSLegacyRawErasureCoderFactory.class.getCanonicalName();
public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY =
IO_ERASURECODE_CODEC + "rs.rawcoders";
public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT =

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoderLegacy.java → hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawDecoder.java
View File

@ -34,12 +34,12 @@
* addressed in HADOOP-11871.
*/
@InterfaceAudience.Private
public class RSRawDecoderLegacy extends RawErasureDecoder {
public class RSLegacyRawDecoder extends RawErasureDecoder {
// To describe and calculate the needed Vandermonde matrix
private int[] errSignature;
private int[] primitivePower;
public RSRawDecoderLegacy(ErasureCoderOptions coderOptions) {
public RSLegacyRawDecoder(ErasureCoderOptions coderOptions) {
super(coderOptions);
if (getNumAllUnits() >= RSUtil.GF.getFieldSize()) {
throw new HadoopIllegalArgumentException(

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoderLegacy.java → hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawEncoder.java
View File

@ -30,10 +30,10 @@
* when possible.
*/
@InterfaceAudience.Private
public class RSRawEncoderLegacy extends RawErasureEncoder {
public class RSLegacyRawEncoder extends RawErasureEncoder {
private int[] generatingPolynomial;
public RSRawEncoderLegacy(ErasureCoderOptions coderOptions) {
public RSLegacyRawEncoder(ErasureCoderOptions coderOptions) {
super(coderOptions);
assert (getNumDataUnits() + getNumParityUnits() < RSUtil.GF.getFieldSize());

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactoryLegacy.java → hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java
View File

@ -24,15 +24,15 @@
* A raw coder factory for the legacy raw Reed-Solomon coder in Java.
*/
@InterfaceAudience.Private
public class RSRawErasureCoderFactoryLegacy implements RawErasureCoderFactory {
public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory {
@Override
public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
return new RSRawEncoderLegacy(coderOptions);
return new RSLegacyRawEncoder(coderOptions);
}
@Override
public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
return new RSRawDecoderLegacy(coderOptions);
return new RSLegacyRawDecoder(coderOptions);
}
}

2
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -677,7 +677,7 @@
<property>
<name>io.erasurecode.codec.rs-legacy.rawcoders</name>
<value>org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy</value>
<value>org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory</value>
<description>
Comma separated raw coder implementations for the rs-legacy codec. The earlier
factory is prior to followings in case of failure of creating raw coders.

12
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
View File

@ -20,9 +20,9 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoderLegacy;
import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoderLegacy;
import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawEncoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
@ -62,10 +62,10 @@ public void testRSDefaultRawCoder() {
// should return default raw coder of rs-legacy codec
encoder = CodecUtil.createRawEncoder(conf,
ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
Assert.assertTrue(encoder instanceof RSRawEncoderLegacy);
Assert.assertTrue(encoder instanceof RSLegacyRawEncoder);
decoder = CodecUtil.createRawDecoder(conf,
ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
Assert.assertTrue(decoder instanceof RSRawDecoderLegacy);
Assert.assertTrue(decoder instanceof RSLegacyRawDecoder);
}
@Test
@ -122,10 +122,10 @@ public void testLegacyCodecFallback() {
// should return default raw coder of rs-legacy codec
RawErasureEncoder encoder = CodecUtil.createRawEncoder(
conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
Assert.assertTrue(encoder instanceof RSRawEncoderLegacy);
Assert.assertTrue(encoder instanceof RSLegacyRawEncoder);
RawErasureDecoder decoder = CodecUtil.createRawDecoder(
conf, ErasureCodeConstants.RS_LEGACY_CODEC_NAME, coderOptions);
Assert.assertTrue(decoder instanceof RSRawDecoderLegacy);
Assert.assertTrue(decoder instanceof RSLegacyRawDecoder);
}
@Test

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderBenchmark.java
View File

@ -60,7 +60,7 @@ private RawErasureCoderBenchmark() {
private static final List<RawErasureCoderFactory> CODER_MAKERS =
Collections.unmodifiableList(
Arrays.asList(new DummyRawErasureCoderFactory(),
new RSRawErasureCoderFactoryLegacy(),
new RSLegacyRawErasureCoderFactory(),
new RSRawErasureCoderFactory(),
new NativeRSRawErasureCoderFactory()));

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestDummyRawCoder.java
View File

@ -29,8 +29,8 @@
public class TestDummyRawCoder extends TestRawCoderBase {
@Before
public void setup() {
encoderClass = DummyRawEncoder.class;
decoderClass = DummyRawDecoder.class;
encoderFactoryClass = DummyRawErasureCoderFactory.class;
decoderFactoryClass = DummyRawErasureCoderFactory.class;
setAllowDump(false);
setChunkSize(baseChunkSize);
}

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeRSRawCoder.java
View File

@ -30,8 +30,8 @@ public class TestNativeRSRawCoder extends TestRSRawCoderBase {
@Before
public void setup() {
Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
this.encoderClass = NativeRSRawEncoder.class;
this.decoderClass = NativeRSRawDecoder.class;
this.encoderFactoryClass = NativeRSRawErasureCoderFactory.class;
this.decoderFactoryClass = NativeRSRawErasureCoderFactory.class;
setAllowDump(true);
}

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestNativeXORRawCoder.java
View File

@ -29,8 +29,8 @@ public class TestNativeXORRawCoder extends TestXORRawCoderBase {
@Before
public void setup() {
Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
this.encoderClass = NativeXORRawEncoder.class;
this.decoderClass = NativeXORRawDecoder.class;
this.encoderFactoryClass = NativeXORRawErasureCoderFactory.class;
this.decoderFactoryClass = NativeXORRawErasureCoderFactory.class;
setAllowDump(true);
}
}

6
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderLegacy.java → hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSLegacyRawCoder.java
View File

@ -22,12 +22,12 @@
/**
* Test the legacy raw Reed-solomon coder implemented in Java.
*/
public class TestRSRawCoderLegacy extends TestRSRawCoderBase {
public class TestRSLegacyRawCoder extends TestRSRawCoderBase {
@Before
public void setup() {
this.encoderClass = RSRawEncoderLegacy.class;
this.decoderClass = RSRawDecoderLegacy.class;
this.encoderFactoryClass = RSLegacyRawErasureCoderFactory.class;
this.decoderFactoryClass = RSLegacyRawErasureCoderFactory.class;
setAllowDump(false); // Change to true to allow verbose dump for debugging
}
}

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
View File

@ -26,8 +26,8 @@ public class TestRSRawCoder extends TestRSRawCoderBase {
@Before
public void setup() {
this.encoderClass = RSRawEncoder.class;
this.decoderClass = RSRawDecoder.class;
this.encoderFactoryClass = RSRawErasureCoderFactory.class;
this.decoderFactoryClass = RSRawErasureCoderFactory.class;
setAllowDump(false);
}
}

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderInteroperable1.java
View File

@ -30,8 +30,8 @@ public class TestRSRawCoderInteroperable1 extends TestRSRawCoderBase {
public void setup() {
Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
this.encoderClass = RSRawEncoder.class;
this.decoderClass = NativeRSRawDecoder.class;
this.encoderFactoryClass = RSRawErasureCoderFactory.class;
this.decoderFactoryClass = NativeRSRawErasureCoderFactory.class;
setAllowDump(true);
}

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderInteroperable2.java
View File

@ -30,8 +30,8 @@ public class TestRSRawCoderInteroperable2 extends TestRSRawCoderBase {
public void setup() {
Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
this.encoderClass = NativeRSRawEncoder.class;
this.decoderClass = RSRawDecoder.class;
this.encoderFactoryClass = NativeRSRawErasureCoderFactory.class;
this.decoderFactoryClass = RSRawErasureCoderFactory.class;
setAllowDump(true);
}

16
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
View File

@ -23,14 +23,12 @@
import org.junit.Assert;
import org.junit.Test;
import java.lang.reflect.Constructor;
/**
* Raw coder test base with utilities.
*/
public abstract class TestRawCoderBase extends TestCoderBase {
protected Class<? extends RawErasureEncoder> encoderClass;
protected Class<? extends RawErasureDecoder> decoderClass;
protected Class<? extends RawErasureCoderFactory> encoderFactoryClass;
protected Class<? extends RawErasureCoderFactory> decoderFactoryClass;
protected RawErasureEncoder encoder;
protected RawErasureDecoder decoder;
@ -234,9 +232,8 @@ protected RawErasureEncoder createEncoder() {
new ErasureCoderOptions(numDataUnits, numParityUnits,
allowChangeInputs, allowDump);
try {
Constructor<? extends RawErasureEncoder> constructor =
encoderClass.getConstructor(ErasureCoderOptions.class);
return constructor.newInstance(coderConf);
RawErasureCoderFactory factory = encoderFactoryClass.newInstance();
return factory.createEncoder(coderConf);
} catch (Exception e) {
throw new RuntimeException("Failed to create encoder", e);
}
@ -251,9 +248,8 @@ protected RawErasureDecoder createDecoder() {
new ErasureCoderOptions(numDataUnits, numParityUnits,
allowChangeInputs, allowDump);
try {
Constructor<? extends RawErasureDecoder> constructor =
decoderClass.getConstructor(ErasureCoderOptions.class);
return constructor.newInstance(coderConf);
RawErasureCoderFactory factory = encoderFactoryClass.newInstance();
return factory.createDecoder(coderConf);
} catch (Exception e) {
throw new RuntimeException("Failed to create decoder", e);
}

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java
View File

@ -26,7 +26,7 @@ public class TestXORRawCoder extends TestXORRawCoderBase {
@Before
public void setup() {
this.encoderClass = XORRawEncoder.class;
this.decoderClass = XORRawDecoder.class;
this.encoderFactoryClass = XORRawErasureCoderFactory.class;
this.decoderFactoryClass = XORRawErasureCoderFactory.class;
}
}

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderInteroperable1.java
View File

@ -29,8 +29,8 @@ public class TestXORRawCoderInteroperable1 extends TestXORRawCoderBase {
@Before
public void setup() {
Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
this.encoderClass = XORRawEncoder.class;
this.decoderClass = NativeXORRawDecoder.class;
this.encoderFactoryClass = XORRawErasureCoderFactory.class;
this.decoderFactoryClass = NativeXORRawErasureCoderFactory.class;
setAllowDump(true);
}
}

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoderInteroperable2.java
View File

@ -29,8 +29,8 @@ public class TestXORRawCoderInteroperable2 extends TestXORRawCoderBase {
@Before
public void setup() {
Assume.assumeTrue(ErasureCodeNative.isNativeCodeLoaded());
this.encoderClass = NativeXORRawEncoder.class;
this.decoderClass = XORRawDecoder.class;
this.encoderFactoryClass = NativeXORRawErasureCoderFactory.class;
this.decoderFactoryClass = XORRawErasureCoderFactory.class;
setAllowDump(true);
}