From 292e367d0772f47e434de4a31af4edf1b07241dc Mon Sep 17 00:00:00 2001 From: drankye Date: Thu, 12 Mar 2015 23:35:22 +0800 Subject: [PATCH] HADOOP-11705. Make erasure coder configurable. Contributed by Kai Zheng --- .../hadoop-common/CHANGES-HDFS-EC-7285.txt | 4 +++ .../coder/AbstractErasureCoder.java | 5 ++- .../rawcoder/AbstractRawErasureCoder.java | 5 ++- .../hadoop/io/erasurecode/TestCoderBase.java | 6 ++++ .../coder/TestErasureCoderBase.java | 36 ++++++++++++++++--- .../rawcoder/TestRawCoderBase.java | 13 +++++-- 6 files changed, 60 insertions(+), 9 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index c17a1bd903..a97dc34b87 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -18,3 +18,7 @@ HADOOP-11646. Erasure Coder API for encoding and decoding of block group ( Kai Zheng via vinayakumarb ) + HADOOP-11705. Make erasure coder configurable. Contributed by Kai Zheng + ( Kai Zheng ) + + diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java index f2cc041e0b..8d3bc34050 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java @@ -17,12 +17,15 @@ */ package org.apache.hadoop.io.erasurecode.coder; +import org.apache.hadoop.conf.Configured; + /** * A common class of basic facilities to be shared by encoder and decoder * * It implements the {@link ErasureCoder} interface. */ -public abstract class AbstractErasureCoder implements ErasureCoder { +public abstract class AbstractErasureCoder + extends Configured implements ErasureCoder { private int numDataUnits; private int numParityUnits; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java index 74d2ab6fc1..e6f3d929da 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java @@ -17,12 +17,15 @@ */ package org.apache.hadoop.io.erasurecode.rawcoder; +import org.apache.hadoop.conf.Configured; + /** * A common class of basic facilities to be shared by encoder and decoder * * It implements the {@link RawErasureCoder} interface. */ -public abstract class AbstractRawErasureCoder implements RawErasureCoder { +public abstract class AbstractRawErasureCoder + extends Configured implements RawErasureCoder { private int numDataUnits; private int numParityUnits; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java index 3c4288c8c5..194413a43a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java @@ -43,6 +43,12 @@ public abstract class TestCoderBase { // may go to different coding implementations. protected boolean usingDirectBuffer = true; + /** + * Prepare before running the case. + * @param numDataUnits + * @param numParityUnits + * @param erasedIndexes + */ protected void prepare(int numDataUnits, int numParityUnits, int[] erasedIndexes) { this.numDataUnits = numDataUnits; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java index ca5c1c9c53..36e061a3ee 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.io.erasurecode.coder; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.erasurecode.ECBlock; import org.apache.hadoop.io.erasurecode.ECChunk; import org.apache.hadoop.io.erasurecode.ECBlockGroup; @@ -29,6 +30,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase { protected Class encoderClass; protected Class decoderClass; + private Configuration conf; protected int numChunksInBlock = 16; /** @@ -45,6 +47,19 @@ public TestBlock(ECChunk[] chunks) { } } + /** + * Prepare before running the case. + * @param conf + * @param numDataUnits + * @param numParityUnits + * @param erasedIndexes + */ + protected void prepare(Configuration conf, int numDataUnits, + int numParityUnits, int[] erasedIndexes) { + this.conf = conf; + super.prepare(numDataUnits, numParityUnits, erasedIndexes); + } + /** * Generating source data, encoding, recovering and then verifying. * RawErasureCoder mainly uses ECChunk to pass input and output data buffers, @@ -56,6 +71,7 @@ protected void testCoding(boolean usingDirectBuffer) { this.usingDirectBuffer = usingDirectBuffer; ErasureEncoder encoder = createEncoder(); + // Generate data and encode ECBlockGroup blockGroup = prepareBlockGroupForEncoding(); // Backup all the source chunks for later recovering because some coders @@ -65,17 +81,25 @@ protected void testCoding(boolean usingDirectBuffer) { // Make a copy of a strip for later comparing TestBlock[] toEraseBlocks = copyDataBlocksToErase(clonedDataBlocks); - ErasureCodingStep codingStep = encoder.encode(blockGroup); - performCodingStep(codingStep); + ErasureCodingStep codingStep; + try { + codingStep = encoder.encode(blockGroup); + performCodingStep(codingStep); + } finally { + encoder.release(); + } // Erase the copied sources eraseSomeDataBlocks(clonedDataBlocks); //Decode blockGroup = new ECBlockGroup(clonedDataBlocks, blockGroup.getParityBlocks()); ErasureDecoder decoder = createDecoder(); - codingStep = decoder.decode(blockGroup); - performCodingStep(codingStep); - + try { + codingStep = decoder.decode(blockGroup); + performCodingStep(codingStep); + } finally { + decoder.release(); + } //Compare compareAndVerify(toEraseBlocks, codingStep.getOutputBlocks()); } @@ -138,6 +162,7 @@ private ErasureEncoder createEncoder() { } encoder.initialize(numDataUnits, numParityUnits, chunkSize); + encoder.setConf(conf); return encoder; } @@ -154,6 +179,7 @@ private ErasureDecoder createDecoder() { } decoder.initialize(numDataUnits, numParityUnits, chunkSize); + decoder.setConf(conf); return decoder; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java index 5f6ccda238..890f632370 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java @@ -49,7 +49,11 @@ protected void testCoding(boolean usingDirectBuffer) { // Make a copy of a strip for later comparing ECChunk[] toEraseDataChunks = copyDataChunksToErase(clonedDataChunks); - encoder.encode(dataChunks, parityChunks); + try { + encoder.encode(dataChunks, parityChunks); + } finally { + encoder.release(); + } // Erase the copied sources eraseSomeDataBlocks(clonedDataChunks); @@ -58,7 +62,12 @@ protected void testCoding(boolean usingDirectBuffer) { parityChunks); ECChunk[] recoveredChunks = prepareOutputChunksForDecoding(); RawErasureDecoder decoder = createDecoder(); - decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks); + try { + decoder.decode(inputChunks, + getErasedIndexesForDecoding(), recoveredChunks); + } finally { + decoder.release(); + } //Compare compareAndVerify(toEraseDataChunks, recoveredChunks);