From e54a74b566f89a424a2f4735a35553ece3bd35d9 Mon Sep 17 00:00:00 2001 From: Zhe Zhang Date: Fri, 3 Apr 2015 15:22:50 -0700 Subject: [PATCH] HADOOP-11740. Combine erasure encoder and decoder interfaces. Contributed by Zhe Zhang. --- .../coder/AbstractErasureDecoder.java | 7 ++-- .../coder/AbstractErasureEncoder.java | 7 ++-- .../io/erasurecode/coder/ErasureCoder.java | 12 ++++++ .../io/erasurecode/coder/ErasureDecoder.java | 41 ------------------- .../io/erasurecode/coder/ErasureEncoder.java | 39 ------------------ .../coder/TestErasureCoderBase.java | 20 ++++----- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 14 ++++++- 7 files changed, 41 insertions(+), 99 deletions(-) delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java delete mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java index 54a6d1ed7a..cd31294f80 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java @@ -23,13 +23,12 @@ /** * An abstract erasure decoder that's to be inherited by new decoders. * - * It implements the {@link ErasureDecoder} interface. + * It implements the {@link ErasureCoder} interface. */ -public abstract class AbstractErasureDecoder extends AbstractErasureCoder - implements ErasureDecoder { +public abstract class AbstractErasureDecoder extends AbstractErasureCoder { @Override - public ErasureCodingStep decode(ECBlockGroup blockGroup) { + public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) { // We may have more than this when considering complicate cases. HADOOP-11550 return prepareDecodingStep(blockGroup); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java index 09b31e5bd2..a836b75e7a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java @@ -23,13 +23,12 @@ /** * An abstract erasure encoder that's to be inherited by new encoders. * - * It implements the {@link ErasureEncoder} interface. + * It implements the {@link ErasureCoder} interface. */ -public abstract class AbstractErasureEncoder extends AbstractErasureCoder - implements ErasureEncoder { +public abstract class AbstractErasureEncoder extends AbstractErasureCoder { @Override - public ErasureCodingStep encode(ECBlockGroup blockGroup) { + public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) { // We may have more than this when considering complicate cases. HADOOP-11550 return prepareEncodingStep(blockGroup); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java index c5922f37b1..fb9015664b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java @@ -18,6 +18,7 @@ package org.apache.hadoop.io.erasurecode.coder; import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.io.erasurecode.ECBlockGroup; /** * An erasure coder to perform encoding or decoding given a group. Generally it @@ -63,6 +64,17 @@ public interface ErasureCoder extends Configurable { */ public int getChunkSize(); + /** + * Calculate the encoding or decoding steps given a block blockGroup. + * + * Note, currently only one coding step is supported. Will support complex + * cases of multiple coding steps. + * + * @param blockGroup the erasure coding block group containing all necessary + * information for codec calculation + */ + public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup); + /** * Tell if native or off-heap buffer is preferred or not. It's for callers to * decide how to allocate coding chunk buffers, either on heap or off heap. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java deleted file mode 100644 index dfd9e545b5..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.io.erasurecode.coder; - - -import org.apache.hadoop.io.erasurecode.ECBlockGroup; - -/** - * Erasure decoder interface to perform decoding given a block group. - * - * It extends {@link ErasureCoder}. - */ -public interface ErasureDecoder extends ErasureCoder { - - /** - * Perform the decoding given a blockGroup. By default it will try the best to - * attempt to recover all the missing blocks according to the codec logic. - * - * Note, currently only one coding step is supported. Will support complex - * cases of multiple coding steps. - * - * @param blockGroup - */ - public ErasureCodingStep decode(ECBlockGroup blockGroup); - -} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java deleted file mode 100644 index e837d229dc..0000000000 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.io.erasurecode.coder; - -import org.apache.hadoop.io.erasurecode.ECBlockGroup; - -/** - * Erasure encoder interface to perform encoding given a block group. - * - * It extends {@link ErasureCoder}. - */ -public interface ErasureEncoder extends ErasureCoder { - - /** - * Calculate the encoding steps given a block blockGroup. - * - * Note, currently only one coding step is supported. Will support complex - * cases of multiple coding steps. - * - * @param blockGroup - */ - public ErasureCodingStep encode(ECBlockGroup blockGroup); - -} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java index b963a594eb..05a62a79f4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java @@ -26,8 +26,8 @@ * Erasure coder test base with utilities. */ public abstract class TestErasureCoderBase extends TestCoderBase { - protected Class encoderClass; - protected Class decoderClass; + protected Class encoderClass; + protected Class decoderClass; protected int numChunksInBlock = 16; @@ -55,7 +55,7 @@ public TestBlock(ECChunk[] chunks) { protected void testCoding(boolean usingDirectBuffer) { this.usingDirectBuffer = usingDirectBuffer; - ErasureEncoder encoder = createEncoder(); + ErasureCoder encoder = createEncoder(); // Generate data and encode ECBlockGroup blockGroup = prepareBlockGroupForEncoding(); @@ -68,7 +68,7 @@ protected void testCoding(boolean usingDirectBuffer) { ErasureCodingStep codingStep; try { - codingStep = encoder.encode(blockGroup); + codingStep = encoder.calculateCoding(blockGroup); performCodingStep(codingStep); } finally { encoder.release(); @@ -78,9 +78,9 @@ protected void testCoding(boolean usingDirectBuffer) { //Decode blockGroup = new ECBlockGroup(clonedDataBlocks, blockGroup.getParityBlocks()); - ErasureDecoder decoder = createDecoder(); + ErasureCoder decoder = createDecoder(); try { - codingStep = decoder.decode(blockGroup); + codingStep = decoder.calculateCoding(blockGroup); performCodingStep(codingStep); } finally { decoder.release(); @@ -138,8 +138,8 @@ protected void compareAndVerify(ECBlock[] erasedBlocks, * Create erasure encoder for test. * @return */ - private ErasureEncoder createEncoder() { - ErasureEncoder encoder; + private ErasureCoder createEncoder() { + ErasureCoder encoder; try { encoder = encoderClass.newInstance(); } catch (Exception e) { @@ -155,8 +155,8 @@ private ErasureEncoder createEncoder() { * Create the erasure decoder for the test. * @return */ - private ErasureDecoder createDecoder() { - ErasureDecoder decoder; + private ErasureCoder createDecoder() { + ErasureCoder decoder; try { decoder = decoderClass.newInstance(); } catch (Exception e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index a686315142..4e60a7c46d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -43,4 +43,16 @@ blocks in NameNode (Jing Zhao) HDFS-8005. Erasure Coding: simplify striped block recovery work computation - and add tests (Jing Zhao) \ No newline at end of file + and add tests (Jing Zhao) + + HDFS-7617. Add unit tests for editlog transactions for EC + (Hui Zheng via Zhe Zhang) + + HADOOP-11782. Correct two thrown messages in ECSchema class + (Xinwei Qin via Kai Zheng) + + HDFS-7839. Erasure coding: implement facilities in NameNode to create and + manage EC zones (Zhe Zhang) + + HADOOP-11740. Combine erasure encoder and decoder interfaces (Zhe Zhang) +