HADOOP-11740. Combine erasure encoder and decoder interfaces. Contributed by Zhe Zhang.
This commit is contained in:
Zhe Zhang
Zhe Zhang
parent
1af8c14862
commit
e54a74b566
@ -23,13 +23,12 @@
|
||||
/**
|
||||
* An abstract erasure decoder that's to be inherited by new decoders.
|
||||
*
|
||||
* It implements the {@link ErasureDecoder} interface.
|
||||
* It implements the {@link ErasureCoder} interface.
|
||||
*/
|
||||
public abstract class AbstractErasureDecoder extends AbstractErasureCoder
|
||||
implements ErasureDecoder {
|
||||
public abstract class AbstractErasureDecoder extends AbstractErasureCoder {
|
||||
|
||||
@Override
|
||||
public ErasureCodingStep decode(ECBlockGroup blockGroup) {
|
||||
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
|
||||
// We may have more than this when considering complicate cases. HADOOP-11550
|
||||
return prepareDecodingStep(blockGroup);
|
||||
}
|
||||
|
||||
@ -23,13 +23,12 @@
|
||||
/**
|
||||
* An abstract erasure encoder that's to be inherited by new encoders.
|
||||
*
|
||||
* It implements the {@link ErasureEncoder} interface.
|
||||
* It implements the {@link ErasureCoder} interface.
|
||||
*/
|
||||
public abstract class AbstractErasureEncoder extends AbstractErasureCoder
|
||||
implements ErasureEncoder {
|
||||
public abstract class AbstractErasureEncoder extends AbstractErasureCoder {
|
||||
|
||||
@Override
|
||||
public ErasureCodingStep encode(ECBlockGroup blockGroup) {
|
||||
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
|
||||
// We may have more than this when considering complicate cases. HADOOP-11550
|
||||
return prepareEncodingStep(blockGroup);
|
||||
}
|
||||
|
||||
@ -18,6 +18,7 @@
|
||||
package org.apache.hadoop.io.erasurecode.coder;
|
||||
|
||||
import org.apache.hadoop.conf.Configurable;
|
||||
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
|
||||
|
||||
/**
|
||||
* An erasure coder to perform encoding or decoding given a group. Generally it
|
||||
@ -63,6 +64,17 @@ public interface ErasureCoder extends Configurable {
|
||||
*/
|
||||
public int getChunkSize();
|
||||
|
||||
/**
|
||||
* Calculate the encoding or decoding steps given a block blockGroup.
|
||||
*
|
||||
* Note, currently only one coding step is supported. Will support complex
|
||||
* cases of multiple coding steps.
|
||||
*
|
||||
* @param blockGroup the erasure coding block group containing all necessary
|
||||
* information for codec calculation
|
||||
*/
|
||||
public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup);
|
||||
|
||||
/**
|
||||
* Tell if native or off-heap buffer is preferred or not. It's for callers to
|
||||
* decide how to allocate coding chunk buffers, either on heap or off heap.
|
||||
|
||||
@ -1,41 +0,0 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.io.erasurecode.coder;
|
||||
|
||||
|
||||
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
|
||||
|
||||
/**
|
||||
* Erasure decoder interface to perform decoding given a block group.
|
||||
*
|
||||
* It extends {@link ErasureCoder}.
|
||||
*/
|
||||
public interface ErasureDecoder extends ErasureCoder {
|
||||
|
||||
/**
|
||||
* Perform the decoding given a blockGroup. By default it will try the best to
|
||||
* attempt to recover all the missing blocks according to the codec logic.
|
||||
*
|
||||
* Note, currently only one coding step is supported. Will support complex
|
||||
* cases of multiple coding steps.
|
||||
*
|
||||
* @param blockGroup
|
||||
*/
|
||||
public ErasureCodingStep decode(ECBlockGroup blockGroup);
|
||||
|
||||
}
|
||||
@ -1,39 +0,0 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.io.erasurecode.coder;
|
||||
|
||||
import org.apache.hadoop.io.erasurecode.ECBlockGroup;
|
||||
|
||||
/**
|
||||
* Erasure encoder interface to perform encoding given a block group.
|
||||
*
|
||||
* It extends {@link ErasureCoder}.
|
||||
*/
|
||||
public interface ErasureEncoder extends ErasureCoder {
|
||||
|
||||
/**
|
||||
* Calculate the encoding steps given a block blockGroup.
|
||||
*
|
||||
* Note, currently only one coding step is supported. Will support complex
|
||||
* cases of multiple coding steps.
|
||||
*
|
||||
* @param blockGroup
|
||||
*/
|
||||
public ErasureCodingStep encode(ECBlockGroup blockGroup);
|
||||
|
||||
}
|
||||
@ -26,8 +26,8 @@
|
||||
* Erasure coder test base with utilities.
|
||||
*/
|
||||
public abstract class TestErasureCoderBase extends TestCoderBase {
|
||||
protected Class<? extends ErasureEncoder> encoderClass;
|
||||
protected Class<? extends ErasureDecoder> decoderClass;
|
||||
protected Class<? extends ErasureCoder> encoderClass;
|
||||
protected Class<? extends ErasureCoder> decoderClass;
|
||||
|
||||
protected int numChunksInBlock = 16;
|
||||
|
||||
@ -55,7 +55,7 @@ public TestBlock(ECChunk[] chunks) {
|
||||
protected void testCoding(boolean usingDirectBuffer) {
|
||||
this.usingDirectBuffer = usingDirectBuffer;
|
||||
|
||||
ErasureEncoder encoder = createEncoder();
|
||||
ErasureCoder encoder = createEncoder();
|
||||
|
||||
// Generate data and encode
|
||||
ECBlockGroup blockGroup = prepareBlockGroupForEncoding();
|
||||
@ -68,7 +68,7 @@ protected void testCoding(boolean usingDirectBuffer) {
|
||||
|
||||
ErasureCodingStep codingStep;
|
||||
try {
|
||||
codingStep = encoder.encode(blockGroup);
|
||||
codingStep = encoder.calculateCoding(blockGroup);
|
||||
performCodingStep(codingStep);
|
||||
} finally {
|
||||
encoder.release();
|
||||
@ -78,9 +78,9 @@ protected void testCoding(boolean usingDirectBuffer) {
|
||||
|
||||
//Decode
|
||||
blockGroup = new ECBlockGroup(clonedDataBlocks, blockGroup.getParityBlocks());
|
||||
ErasureDecoder decoder = createDecoder();
|
||||
ErasureCoder decoder = createDecoder();
|
||||
try {
|
||||
codingStep = decoder.decode(blockGroup);
|
||||
codingStep = decoder.calculateCoding(blockGroup);
|
||||
performCodingStep(codingStep);
|
||||
} finally {
|
||||
decoder.release();
|
||||
@ -138,8 +138,8 @@ protected void compareAndVerify(ECBlock[] erasedBlocks,
|
||||
* Create erasure encoder for test.
|
||||
* @return
|
||||
*/
|
||||
private ErasureEncoder createEncoder() {
|
||||
ErasureEncoder encoder;
|
||||
private ErasureCoder createEncoder() {
|
||||
ErasureCoder encoder;
|
||||
try {
|
||||
encoder = encoderClass.newInstance();
|
||||
} catch (Exception e) {
|
||||
@ -155,8 +155,8 @@ private ErasureEncoder createEncoder() {
|
||||
* Create the erasure decoder for the test.
|
||||
* @return
|
||||
*/
|
||||
private ErasureDecoder createDecoder() {
|
||||
ErasureDecoder decoder;
|
||||
private ErasureCoder createDecoder() {
|
||||
ErasureCoder decoder;
|
||||
try {
|
||||
decoder = decoderClass.newInstance();
|
||||
} catch (Exception e) {
|
||||
|
||||
@ -43,4 +43,16 @@
|
||||
blocks in NameNode (Jing Zhao)
|
||||
|
||||
HDFS-8005. Erasure Coding: simplify striped block recovery work computation
|
||||
and add tests (Jing Zhao)
|
||||
and add tests (Jing Zhao)
|
||||
|
||||
HDFS-7617. Add unit tests for editlog transactions for EC
|
||||
(Hui Zheng via Zhe Zhang)
|
||||
|
||||
HADOOP-11782. Correct two thrown messages in ECSchema class
|
||||
(Xinwei Qin via Kai Zheng)
|
||||
|
||||
HDFS-7839. Erasure coding: implement facilities in NameNode to create and
|
||||
manage EC zones (Zhe Zhang)
|
||||
|
||||
HADOOP-11740. Combine erasure encoder and decoder interfaces (Zhe Zhang)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user