HDFS-8074 Define a system-wide default EC schema. Contributed by Kai Zheng

This commit is contained in:
Kai Zheng 2015-04-09 01:30:02 +08:00 committed by Zhe Zhang
parent 2ca3e38feb
commit 1a37b0fe9a
4 changed files with 120 additions and 8 deletions

View File

@ -27,11 +27,6 @@ You can modify and remove those not used yet, or add new ones.
--> -->
<schemas> <schemas>
<schema name="RS-6-3">
<k>6</k>
<m>3</m>
<codec>RS</codec>
</schema>
<schema name="RS-10-4"> <schema name="RS-10-4">
<k>10</k> <k>10</k>
<m>4</m> <m>4</m>

View File

@ -23,12 +23,12 @@
/** /**
* Erasure coding schema to housekeeper relevant information. * Erasure coding schema to housekeeper relevant information.
*/ */
public class ECSchema { public final class ECSchema {
public static final String NUM_DATA_UNITS_KEY = "k"; public static final String NUM_DATA_UNITS_KEY = "k";
public static final String NUM_PARITY_UNITS_KEY = "m"; public static final String NUM_PARITY_UNITS_KEY = "m";
public static final String CODEC_NAME_KEY = "codec"; public static final String CODEC_NAME_KEY = "codec";
public static final String CHUNK_SIZE_KEY = "chunkSize"; public static final String CHUNK_SIZE_KEY = "chunkSize";
public static final int DEFAULT_CHUNK_SIZE = 64 * 1024; // 64K public static final int DEFAULT_CHUNK_SIZE = 256 * 1024; // 256K
private String schemaName; private String schemaName;
private String codecName; private String codecName;
@ -81,6 +81,18 @@ public ECSchema(String schemaName, Map<String, String> options) {
initWith(codecName, dataUnits, parityUnits, options); initWith(codecName, dataUnits, parityUnits, options);
} }
/**
* Constructor with key parameters provided.
* @param schemaName
* @param codecName
* @param numDataUnits
* @param numParityUnits
*/
public ECSchema(String schemaName, String codecName,
int numDataUnits, int numParityUnits) {
this(schemaName, codecName, numDataUnits, numParityUnits, null);
}
/** /**
* Constructor with key parameters provided. Note the options may contain * Constructor with key parameters provided. Note the options may contain
* additional information for the erasure codec to interpret further. * additional information for the erasure codec to interpret further.
@ -200,4 +212,45 @@ public String toString() {
return sb.toString(); return sb.toString();
} }
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
ECSchema ecSchema = (ECSchema) o;
if (numDataUnits != ecSchema.numDataUnits) {
return false;
}
if (numParityUnits != ecSchema.numParityUnits) {
return false;
}
if (chunkSize != ecSchema.chunkSize) {
return false;
}
if (!schemaName.equals(ecSchema.schemaName)) {
return false;
}
if (!codecName.equals(ecSchema.codecName)) {
return false;
}
return options.equals(ecSchema.options);
}
@Override
public int hashCode() {
int result = schemaName.hashCode();
result = 31 * result + codecName.hashCode();
result = 31 * result + options.hashCode();
result = 31 * result + numDataUnits;
result = 31 * result + numParityUnits;
result = 31 * result + chunkSize;
return result;
}
} }

View File

@ -53,3 +53,5 @@
HDFS-8023. Erasure Coding: retrieve eraure coding schema for a file from HDFS-8023. Erasure Coding: retrieve eraure coding schema for a file from
NameNode (vinayakumarb) NameNode (vinayakumarb)
HDFS-8074. Define a system-wide default EC schema. (Kai Zheng)

View File

@ -0,0 +1,62 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.ECSchema;
/**
* This manages EC schemas predefined and activated in the system. It loads from
* predefined ones in XML and syncs with persisted ones in NameNode image.
*
* This class is instantiated by the FSNamesystem.
*/
@InterfaceAudience.LimitedPrivate({"HDFS"})
public final class ECSchemaManager {
private static final int DEFAULT_DATA_BLOCKS = 6;
private static final int DEFAULT_PARITY_BLOCKS = 3;
private static final String DEFAULT_CODEC_NAME = "rs";
private static final String DEFAULT_SCHEMA_NAME = "SYS-DEFAULT-RS-6-3";
private static ECSchema SYS_DEFAULT_SCHEMA = new ECSchema(DEFAULT_SCHEMA_NAME,
DEFAULT_CODEC_NAME, DEFAULT_DATA_BLOCKS, DEFAULT_PARITY_BLOCKS);
/**
* Get system-wide default EC schema, which can be used by default when no
* schema is specified for an EC zone.
* @return schema
*/
public static ECSchema getSystemDefaultSchema() {
return SYS_DEFAULT_SCHEMA;
}
/**
* Tell the specified schema is the system default one or not.
* @param schema
* @return true if it's the default false otherwise
*/
public static boolean isSystemDefault(ECSchema schema) {
if (schema == null) {
throw new IllegalArgumentException("Invalid schema parameter");
}
// schema name is the identifier, but for safety we check all properties.
return SYS_DEFAULT_SCHEMA.equals(schema);
}
}