big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-11841. Remove unused ecschema-def.xml files.

Browse Source
This commit is contained in:
Tsz-Wo Nicholas Sze 2015-04-17 16:07:07 -07:00 committed by Zhe Zhang
parent 64f8f0a145
commit 909632dd90
5 changed files with 25 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
View File

@ -42,3 +42,5 @@
( Kai Zheng via vinayakumarb )
HADOOP-11818. Minor improvements for erasurecode classes. (Rakesh R via Kai Zheng)
HADOOP-11841. Remove unused ecschema-def.xml files. (szetszwo)

35
hadoop-common-project/hadoop-common/src/main/conf/ecschema-def.xml
View File

@ -1,35 +0,0 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
Please define your EC schemas here. Note, once these schemas are loaded
and referenced by EC storage policies, any change to them will be ignored.
You can modify and remove those not used yet, or add new ones.
-->
<schemas>
<schema name="RS-10-4">
<k>10</k>
<m>4</m>
<codec>RS</codec>
</schema>
</schemas>

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
View File

@ -143,11 +143,6 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
/** Supported erasure codec classes */
public static final String IO_ERASURECODE_CODECS_KEY = "io.erasurecode.codecs";
public static final String IO_ERASURECODE_SCHEMA_FILE_KEY =
"io.erasurecode.schema.file";
public static final String IO_ERASURECODE_SCHEMA_FILE_DEFAULT =
"ecschema-def.xml";
/** Use XOR raw coder when possible for the RS codec */
public static final String IO_ERASURECODE_CODEC_RS_USEXOR_KEY =
"io.erasurecode.codec.rs.usexor";

36
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/SchemaLoader.java
View File

@ -17,20 +17,27 @@
*/
package org.apache.hadoop.io.erasurecode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.w3c.dom.*;
import org.xml.sax.SAXException;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.dom.Text;
import org.xml.sax.SAXException;
/**
* A EC schema loading utility that loads predefined EC schemas from XML file
@ -42,8 +49,8 @@ public class SchemaLoader {
* Load predefined ec schemas from configuration file. This file is
* expected to be in the XML format.
*/
public List<ECSchema> loadSchema(Configuration conf) {
File confFile = getSchemaFile(conf);
public List<ECSchema> loadSchema(String schemaFilePath) {
File confFile = getSchemaFile(schemaFilePath);
if (confFile == null) {
LOG.warn("Not found any predefined EC schema file");
return Collections.emptyList();
@ -100,10 +107,7 @@ private List<ECSchema> loadSchema(File schemaFile)
* Path to the XML file containing predefined ec schemas. If the path is
* relative, it is searched for in the classpath.
*/
private File getSchemaFile(Configuration conf) {
String schemaFilePath = conf.get(
CommonConfigurationKeys.IO_ERASURECODE_SCHEMA_FILE_KEY,
CommonConfigurationKeys.IO_ERASURECODE_SCHEMA_FILE_DEFAULT);
private File getSchemaFile(String schemaFilePath) {
File schemaFile = new File(schemaFilePath);
if (! schemaFile.isAbsolute()) {
URL url = Thread.currentThread().getContextClassLoader()

12
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestSchemaLoader.java
View File

@ -17,16 +17,14 @@
*/
package org.apache.hadoop.io.erasurecode;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.List;
import static org.junit.Assert.assertEquals;
import org.junit.Test;
public class TestSchemaLoader {
@ -54,12 +52,8 @@ public void testLoadSchema() throws Exception {
out.println("</schemas>");
out.close();
Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.IO_ERASURECODE_SCHEMA_FILE_KEY,
SCHEMA_FILE);
SchemaLoader schemaLoader = new SchemaLoader();
List<ECSchema> schemas = schemaLoader.loadSchema(conf);
List<ECSchema> schemas = schemaLoader.loadSchema(SCHEMA_FILE);
assertEquals(2, schemas.size());