big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6192. Create unit test to automatically compare MR related classes and mapred-default.xml (rchiang via rkanter)

Browse Source
This commit is contained in:
Robert Kanter 2015-05-04 17:48:10 -07:00
parent 0100b15501
commit 9809a16d3c
3 changed files with 135 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
View File

@ -146,6 +146,12 @@ public abstract class TestConfigurationFieldsBase {
*/
private Set<String> xmlFieldsMissingInConfiguration = null;
/**
* Member variable for debugging base class operation
*/
protected boolean configDebug = false;
protected boolean xmlDebug = false;
/**
* Abstract method to be used by subclasses for initializing base
* members.
@ -168,13 +174,16 @@ public abstract class TestConfigurationFieldsBase {
HashMap<String,String> retVal = new HashMap<String,String>();
// Setup regexp for valid properties
String propRegex = "^[A-Za-z_-]+(\\.[A-Za-z_-]+)+$";
String propRegex = "^[A-Za-z][A-Za-z0-9_-]+(\\.[A-Za-z0-9_-]+)+$";
Pattern p = Pattern.compile(propRegex);
// Iterate through class member variables
int totalFields = 0;
String value;
for (Field f : fields) {
if (configDebug) {
System.out.println("Field: " + f);
}
// Filter out anything that isn't "public static final"
if (!Modifier.isStatic(f.getModifiers()) ||
!Modifier.isPublic(f.getModifiers()) ||
@ -192,6 +201,9 @@ public abstract class TestConfigurationFieldsBase {
} catch (IllegalAccessException iaException) {
continue;
}
if (configDebug) {
System.out.println(" Value: " + value);
}
// Special Case: Detect and ignore partial properties (ending in x)
// or file properties (ending in .xml)
if (value.endsWith(".xml") ||
@ -221,11 +233,23 @@ public abstract class TestConfigurationFieldsBase {
// something like: blah.blah2(.blah3.blah4...)
Matcher m = p.matcher(value);
if (!m.find()) {
if (configDebug) {
System.out.println(" Passes Regex: false");
}
continue;
}
if (configDebug) {
System.out.println(" Passes Regex: true");
}
// Save member variable/value as hash
retVal.put(value,f.getName());
if (!retVal.containsKey(value)) {
retVal.put(value,f.getName());
} else {
if (configDebug) {
System.out.println("ERROR: Already found key for property " + value);
}
}
}
return retVal;
@ -256,6 +280,9 @@ public abstract class TestConfigurationFieldsBase {
// Ignore known xml props
if (xmlPropsToSkipCompare != null) {
if (xmlPropsToSkipCompare.contains(key)) {
if (xmlDebug) {
System.out.println(" Skipping Full Key: " + key);
}
continue;
}
}
@ -270,14 +297,23 @@ public abstract class TestConfigurationFieldsBase {
}
}
if (skipPrefix) {
if (xmlDebug) {
System.out.println(" Skipping Prefix Key: " + key);
}
continue;
}
if (conf.onlyKeyExists(key)) {
retVal.put(key,null);
if (xmlDebug) {
System.out.println(" XML Key,Null Value: " + key);
}
} else {
String value = conf.get(key);
if (value!=null) {
retVal.put(key,entry.getValue());
if (xmlDebug) {
System.out.println(" XML Key,Valid Value: " + key);
}
}
}
kvItr.remove();
@ -312,6 +348,10 @@ public void setupTestConfigurationFields() throws Exception {
// Create class member/value map
configurationMemberVariables = new HashMap<String,String>();
if (configDebug) {
System.out.println("Reading configuration classes");
System.out.println("");
}
for (Class c : configurationClasses) {
Field[] fields = c.getDeclaredFields();
Map<String,String> memberMap =
@ -320,9 +360,23 @@ public void setupTestConfigurationFields() throws Exception {
configurationMemberVariables.putAll(memberMap);
}
}
if (configDebug) {
System.out.println("");
System.out.println("=====");
System.out.println("");
}
// Create XML key/value map
if (xmlDebug) {
System.out.println("Reading XML property files");
System.out.println("");
}
xmlKeyValueMap = extractPropertiesFromXml(xmlFilename);
if (xmlDebug) {
System.out.println("");
System.out.println("=====");
System.out.println("");
}
// Find class members not in the XML file
configurationFieldsMissingInXmlFile = compareConfigurationToXmlFields

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -304,6 +304,9 @@ Release 2.8.0 - UNRELEASED
mapreduce.tasktracker.taskmemorymanager.monitoringinterval.
(J.Andreina via aajisaka)
MAPREDUCE-6192. Create unit test to automatically compare
MR related classes and mapred-default.xml (rchiang via rkanter)
OPTIMIZATIONS
BUG FIXES

76
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestMapreduceConfigFields.java Normal file
View File

@ -0,0 +1,76 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapreduce;
import java.util.HashSet;
import org.apache.hadoop.conf.TestConfigurationFieldsBase;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
/**
* Unit test class to compare the following MR Configuration classes:
* <p></p>
* {@link org.apache.hadoop.mapreduce.MRJobConfig}
* {@link org.apache.hadoop.mapreduce.MRConfig}
* {@link org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig}
* {@link org.apache.hadoop.mapred.ShuffleHandler}
* {@link org.apache.hadoop.mapreduce.lib.output.FileOutputFormat}
* {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}
* {@link org.apache.hadoop.mapreduce.Job}
* {@link org.apache.hadoop.mapreduce.lib.input.NLineInputFormat}
* {@link org.apache.hadoop.mapred.JobConf}
* <p></p>
* against mapred-default.xml for missing properties. Currently only
* throws an error if the class is missing a property.
* <p></p>
* Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase}
* for how this class works.
*/
public class TestMapreduceConfigFields extends TestConfigurationFieldsBase {
@SuppressWarnings("deprecation")
@Override
public void initializeMemberVariables() {
xmlFilename = new String("mapred-default.xml");
configurationClasses = new Class[] { MRJobConfig.class, MRConfig.class,
JHAdminConfig.class, ShuffleHandler.class, FileOutputFormat.class,
FileInputFormat.class, Job.class, NLineInputFormat.class,
JobConf.class, FileOutputCommitter.class };
// Initialize used variables
configurationPropsToSkipCompare = new HashSet<String>();
// Set error modes
errorIfMissingConfigProps = true;
errorIfMissingXmlProps = false;
// Ignore deprecated MR1 properties in JobConf
configurationPropsToSkipCompare
.add(JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY);
configurationPropsToSkipCompare
.add(JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY);
}
}