big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6662. Clear ASF Warnings on test data files. Contributed by Vinayakumar B

Browse Source
This commit is contained in:
Rohith Sharma K S 2016-03-29 10:32:21 +08:00
parent 948b758070
commit 0ef8bbfd87
9 changed files with 44 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
View File

@ -71,8 +71,9 @@ public static void setUpClass() throws Exception {
Configuration conf = new HdfsConfiguration(); Configuration conf = new HdfsConfiguration();
Configuration conf2 = new HdfsConfiguration(); Configuration conf2 = new HdfsConfiguration();
dfsCluster = new MiniDFSCluster.Builder(conf).build(); dfsCluster = new MiniDFSCluster.Builder(conf).build();
conf2.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, conf2.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, conf.get(
conf.get(MiniDFSCluster.HDFS_MINIDFS_BASEDIR) + "_2"); MiniDFSCluster.HDFS_MINIDFS_BASEDIR, MiniDFSCluster.getBaseDirectory())
+ "_2");
dfsCluster2 = new MiniDFSCluster.Builder(conf2).build(); dfsCluster2 = new MiniDFSCluster.Builder(conf2).build();
} }

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestFieldSelection.java
View File

@ -44,8 +44,9 @@ public static void launch() throws Exception {
FileSystem fs = FileSystem.get(conf); FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 10; int numOfInputLines = 10;
Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test"); String baseDir = System.getProperty("test.build.data", "build/test/data");
Path INPUT_DIR = new Path("build/test/input_for_field_selection_test"); Path OUTPUT_DIR = new Path(baseDir + "/output_for_field_selection_test");
Path INPUT_DIR = new Path(baseDir + "/input_for_field_selection_test");
String inputFile = "input.txt"; String inputFile = "input.txt";
fs.delete(INPUT_DIR, true); fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR); fs.mkdirs(INPUT_DIR);

19
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
View File

@ -97,9 +97,10 @@ public void testMRTimelineEventHandling() throws Exception {
+ cluster.getApplicationHistoryServer().getPort()); + cluster.getApplicationHistoryServer().getPort());
TimelineStore ts = cluster.getApplicationHistoryServer() TimelineStore ts = cluster.getApplicationHistoryServer()
.getTimelineStore(); .getTimelineStore();
String localPathRoot = System.getProperty("test.build.data",
Path inDir = new Path("input"); "build/test/data");
Path outDir = new Path("output"); Path inDir = new Path(localPathRoot, "input");
Path outDir = new Path(localPathRoot, "output");
RunningJob job = RunningJob job =
UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir); UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
Assert.assertEquals(JobStatus.SUCCEEDED, Assert.assertEquals(JobStatus.SUCCEEDED,
@ -155,8 +156,10 @@ public void testMapreduceJobTimelineServiceEnabled()
TimelineStore ts = cluster.getApplicationHistoryServer() TimelineStore ts = cluster.getApplicationHistoryServer()
.getTimelineStore(); .getTimelineStore();
Path inDir = new Path("input"); String localPathRoot = System.getProperty("test.build.data",
Path outDir = new Path("output"); "build/test/data");
Path inDir = new Path(localPathRoot, "input");
Path outDir = new Path(localPathRoot, "output");
RunningJob job = RunningJob job =
UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir); UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
Assert.assertEquals(JobStatus.SUCCEEDED, Assert.assertEquals(JobStatus.SUCCEEDED,
@ -195,8 +198,10 @@ public void testMapreduceJobTimelineServiceEnabled()
TimelineStore ts = cluster.getApplicationHistoryServer() TimelineStore ts = cluster.getApplicationHistoryServer()
.getTimelineStore(); .getTimelineStore();
Path inDir = new Path("input"); String localPathRoot = System.getProperty("test.build.data",
Path outDir = new Path("output"); "build/test/data");
Path inDir = new Path(localPathRoot, "input");
Path outDir = new Path(localPathRoot, "output");
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false);
RunningJob job = RunningJob job =

16
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java
View File

@ -436,8 +436,10 @@ public void testMapRedExecutionEnv() {
JobConf conf = new JobConf(mr.getConfig()); JobConf conf = new JobConf(mr.getConfig());
// initialize input, output directories // initialize input, output directories
Path inDir = new Path("input"); Path rootDir = new Path(System.getProperty("test.build.data",
Path outDir = new Path("output"); "build/test/data"));
Path inDir = new Path(rootDir, "input");
Path outDir = new Path(rootDir, "output");
String input = "The input"; String input = "The input";
// set config to use the ExecutionEnvCheckMapClass map class // set config to use the ExecutionEnvCheckMapClass map class
@ -462,9 +464,10 @@ public void testMapRedExecutionEnv() {
public void testTaskEnv(){ public void testTaskEnv(){
try { try {
JobConf conf = new JobConf(mr.getConfig()); JobConf conf = new JobConf(mr.getConfig());
String baseDir = System.getProperty("test.build.data", "build/test/data");
// initialize input, output directories // initialize input, output directories
Path inDir = new Path("testing/wc/input1"); Path inDir = new Path(baseDir + "/testing/wc/input1");
Path outDir = new Path("testing/wc/output1"); Path outDir = new Path(baseDir + "/testing/wc/output1");
FileSystem outFs = outDir.getFileSystem(conf); FileSystem outFs = outDir.getFileSystem(conf);
runTestTaskEnv(conf, inDir, outDir, false); runTestTaskEnv(conf, inDir, outDir, false);
outFs.delete(outDir, true); outFs.delete(outDir, true);
@ -485,9 +488,10 @@ public void testTaskEnv(){
public void testTaskOldEnv(){ public void testTaskOldEnv(){
try { try {
JobConf conf = new JobConf(mr.getConfig()); JobConf conf = new JobConf(mr.getConfig());
String baseDir = System.getProperty("test.build.data", "build/test/data");
// initialize input, output directories // initialize input, output directories
Path inDir = new Path("testing/wc/input1"); Path inDir = new Path(baseDir + "/testing/wc/input1");
Path outDir = new Path("testing/wc/output1"); Path outDir = new Path(baseDir + "/testing/wc/output1");
FileSystem outFs = outDir.getFileSystem(conf); FileSystem outFs = outDir.getFileSystem(conf);
runTestTaskEnv(conf, inDir, outDir, true); runTestTaskEnv(conf, inDir, outDir, true);
outFs.delete(outDir, true); outFs.delete(outDir, true);

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestOldCombinerGrouping.java
View File

@ -37,8 +37,9 @@
import java.util.UUID; import java.util.UUID;
public class TestOldCombinerGrouping { public class TestOldCombinerGrouping {
private static String TEST_ROOT_DIR = private static String TEST_ROOT_DIR = new File(System.getProperty(
new File("build", UUID.randomUUID().toString()).getAbsolutePath(); "test.build.data", "build/test/data"), UUID.randomUUID().toString())
.getAbsolutePath();
public static class Map implements public static class Map implements
Mapper<LongWritable, Text, Text, LongWritable> { Mapper<LongWritable, Text, Text, LongWritable> {

7
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/lib/aggregate/TestAggregates.java
View File

@ -46,8 +46,9 @@ public static void launch() throws Exception {
FileSystem fs = FileSystem.get(conf); FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 20; int numOfInputLines = 20;
Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test"); String baseDir = System.getProperty("test.build.data", "build/test/data");
Path INPUT_DIR = new Path("build/test/input_for_aggregates_test"); Path OUTPUT_DIR = new Path(baseDir + "/output_for_aggregates_test");
Path INPUT_DIR = new Path(baseDir + "/input_for_aggregates_test");
String inputFile = "input.txt"; String inputFile = "input.txt";
fs.delete(INPUT_DIR, true); fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR); fs.mkdirs(INPUT_DIR);
@ -115,7 +116,7 @@ public static void launch() throws Exception {
outdata = outdata.substring(0, expectedOutput.toString().length()); outdata = outdata.substring(0, expectedOutput.toString().length());
assertEquals(expectedOutput.toString(),outdata); assertEquals(expectedOutput.toString(),outdata);
//fs.delete(OUTPUT_DIR); fs.delete(OUTPUT_DIR, true);
fs.delete(INPUT_DIR, true); fs.delete(INPUT_DIR, true);
} }

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestNewCombinerGrouping.java
View File

@ -39,8 +39,9 @@
import java.util.UUID; import java.util.UUID;
public class TestNewCombinerGrouping { public class TestNewCombinerGrouping {
private static String TEST_ROOT_DIR = private static String TEST_ROOT_DIR = new File(System.getProperty(
new File("build", UUID.randomUUID().toString()).getAbsolutePath(); "test.build.data", "build/test/data"), UUID.randomUUID().toString())
.getAbsolutePath();
public static class Map extends public static class Map extends
Mapper<LongWritable, Text, Text, LongWritable> { Mapper<LongWritable, Text, Text, LongWritable> {

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/aggregate/TestMapReduceAggregates.java
View File

@ -51,8 +51,9 @@ public static void launch() throws Exception {
FileSystem fs = FileSystem.get(conf); FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 20; int numOfInputLines = 20;
Path OUTPUT_DIR = new Path("build/test/output_for_aggregates_test"); String baseDir = System.getProperty("test.build.data", "build/test/data");
Path INPUT_DIR = new Path("build/test/input_for_aggregates_test"); Path OUTPUT_DIR = new Path(baseDir + "/output_for_aggregates_test");
Path INPUT_DIR = new Path(baseDir + "/input_for_aggregates_test");
String inputFile = "input.txt"; String inputFile = "input.txt";
fs.delete(INPUT_DIR, true); fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR); fs.mkdirs(INPUT_DIR);

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
View File

@ -1772,7 +1772,9 @@ public void testForNonDefaultFileSystem() throws Throwable {
// default fs path // default fs path
assertEquals(DUMMY_FS_URI, FileSystem.getDefaultUri(conf).toString()); assertEquals(DUMMY_FS_URI, FileSystem.getDefaultUri(conf).toString());
// add a local file // add a local file
Path localPath = new Path("testFile1"); String localPathRoot = System.getProperty("test.build.data",
"build/test/data");
Path localPath = new Path(localPathRoot, "testFile1");
FileSystem lfs = FileSystem.getLocal(conf); FileSystem lfs = FileSystem.getLocal(conf);
FSDataOutputStream dos = lfs.create(localPath); FSDataOutputStream dos = lfs.create(localPath);
dos.writeChars("Local file for CFIF"); dos.writeChars("Local file for CFIF");