big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-5810. Removed the faulty and failing streaming test TestStreamingTaskLog. Contributed by Akira Ajisaka.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1582896 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2014-03-28 22:16:55 +00:00
parent c3dbaea6fd
commit 86ef362a7d
2 changed files with 3 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -269,6 +269,9 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5805. Fixed MapReduce JobHistory encoding of queue-name to escape
hyphens and thus avoid parsing errors. (Akira AJISAKA via vinodkv)
MAPREDUCE-5810. Removed the faulty and failing streaming test
TestStreamingTaskLog. (Akira Ajisaka via vinodkv)
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES

162
hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
View File

@ -1,162 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.streaming;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapreduce.MapReduceTestUtil;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.util.Shell;
import org.junit.Test;
import static org.junit.Assert.*;
/**
* This tests the environment set by TT for the child of task jvm.
* This will launch a streaming job with a shell script as mapper.
*/
public class TestStreamingTaskLog {
String input = "the dummy input";
Path inputPath = new Path("target/inDir");
Path outputPath = new Path("target/outDir");
String map = null;
MiniMRCluster mr = null;
FileSystem fs = null;
final long USERLOG_LIMIT_KB = 5;//consider 5kb as logSize
String[] genArgs() {
List<String> args = new ArrayList<String>();
for (Map.Entry<String, String> entry : mr.createJobConf()) {
args.add("-jobconf");
args.add(entry.getKey() + "=" + entry.getValue());
}
String[] argv = new String[] {
"-input", inputPath.toString(),
"-output", outputPath.toString(),
"-mapper", map,
"-reducer", StreamJob.REDUCE_NONE,
"-jobconf", "mapred.map.tasks=1",
"-jobconf", "keep.failed.task.files=true",
"-jobconf", "mapreduce.task.userlog.limit.kb=" + USERLOG_LIMIT_KB,
"-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp"),
"-jobconf", "mapred.jar=" + TestStreaming.STREAMING_JAR,
};
for (String arg : argv) {
args.add(arg);
}
argv = args.toArray(new String[args.size()]);
return argv;
}
/**
* This test validates the setting of HADOOP_ROOT_LOGGER to 'INFO,TLA' and the
* dependent properties
* (a) hadoop.tasklog.taskid and
* (b) hadoop.tasklog.totalLogFileSize
* for the children of java tasks in streaming jobs.
*/
@Test (timeout = 120000)
public void testStreamingTaskLogWithHadoopCmd() {
try {
final int numSlaves = 1;
JobConf conf = new JobConf();
fs = FileSystem.getLocal(conf);
Path testDir = new Path(System.getProperty("test.build.data","/tmp"));
if (fs.exists(testDir)) {
fs.delete(testDir, true);
}
fs.mkdirs(testDir);
File scriptFile = createScript(testDir.toString() +
(Shell.WINDOWS ? "/testTaskLog.cmd" : "/testTaskLog.sh"));
conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, false);
mr = new MiniMRCluster(numSlaves, fs.getUri().toString(), 1, null, null, conf);
writeInputFile(fs, inputPath);
map = Shell.WINDOWS ? "cmd /c " + scriptFile.getAbsolutePath() :
scriptFile.getAbsolutePath();
runStreamJobAndValidateEnv();
fs.delete(outputPath, true);
assertFalse("output not cleaned up", fs.exists(outputPath));
} catch(IOException e) {
fail(e.toString());
} finally {
if (mr != null) {
mr.shutdown();
}
}
}
private File createScript(String script) throws IOException {
File scriptFile = new File(script);
UtilTest.recursiveDelete(scriptFile);
FileOutputStream in = new FileOutputStream(scriptFile);
if (Shell.WINDOWS) {
in.write("@echo %HADOOP_ROOT_LOGGER% %HADOOP_CLIENT_OPTS%".getBytes());
} else {
in.write(("cat > /dev/null 2>&1\n" +
"echo $HADOOP_ROOT_LOGGER $HADOOP_CLIENT_OPTS").getBytes());
}
in.close();
Shell.execCommand(Shell.getSetPermissionCommand("+x", false,
scriptFile.getAbsolutePath()));
return scriptFile;
}
private void writeInputFile(FileSystem fs, Path dir) throws IOException {
DataOutputStream out = fs.create(new Path(dir, "part0"));
out.writeBytes(input);
out.close();
}
/**
* Runs the streaming job and validates the output.
* @throws IOException
*/
private void runStreamJobAndValidateEnv() throws IOException {
int returnStatus = -1;
boolean mayExit = false;
StreamJob job = new StreamJob(genArgs(), mayExit);
returnStatus = job.go();
assertEquals("StreamJob failed.", 0, returnStatus);
// validate environment variables set for the child(script) of java process
String env = MapReduceTestUtil.readOutput(outputPath, mr.createJobConf());
long logSize = USERLOG_LIMIT_KB * 1024;
assertTrue("environment set for child is wrong", env.contains("INFO,CLA")
&& env.contains("-Dyarn.app.container.log.dir=")
&& env.contains("-Dyarn.app.container.log.filesize=" + logSize)
&& env.contains("-Dlog4j.configuration="));
}
}