big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-7305. [JDK 11] TestMRJobsWithProfiler fails. (#2463)

Browse Source 
(cherry picked from commit 2045a9df13)
This commit is contained in:
Akira Ajisaka 2020-11-18 14:15:11 +09:00
parent 22039a14ff
commit 71c106b412
No known key found for this signature in database
GPG Key ID: C1EDBB9CA400FD50
1 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
View File

@ -35,6 +35,7 @@
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@ -43,6 +44,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.junit.Assume.assumeFalse;
public class TestMRJobsWithProfiler {
private static final Logger LOG =
@ -107,6 +110,8 @@ public static void tearDown() {
@Test (timeout = 150000)
public void testDefaultProfiler() throws Exception {
assumeFalse("The hprof agent has been removed since Java 9. Skipping.",
Shell.isJavaVersionAtLeast(9));
LOG.info("Starting testDefaultProfiler");
testProfilerInternal(true);
}
@ -132,13 +137,21 @@ private void testProfilerInternal(boolean useDefault) throws Exception {
sleepConf.setProfileTaskRange(false, String.valueOf(PROFILED_TASK_ID));
if (!useDefault) {
// use hprof for map to profile.out
sleepConf.set(MRJobConfig.TASK_MAP_PROFILE_PARAMS,
"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,"
+ "file=%s");
if (Shell.isJavaVersionAtLeast(9)) {
// use JDK Flight Recorder
sleepConf.set(MRJobConfig.TASK_MAP_PROFILE_PARAMS,
"-XX:StartFlightRecording=dumponexit=true,filename=%s");
sleepConf.set(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS,
"-XX:StartFlightRecording=dumponexit=true,filename=%s");
} else {
// use hprof for map to profile.out
sleepConf.set(MRJobConfig.TASK_MAP_PROFILE_PARAMS,
"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,"
+ "file=%s");
// use Xprof for reduce to stdout
sleepConf.set(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, "-Xprof");
// use Xprof for reduce to stdout
sleepConf.set(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, "-Xprof");
}
}
sleepJob.setConf(sleepConf);
@ -211,6 +224,11 @@ private void testProfilerInternal(boolean useDefault) throws Exception {
Assert.assertEquals(4, taLogDirs.size()); // all 4 attempts found
// Skip checking the contents because the JFR dumps binary files
if (Shell.isJavaVersionAtLeast(9)) {
return;
}
for (Map.Entry<TaskAttemptID,Path> dirEntry : taLogDirs.entrySet()) {
final TaskAttemptID tid = dirEntry.getKey();
final Path profilePath = new Path(dirEntry.getValue(),