big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-7305. [JDK 11] TestMRJobsWithProfiler fails. (#2463)

Browse Source
This commit is contained in:
Akira Ajisaka 2020-11-18 14:15:11 +09:00 committed by GitHub
parent a7b923c80c
commit 2045a9df13
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithProfiler.java
View File

@ -35,6 +35,7 @@
import org.apache.hadoop.mapreduce.*; import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@ -43,6 +44,8 @@
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import static org.junit.Assume.assumeFalse;
public class TestMRJobsWithProfiler { public class TestMRJobsWithProfiler {
private static final Logger LOG = private static final Logger LOG =
@ -107,6 +110,8 @@ public static void tearDown() {
@Test (timeout = 150000) @Test (timeout = 150000)
public void testDefaultProfiler() throws Exception { public void testDefaultProfiler() throws Exception {
assumeFalse("The hprof agent has been removed since Java 9. Skipping.",
Shell.isJavaVersionAtLeast(9));
LOG.info("Starting testDefaultProfiler"); LOG.info("Starting testDefaultProfiler");
testProfilerInternal(true); testProfilerInternal(true);
} }
@ -132,13 +137,21 @@ private void testProfilerInternal(boolean useDefault) throws Exception {
sleepConf.setProfileTaskRange(false, String.valueOf(PROFILED_TASK_ID)); sleepConf.setProfileTaskRange(false, String.valueOf(PROFILED_TASK_ID));
if (!useDefault) { if (!useDefault) {
// use hprof for map to profile.out if (Shell.isJavaVersionAtLeast(9)) {
sleepConf.set(MRJobConfig.TASK_MAP_PROFILE_PARAMS, // use JDK Flight Recorder
"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n," sleepConf.set(MRJobConfig.TASK_MAP_PROFILE_PARAMS,
+ "file=%s"); "-XX:StartFlightRecording=dumponexit=true,filename=%s");
sleepConf.set(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS,
"-XX:StartFlightRecording=dumponexit=true,filename=%s");
} else {
// use hprof for map to profile.out
sleepConf.set(MRJobConfig.TASK_MAP_PROFILE_PARAMS,
"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,"
+ "file=%s");
// use Xprof for reduce to stdout // use Xprof for reduce to stdout
sleepConf.set(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, "-Xprof"); sleepConf.set(MRJobConfig.TASK_REDUCE_PROFILE_PARAMS, "-Xprof");
}
} }
sleepJob.setConf(sleepConf); sleepJob.setConf(sleepConf);
@ -211,6 +224,11 @@ private void testProfilerInternal(boolean useDefault) throws Exception {
Assert.assertEquals(4, taLogDirs.size()); // all 4 attempts found Assert.assertEquals(4, taLogDirs.size()); // all 4 attempts found
// Skip checking the contents because the JFR dumps binary files
if (Shell.isJavaVersionAtLeast(9)) {
return;
}
for (Map.Entry<TaskAttemptID,Path> dirEntry : taLogDirs.entrySet()) { for (Map.Entry<TaskAttemptID,Path> dirEntry : taLogDirs.entrySet()) {
final TaskAttemptID tid = dirEntry.getKey(); final TaskAttemptID tid = dirEntry.getKey();
final Path profilePath = new Path(dirEntry.getValue(), final Path profilePath = new Path(dirEntry.getValue(),