big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "MAPREDUCE-5875. Make Counter limits consistent across JobClient, MRAppMaster, and YarnChild. (Gera Shegalov via kasha)"

Browse Source 
This reverts commit e8a31f2e1c.
This commit is contained in:
Junping Du 2017-07-31 14:09:16 -07:00
parent 3e23415a92
commit fbb7d6bcbb
7 changed files with 21 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
View File

@ -62,7 +62,6 @@
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.hadoop.mapreduce.jobhistory.AMStartedEvent;
import org.apache.hadoop.mapreduce.jobhistory.EventReader;
import org.apache.hadoop.mapreduce.jobhistory.EventType;
@ -1281,8 +1280,6 @@ protected void serviceStart() throws Exception {
// finally set the job classloader
MRApps.setClassLoader(jobClassLoader, getConfig());
// set job classloader if configured
Limits.init(getConfig());
if (initFailed) {
JobEvent initFailedEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT_FAILED);

16
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
View File

@ -213,15 +213,15 @@ public FileSystem run() throws IOException, InterruptedException {
public Job getJob(JobID jobId) throws IOException, InterruptedException {
JobStatus status = client.getJobStatus(jobId);
if (status != null) {
final JobConf conf = new JobConf();
final Path jobPath = new Path(client.getFilesystemName(),
status.getJobFile());
final FileSystem fs = FileSystem.get(jobPath.toUri(), getConf());
JobConf conf;
try {
conf.addResource(fs.open(jobPath), jobPath.toString());
} catch (FileNotFoundException fnf) {
if (LOG.isWarnEnabled()) {
LOG.warn("Job conf missing on cluster", fnf);
conf = new JobConf(status.getJobFile());
} catch (RuntimeException ex) {
// If job file doesn't exist it means we can't find the job
if (ex.getCause() instanceof FileNotFoundException) {
return null;
} else {
throw ex;
}
}
return Job.getInstance(this, status, conf);

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
View File

@ -52,7 +52,6 @@
import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName;
import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
import org.apache.hadoop.mapreduce.security.TokenCache;
@ -246,7 +245,6 @@ JobStatus submitJobInternal(Job job, Cluster cluster)
// Write job file to submit dir
writeConf(conf, submitJobFile);
Limits.reset(conf);
//
// Now, actually submit the job (using the submit name)

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
View File

@ -123,9 +123,4 @@ public synchronized void checkGroups(int size) {
public synchronized LimitExceededException violation() {
return firstViolation;
}
public static synchronized void reset(Configuration conf) {
isInited = false;
init(conf);
}
}

16
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java
View File

@ -17,7 +17,6 @@
*/
package org.apache.hadoop.mapreduce.jobhistory;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.util.HashMap;
@ -25,8 +24,6 @@
import java.util.Set;
import java.util.TreeSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@ -36,7 +33,6 @@
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
import org.apache.hadoop.mapreduce.util.HostUtil;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@ -49,7 +45,6 @@
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class HistoryViewer {
private static final Log LOG = LogFactory.getLog(HistoryViewer.class);
private FileSystem fs;
private JobInfo job;
private HistoryViewerPrinter jhvp;
@ -89,17 +84,6 @@ public HistoryViewer(String historyFile, Configuration conf, boolean printAll,
System.err.println("Ignore unrecognized file: " + jobFile.getName());
throw new IOException(errorMsg);
}
final Path jobConfPath = new Path(jobFile.getParent(), jobDetails[0]
+ "_" + jobDetails[1] + "_" + jobDetails[2] + "_conf.xml");
final Configuration jobConf = new Configuration(conf);
try {
jobConf.addResource(fs.open(jobConfPath), jobConfPath.toString());
Limits.reset(conf);
} catch (FileNotFoundException fnf) {
if (LOG.isWarnEnabled()) {
LOG.warn("Missing job conf in history", fnf);
}
}
JobHistoryParser parser = new JobHistoryParser(fs, jobFile);
job = parser.parse();
String scheme = WebAppUtils.getHttpSchemePrefix(fs.getConf());

15
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.mapreduce.v2.hs;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.UnknownHostException;
import java.util.ArrayList;
@ -35,7 +34,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobACLsManager;
import org.apache.hadoop.mapred.TaskCompletionEvent;
@ -43,7 +41,6 @@
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
@ -353,19 +350,7 @@ protected synchronized void loadFullHistoryData(boolean loadTasks,
if (historyFileAbsolute != null) {
JobHistoryParser parser = null;
try {
final FileSystem fs = historyFileAbsolute.getFileSystem(conf);
parser = createJobHistoryParser(historyFileAbsolute);
final Path jobConfPath = new Path(historyFileAbsolute.getParent(),
JobHistoryUtils.getIntermediateConfFileName(jobId));
final Configuration conf = new Configuration();
try {
conf.addResource(fs.open(jobConfPath), jobConfPath.toString());
Limits.reset(conf);
} catch (FileNotFoundException fnf) {
if (LOG.isWarnEnabled()) {
LOG.warn("Missing job conf in history", fnf);
}
}
this.jobInfo = parser.parse();
} catch (IOException e) {
throw new YarnRuntimeException("Could not load history file "

87
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
View File

@ -55,14 +55,10 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TaskLog;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.Job;
@ -114,7 +110,6 @@ public class TestMRJobs {
EnumSet.of(RMAppState.FINISHED, RMAppState.FAILED, RMAppState.KILLED);
private static final int NUM_NODE_MGRS = 3;
private static final String TEST_IO_SORT_MB = "11";
private static final String TEST_GROUP_MAX = "200";
private static final int DEFAULT_REDUCES = 2;
protected int numSleepReducers = DEFAULT_REDUCES;
@ -466,58 +461,31 @@ private void waitForPriorityToUpdate(Job job, JobPriority expectedStatus)
}
@Test(timeout = 300000)
public void testConfVerificationWithClassloader() throws Exception {
testConfVerification(true, false, false, false);
public void testJobClassloader() throws IOException, InterruptedException,
ClassNotFoundException {
testJobClassloader(false);
}
@Test(timeout = 300000)
public void testConfVerificationWithClassloaderCustomClasses()
throws Exception {
testConfVerification(true, true, false, false);
public void testJobClassloaderWithCustomClasses() throws IOException,
InterruptedException, ClassNotFoundException {
testJobClassloader(true);
}
@Test(timeout = 300000)
public void testConfVerificationWithOutClassloader() throws Exception {
testConfVerification(false, false, false, false);
}
@Test(timeout = 300000)
public void testConfVerificationWithJobClient() throws Exception {
testConfVerification(false, false, true, false);
}
@Test(timeout = 300000)
public void testConfVerificationWithJobClientLocal() throws Exception {
testConfVerification(false, false, true, true);
}
private void testConfVerification(boolean useJobClassLoader,
boolean useCustomClasses, boolean useJobClientForMonitring,
boolean useLocal) throws Exception {
LOG.info("\n\n\nStarting testConfVerification()"
+ " jobClassloader=" + useJobClassLoader
+ " customClasses=" + useCustomClasses
+ " jobClient=" + useJobClientForMonitring
+ " localMode=" + useLocal);
private void testJobClassloader(boolean useCustomClasses) throws IOException,
InterruptedException, ClassNotFoundException {
LOG.info("\n\n\nStarting testJobClassloader()"
+ " useCustomClasses=" + useCustomClasses);
if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
+ " not found. Not running test.");
return;
}
final Configuration clusterConfig;
if (useLocal) {
clusterConfig = new Configuration();
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
} else {
clusterConfig = mrCluster.getConfig();
}
final JobClient jc = new JobClient(clusterConfig);
final Configuration sleepConf = new Configuration(clusterConfig);
final Configuration sleepConf = new Configuration(mrCluster.getConfig());
// set master address to local to test that local mode applied iff framework == local
sleepConf.set(MRConfig.MASTER_ADDRESS, "local");
sleepConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER,
useJobClassLoader);
sleepConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
if (useCustomClasses) {
// to test AM loading user classes such as output format class, we want
// to blacklist them from the system classes (they need to be prepended
@ -535,7 +503,6 @@ private void testConfVerification(boolean useJobClassLoader,
sleepConf.set(MRJobConfig.MAP_LOG_LEVEL, Level.ALL.toString());
sleepConf.set(MRJobConfig.REDUCE_LOG_LEVEL, Level.ALL.toString());
sleepConf.set(MRJobConfig.MAP_JAVA_OPTS, "-verbose:class");
sleepConf.set(MRJobConfig.COUNTER_GROUPS_MAX_KEY, TEST_GROUP_MAX);
final SleepJob sleepJob = new SleepJob();
sleepJob.setConf(sleepConf);
final Job job = sleepJob.createJob(1, 1, 10, 1, 10, 1);
@ -553,26 +520,7 @@ private void testConfVerification(boolean useJobClassLoader,
jobConf.setBoolean(MRJobConfig.MAP_SPECULATIVE, true);
}
job.submit();
final boolean succeeded;
if (useJobClientForMonitring && !useLocal) {
// We can't use getJobID in useLocal case because JobClient and Job
// point to different instances of LocalJobRunner
//
final JobID mapredJobID = JobID.downgrade(job.getJobID());
RunningJob runningJob = null;
do {
Thread.sleep(10);
runningJob = jc.getJob(mapredJobID);
} while (runningJob == null);
Assert.assertEquals("Unexpected RunningJob's "
+ MRJobConfig.COUNTER_GROUPS_MAX_KEY,
TEST_GROUP_MAX, runningJob.getConfiguration()
.get(MRJobConfig.COUNTER_GROUPS_MAX_KEY));
runningJob.waitForCompletion();
succeeded = runningJob.isSuccessful();
} else {
succeeded = job.waitForCompletion(true);
}
boolean succeeded = job.waitForCompletion(true);
Assert.assertTrue("Job status: " + job.getStatus().getFailureInfo(),
succeeded);
}
@ -1366,14 +1314,5 @@ protected void setup(Context context)
+ ", actual: " + ioSortMb);
}
}
@Override
public void map(IntWritable key, IntWritable value, Context context) throws IOException, InterruptedException {
super.map(key, value, context);
for (int i = 0; i < 100; i++) {
context.getCounter("testCounterGroup-" + i,
"testCounter").increment(1);
}
}
}
}