diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index 1445481f70..8c9f6054d1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -62,7 +62,6 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TypeConverter; -import org.apache.hadoop.mapreduce.counters.Limits; import org.apache.hadoop.mapreduce.jobhistory.AMStartedEvent; import org.apache.hadoop.mapreduce.jobhistory.EventReader; import org.apache.hadoop.mapreduce.jobhistory.EventType; @@ -1281,8 +1280,6 @@ protected void serviceStart() throws Exception { // finally set the job classloader MRApps.setClassLoader(jobClassLoader, getConfig()); - // set job classloader if configured - Limits.init(getConfig()); if (initFailed) { JobEvent initFailedEvent = new JobEvent(job.getID(), JobEventType.JOB_INIT_FAILED); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java index fbf6806bdb..4245daf6ac 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java @@ -213,15 +213,15 @@ public FileSystem run() throws IOException, InterruptedException { public Job getJob(JobID jobId) throws IOException, InterruptedException { JobStatus status = client.getJobStatus(jobId); if (status != null) { - final JobConf conf = new JobConf(); - final Path jobPath = new Path(client.getFilesystemName(), - status.getJobFile()); - final FileSystem fs = FileSystem.get(jobPath.toUri(), getConf()); + JobConf conf; try { - conf.addResource(fs.open(jobPath), jobPath.toString()); - } catch (FileNotFoundException fnf) { - if (LOG.isWarnEnabled()) { - LOG.warn("Job conf missing on cluster", fnf); + conf = new JobConf(status.getJobFile()); + } catch (RuntimeException ex) { + // If job file doesn't exist it means we can't find the job + if (ex.getCause() instanceof FileNotFoundException) { + return null; + } else { + throw ex; } } return Job.getInstance(this, status, conf); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java index 6ade376f01..e5ff26d93d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java @@ -52,7 +52,6 @@ import static org.apache.hadoop.mapred.QueueManager.toFullPropertyName; -import org.apache.hadoop.mapreduce.counters.Limits; import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.protocol.ClientProtocol; import org.apache.hadoop.mapreduce.security.TokenCache; @@ -246,7 +245,6 @@ JobStatus submitJobInternal(Job job, Cluster cluster) // Write job file to submit dir writeConf(conf, submitJobFile); - Limits.reset(conf); // // Now, actually submit the job (using the submit name) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java index 3821694b2f..34b0fae6e8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java @@ -123,9 +123,4 @@ public synchronized void checkGroups(int size) { public synchronized LimitExceededException violation() { return firstViolation; } - - public static synchronized void reset(Configuration conf) { - isInited = false; - init(conf); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java index 25c0630456..5f10fdfa97 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.mapreduce.jobhistory; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; import java.util.HashMap; @@ -25,8 +24,6 @@ import java.util.Set; import java.util.TreeSet; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -36,7 +33,6 @@ import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.counters.Limits; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; import org.apache.hadoop.mapreduce.util.HostUtil; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -49,7 +45,6 @@ @InterfaceAudience.Private @InterfaceStability.Unstable public class HistoryViewer { - private static final Log LOG = LogFactory.getLog(HistoryViewer.class); private FileSystem fs; private JobInfo job; private HistoryViewerPrinter jhvp; @@ -89,17 +84,6 @@ public HistoryViewer(String historyFile, Configuration conf, boolean printAll, System.err.println("Ignore unrecognized file: " + jobFile.getName()); throw new IOException(errorMsg); } - final Path jobConfPath = new Path(jobFile.getParent(), jobDetails[0] - + "_" + jobDetails[1] + "_" + jobDetails[2] + "_conf.xml"); - final Configuration jobConf = new Configuration(conf); - try { - jobConf.addResource(fs.open(jobConfPath), jobConfPath.toString()); - Limits.reset(conf); - } catch (FileNotFoundException fnf) { - if (LOG.isWarnEnabled()) { - LOG.warn("Missing job conf in history", fnf); - } - } JobHistoryParser parser = new JobHistoryParser(fs, jobFile); job = parser.parse(); String scheme = WebAppUtils.getHttpSchemePrefix(fs.getConf()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java index 4deb9ae2f0..bbb126dff8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java @@ -18,7 +18,6 @@ package org.apache.hadoop.mapreduce.v2.hs; -import java.io.FileNotFoundException; import java.io.IOException; import java.net.UnknownHostException; import java.util.ArrayList; @@ -35,7 +34,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobACLsManager; import org.apache.hadoop.mapred.TaskCompletionEvent; @@ -43,7 +41,6 @@ import org.apache.hadoop.mapreduce.JobACL; import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.mapreduce.TypeConverter; -import org.apache.hadoop.mapreduce.counters.Limits; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo; @@ -353,19 +350,7 @@ protected synchronized void loadFullHistoryData(boolean loadTasks, if (historyFileAbsolute != null) { JobHistoryParser parser = null; try { - final FileSystem fs = historyFileAbsolute.getFileSystem(conf); parser = createJobHistoryParser(historyFileAbsolute); - final Path jobConfPath = new Path(historyFileAbsolute.getParent(), - JobHistoryUtils.getIntermediateConfFileName(jobId)); - final Configuration conf = new Configuration(); - try { - conf.addResource(fs.open(jobConfPath), jobConfPath.toString()); - Limits.reset(conf); - } catch (FileNotFoundException fnf) { - if (LOG.isWarnEnabled()) { - LOG.warn("Missing job conf in history", fnf); - } - } this.jobInfo = parser.parse(); } catch (IOException e) { throw new YarnRuntimeException("Could not load history file " diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java index 7a0c43e14e..c6d2168261 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java @@ -55,14 +55,10 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.JobID; -import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.mapred.TaskLog; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; @@ -114,7 +110,6 @@ public class TestMRJobs { EnumSet.of(RMAppState.FINISHED, RMAppState.FAILED, RMAppState.KILLED); private static final int NUM_NODE_MGRS = 3; private static final String TEST_IO_SORT_MB = "11"; - private static final String TEST_GROUP_MAX = "200"; private static final int DEFAULT_REDUCES = 2; protected int numSleepReducers = DEFAULT_REDUCES; @@ -466,58 +461,31 @@ private void waitForPriorityToUpdate(Job job, JobPriority expectedStatus) } @Test(timeout = 300000) - public void testConfVerificationWithClassloader() throws Exception { - testConfVerification(true, false, false, false); + public void testJobClassloader() throws IOException, InterruptedException, + ClassNotFoundException { + testJobClassloader(false); } @Test(timeout = 300000) - public void testConfVerificationWithClassloaderCustomClasses() - throws Exception { - testConfVerification(true, true, false, false); + public void testJobClassloaderWithCustomClasses() throws IOException, + InterruptedException, ClassNotFoundException { + testJobClassloader(true); } - @Test(timeout = 300000) - public void testConfVerificationWithOutClassloader() throws Exception { - testConfVerification(false, false, false, false); - } - - @Test(timeout = 300000) - public void testConfVerificationWithJobClient() throws Exception { - testConfVerification(false, false, true, false); - } - - @Test(timeout = 300000) - public void testConfVerificationWithJobClientLocal() throws Exception { - testConfVerification(false, false, true, true); - } - - private void testConfVerification(boolean useJobClassLoader, - boolean useCustomClasses, boolean useJobClientForMonitring, - boolean useLocal) throws Exception { - LOG.info("\n\n\nStarting testConfVerification()" - + " jobClassloader=" + useJobClassLoader - + " customClasses=" + useCustomClasses - + " jobClient=" + useJobClientForMonitring - + " localMode=" + useLocal); + private void testJobClassloader(boolean useCustomClasses) throws IOException, + InterruptedException, ClassNotFoundException { + LOG.info("\n\n\nStarting testJobClassloader()" + + " useCustomClasses=" + useCustomClasses); if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) { LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR + " not found. Not running test."); return; } - final Configuration clusterConfig; - if (useLocal) { - clusterConfig = new Configuration(); - conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); - } else { - clusterConfig = mrCluster.getConfig(); - } - final JobClient jc = new JobClient(clusterConfig); - final Configuration sleepConf = new Configuration(clusterConfig); + final Configuration sleepConf = new Configuration(mrCluster.getConfig()); // set master address to local to test that local mode applied iff framework == local sleepConf.set(MRConfig.MASTER_ADDRESS, "local"); - sleepConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, - useJobClassLoader); + sleepConf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true); if (useCustomClasses) { // to test AM loading user classes such as output format class, we want // to blacklist them from the system classes (they need to be prepended @@ -535,7 +503,6 @@ private void testConfVerification(boolean useJobClassLoader, sleepConf.set(MRJobConfig.MAP_LOG_LEVEL, Level.ALL.toString()); sleepConf.set(MRJobConfig.REDUCE_LOG_LEVEL, Level.ALL.toString()); sleepConf.set(MRJobConfig.MAP_JAVA_OPTS, "-verbose:class"); - sleepConf.set(MRJobConfig.COUNTER_GROUPS_MAX_KEY, TEST_GROUP_MAX); final SleepJob sleepJob = new SleepJob(); sleepJob.setConf(sleepConf); final Job job = sleepJob.createJob(1, 1, 10, 1, 10, 1); @@ -553,26 +520,7 @@ private void testConfVerification(boolean useJobClassLoader, jobConf.setBoolean(MRJobConfig.MAP_SPECULATIVE, true); } job.submit(); - final boolean succeeded; - if (useJobClientForMonitring && !useLocal) { - // We can't use getJobID in useLocal case because JobClient and Job - // point to different instances of LocalJobRunner - // - final JobID mapredJobID = JobID.downgrade(job.getJobID()); - RunningJob runningJob = null; - do { - Thread.sleep(10); - runningJob = jc.getJob(mapredJobID); - } while (runningJob == null); - Assert.assertEquals("Unexpected RunningJob's " - + MRJobConfig.COUNTER_GROUPS_MAX_KEY, - TEST_GROUP_MAX, runningJob.getConfiguration() - .get(MRJobConfig.COUNTER_GROUPS_MAX_KEY)); - runningJob.waitForCompletion(); - succeeded = runningJob.isSuccessful(); - } else { - succeeded = job.waitForCompletion(true); - } + boolean succeeded = job.waitForCompletion(true); Assert.assertTrue("Job status: " + job.getStatus().getFailureInfo(), succeeded); } @@ -1366,14 +1314,5 @@ protected void setup(Context context) + ", actual: " + ioSortMb); } } - - @Override - public void map(IntWritable key, IntWritable value, Context context) throws IOException, InterruptedException { - super.map(key, value, context); - for (int i = 0; i < 100; i++) { - context.getCounter("testCounterGroup-" + i, - "testCounter").increment(1); - } - } } }