From db807057193311fc06caf912e13000193edd1bd8 Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Wed, 29 Jan 2014 23:51:40 +0000 Subject: [PATCH] MAPREDUCE-5732. Report proper queue when job has been automatically placed (Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1562641 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../jobhistory/JobHistoryEventHandler.java | 6 +++ .../hadoop/mapreduce/v2/app/job/Job.java | 4 +- .../mapreduce/v2/app/job/impl/JobImpl.java | 10 ++++- .../mapreduce/v2/app/rm/RMCommunicator.java | 7 +++- .../mapreduce/jobhistory/TestEvents.java | 9 +++++ .../apache/hadoop/mapreduce/v2/app/MRApp.java | 30 ++++++++++---- .../mapreduce/v2/app/MockAppContext.java | 1 + .../hadoop/mapreduce/v2/app/MockJobs.java | 5 +++ .../v2/app/TestRuntimeEstimators.java | 5 +++ .../src/main/avro/Events.avpr | 9 +++++ .../mapreduce/jobhistory/EventReader.java | 2 + .../jobhistory/JobHistoryParser.java | 7 ++++ .../hadoop/mapreduce/v2/hs/CompletedJob.java | 5 +++ .../hadoop/mapreduce/v2/hs/PartialJob.java | 5 +++ .../mapreduce/v2/hs/TestJobHistoryEvents.java | 40 +++++++++++++++++++ .../v2/hs/webapp/TestHsWebServicesAcls.java | 4 ++ 17 files changed, 140 insertions(+), 12 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 596b51034e..53931dddb4 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -209,6 +209,9 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5464. Add analogs of the SLOTS_MILLIS counters that jive with the YARN resource model (Sandy Ryza) + MAPREDUCE-5732. Report proper queue when job has been automatically placed + (Sandy Ryza) + OPTIMIZATIONS MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index 5c1e7d6824..3de5ba7e76 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -525,6 +525,12 @@ protected void handleEvent(JobHistoryEvent event) { JobInitedEvent jie = (JobInitedEvent) event.getHistoryEvent(); mi.getJobIndexInfo().setJobStartTime(jie.getLaunchTime()); } + + if (event.getHistoryEvent().getEventType() == EventType.JOB_QUEUE_CHANGED) { + JobQueueChangeEvent jQueueEvent = + (JobQueueChangeEvent) event.getHistoryEvent(); + mi.getJobIndexInfo().setQueueName(jQueueEvent.getJobQueueName()); + } // If this is JobFinishedEvent, close the writer and setup the job-index if (event.getHistoryEvent().getEventType() == EventType.JOB_FINISHED) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java index b14abcc6d5..a40e5d2ddc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java @@ -39,7 +39,7 @@ /** - * Main interface to interact with the job. Provides only getters. + * Main interface to interact with the job. */ public interface Job { @@ -98,4 +98,6 @@ public interface Job { List getAMInfos(); boolean checkAccess(UserGroupInformation callerUGI, JobACL jobOperation); + + public void setQueueName(String queueName); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index 5f08303b36..88dc99eb36 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -59,6 +59,7 @@ import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo; import org.apache.hadoop.mapreduce.jobhistory.JobInfoChangeEvent; import org.apache.hadoop.mapreduce.jobhistory.JobInitedEvent; +import org.apache.hadoop.mapreduce.jobhistory.JobQueueChangeEvent; import org.apache.hadoop.mapreduce.jobhistory.JobSubmittedEvent; import org.apache.hadoop.mapreduce.jobhistory.JobUnsuccessfulCompletionEvent; import org.apache.hadoop.mapreduce.lib.chain.ChainMapper; @@ -181,7 +182,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, private final EventHandler eventHandler; private final MRAppMetrics metrics; private final String userName; - private final String queueName; + private String queueName; private final long appSubmitTime; private final AppContext appContext; @@ -1123,6 +1124,13 @@ public String getQueueName() { return queueName; } + @Override + public void setQueueName(String queueName) { + this.queueName = queueName; + JobQueueChangeEvent jqce = new JobQueueChangeEvent(oldJobId, queueName); + eventHandler.handle(new JobHistoryEvent(jobId, jqce)); + } + /* * (non-Javadoc) * @see org.apache.hadoop.mapreduce.v2.app.job.Job#getConfFile() diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java index f09ac744d5..a5c2306f71 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java @@ -109,11 +109,11 @@ protected void serviceInit(Configuration conf) throws Exception { @Override protected void serviceStart() throws Exception { scheduler= createSchedulerProxy(); - register(); - startAllocatorThread(); JobID id = TypeConverter.fromYarn(this.applicationId); JobId jobId = TypeConverter.toYarn(id); job = context.getJob(jobId); + register(); + startAllocatorThread(); super.serviceStart(); } @@ -161,6 +161,9 @@ protected void register() { } this.applicationACLs = response.getApplicationACLs(); LOG.info("maxContainerCapability: " + maxContainerCapability.getMemory()); + String queue = response.getQueue(); + LOG.info("queue: " + queue); + job.setQueueName(queue); } catch (Exception are) { LOG.error("Exception while registering", are); throw new YarnRuntimeException(are); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java index 8f1e4b9c2b..d383f2912f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java @@ -81,6 +81,15 @@ public void testJobPriorityChange() throws Exception { assertEquals(test.getPriority(), JobPriority.LOW); } + + @Test(timeout = 10000) + public void testJobQueueChange() throws Exception { + org.apache.hadoop.mapreduce.JobID jid = new JobID("001", 1); + JobQueueChangeEvent test = new JobQueueChangeEvent(jid, + "newqueue"); + assertEquals(test.getJobId().toString(), jid.toString()); + assertEquals(test.getJobQueueName(), "newqueue"); + } /** * simple test TaskUpdatedEvent and TaskUpdated diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java index 7f698c7c03..93b6efdfb3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java @@ -117,6 +117,9 @@ public class MRApp extends MRAppMaster { private File testWorkDir; private Path testAbsPath; private ClusterInfo clusterInfo; + + // Queue to pretend the RM assigned us + private String assignedQueue; public static String NM_HOST = "localhost"; public static int NM_PORT = 1234; @@ -133,7 +136,7 @@ public class MRApp extends MRAppMaster { public MRApp(int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart, Clock clock) { - this(maps, reduces, autoComplete, testName, cleanOnStart, 1, clock); + this(maps, reduces, autoComplete, testName, cleanOnStart, 1, clock, null); } public MRApp(int maps, int reduces, boolean autoComplete, String testName, @@ -146,6 +149,12 @@ public MRApp(int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart) { this(maps, reduces, autoComplete, testName, cleanOnStart, 1); } + + public MRApp(int maps, int reduces, boolean autoComplete, String testName, + boolean cleanOnStart, String assignedQueue) { + this(maps, reduces, autoComplete, testName, cleanOnStart, 1, + new SystemClock(), assignedQueue); + } public MRApp(int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart, boolean unregistered) { @@ -178,7 +187,7 @@ private static ContainerId getContainerId(ApplicationId applicationId, public MRApp(int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart, int startCount) { this(maps, reduces, autoComplete, testName, cleanOnStart, startCount, - new SystemClock()); + new SystemClock(), null); } public MRApp(int maps, int reduces, boolean autoComplete, String testName, @@ -191,33 +200,34 @@ public MRApp(int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart, int startCount, Clock clock, boolean unregistered) { this(getApplicationAttemptId(applicationId, startCount), getContainerId( applicationId, startCount), maps, reduces, autoComplete, testName, - cleanOnStart, startCount, clock, unregistered); + cleanOnStart, startCount, clock, unregistered, null); } public MRApp(int maps, int reduces, boolean autoComplete, String testName, - boolean cleanOnStart, int startCount, Clock clock) { + boolean cleanOnStart, int startCount, Clock clock, String assignedQueue) { this(getApplicationAttemptId(applicationId, startCount), getContainerId( applicationId, startCount), maps, reduces, autoComplete, testName, - cleanOnStart, startCount, clock, true); + cleanOnStart, startCount, clock, true, assignedQueue); } public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId, int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart, int startCount, boolean unregistered) { this(appAttemptId, amContainerId, maps, reduces, autoComplete, testName, - cleanOnStart, startCount, new SystemClock(), unregistered); + cleanOnStart, startCount, new SystemClock(), unregistered, null); } public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId, int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart, int startCount) { this(appAttemptId, amContainerId, maps, reduces, autoComplete, testName, - cleanOnStart, startCount, new SystemClock(), true); + cleanOnStart, startCount, new SystemClock(), true, null); } public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId, int maps, int reduces, boolean autoComplete, String testName, - boolean cleanOnStart, int startCount, Clock clock, boolean unregistered) { + boolean cleanOnStart, int startCount, Clock clock, boolean unregistered, + String assignedQueue) { super(appAttemptId, amContainerId, NM_HOST, NM_PORT, NM_HTTP_PORT, clock, System .currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); this.testWorkDir = new File("target", testName); @@ -239,6 +249,7 @@ public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId, // If safeToReportTerminationToUser is set to true, we can verify whether // the job can reaches the final state when MRAppMaster shuts down. this.successfullyUnregistered.set(unregistered); + this.assignedQueue = assignedQueue; } @Override @@ -285,6 +296,9 @@ public Job submit(Configuration conf, boolean mapSpeculative, start(); DefaultMetricsSystem.shutdown(); Job job = getContext().getAllJobs().values().iterator().next(); + if (assignedQueue != null) { + job.setQueueName(assignedQueue); + } // Write job.xml String jobFile = MRApps.getJobFile(conf, user, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java index d33e734f83..511731a9ba 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockAppContext.java @@ -39,6 +39,7 @@ public class MockAppContext implements AppContext { final Map jobs; final long startTime = System.currentTimeMillis(); Set blacklistedNodes; + String queue; public MockAppContext(int appid) { appID = MockJobs.newAppID(appid); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java index eb4919c9cd..19ac0db98d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java @@ -629,6 +629,11 @@ public Configuration loadConfFile() throws IOException { jobConf.addResource(fc.open(configFile), configFile.toString()); return jobConf; } + + @Override + public void setQueueName(String queueName) { + // do nothing + } }; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java index 3d555f2b7f..6fadf350f8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java @@ -505,6 +505,11 @@ public List getAMInfos() { public Configuration loadConfFile() { throw new UnsupportedOperationException(); } + + @Override + public void setQueueName(String queueName) { + // do nothing + } } /* diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr index 64ecf37659..fa194338cf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr @@ -122,6 +122,13 @@ ] }, + {"type": "record", "name": "JobQueueChange", + "fields": [ + {"name": "jobid", "type": "string"}, + {"name": "jobQueueName", "type": "string"} + ] + }, + {"type": "record", "name": "JobUnsuccessfulCompletion", "fields": [ {"name": "jobid", "type": "string"}, @@ -267,6 +274,7 @@ "JOB_FINISHED", "JOB_PRIORITY_CHANGED", "JOB_STATUS_CHANGED", + "JOB_QUEUE_CHANGED", "JOB_FAILED", "JOB_KILLED", "JOB_ERROR", @@ -306,6 +314,7 @@ "JobInited", "AMStarted", "JobPriorityChange", + "JobQueueChange", "JobStatusChanged", "JobSubmitted", "JobUnsuccessfulCompletion", diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java index 37052b42c0..43d25cc82a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java @@ -98,6 +98,8 @@ public HistoryEvent getNextEvent() throws IOException { result = new JobFinishedEvent(); break; case JOB_PRIORITY_CHANGED: result = new JobPriorityChangeEvent(); break; + case JOB_QUEUE_CHANGED: + result = new JobQueueChangeEvent(); break; case JOB_STATUS_CHANGED: result = new JobStatusChangedEvent(); break; case JOB_FAILED: diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java index d683505022..9d6f579c44 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java @@ -183,6 +183,9 @@ public void handleEvent(HistoryEvent event) { case JOB_PRIORITY_CHANGED: handleJobPriorityChangeEvent((JobPriorityChangeEvent) event); break; + case JOB_QUEUE_CHANGED: + handleJobQueueChangeEvent((JobQueueChangeEvent) event); + break; case JOB_FAILED: case JOB_KILLED: case JOB_ERROR: @@ -385,6 +388,10 @@ private void handleJobFinishedEvent(JobFinishedEvent event) { private void handleJobPriorityChangeEvent(JobPriorityChangeEvent event) { info.priority = event.getPriority(); } + + private void handleJobQueueChangeEvent(JobQueueChangeEvent event) { + info.jobQueueName = event.getJobQueueName(); + } private void handleJobInitedEvent(JobInitedEvent event) { info.launchTime = event.getLaunchTime(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java index 3f0644ab37..79b9275467 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java @@ -453,4 +453,9 @@ public List getAMInfos() { } return amInfos; } + + @Override + public void setQueueName(String queueName) { + throw new UnsupportedOperationException("Can't set job's queue name in history"); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java index ca12cb503a..0725f46582 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java @@ -190,5 +190,10 @@ public Map getJobACLs() { public List getAMInfos() { return null; } + + @Override + public void setQueueName(String queueName) { + throw new UnsupportedOperationException("Can't set job's queue name in history"); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java index 54a4a25db3..6078e07e55 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java @@ -155,6 +155,41 @@ public void testJobHistoryEventHandlerIsFirstServiceToStop() { Assert.assertEquals("JobHistoryEventHandler", services[services.length - 1].getName()); } + + @Test + public void testAssignedQueue() throws Exception { + Configuration conf = new Configuration(); + MRApp app = new MRAppWithHistory(2, 1, true, this.getClass().getName(), + true, "assignedQueue"); + app.submit(conf); + Job job = app.getContext().getAllJobs().values().iterator().next(); + JobId jobId = job.getID(); + LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString()); + app.waitForState(job, JobState.SUCCEEDED); + + //make sure all events are flushed + app.waitForState(Service.STATE.STOPPED); + /* + * Use HistoryContext to read logged events and verify the number of + * completed maps + */ + HistoryContext context = new JobHistory(); + // test start and stop states + ((JobHistory)context).init(conf); + ((JobHistory)context).start(); + Assert.assertTrue( context.getStartTime()>0); + Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STARTED); + + // get job before stopping JobHistory + Job parsedJob = context.getJob(jobId); + + // stop JobHistory + ((JobHistory)context).stop(); + Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STOPPED); + + Assert.assertEquals("QueueName not correct", "assignedQueue", + parsedJob.getQueueName()); + } private void verifyTask(Task task) { Assert.assertEquals("Task state not currect", TaskState.SUCCEEDED, @@ -184,6 +219,11 @@ public MRAppWithHistory(int maps, int reduces, boolean autoComplete, super(maps, reduces, autoComplete, testName, cleanOnStart); } + public MRAppWithHistory(int maps, int reduces, boolean autoComplete, + String testName, boolean cleanOnStart, String assignedQueue) { + super(maps, reduces, autoComplete, testName, cleanOnStart, assignedQueue); + } + @Override protected EventHandler createJobHistoryHandler( AppContext context) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java index 63f7fb0789..fd87b9494b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java @@ -415,5 +415,9 @@ public boolean checkAccess(UserGroupInformation callerUGI, return aclsMgr.checkAccess(callerUGI, jobOperation, this.getUserName(), jobAcls.get(jobOperation)); } + + @Override + public void setQueueName(String queueName) { + } } }