From ad3d3f54d5f9d2b3648da43481c72bc3f6748ace Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 9 Mar 2012 18:53:19 +0000 Subject: [PATCH] MAPREDUCE-3348. Fixed a bug in MR client to redirect to JobHistoryServer correctly when RM forgets the app. Contributed by Devaraj K. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1298978 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../hadoop/mapred/ClientServiceDelegate.java | 4 +-- .../resourcemanager/ClientRMService.java | 10 +++++-- .../resourcemanager/TestClientRMService.java | 30 +++++++++++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 5b962d8079..3b2470d270 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -137,6 +137,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-3578. Starting nodemanager as root gives "Unknown -jvm option" (tomwhite) + MAPREDUCE-3348. Fixed a bug in MR client to redirect to JobHistoryServer + correctly when RM forgets the app. (Devaraj K via vinodkv) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java index 83ed5c264a..7216c09071 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java @@ -149,7 +149,7 @@ private MRClientProtocol getProxy() throws YarnRemoteException { || YarnApplicationState.RUNNING == application .getYarnApplicationState()) { if (application == null) { - LOG.debug("Could not get Job info from RM for job " + jobId + LOG.info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server."); return checkAndGetHSProxy(null, JobState.NEW); } @@ -216,7 +216,7 @@ public MRClientProtocol run() throws IOException { } application = rm.getApplicationReport(appId); if (application == null) { - LOG.debug("Could not get Job info from RM for job " + jobId + LOG.info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server."); return checkAndGetHSProxy(null, JobState.RUNNING); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 433c8afc1b..edaacf6b6f 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -199,6 +199,10 @@ public GetNewApplicationResponse getNewApplication( return response; } + /** + * It gives response which includes application report if the application + * present otherwise gives response with application report as null. + */ @Override public GetApplicationReportResponse getApplicationReport( GetApplicationReportRequest request) throws YarnRemoteException { @@ -214,8 +218,10 @@ public GetApplicationReportResponse getApplicationReport( RMApp application = this.rmContext.getRMApps().get(applicationId); if (application == null) { - throw RPCUtil.getRemoteException("Trying to get information for an " - + "absent application " + applicationId); + // If the RM doesn't have the application, provide the response with + // application report as null and let the clients to handle. + return recordFactory + .newRecordInstance(GetApplicationReportResponse.class); } boolean allowAccess = checkAccess(callerUGI, application.getUser(), diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 5ea88814c1..89f3b1f61d 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -18,8 +18,12 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.net.InetSocketAddress; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; import junit.framework.Assert; @@ -27,12 +31,20 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.ClientRMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.exceptions.YarnRemoteException; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.util.Records; import org.junit.Test; + public class TestClientRMService { private static final Log LOG = LogFactory.getLog(TestClientRMService.class); @@ -79,4 +91,22 @@ protected ClientRMService createClientRMService() { Assert.assertFalse("Node is expected to be unhealthy!", nodeReports.get(0) .getNodeHealthStatus().getIsNodeHealthy()); } + + @Test + public void testGetApplicationReport() throws YarnRemoteException { + RMContext rmContext = mock(RMContext.class); + when(rmContext.getRMApps()).thenReturn( + new ConcurrentHashMap()); + ClientRMService rmService = new ClientRMService(rmContext, null, null, + null, null); + RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); + GetApplicationReportRequest request = recordFactory + .newRecordInstance(GetApplicationReportRequest.class); + request.setApplicationId(recordFactory + .newRecordInstance(ApplicationId.class)); + GetApplicationReportResponse applicationReport = rmService + .getApplicationReport(request); + Assert.assertNull("It should return null as application report for absent application.", + applicationReport.getApplicationReport()); + } }