From 45b2bce8714ecbcb17c6b487208a48d26eed7824 Mon Sep 17 00:00:00 2001 From: Mahadev Konar Date: Fri, 21 Oct 2011 18:35:35 +0000 Subject: [PATCH] MAPREDUCE-3163. JobClient spews errors when killing MR2 job. (mahadev) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1187491 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapred/ClientServiceDelegate.java | 21 +++++++++---------- .../resourcemanager/rmapp/RMAppImpl.java | 3 ++- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 664aa52156..f06087832e 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -416,6 +416,9 @@ Release 0.23.0 - Unreleased MAPREDUCE-3144. Augmented JobHistory with the information needed for serving aggregated logs. (Siddharth Seth via vinodkv) + + MAPREDUCE-3163. JobClient spews errors when killing MR2 job. + (mahadev) OPTIMIZATIONS diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java index 20817af8e1..b8bf1676b0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java @@ -56,21 +56,18 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobState; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.SecurityInfo; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; -import org.apache.hadoop.yarn.security.SchedulerSecurityInfo; public class ClientServiceDelegate { private static final Log LOG = LogFactory.getLog(ClientServiceDelegate.class); @@ -128,7 +125,7 @@ private MRClientProtocol getProxy() throws YarnRemoteException { if (!forceRefresh && realProxy != null) { return realProxy; } - //TODO RM NPEs for unknown jobs. History may still be aware. + // Possibly allow nulls through the PB tunnel, otherwise deal with an exception // and redirect to the history server. ApplicationReport application = rm.getApplicationReport(appId); @@ -136,7 +133,7 @@ private MRClientProtocol getProxy() throws YarnRemoteException { trackingUrl = application.getTrackingUrl(); } String serviceAddr = null; - while (application == null || YarnApplicationState.RUNNING.equals(application.getYarnApplicationState())) { + while (application == null || YarnApplicationState.RUNNING == application.getYarnApplicationState()) { if (application == null) { LOG.info("Could not get Job info from RM for job " + jobId + ". Redirecting to job history server."); @@ -196,7 +193,6 @@ private MRClientProtocol getProxy() throws YarnRemoteException { * block on it. This is to be able to return job status * on an allocating Application. */ - String user = application.getUser(); if (user == null) { throw RPCUtil.getRemoteException("User is not set in the application report"); @@ -269,16 +265,19 @@ private synchronized Object invoke(String method, Class argClass, throw yre; } catch (InvocationTargetException e) { if (e.getTargetException() instanceof YarnRemoteException) { - LOG.warn("Exception thrown by remote end.", e - .getTargetException()); + LOG.warn("Error from remote end: " + e + .getTargetException().getLocalizedMessage()); + LOG.debug("Tracing remote error ", e.getTargetException()); throw (YarnRemoteException) e.getTargetException(); } - LOG.info("Failed to contact AM/History for job " + jobId - + " Will retry..", e.getTargetException()); + LOG.info("Failed to contact AM/History for job " + jobId + + " retrying.."); + LOG.debug("Failed exception on AM/History contact", + e.getTargetException()); forceRefresh = true; } catch (Exception e) { LOG.info("Failed to contact AM/History for job " + jobId - + " Will retry..", e); + + " Will retry.."); LOG.debug("Failing to contact application master", e); forceRefresh = true; } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 7c943f1562..bbcbeabfc0 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -212,7 +212,8 @@ public FinalApplicationStatus getFinalApplicationStatus() { && currentAttempt.getFinalApplicationStatus() != null) { return currentAttempt.getFinalApplicationStatus(); } - return createFinalApplicationStatus(this.stateMachine.getCurrentState()); + return + createFinalApplicationStatus(this.stateMachine.getCurrentState()); } finally { this.readLock.unlock(); }