MAPREDUCE-2735. Add an applications summary log to ResourceManager. Contributed by Thomas Graves.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1164301 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/src/main/conf/log4j.properties

@@ -149,3 +149,25 @@ log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
 #log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd
 #log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
 #log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+#
+# Yarn ResourceManager Application Summary Log 
+#
+# Set the ResourceManager summary log filename
+#yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log
+# Set the ResourceManager summary log level and appender
+#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
+
+# Appender for ResourceManager Application Summary Log - rolled daily
+# Requires the following properties to be set
+#    - hadoop.log.dir (Hadoop Log directory)
+#    - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
+#    - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
+
+#log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
+#log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
+#log4j.appender.RMSUMMARY=org.apache.log4j.DailyRollingFileAppender
+#log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}
+#log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
+#log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
+#log4j.appender.RMSUMMARY.DatePattern=.yyyy-MM-dd

hadoop-mapreduce-project/CHANGES.txt

@@ -230,6 +230,9 @@ Release 0.23.0 - Unreleased
     making shuffle service port to be truely configurable. (Robert Evans via
     vinodkv)
 
+    MAPREDUCE-2735. Add an applications summary log to ResourceManager.
+    (Thomas Graves via acmurthy) 
+
   OPTIMIZATIONS
 
     MAPREDUCE-2026. Make JobTracker.getJobCounters() and

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java

@@ -34,11 +34,14 @@
 import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.util.StringUtils;
 
 /**
  * This class manages the list of applications for the resource manager. 
@@ -69,6 +72,86 @@ public RMAppManager(RMContext context, ClientToAMSecretManager
         RMConfig.DEFAULT_EXPIRE_APPLICATIONS_COMPLETED_MAX));
   }
 
+  /**
+   *  This class is for logging the application summary.
+   */
+  static class ApplicationSummary {
+    static final Log LOG = LogFactory.getLog(ApplicationSummary.class);
+
+    // Escape sequences 
+    static final char EQUALS = '=';
+    static final char[] charsToEscape =
+      {StringUtils.COMMA, EQUALS, StringUtils.ESCAPE_CHAR};
+
+    static class SummaryBuilder {
+      final StringBuilder buffer = new StringBuilder();
+
+      // A little optimization for a very common case
+      SummaryBuilder add(String key, long value) {
+        return _add(key, Long.toString(value));
+      }
+
+      <T> SummaryBuilder add(String key, T value) {
+        return _add(key, StringUtils.escapeString(String.valueOf(value),
+                    StringUtils.ESCAPE_CHAR, charsToEscape));
+      }
+
+      SummaryBuilder add(SummaryBuilder summary) {
+        if (buffer.length() > 0) buffer.append(StringUtils.COMMA);
+        buffer.append(summary.buffer);
+        return this;
+      }
+
+      SummaryBuilder _add(String key, String value) {
+        if (buffer.length() > 0) buffer.append(StringUtils.COMMA);
+        buffer.append(key).append(EQUALS).append(value);
+        return this;
+      }
+
+      @Override public String toString() {
+        return buffer.toString();
+      }
+    }
+
+    /**
+     * create a summary of the application's runtime.
+     * 
+     * @param app {@link RMApp} whose summary is to be created, cannot
+     *            be <code>null</code>.
+     */
+    public static SummaryBuilder createAppSummary(RMApp app) {
+      String trackingUrl = "N/A";
+      String host = "N/A";
+      RMAppAttempt attempt = app.getCurrentAppAttempt();
+      if (attempt != null) {
+        trackingUrl = attempt.getTrackingUrl();
+        host = attempt.getHost();
+      }
+      SummaryBuilder summary = new SummaryBuilder()
+          .add("appId", app.getApplicationId())
+          .add("name", app.getName())
+          .add("user", app.getUser())
+          .add("queue", app.getQueue())
+          .add("state", app.getState())
+          .add("trackingUrl", trackingUrl)
+          .add("appMasterHost", host)
+          .add("startTime", app.getStartTime())
+          .add("finishTime", app.getFinishTime());
+      return summary;
+    }
+
+    /**
+     * Log a summary of the application's runtime.
+     * 
+     * @param app {@link RMApp} whose summary is to be logged
+     */
+    public static void logAppSummary(RMApp app) {
+      if (app != null) {
+        LOG.info(createAppSummary(app));
+      }
+    }
+  }
+
   protected void setCompletedAppsMax(int max) {
     this.completedAppsMax = max;
   }
@@ -154,6 +237,7 @@ public void handle(RMAppManagerEvent event) {
       case APP_COMPLETED: 
       {
         addCompletedApp(appID);
+        ApplicationSummary.logAppSummary(rmContext.getRMApps().get(appID));
         checkAppNumCompletedLimit(); 
       } 
       break;

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java

@@ -36,6 +36,9 @@ public class MockRMApp implements RMApp {
   RMAppState state = RMAppState.NEW;
   int failCount = 0;
   ApplicationId id;
+  String url = null;
+  StringBuilder diagnostics = new StringBuilder();
+  RMAppAttempt attempt;
 
   public MockRMApp(int newid, long time, RMAppState newState) {
     finish = time;
@@ -58,11 +61,19 @@ public RMAppState getState() {
     return state;
   }
 
+  public void setState(RMAppState state) {
+    this.state = state;
+  }
+
   @Override
   public String getUser() {
     return user;
   }
 
+  public void setUser(String user) {
+    this.user = user;
+  }
+
   @Override
   public float getProgress() {
     return (float) 0.0;
@@ -78,14 +89,26 @@ public String getQueue() {
     return queue;
   }
 
+  public void setQueue(String queue) {
+    this.queue = queue;
+  }
+
   @Override
   public String getName() {
     return name;
   }
 
+  public void setName(String name) {
+    this.name = name;
+  }
+
   @Override
   public RMAppAttempt getCurrentAppAttempt() {
-    throw new UnsupportedOperationException("Not supported yet.");
+    return attempt;
+  }
+
+  public void setCurrentAppAttempt(RMAppAttempt attempt) {
+    this.attempt = attempt;
   }
 
   @Override
@@ -103,19 +126,35 @@ public long getFinishTime() {
     return finish;
   }
 
+  public void setFinishTime(long time) {
+    this.finish = time;
+  }
+
   @Override
   public long getStartTime() {
     return start;
   }
 
+  public void setStartTime(long time) {
+    this.start = time;
+  }
+
   @Override
   public String getTrackingUrl() {
-    throw new UnsupportedOperationException("Not supported yet.");
+    return url;
+  }
+
+  public void setTrackingUrl(String url) {
+    this.url = url;
   }
 
   @Override
   public StringBuilder getDiagnostics() {
-    throw new UnsupportedOperationException("Not supported yet.");
+    return diagnostics;
+  }
+
+  public void setDiagnostics(String diag) {
+    this.diagnostics  = new StringBuilder(diag);
   }
 
   public void handle(RMAppEvent event) {