diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties index e3338c783a..ede0c93480 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties +++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties @@ -149,3 +149,25 @@ log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false #log4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd #log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout #log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + +# +# Yarn ResourceManager Application Summary Log +# +# Set the ResourceManager summary log filename +#yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log +# Set the ResourceManager summary log level and appender +#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY + +# Appender for ResourceManager Application Summary Log - rolled daily +# Requires the following properties to be set +# - hadoop.log.dir (Hadoop Log directory) +# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) +# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) + +#log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} +#log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false +#log4j.appender.RMSUMMARY=org.apache.log4j.DailyRollingFileAppender +#log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} +#log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout +#log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n +#log4j.appender.RMSUMMARY.DatePattern=.yyyy-MM-dd diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 33a88e6772..9317584a62 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -230,6 +230,9 @@ Release 0.23.0 - Unreleased making shuffle service port to be truely configurable. (Robert Evans via vinodkv) + MAPREDUCE-2735. Add an applications summary log to ResourceManager. + (Thomas Graves via acmurthy) + OPTIMIZATIONS MAPREDUCE-2026. Make JobTracker.getJobCounters() and diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 3219d8220d..5ce9c015b2 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -34,11 +34,14 @@ import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; +import org.apache.hadoop.util.StringUtils; /** * This class manages the list of applications for the resource manager. @@ -69,6 +72,86 @@ public RMAppManager(RMContext context, ClientToAMSecretManager RMConfig.DEFAULT_EXPIRE_APPLICATIONS_COMPLETED_MAX)); } + /** + * This class is for logging the application summary. + */ + static class ApplicationSummary { + static final Log LOG = LogFactory.getLog(ApplicationSummary.class); + + // Escape sequences + static final char EQUALS = '='; + static final char[] charsToEscape = + {StringUtils.COMMA, EQUALS, StringUtils.ESCAPE_CHAR}; + + static class SummaryBuilder { + final StringBuilder buffer = new StringBuilder(); + + // A little optimization for a very common case + SummaryBuilder add(String key, long value) { + return _add(key, Long.toString(value)); + } + + SummaryBuilder add(String key, T value) { + return _add(key, StringUtils.escapeString(String.valueOf(value), + StringUtils.ESCAPE_CHAR, charsToEscape)); + } + + SummaryBuilder add(SummaryBuilder summary) { + if (buffer.length() > 0) buffer.append(StringUtils.COMMA); + buffer.append(summary.buffer); + return this; + } + + SummaryBuilder _add(String key, String value) { + if (buffer.length() > 0) buffer.append(StringUtils.COMMA); + buffer.append(key).append(EQUALS).append(value); + return this; + } + + @Override public String toString() { + return buffer.toString(); + } + } + + /** + * create a summary of the application's runtime. + * + * @param app {@link RMApp} whose summary is to be created, cannot + * be null. + */ + public static SummaryBuilder createAppSummary(RMApp app) { + String trackingUrl = "N/A"; + String host = "N/A"; + RMAppAttempt attempt = app.getCurrentAppAttempt(); + if (attempt != null) { + trackingUrl = attempt.getTrackingUrl(); + host = attempt.getHost(); + } + SummaryBuilder summary = new SummaryBuilder() + .add("appId", app.getApplicationId()) + .add("name", app.getName()) + .add("user", app.getUser()) + .add("queue", app.getQueue()) + .add("state", app.getState()) + .add("trackingUrl", trackingUrl) + .add("appMasterHost", host) + .add("startTime", app.getStartTime()) + .add("finishTime", app.getFinishTime()); + return summary; + } + + /** + * Log a summary of the application's runtime. + * + * @param app {@link RMApp} whose summary is to be logged + */ + public static void logAppSummary(RMApp app) { + if (app != null) { + LOG.info(createAppSummary(app)); + } + } + } + protected void setCompletedAppsMax(int max) { this.completedAppsMax = max; } @@ -154,6 +237,7 @@ public void handle(RMAppManagerEvent event) { case APP_COMPLETED: { addCompletedApp(appID); + ApplicationSummary.logAppSummary(rmContext.getRMApps().get(appID)); checkAppNumCompletedLimit(); } break; diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 048af90d39..f8f34ec592 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -36,6 +36,9 @@ public class MockRMApp implements RMApp { RMAppState state = RMAppState.NEW; int failCount = 0; ApplicationId id; + String url = null; + StringBuilder diagnostics = new StringBuilder(); + RMAppAttempt attempt; public MockRMApp(int newid, long time, RMAppState newState) { finish = time; @@ -58,11 +61,19 @@ public RMAppState getState() { return state; } + public void setState(RMAppState state) { + this.state = state; + } + @Override public String getUser() { return user; } + public void setUser(String user) { + this.user = user; + } + @Override public float getProgress() { return (float) 0.0; @@ -78,14 +89,26 @@ public String getQueue() { return queue; } + public void setQueue(String queue) { + this.queue = queue; + } + @Override public String getName() { return name; } + public void setName(String name) { + this.name = name; + } + @Override public RMAppAttempt getCurrentAppAttempt() { - throw new UnsupportedOperationException("Not supported yet."); + return attempt; + } + + public void setCurrentAppAttempt(RMAppAttempt attempt) { + this.attempt = attempt; } @Override @@ -103,19 +126,35 @@ public long getFinishTime() { return finish; } + public void setFinishTime(long time) { + this.finish = time; + } + @Override public long getStartTime() { return start; } + public void setStartTime(long time) { + this.start = time; + } + @Override public String getTrackingUrl() { - throw new UnsupportedOperationException("Not supported yet."); + return url; + } + + public void setTrackingUrl(String url) { + this.url = url; } @Override public StringBuilder getDiagnostics() { - throw new UnsupportedOperationException("Not supported yet."); + return diagnostics; + } + + public void setDiagnostics(String diag) { + this.diagnostics = new StringBuilder(diag); } public void handle(RMAppEvent event) {