From 4aed2d8e91c7dccc78fbaffc409d3076c3316289 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Mon, 13 Oct 2014 10:53:37 -0700 Subject: [PATCH] YARN-2651. Spun off LogRollingInterval from LogAggregationContext. Contributed by Xuan Gong. --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../api/records/LogAggregationContext.java | 28 +------------------ .../hadoop/yarn/conf/YarnConfiguration.java | 7 +++++ .../src/main/proto/yarn_protos.proto | 1 - .../impl/pb/LogAggregationContextPBImpl.java | 15 ---------- .../src/main/resources/yarn-default.xml | 10 +++++++ .../logaggregation/AppLogAggregatorImpl.java | 8 ++++-- .../TestContainerManagerRecovery.java | 5 +--- .../TestLogAggregationService.java | 4 ++- .../capacity/TestContainerAllocation.java | 4 +-- 10 files changed, 31 insertions(+), 54 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 9e61b2d68f..782cd13329 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -630,6 +630,9 @@ Release 2.6.0 - UNRELEASED YARN-2667. Fix the release audit warning caused by hadoop-yarn-registry (Yi Liu via jlowe) + YARN-2651. Spun off LogRollingInterval from LogAggregationContext. (Xuan Gong + via zjshen) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java index 9a0a15774f..46c1809b41 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java @@ -37,13 +37,6 @@ * which match the defined exclude pattern and those log files * will not be uploaded. If the log file name matches both the * include and the exclude pattern, this file will be excluded eventually - *
  • rollingIntervalSeconds. The default value is -1. By default, - * the logAggregationService only uploads container logs when - * the application is finished. This configure defines - * how often the logAggregationSerivce uploads container logs in seconds. - * By setting this configure, the logAggregationSerivce can upload container - * logs periodically when the application is running. - *
    • * *

    * @@ -57,11 +50,10 @@ public abstract class LogAggregationContext { @Public @Unstable public static LogAggregationContext newInstance(String includePattern, - String excludePattern, long rollingIntervalSeconds) { + String excludePattern) { LogAggregationContext context = Records.newRecord(LogAggregationContext.class); context.setIncludePattern(includePattern); context.setExcludePattern(excludePattern); - context.setRollingIntervalSeconds(rollingIntervalSeconds); return context; } @@ -100,22 +92,4 @@ public static LogAggregationContext newInstance(String includePattern, @Public @Unstable public abstract void setExcludePattern(String excludePattern); - - /** - * Get rollingIntervalSeconds - * - * @return the rollingIntervalSeconds - */ - @Public - @Unstable - public abstract long getRollingIntervalSeconds(); - - /** - * Set rollingIntervalSeconds - * - * @param rollingIntervalSeconds - */ - @Public - @Unstable - public abstract void setRollingIntervalSeconds(long rollingIntervalSeconds); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index dfb7a2cbf8..450fd5b32b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -711,6 +711,13 @@ public class YarnConfiguration extends Configuration { + "log.retain-seconds"; public static final long DEFAULT_NM_LOG_RETAIN_SECONDS = 3 * 60 * 60; + /** + * Define how often NMs wake up and upload log files + */ + public static final String NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = + NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds"; + public static final long + DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1; /** * Number of threads used in log cleanup. Only applicable if Log aggregation * is disabled diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 8db451d865..136192da3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -302,7 +302,6 @@ message ApplicationSubmissionContextProto { message LogAggregationContextProto { optional string include_pattern = 1 [default = ".*"]; optional string exclude_pattern = 2 [default = ""]; - optional int64 rolling_interval_seconds = 3 [default = -1]; } enum ApplicationAccessTypeProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java index 4406ef9fce..dc7a21d2a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java @@ -116,19 +116,4 @@ public void setExcludePattern(String excludePattern) { } builder.setExcludePattern(excludePattern); } - - @Override - public long getRollingIntervalSeconds() { - LogAggregationContextProtoOrBuilder p = viaProto ? proto : builder; - if (! p.hasRollingIntervalSeconds()) { - return -1; - } - return p.getRollingIntervalSeconds(); - } - - @Override - public void setRollingIntervalSeconds(long rollingIntervalSeconds) { - maybeInitBuilder(); - builder.setRollingIntervalSeconds(rollingIntervalSeconds); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 5d53191799..0c07337b5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1523,4 +1523,14 @@ Client + + Defines how often NMs wake up to upload log files. + The default value is -1. By default, the logs will be uploaded when + the application is finished. By setting this configure, logs can be uploaded + periodically when the application is running. The minimum rolling-interval-seconds + can be set is 3600. + + yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds + -1 + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index 63f7c66ede..98e3cafe84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -148,9 +148,11 @@ public AppLogAggregatorImpl(Dispatcher dispatcher, } else { this.retentionSize = configuredRentionSize; } - long configuredRollingMonitorInterval = - this.logAggregationContext == null ? -1 : this.logAggregationContext - .getRollingIntervalSeconds(); + long configuredRollingMonitorInterval = conf.getLong( + YarnConfiguration + .NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, + YarnConfiguration + .DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS); boolean debug_mode = conf.getBoolean(NM_LOG_AGGREGATION_DEBUG_ENABLED, DEFAULT_NM_LOG_AGGREGATION_DEBUG_ENABLED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java index 2c69843fd7..7850a1c665 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java @@ -130,8 +130,7 @@ public void testApplicationRecovery() throws Exception { containerTokens, acls); // create the logAggregationContext LogAggregationContext logAggregationContext = - LogAggregationContext.newInstance("includePattern", "excludePattern", - 1000); + LogAggregationContext.newInstance("includePattern", "excludePattern"); StartContainersResponse startResponse = startContainer(context, cm, cid, clc, logAggregationContext); assertTrue(startResponse.getFailedRequests().isEmpty()); @@ -168,8 +167,6 @@ public void testApplicationRecovery() throws Exception { LogAggregationContext recovered = ((ApplicationImpl) app).getLogAggregationContext(); assertNotNull(recovered); - assertEquals(logAggregationContext.getRollingIntervalSeconds(), - recovered.getRollingIntervalSeconds()); assertEquals(logAggregationContext.getIncludePattern(), recovered.getIncludePattern()); assertEquals(logAggregationContext.getExcludePattern(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 2c0f349e7f..ab86a18df5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -1235,10 +1235,12 @@ private void testLogAggregationService(boolean retentionSizeLimitation) throws Exception { LogAggregationContext logAggregationContextWithInterval = Records.newRecord(LogAggregationContext.class); - logAggregationContextWithInterval.setRollingIntervalSeconds(5000); this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath()); + this.conf.setLong( + YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, + 3600); if (retentionSizeLimitation) { // set the retention size as 1. The number of logs for one application // in one NM should be 1. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 85ef381a01..365c7541d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -212,16 +212,14 @@ public void testLogAggregationContextPassedIntoContainerToken() .assertNull(getLogAggregationContextFromContainerToken(rm1, nm1, null)); // create a not-null LogAggregationContext - final int interval = 2000; LogAggregationContext logAggregationContext = LogAggregationContext.newInstance( - "includePattern", "excludePattern", interval); + "includePattern", "excludePattern"); LogAggregationContext returned = getLogAggregationContextFromContainerToken(rm1, nm2, logAggregationContext); Assert.assertEquals("includePattern", returned.getIncludePattern()); Assert.assertEquals("excludePattern", returned.getExcludePattern()); - Assert.assertEquals(interval, returned.getRollingIntervalSeconds()); rm1.stop(); }