big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2651. Spun off LogRollingInterval from LogAggregationContext. Contributed by Xuan Gong.

Browse Source
This commit is contained in:
Zhijie Shen 2014-10-13 10:53:37 -07:00
parent 53100318ea
commit 4aed2d8e91
10 changed files with 31 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -630,6 +630,9 @@ Release 2.6.0 - UNRELEASED
YARN-2667. Fix the release audit warning caused by hadoop-yarn-registry
(Yi Liu via jlowe)
YARN-2651. Spun off LogRollingInterval from LogAggregationContext. (Xuan Gong
via zjshen)
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
View File

@ -37,13 +37,6 @@
* which match the defined exclude pattern and those log files
* will not be uploaded. If the log file name matches both the
* include and the exclude pattern, this file will be excluded eventually</li>
* <li>rollingIntervalSeconds. The default value is -1. By default,
* the logAggregationService only uploads container logs when
* the application is finished. This configure defines
* how often the logAggregationSerivce uploads container logs in seconds.
* By setting this configure, the logAggregationSerivce can upload container
* logs periodically when the application is running.
* </li>
* </ul>
* </p>
*
@ -57,11 +50,10 @@ public abstract class LogAggregationContext {
@Public
@Unstable
public static LogAggregationContext newInstance(String includePattern,
String excludePattern, long rollingIntervalSeconds) {
String excludePattern) {
LogAggregationContext context = Records.newRecord(LogAggregationContext.class);
context.setIncludePattern(includePattern);
context.setExcludePattern(excludePattern);
context.setRollingIntervalSeconds(rollingIntervalSeconds);
return context;
}
@ -100,22 +92,4 @@ public static LogAggregationContext newInstance(String includePattern,
@Public
@Unstable
public abstract void setExcludePattern(String excludePattern);
/**
* Get rollingIntervalSeconds
*
* @return the rollingIntervalSeconds
*/
@Public
@Unstable
public abstract long getRollingIntervalSeconds();
/**
* Set rollingIntervalSeconds
*
* @param rollingIntervalSeconds
*/
@Public
@Unstable
public abstract void setRollingIntervalSeconds(long rollingIntervalSeconds);
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -711,6 +711,13 @@ public class YarnConfiguration extends Configuration {
+ "log.retain-seconds";
public static final long DEFAULT_NM_LOG_RETAIN_SECONDS = 3 * 60 * 60;
/**
* Define how often NMs wake up and upload log files
*/
public static final String NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS =
NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds";
public static final long
DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1;
/**
* Number of threads used in log cleanup. Only applicable if Log aggregation
* is disabled

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -302,7 +302,6 @@ message ApplicationSubmissionContextProto {
message LogAggregationContextProto {
optional string include_pattern = 1 [default = ".*"];
optional string exclude_pattern = 2 [default = ""];
optional int64 rolling_interval_seconds = 3 [default = -1];
}
enum ApplicationAccessTypeProto {

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java
View File

@ -116,19 +116,4 @@ public void setExcludePattern(String excludePattern) {
}
builder.setExcludePattern(excludePattern);
}
@Override
public long getRollingIntervalSeconds() {
LogAggregationContextProtoOrBuilder p = viaProto ? proto : builder;
if (! p.hasRollingIntervalSeconds()) {
return -1;
}
return p.getRollingIntervalSeconds();
}
@Override
public void setRollingIntervalSeconds(long rollingIntervalSeconds) {
maybeInitBuilder();
builder.setRollingIntervalSeconds(rollingIntervalSeconds);
}
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -1523,4 +1523,14 @@
<value>Client</value>
</property>
<property>
<description>Defines how often NMs wake up to upload log files.
The default value is -1. By default, the logs will be uploaded when
the application is finished. By setting this configure, logs can be uploaded
periodically when the application is running. The minimum rolling-interval-seconds
can be set is 3600.
</description>
<name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
<value>-1</value>
</property>
</configuration>

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
View File

@ -148,9 +148,11 @@ public AppLogAggregatorImpl(Dispatcher dispatcher,
} else {
this.retentionSize = configuredRentionSize;
}
long configuredRollingMonitorInterval =
this.logAggregationContext == null ? -1 : this.logAggregationContext
.getRollingIntervalSeconds();
long configuredRollingMonitorInterval = conf.getLong(
YarnConfiguration
.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS,
YarnConfiguration
.DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS);
boolean debug_mode =
conf.getBoolean(NM_LOG_AGGREGATION_DEBUG_ENABLED,
DEFAULT_NM_LOG_AGGREGATION_DEBUG_ENABLED);

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
View File

@ -130,8 +130,7 @@ public void testApplicationRecovery() throws Exception {
containerTokens, acls);
// create the logAggregationContext
LogAggregationContext logAggregationContext =
LogAggregationContext.newInstance("includePattern", "excludePattern",
1000);
LogAggregationContext.newInstance("includePattern", "excludePattern");
StartContainersResponse startResponse = startContainer(context, cm, cid,
clc, logAggregationContext);
assertTrue(startResponse.getFailedRequests().isEmpty());
@ -168,8 +167,6 @@ public void testApplicationRecovery() throws Exception {
LogAggregationContext recovered =
((ApplicationImpl) app).getLogAggregationContext();
assertNotNull(recovered);
assertEquals(logAggregationContext.getRollingIntervalSeconds(),
recovered.getRollingIntervalSeconds());
assertEquals(logAggregationContext.getIncludePattern(),
recovered.getIncludePattern());
assertEquals(logAggregationContext.getExcludePattern(),

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
View File

@ -1235,10 +1235,12 @@ private void testLogAggregationService(boolean retentionSizeLimitation)
throws Exception {
LogAggregationContext logAggregationContextWithInterval =
Records.newRecord(LogAggregationContext.class);
logAggregationContextWithInterval.setRollingIntervalSeconds(5000);
this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
this.remoteRootLogDir.getAbsolutePath());
this.conf.setLong(
YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS,
3600);
if (retentionSizeLimitation) {
// set the retention size as 1. The number of logs for one application
// in one NM should be 1.

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
View File

@ -212,16 +212,14 @@ public void testLogAggregationContextPassedIntoContainerToken()
.assertNull(getLogAggregationContextFromContainerToken(rm1, nm1, null));
// create a not-null LogAggregationContext
final int interval = 2000;
LogAggregationContext logAggregationContext =
LogAggregationContext.newInstance(
"includePattern", "excludePattern", interval);
"includePattern", "excludePattern");
LogAggregationContext returned =
getLogAggregationContextFromContainerToken(rm1, nm2,
logAggregationContext);
Assert.assertEquals("includePattern", returned.getIncludePattern());
Assert.assertEquals("excludePattern", returned.getExcludePattern());
Assert.assertEquals(interval, returned.getRollingIntervalSeconds());
rm1.stop();
}