big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-6931. Make the aggregation interval in AppLevelTimelineCollector configurable. (Abhishek Modi via Haibo Chen)

Browse Source
This commit is contained in:
Haibo Chen 2018-06-12 10:03:07 -07:00
parent e9ea902299
commit 24a89825f0
3 changed files with 27 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -2664,6 +2664,15 @@ public class YarnConfiguration extends Configuration {
public static final String TIMELINE_SERVICE_READ_AUTH_ENABLED =
TIMELINE_SERVICE_PREFIX + "read.authentication.enabled";
/**
* The name for setting that controls how often in-memory app level
* aggregation is kicked off in timeline collector.
*/
public static final String TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS =
TIMELINE_SERVICE_PREFIX + "app-aggregation-interval-secs";
public static final int
DEFAULT_TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS = 15;
/**
* The default setting for authentication checks for reading timeline
* service v2 data.

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -2545,6 +2545,15 @@
<value>259200000</value>
</property>
<property>
<description>
The setting that controls how often in-memory app level
aggregation is kicked off in timeline collector.
</description>
<name>yarn.timeline-service.app-aggregation-interval-secs</name>
<value>15</value>
</property>
<property>
<description>
The default hdfs location for flowrun coprocessor jar.

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollectorWithAgg.java
View File

@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@ -50,7 +51,7 @@ public class AppLevelTimelineCollectorWithAgg
LoggerFactory.getLogger(TimelineCollector.class);
private final static int AGGREGATION_EXECUTOR_NUM_THREADS = 1;
private final static int AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS = 15;
private int aggregationExecutorIntervalSecs;
private static Set<String> entityTypesSkipAggregation
= initializeSkipSet();
@ -71,6 +72,11 @@ public class AppLevelTimelineCollectorWithAgg
@Override
protected void serviceInit(Configuration conf) throws Exception {
aggregationExecutorIntervalSecs = conf.getInt(
YarnConfiguration.TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS,
YarnConfiguration.
DEFAULT_TIMELINE_SERVICE_AGGREGATION_INTERVAL_SECS
);
super.serviceInit(conf);
}
@ -84,10 +90,8 @@ public class AppLevelTimelineCollectorWithAgg
.build());
appAggregator = new AppLevelAggregator();
appAggregationExecutor.scheduleAtFixedRate(appAggregator,
AppLevelTimelineCollectorWithAgg.
AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS,
AppLevelTimelineCollectorWithAgg.
AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS,
aggregationExecutorIntervalSecs,
aggregationExecutorIntervalSecs,
TimeUnit.SECONDS);
super.serviceStart();
}