From ed13cf84068ad50a78e66211132c1f08238fb65b Mon Sep 17 00:00:00 2001 From: bibinchundatt Date: Fri, 22 Feb 2019 21:01:34 +0530 Subject: [PATCH] YARN-9317. Avoid repeated YarnConfiguration#timelineServiceV2Enabled check. Contributed by Prabhu Joseph --- .../server/nodemanager/NodeStatusUpdaterImpl.java | 6 +++++- .../containermanager/ContainerManagerImpl.java | 15 +++++++++------ .../resourcemanager/ApplicationMasterService.java | 6 +++++- .../server/resourcemanager/ClientRMService.java | 6 +++++- .../resourcemanager/DefaultAMSProcessor.java | 6 ++++-- .../yarn/server/resourcemanager/RMAppManager.java | 5 ++++- .../resourcemanager/ResourceTrackerService.java | 9 +++++---- .../resourcemanager/amlauncher/AMLauncher.java | 5 ++++- 8 files changed, 41 insertions(+), 17 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 5762ffa9ea..1f13eb8788 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -160,6 +160,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements private NodeLabelsProvider nodeLabelsProvider; private NodeAttributesProvider nodeAttributesProvider; private long tokenSequenceNo; + private boolean timelineServiceV2Enabled; public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) { @@ -254,6 +255,9 @@ protected void serviceInit(Configuration conf) throws Exception { this.logAggregationEnabled = conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); + } @Override @@ -1411,7 +1415,7 @@ public void run() { newResource.toString()); } } - if (YarnConfiguration.timelineServiceV2Enabled(context.getConf())) { + if (timelineServiceV2Enabled) { updateTimelineCollectorData(response); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 2ca63ae7e9..f3f5f8b928 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -230,6 +230,7 @@ private enum ReInitOp { // NM metrics publisher is set only if the timeline service v.2 is enabled private NMTimelinePublisher nmMetricsPublisher; + private boolean timelineServiceV2Enabled; public ContainerManagerImpl(Context context, ContainerExecutor exec, DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater, @@ -267,11 +268,13 @@ public ContainerManagerImpl(Context context, ContainerExecutor exec, // initialize the metrics publisher if the timeline service v.2 is enabled // and the system publisher is enabled Configuration conf = context.getConf(); - if (YarnConfiguration.timelineServiceV2Enabled(conf) && - YarnConfiguration.systemMetricsPublisherEnabled(conf)) { - LOG.info("YARN system metrics publishing service is enabled"); - nmMetricsPublisher = createNMTimelinePublisher(context); - context.setNMTimelinePublisher(nmMetricsPublisher); + if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) { + LOG.info("YARN system metrics publishing service is enabled"); + nmMetricsPublisher = createNMTimelinePublisher(context); + context.setNMTimelinePublisher(nmMetricsPublisher); + } + this.timelineServiceV2Enabled = true; } this.containersMonitor = createContainersMonitor(exec); addService(this.containersMonitor); @@ -1191,7 +1194,7 @@ protected void startContainerInternal( private FlowContext getFlowContext(ContainerLaunchContext launchContext, ApplicationId applicationID) { FlowContext flowContext = null; - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { String flowName = launchContext.getEnvironment() .get(TimelineUtils.FLOW_NAME_TAG_PREFIX); String flowVersion = launchContext.getEnvironment() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 70b7498b9e..7bc4260ca0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -95,6 +95,7 @@ public class ApplicationMasterService extends AbstractService implements new ConcurrentHashMap(); protected final RMContext rmContext; private final AMSProcessingChain amsProcessingChain; + private boolean timelineServiceV2Enabled; public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) { @@ -212,6 +213,9 @@ protected void serviceStart() throws Exception { YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS, server.getListenerAddress()); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); + super.serviceStart(); } @@ -302,7 +306,7 @@ public FinishApplicationMasterResponse finishApplicationMaster( rmContext.getRMApps().get(applicationAttemptId.getApplicationId()); // Remove collector address when app get finished. - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { ((RMAppImpl) rmApp).removeCollectorData(); } // checking whether the app exits in RMStateStore at first not to throw diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 2da000c8d2..757c7b88ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -236,6 +236,7 @@ public class ClientRMService extends AbstractService implements RMAppState.ACCEPTED, RMAppState.RUNNING); private ResourceProfilesManager resourceProfilesManager; + private boolean timelineServiceV2Enabled; public ClientRMService(RMContext rmContext, YarnScheduler scheduler, RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager, @@ -306,6 +307,9 @@ protected void serviceStart() throws Exception { YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, server.getListenerAddress()); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); + super.serviceStart(); } @@ -585,7 +589,7 @@ public SubmitApplicationResponse submitApplication( throw RPCUtil.getRemoteException(ie); } - if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) { + if (timelineServiceV2Enabled) { // Sanity check for flow run String value = null; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java index b2c5ef3773..9e05edae44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java @@ -115,12 +115,15 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor { private RMContext rmContext; private ResourceProfilesManager resourceProfilesManager; + private boolean timelineServiceV2Enabled; @Override public void init(ApplicationMasterServiceContext amsContext, ApplicationMasterServiceProcessor nextProcessor) { this.rmContext = (RMContext)amsContext; this.resourceProfilesManager = rmContext.getResourceProfilesManager(); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(rmContext.getYarnConfiguration()); } @Override @@ -326,8 +329,7 @@ public void allocate(ApplicationAttemptId appAttemptId, response.setNumClusterNodes(getScheduler().getNumClusterNodes()); // add collector address for this application - if (YarnConfiguration.timelineServiceV2Enabled( - getRmContext().getYarnConfiguration())) { + if (timelineServiceV2Enabled) { CollectorInfo collectorInfo = app.getCollectorInfo(); if (collectorInfo != null) { response.setCollectorInfo(collectorInfo); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 3c5436c02a..0679b0eec6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -95,6 +95,7 @@ public class RMAppManager implements EventHandler, private final ApplicationACLsManager applicationACLsManager; private Configuration conf; private YarnAuthorizationProvider authorizer; + private boolean timelineServiceV2Enabled; public RMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService masterService, @@ -115,6 +116,8 @@ public RMAppManager(RMContext context, this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory; } this.authorizer = YarnAuthorizationProvider.getInstance(conf); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); } /** @@ -493,7 +496,7 @@ private RMAppImpl createAndPopulateNewRMApp( throw new YarnException(message); } - if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (timelineServiceV2Enabled) { // Start timeline collector for the submitted app application.startTimelineCollector(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index c50950bbd9..9365096fa6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -128,6 +128,7 @@ public class ResourceTrackerService extends AbstractService implements private final AtomicLong timelineCollectorVersion = new AtomicLong(0); private boolean checkIpHostnameInRegistration; + private boolean timelineServiceV2Enabled; public ResourceTrackerService(RMContext rmContext, NodesListManager nodesListManager, @@ -177,6 +178,8 @@ protected void serviceInit(Configuration conf) throws Exception { minimumNodeManagerVersion = conf.get( YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION, YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION); + timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); if (YarnConfiguration.areNodeLabelsEnabled(conf)) { isDistributedNodeLabelsConf = @@ -621,9 +624,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) NodeAction.SHUTDOWN, message); } - boolean timelineV2Enabled = - YarnConfiguration.timelineServiceV2Enabled(getConfig()); - if (timelineV2Enabled) { + if (timelineServiceV2Enabled) { // Check & update collectors info from request. updateAppCollectorsMap(request); } @@ -639,7 +640,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) populateTokenSequenceNo(request, nodeHeartBeatResponse); - if (timelineV2Enabled) { + if (timelineServiceV2Enabled) { // Return collectors' map that NM needs to know setAppCollectorsMapToResponse(rmNode.getRunningApps(), nodeHeartBeatResponse); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index a734cb66b6..3ce014dc05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -84,6 +84,7 @@ public class AMLauncher implements Runnable { private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; + private boolean timelineServiceV2Enabled; @SuppressWarnings("rawtypes") private final EventHandler handler; @@ -96,6 +97,8 @@ public AMLauncher(RMContext rmContext, RMAppAttempt application, this.rmContext = rmContext; this.handler = rmContext.getDispatcher().getEventHandler(); this.masterContainer = application.getMasterContainer(); + this.timelineServiceV2Enabled = YarnConfiguration. + timelineServiceV2Enabled(conf); } private void connect() throws IOException { @@ -268,7 +271,7 @@ protected void setupTokens( } private void setFlowContext(ContainerLaunchContext container) { - if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (timelineServiceV2Enabled) { Map environment = container.getEnvironment(); ApplicationId applicationId = application.getAppAttemptId().getApplicationId();