From 34f02f07d5bf9fbfa1d07d1b219225c298539105 Mon Sep 17 00:00:00 2001 From: Li Lu Date: Wed, 20 Jan 2016 01:05:52 -0800 Subject: [PATCH] Rebase to latest trunk --- hadoop-tools/hadoop-aws/pom.xml | 8 ++++++++ .../client/api/impl/TimelineClientImpl.java | 4 ++-- .../metrics/TimelineServiceV1Publisher.java | 17 +++++++++++++++-- .../metrics/TimelineServiceV2Publisher.java | 18 +++++++++++++++--- .../TestResourceTrackerService.java | 12 ++++++++---- .../hadoop-yarn-server-timelineservice/pom.xml | 6 ------ 6 files changed, 48 insertions(+), 17 deletions(-) diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 7c25e608c7..83ee58b975 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -250,6 +250,14 @@ hadoop-yarn-server-tests test test-jar + + + + joda-time + joda-time + + org.apache.hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java index b15278c1b2..f36167dc56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java @@ -335,8 +335,8 @@ protected void serviceInit(Configuration conf) throws Exception { @Override protected void serviceStart() throws Exception { timelineWriter = createTimelineWriter( - configuration, authUgi, client, constructResURI(getConfig(), - getTimelineServiceAddress(), false)); + configuration, authUgi, client, + constructResURI(getConfig(), timelineServiceAddress, false)); } protected TimelineWriter createTimelineWriter(Configuration conf, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java index c890577f04..98a3b3187d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java @@ -88,6 +88,17 @@ public void appCreated(RMApp app, long createdTime) { app.getAmNodeLabelExpression()); entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION, app.getAppNodeLabelExpression()); + if (app.getCallerContext() != null) { + if (app.getCallerContext().getContext() != null) { + entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT, + app.getCallerContext().getContext()); + } + if (app.getCallerContext().getSignature() != null) { + entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_SIGNATURE, + app.getCallerContext().getSignature()); + } + } + entity.setOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE); @@ -200,8 +211,10 @@ public void appAttemptRegistered(RMAppAttempt appAttempt, appAttempt.getHost()); eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, appAttempt.getRpcPort()); - eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, - appAttempt.getMasterContainer().getId().toString()); + if (appAttempt.getMasterContainer() != null) { + eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, + appAttempt.getMasterContainer().getId().toString()); + } tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); getDispatcher().getEventHandler().handle( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index 0a8d7d7c46..b66ce5b88e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -118,8 +118,18 @@ public void appCreated(RMApp app, long createdTime) { entity.getConfigs().put( ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION, app.getAppNodeLabelExpression()); - entity.setInfo(entityInfo); + if (app.getCallerContext() != null) { + if (app.getCallerContext().getContext() != null) { + entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT, + app.getCallerContext().getContext()); + } + if (app.getCallerContext().getSignature() != null) { + entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_SIGNATURE, + app.getCallerContext().getSignature()); + } + } + entity.setInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE); tEvent.setTimestamp(createdTime); @@ -240,8 +250,10 @@ public void appAttemptRegistered(RMAppAttempt appAttempt, appAttempt.getHost()); eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, appAttempt.getRpcPort()); - eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, - appAttempt.getMasterContainer().getId().toString()); + if (appAttempt.getMasterContainer() != null) { + eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, + appAttempt.getMasterContainer().getId().toString()); + } tEvent.setInfo(eventInfo); entity.addEvent(tEvent); getDispatcher().getEventHandler().handle( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 809ef4aaac..498112e008 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -915,8 +915,10 @@ public void testNodeHeartbeatForAppCollectorsMap() throws Exception { statusList.add(status1); NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true, "", System.currentTimeMillis()); - node1.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeHealth, - statusList, null, nodeHeartbeat1)); + NodeStatus nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0, + statusList, null, nodeHealth, null, null, null); + node1.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeStatus, + nodeHeartbeat1)); Assert.assertEquals(1, node1.getRunningApps().size()); Assert.assertEquals(app1.getApplicationId(), node1.getRunningApps().get(0)); @@ -930,8 +932,10 @@ public void testNodeHeartbeatForAppCollectorsMap() throws Exception { ContainerState.RUNNING, "", 0); statusList = new ArrayList(); statusList.add(status2); - node2.handle(new RMNodeStatusEvent(nm2.getNodeId(), nodeHealth, - statusList, null, nodeHeartbeat2)); + nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0, + statusList, null, nodeHealth, null, null, null); + node2.handle(new RMNodeStatusEvent(nm2.getNodeId(), nodeStatus, + nodeHeartbeat2)); Assert.assertEquals(1, node2.getRunningApps().size()); Assert.assertEquals(app2.getApplicationId(), node2.getRunningApps().get(0)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml index b7608d6edc..3d17d98bf0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml @@ -124,12 +124,6 @@ org.apache.phoenix phoenix-core - - - joda-time - joda-time - - org.apache.hbase