diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index bb46de0bf4..e6858a2b85 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1133,11 +1133,6 @@
test
true
-
-
- jdk.tools
- jdk.tools
-
org.jruby
jruby-complete
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index a01f495a6e..dbbaed37cd 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -48,11 +48,10 @@ function hadoop_usage
hadoop_add_subcommand "rmadmin" "admin tools"
hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools"
hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager daemon"
- hadoop_add_subcommand "timelineserver" "run the timeline server"
hadoop_add_subcommand "timelinereader" "run the timeline reader server"
+ hadoop_add_subcommand "timelineserver" "run the timeline server"
hadoop_add_subcommand "top" "view cluster information"
hadoop_add_subcommand "version" "print the version"
- hadoop_generate_usage "${MYNAME}" true
hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true
}
@@ -173,6 +172,10 @@ function yarncmd_case
hadoop_debug "Append YARN_SHAREDCACHEMANAGER_OPTS onto HADOOP_OPTS"
HADOOP_OPTS="${HADOOP_OPTS} ${YARN_SHAREDCACHEMANAGER_OPTS}"
;;
+ timelinereader)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
+ ;;
timelineserver)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
@@ -184,10 +187,6 @@ function yarncmd_case
HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}"
fi
;;
- timelinereader)
- supportdaemonization="true"
- CLASS='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
- ;;
version)
HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 6e0e7601a0..b37b38e4f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -81,7 +81,6 @@
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 663a2e1549..89e81ca6ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -154,7 +154,6 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
-import org.apache.hadoop.yarn.util.resource.Resources;
public class ContainerManagerImpl extends CompositeService implements
ContainerManager {
@@ -351,9 +350,8 @@ private void recoverApplication(ContainerManagerApplicationProto p)
LOG.info("Recovering application " + appId);
//TODO: Recover flow and flow run ID
- ApplicationImpl app = new ApplicationImpl(
- dispatcher, p.getUser(), null, appId, creds, context,
- p.getAppLogAggregationInitedTime());
+ ApplicationImpl app = new ApplicationImpl(dispatcher, p.getUser(), appId,
+ creds, context, p.getAppLogAggregationInitedTime());
context.getApplications().put(appId, app);
app.handle(new ApplicationInitEvent(appId, acls, logAggregationContext));
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index 5a5d30cce1..531693e1ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -40,7 +40,6 @@
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.LogAggregationContextPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.proto.YarnProtos;
@@ -118,10 +117,19 @@ public ApplicationImpl(Dispatcher dispatcher, String user,
Context context, long recoveredLogInitedTime) {
this.dispatcher = dispatcher;
this.user = user;
- this.flowContext = flowContext;
this.appId = appId;
this.credentials = credentials;
this.aclsManager = context.getApplicationACLsManager();
+ Configuration conf = context.getConf();
+ if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ if (flowContext == null) {
+ throw new IllegalArgumentException("flow context cannot be null");
+ }
+ this.flowContext = flowContext;
+ if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
+ context.getNMTimelinePublisher().createTimelineClient(appId);
+ }
+ }
this.context = context;
this.appStateStore = context.getNMStateStore();
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
@@ -136,16 +144,6 @@ public ApplicationImpl(Dispatcher dispatcher, String user,
Credentials credentials, Context context) {
this(dispatcher, user, flowContext, appId, credentials,
context, -1);
- Configuration conf = context.getConf();
- if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
- if (flowContext == null) {
- throw new IllegalArgumentException("flow context cannot be null");
- }
- this.flowContext = flowContext;
- if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
- context.getNMTimelinePublisher().createTimelineClient(appId);
- }
- }
}
/**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
index ed8cf06d51..61e7d650ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -181,6 +181,8 @@ public void appStateUpdated(RMApp app, YarnApplicationState appState,
SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId()));
}
+ @SuppressWarnings("unchecked")
+ @Override
public void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime) {
TimelineEntity entity = createApplicationEntity(app.getApplicationId());
TimelineEvent tEvent = new TimelineEvent();
@@ -248,6 +250,10 @@ public void appAttemptFinished(RMAppAttempt appAttempt,
app.getFinalApplicationStatus().toString());
eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, RMServerUtils
.createApplicationAttemptState(appAttemtpState).toString());
+ if (appAttempt.getMasterContainer() != null) {
+ eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
+ appAttempt.getMasterContainer().getId().toString());
+ }
tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent);
@@ -301,6 +307,12 @@ public void containerFinished(RMContainer container, long finishedTime) {
container.getContainerExitStatus());
eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO,
container.getContainerState().toString());
+ Map entityInfo = new HashMap();
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO,
+ container.getAllocatedNode().getHost());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO,
+ container.getAllocatedNode().getPort());
+ entity.setOtherInfo(entityInfo);
tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 955570fc19..a248199381 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -196,10 +196,14 @@ public void appStateUpdated(RMApp app, YarnApplicationState appState,
@Override
public void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime) {
ApplicationEntity entity = createApplicationEntity(app.getApplicationId());
+ TimelineEvent tEvent = new TimelineEvent();
Map entityInfo = new HashMap();
entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
- appViewACLs);
+ (appViewACLs == null) ? "" : appViewACLs);
entity.setInfo(entityInfo);
+ tEvent.setId(ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE);
+ tEvent.setTimestamp(updatedTime);
+ entity.addEvent(tEvent);
getDispatcher().getEventHandler().handle(new TimelineV2PublishEvent(
SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId()));
@@ -285,6 +289,10 @@ public void appAttemptFinished(RMAppAttempt appAttempt,
app.getFinalApplicationStatus().toString());
eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, RMServerUtils
.createApplicationAttemptState(appAttemtpState).toString());
+ if (appAttempt.getMasterContainer() != null) {
+ eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
+ appAttempt.getMasterContainer().getId().toString());
+ }
tEvent.setInfo(eventInfo);
entity.addEvent(tEvent);
@@ -353,6 +361,12 @@ public void containerFinished(RMContainer container, long finishedTime) {
container.getContainerExitStatus());
eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO,
container.getContainerState().toString());
+ Map entityInfo = new HashMap();
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO,
+ container.getAllocatedNode().getHost());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO,
+ container.getAllocatedNode().getPort());
+ entity.setInfo(entityInfo);
tEvent.setInfo(eventInfo);
entity.addEvent(tEvent);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
index a8e519538b..5641572b54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml
@@ -22,12 +22,12 @@
hadoop-yarn-server
org.apache.hadoop
- 3.0.0-SNAPSHOT
+ 3.0.0-alpha1-SNAPSHOT
4.0.0
org.apache.hadoop
hadoop-yarn-server-timelineservice-hbase-tests
- 3.0.0-SNAPSHOT
+ 3.0.0-alpha1-SNAPSHOT
Apache Hadoop YARN Timeline Service HBase tests