big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-3333. Rename TimelineAggregator etc. to TimelineCollector. Contributed by Sangjin Lee

Browse Source
This commit is contained in:
Junping Du 2015-03-19 11:49:07 -07:00 committed by Sangjin Lee
parent 9b56364080
commit 2188a07e5b
65 changed files with 1110 additions and 995 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
View File

@ -47,6 +47,10 @@
<groupId>commons-logging</groupId> <groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId> <artifactId>commons-logging</artifactId>
</dependency> </dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency --> <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency> <dependency>

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
View File

@ -127,7 +127,7 @@ public static AllocateResponse newInstance(int responseId,
response.setAMRMToken(amRMToken); response.setAMRMToken(amRMToken);
return response; return response;
} }
@Public @Public
@Unstable @Unstable
public static AllocateResponse newInstance(int responseId, public static AllocateResponse newInstance(int responseId,
@ -137,13 +137,13 @@ public static AllocateResponse newInstance(int responseId,
PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken, PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken,
List<Container> increasedContainers, List<Container> increasedContainers,
List<Container> decreasedContainers, List<Container> decreasedContainers,
String aggregatorAddr) { String collectorAddr) {
AllocateResponse response = AllocateResponse response =
newInstance(responseId, completedContainers, allocatedContainers, newInstance(responseId, completedContainers, allocatedContainers,
updatedNodes, availResources, command, numClusterNodes, preempt, updatedNodes, availResources, command, numClusterNodes, preempt,
nmTokens, increasedContainers, decreasedContainers); nmTokens, increasedContainers, decreasedContainers);
response.setAMRMToken(amRMToken); response.setAMRMToken(amRMToken);
response.setAggregatorAddr(aggregatorAddr); response.setCollectorAddr(collectorAddr);
return response; return response;
} }
@ -349,16 +349,16 @@ public abstract void setDecreasedContainers(
public abstract void setApplicationPriority(Priority priority); public abstract void setApplicationPriority(Priority priority);
/** /**
* The address of aggregator that belong to this app * The address of collector that belong to this app
* *
* @return The address of aggregator that belong to this attempt * @return The address of collector that belong to this attempt
*/ */
@Public @Public
@Unstable @Unstable
public abstract String getAggregatorAddr(); public abstract String getCollectorAddr();
@Private @Private
@Unstable @Unstable
public abstract void setAggregatorAddr(String aggregatorAddr); public abstract void setCollectorAddr(String collectorAddr);
} }

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java
View File

@ -47,7 +47,7 @@ public TimelineWriteResponse() {
/** /**
* Get a list of {@link TimelineWriteError} instances * Get a list of {@link TimelineWriteError} instances
* *
* @return a list of {@link TimelineWriteError} instances * @return a list of {@link TimelineWriteError} instances
*/ */
@XmlElement(name = "errors") @XmlElement(name = "errors")
@ -57,7 +57,7 @@ public List<TimelineWriteError> getErrors() {
/** /**
* Add a single {@link TimelineWriteError} instance into the existing list * Add a single {@link TimelineWriteError} instance into the existing list
* *
* @param error * @param error
* a single {@link TimelineWriteError} instance * a single {@link TimelineWriteError} instance
*/ */
@ -67,7 +67,7 @@ public void addError(TimelineWriteError error) {
/** /**
* Add a list of {@link TimelineWriteError} instances into the existing list * Add a list of {@link TimelineWriteError} instances into the existing list
* *
* @param errors * @param errors
* a list of {@link TimelineWriteError} instances * a list of {@link TimelineWriteError} instances
*/ */
@ -77,7 +77,7 @@ public void addErrors(List<TimelineWriteError> errors) {
/** /**
* Set the list to the given list of {@link TimelineWriteError} instances * Set the list to the given list of {@link TimelineWriteError} instances
* *
* @param errors * @param errors
* a list of {@link TimelineWriteError} instances * a list of {@link TimelineWriteError} instances
*/ */
@ -107,7 +107,7 @@ public static class TimelineWriteError {
/** /**
* Get the entity Id * Get the entity Id
* *
* @return the entity Id * @return the entity Id
*/ */
@XmlElement(name = "entity") @XmlElement(name = "entity")
@ -117,7 +117,7 @@ public String getEntityId() {
/** /**
* Set the entity Id * Set the entity Id
* *
* @param entityId * @param entityId
* the entity Id * the entity Id
*/ */
@ -127,7 +127,7 @@ public void setEntityId(String entityId) {
/** /**
* Get the entity type * Get the entity type
* *
* @return the entity type * @return the entity type
*/ */
@XmlElement(name = "entitytype") @XmlElement(name = "entitytype")
@ -137,7 +137,7 @@ public String getEntityType() {
/** /**
* Set the entity type * Set the entity type
* *
* @param entityType * @param entityType
* the entity type * the entity type
*/ */
@ -147,7 +147,7 @@ public void setEntityType(String entityType) {
/** /**
* Get the error code * Get the error code
* *
* @return an error code * @return an error code
*/ */
@XmlElement(name = "errorcode") @XmlElement(name = "errorcode")
@ -157,7 +157,7 @@ public int getErrorCode() {
/** /**
* Set the error code to the given error code * Set the error code to the given error code
* *
* @param errorCode * @param errorCode
* an error code * an error code
*/ */

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -814,10 +814,10 @@ public static boolean isAclEnabled(Configuration conf) {
public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20; public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20;
/** Number of threads container manager uses.*/ /** Number of threads container manager uses.*/
public static final String NM_AGGREGATOR_SERVICE_THREAD_COUNT = public static final String NM_COLLECTOR_SERVICE_THREAD_COUNT =
NM_PREFIX + "aggregator-service.thread-count"; NM_PREFIX + "collector-service.thread-count";
public static final int DEFAULT_NM_AGGREGATOR_SERVICE_THREAD_COUNT = 5; public static final int DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT = 5;
/** Number of threads used in cleanup.*/ /** Number of threads used in cleanup.*/
public static final String NM_DELETE_THREAD_COUNT = public static final String NM_DELETE_THREAD_COUNT =
NM_PREFIX + "delete.thread-count"; NM_PREFIX + "delete.thread-count";
@ -845,13 +845,13 @@ public static boolean isAclEnabled(Configuration conf) {
public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" + public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" +
DEFAULT_NM_LOCALIZER_PORT; DEFAULT_NM_LOCALIZER_PORT;
/** Address where the aggregator service IPC is.*/ /** Address where the collector service IPC is.*/
public static final String NM_AGGREGATOR_SERVICE_ADDRESS = public static final String NM_COLLECTOR_SERVICE_ADDRESS =
NM_PREFIX + "aggregator-service.address"; NM_PREFIX + "collector-service.address";
public static final int DEFAULT_NM_AGGREGATOR_SERVICE_PORT = 8048; public static final int DEFAULT_NM_COLLECTOR_SERVICE_PORT = 8048;
public static final String DEFAULT_NM_AGGREGATOR_SERVICE_ADDRESS = public static final String DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS =
"0.0.0.0:" + DEFAULT_NM_LOCALIZER_PORT; "0.0.0.0:" + DEFAULT_NM_LOCALIZER_PORT;
/** Interval in between cache cleanups.*/ /** Interval in between cache cleanups.*/
public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS =
NM_PREFIX + "localizer.cache.cleanup.interval-ms"; NM_PREFIX + "localizer.cache.cleanup.interval-ms";

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
View File

@ -89,7 +89,7 @@ message AllocateResponseProto {
repeated ContainerProto decreased_containers = 11; repeated ContainerProto decreased_containers = 11;
optional hadoop.common.TokenProto am_rm_token = 12; optional hadoop.common.TokenProto am_rm_token = 12;
optional PriorityProto application_priority = 13; optional PriorityProto application_priority = 13;
optional string aggregator_addr = 14; optional string collector_addr = 14;
} }
enum SchedulerResourceTypes { enum SchedulerResourceTypes {

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
View File

@ -70,6 +70,15 @@
<groupId>commons-io</groupId> <groupId>commons-io</groupId>
<artifactId>commons-io</artifactId> <artifactId>commons-io</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timelineservice</artifactId>
<scope>test</scope>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency --> <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
View File

@ -40,10 +40,10 @@
import java.util.Vector; import java.util.Vector;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.GnuParser;
@ -222,12 +222,12 @@ public static enum DSEntity {
private int appMasterRpcPort = -1; private int appMasterRpcPort = -1;
// Tracking url to which app master publishes info for clients to monitor // Tracking url to which app master publishes info for clients to monitor
private String appMasterTrackingUrl = ""; private String appMasterTrackingUrl = "";
private boolean newTimelineService = false; private boolean newTimelineService = false;
// For posting entities in new timeline service in a non-blocking way // For posting entities in new timeline service in a non-blocking way
// TODO replace with event loop in TimelineClient. // TODO replace with event loop in TimelineClient.
private static ExecutorService threadPool = private static ExecutorService threadPool =
Executors.newCachedThreadPool( Executors.newCachedThreadPool(
new ThreadFactoryBuilder().setNameFormat("TimelineService #%d") new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
.build()); .build());
@ -331,9 +331,9 @@ public static void main(String[] args) {
} }
appMaster.run(); appMaster.run();
result = appMaster.finish(); result = appMaster.finish();
threadPool.shutdown(); threadPool.shutdown();
while (!threadPool.isTerminated()) { // wait for all posting thread to finish while (!threadPool.isTerminated()) { // wait for all posting thread to finish
try { try {
if (!threadPool.awaitTermination(30, TimeUnit.SECONDS)) { if (!threadPool.awaitTermination(30, TimeUnit.SECONDS)) {
@ -427,7 +427,7 @@ public boolean init(String[] args) throws ParseException, IOException {
opts.addOption("container_retry_interval", true, opts.addOption("container_retry_interval", true,
"Interval between each retry, unit is milliseconds"); "Interval between each retry, unit is milliseconds");
opts.addOption("debug", false, "Dump out debug information"); opts.addOption("debug", false, "Dump out debug information");
opts.addOption("timeline_service_version", true, opts.addOption("timeline_service_version", true,
"Version for timeline service"); "Version for timeline service");
opts.addOption("help", false, "Print usage"); opts.addOption("help", false, "Print usage");
CommandLine cliParser = new GnuParser().parse(opts, args); CommandLine cliParser = new GnuParser().parse(opts, args);
@ -583,7 +583,7 @@ public boolean init(String[] args) throws ParseException, IOException {
if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
if (cliParser.hasOption("timeline_service_version")) { if (cliParser.hasOption("timeline_service_version")) {
String timelineServiceVersion = String timelineServiceVersion =
cliParser.getOptionValue("timeline_service_version", "v1"); cliParser.getOptionValue("timeline_service_version", "v1");
if (timelineServiceVersion.trim().equalsIgnoreCase("v1")) { if (timelineServiceVersion.trim().equalsIgnoreCase("v1")) {
newTimelineService = false; newTimelineService = false;
@ -655,7 +655,7 @@ public void run() throws YarnException, IOException, InterruptedException {
amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
amRMClient.init(conf); amRMClient.init(conf);
amRMClient.start(); amRMClient.start();
containerListener = createNMCallbackHandler(); containerListener = createNMCallbackHandler();
nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync = new NMClientAsyncImpl(containerListener);
nmClientAsync.init(conf); nmClientAsync.init(conf);
@ -840,7 +840,7 @@ protected boolean finish() {
if(timelineClient != null) { if(timelineClient != null) {
timelineClient.stop(); timelineClient.stop();
} }
return success; return success;
} }
@ -1392,11 +1392,11 @@ Thread createLaunchContainerThread(Container allocatedContainer,
} }
private static void publishContainerStartEventOnNewTimelineService( private static void publishContainerStartEventOnNewTimelineService(
final TimelineClient timelineClient, final Container container, final TimelineClient timelineClient, final Container container,
final String domainId, final UserGroupInformation ugi) { final String domainId, final UserGroupInformation ugi) {
Runnable publishWrapper = new Runnable() { Runnable publishWrapper = new Runnable() {
public void run() { public void run() {
publishContainerStartEventOnNewTimelineServiceBase(timelineClient, publishContainerStartEventOnNewTimelineServiceBase(timelineClient,
container, domainId, ugi); container, domainId, ugi);
} }
}; };
@ -1406,14 +1406,14 @@ public void run() {
private static void publishContainerStartEventOnNewTimelineServiceBase( private static void publishContainerStartEventOnNewTimelineServiceBase(
final TimelineClient timelineClient, Container container, String domainId, final TimelineClient timelineClient, Container container, String domainId,
UserGroupInformation ugi) { UserGroupInformation ugi) {
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity(); new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.setId(container.getId().toString()); entity.setId(container.getId().toString());
entity.setType(DSEntity.DS_CONTAINER.toString()); entity.setType(DSEntity.DS_CONTAINER.toString());
//entity.setDomainId(domainId); //entity.setDomainId(domainId);
entity.addInfo("user", ugi.getShortUserName()); entity.addInfo("user", ugi.getShortUserName());
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event = org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent(); new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent();
event.setTimestamp(System.currentTimeMillis()); event.setTimestamp(System.currentTimeMillis());
event.setId(DSEvent.DS_CONTAINER_START.toString()); event.setId(DSEvent.DS_CONTAINER_START.toString());
@ -1435,29 +1435,29 @@ public TimelinePutResponse run() throws Exception {
e instanceof UndeclaredThrowableException ? e.getCause() : e); e instanceof UndeclaredThrowableException ? e.getCause() : e);
} }
} }
private static void publishContainerEndEventOnNewTimelineService( private static void publishContainerEndEventOnNewTimelineService(
final TimelineClient timelineClient, final ContainerStatus container, final TimelineClient timelineClient, final ContainerStatus container,
final String domainId, final UserGroupInformation ugi) { final String domainId, final UserGroupInformation ugi) {
Runnable publishWrapper = new Runnable() { Runnable publishWrapper = new Runnable() {
public void run() { public void run() {
publishContainerEndEventOnNewTimelineServiceBase(timelineClient, publishContainerEndEventOnNewTimelineServiceBase(timelineClient,
container, domainId, ugi); container, domainId, ugi);
} }
}; };
threadPool.execute(publishWrapper); threadPool.execute(publishWrapper);
} }
private static void publishContainerEndEventOnNewTimelineServiceBase( private static void publishContainerEndEventOnNewTimelineServiceBase(
final TimelineClient timelineClient, final ContainerStatus container, final TimelineClient timelineClient, final ContainerStatus container,
final String domainId, final UserGroupInformation ugi) { final String domainId, final UserGroupInformation ugi) {
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity(); new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.setId(container.getContainerId().toString()); entity.setId(container.getContainerId().toString());
entity.setType(DSEntity.DS_CONTAINER.toString()); entity.setType(DSEntity.DS_CONTAINER.toString());
//entity.setDomainId(domainId); //entity.setDomainId(domainId);
entity.addInfo("user", ugi.getShortUserName()); entity.addInfo("user", ugi.getShortUserName());
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event = org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent(); new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent();
event.setTimestamp(System.currentTimeMillis()); event.setTimestamp(System.currentTimeMillis());
event.setId(DSEvent.DS_CONTAINER_END.toString()); event.setId(DSEvent.DS_CONTAINER_END.toString());
@ -1482,28 +1482,28 @@ public TimelinePutResponse run() throws Exception {
private static void publishApplicationAttemptEventOnNewTimelineService( private static void publishApplicationAttemptEventOnNewTimelineService(
final TimelineClient timelineClient, final String appAttemptId, final TimelineClient timelineClient, final String appAttemptId,
final DSEvent appEvent, final String domainId, final DSEvent appEvent, final String domainId,
final UserGroupInformation ugi) { final UserGroupInformation ugi) {
Runnable publishWrapper = new Runnable() { Runnable publishWrapper = new Runnable() {
public void run() { public void run() {
publishApplicationAttemptEventOnNewTimelineServiceBase(timelineClient, publishApplicationAttemptEventOnNewTimelineServiceBase(timelineClient,
appAttemptId, appEvent, domainId, ugi); appAttemptId, appEvent, domainId, ugi);
} }
}; };
threadPool.execute(publishWrapper); threadPool.execute(publishWrapper);
} }
private static void publishApplicationAttemptEventOnNewTimelineServiceBase( private static void publishApplicationAttemptEventOnNewTimelineServiceBase(
final TimelineClient timelineClient, String appAttemptId, final TimelineClient timelineClient, String appAttemptId,
DSEvent appEvent, String domainId, UserGroupInformation ugi) { DSEvent appEvent, String domainId, UserGroupInformation ugi) {
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity = final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity(); new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.setId(appAttemptId); entity.setId(appAttemptId);
entity.setType(DSEntity.DS_APP_ATTEMPT.toString()); entity.setType(DSEntity.DS_APP_ATTEMPT.toString());
//entity.setDomainId(domainId); //entity.setDomainId(domainId);
entity.addInfo("user", ugi.getShortUserName()); entity.addInfo("user", ugi.getShortUserName());
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event = org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent(); new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent();
event.setId(appEvent.toString()); event.setId(appEvent.toString());
event.setTimestamp(System.currentTimeMillis()); event.setTimestamp(System.currentTimeMillis());

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
View File

@ -189,7 +189,7 @@ public class Client {
// Command line options // Command line options
private Options opts; private Options opts;
private String timelineServiceVersion; private String timelineServiceVersion;
private static final String shellCommandPath = "shellCommands"; private static final String shellCommandPath = "shellCommands";
@ -373,11 +373,11 @@ public boolean init(String[] args) throws ParseException {
throw new IllegalArgumentException("Invalid virtual cores specified for application master, exiting." throw new IllegalArgumentException("Invalid virtual cores specified for application master, exiting."
+ " Specified virtual cores=" + amVCores); + " Specified virtual cores=" + amVCores);
} }
if (cliParser.hasOption("timeline_service_version")) { if (cliParser.hasOption("timeline_service_version")) {
timelineServiceVersion = timelineServiceVersion =
cliParser.getOptionValue("timeline_service_version", "v1"); cliParser.getOptionValue("timeline_service_version", "v1");
if (! (timelineServiceVersion.trim().equalsIgnoreCase("v1") || if (! (timelineServiceVersion.trim().equalsIgnoreCase("v1") ||
timelineServiceVersion.trim().equalsIgnoreCase("v2"))) { timelineServiceVersion.trim().equalsIgnoreCase("v2"))) {
throw new IllegalArgumentException( throw new IllegalArgumentException(
"timeline_service_version is not set properly, should be 'v1' or 'v2'"); "timeline_service_version is not set properly, should be 'v1' or 'v2'");

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
View File

@ -79,7 +79,7 @@
import org.apache.hadoop.yarn.server.timeline.TimelineVersionWatcher; import org.apache.hadoop.yarn.server.timeline.TimelineVersionWatcher;
import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.server.timelineservice.aggregator.PerNodeTimelineAggregatorsAuxService; import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
@ -100,7 +100,7 @@ public class TestDistributedShell {
protected YarnConfiguration conf = null; protected YarnConfiguration conf = null;
private static final int NUM_NMS = 1; private static final int NUM_NMS = 1;
private static final float DEFAULT_TIMELINE_VERSION = 1.0f; private static final float DEFAULT_TIMELINE_VERSION = 1.0f;
private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_aggregator"; private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";
protected final static String APPMASTER_JAR = protected final static String APPMASTER_JAR =
JarFinder.getJar(ApplicationMaster.class); JarFinder.getJar(ApplicationMaster.class);
@ -161,7 +161,7 @@ private void setupInternal(int numNodeManager, float timelineVersion)
// enable aux-service based timeline aggregators // enable aux-service based timeline aggregators
conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME); conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME
+ ".class", PerNodeTimelineAggregatorsAuxService.class.getName()); + ".class", PerNodeTimelineCollectorsAuxService.class.getName());
} else { } else {
Assert.fail("Wrong timeline version number: " + timelineVersion); Assert.fail("Wrong timeline version number: " + timelineVersion);
} }

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
View File

@ -51,7 +51,7 @@
public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
AbstractService { AbstractService {
private static final Log LOG = LogFactory.getLog(AMRMClient.class); private static final Log LOG = LogFactory.getLog(AMRMClient.class);
private TimelineClient timelineClient; private TimelineClient timelineClient;
/** /**
@ -468,7 +468,7 @@ public NMTokenCache getNMTokenCache() {
public void registerTimelineClient(TimelineClient timelineClient) { public void registerTimelineClient(TimelineClient timelineClient) {
this.timelineClient = timelineClient; this.timelineClient = timelineClient;
} }
/** /**
* Get registered timeline client. * Get registered timeline client.
* @return * @return
@ -476,7 +476,7 @@ public void registerTimelineClient(TimelineClient timelineClient) {
public TimelineClient getRegisteredTimeineClient() { public TimelineClient getRegisteredTimeineClient() {
return this.timelineClient; return this.timelineClient;
} }
/** /**
* Wait for <code>check</code> to return true for each 1000 ms. * Wait for <code>check</code> to return true for each 1000 ms.
* See also {@link #waitFor(com.google.common.base.Supplier, int)} * See also {@link #waitFor(com.google.common.base.Supplier, int)}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
View File

@ -293,7 +293,7 @@ public abstract void requestContainerResourceChange(
* @return Current number of nodes in the cluster * @return Current number of nodes in the cluster
*/ */
public abstract int getClusterNodeCount(); public abstract int getClusterNodeCount();
/** /**
* Register TimelineClient to AMRMClient. * Register TimelineClient to AMRMClient.
* @param timelineClient * @param timelineClient
@ -301,7 +301,7 @@ public abstract void requestContainerResourceChange(
public void registerTimelineClient(TimelineClient timelineClient) { public void registerTimelineClient(TimelineClient timelineClient) {
client.registerTimelineClient(timelineClient); client.registerTimelineClient(timelineClient);
} }
/** /**
* Get registered timeline client. * Get registered timeline client.
* @return * @return

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
View File

@ -67,8 +67,8 @@ public class AMRMClientAsyncImpl<T extends ContainerRequest>
private volatile boolean keepRunning; private volatile boolean keepRunning;
private volatile float progress; private volatile float progress;
private volatile String aggregatorAddr; private volatile String collectorAddr;
private volatile Throwable savedException; private volatile Throwable savedException;
/** /**
@ -354,15 +354,15 @@ public void run() {
if (!allocated.isEmpty()) { if (!allocated.isEmpty()) {
handler.onContainersAllocated(allocated); handler.onContainersAllocated(allocated);
} }
String aggregatorAddress = response.getAggregatorAddr(); String collectorAddress = response.getCollectorAddr();
TimelineClient timelineClient = client.getRegisteredTimeineClient(); TimelineClient timelineClient = client.getRegisteredTimeineClient();
if (timelineClient != null && aggregatorAddress != null if (timelineClient != null && collectorAddress != null
&& !aggregatorAddress.isEmpty()) { && !collectorAddress.isEmpty()) {
if (aggregatorAddr == null || if (collectorAddr == null ||
!aggregatorAddr.equals(aggregatorAddress)) { !collectorAddr.equals(collectorAddress)) {
aggregatorAddr = aggregatorAddress; collectorAddr = collectorAddress;
timelineClient.setTimelineServiceAddress(aggregatorAddress); timelineClient.setTimelineServiceAddress(collectorAddress);
} }
} }
progress = handler.getProgress(); progress = handler.getProgress();

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
View File

@ -384,22 +384,22 @@ public synchronized void setAMRMToken(Token amRMToken) {
} }
this.amrmToken = amRMToken; this.amrmToken = amRMToken;
} }
@Override @Override
public String getAggregatorAddr() { public String getCollectorAddr() {
AllocateResponseProtoOrBuilder p = viaProto ? proto : builder; AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
return p.getAggregatorAddr(); return p.getCollectorAddr();
} }
@Override @Override
public void setAggregatorAddr(String aggregatorAddr) { public void setCollectorAddr(String collectorAddr) {
maybeInitBuilder(); maybeInitBuilder();
if (aggregatorAddr == null) { if (collectorAddr == null) {
builder.clearAggregatorAddr(); builder.clearCollectorAddr();
return; return;
} }
builder.setAggregatorAddr(aggregatorAddr); builder.setCollectorAddr(collectorAddr);
} }
@Override @Override

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
View File

@ -242,5 +242,5 @@ public abstract void putEntitiesAsync(
* the timeline service address * the timeline service address
*/ */
public abstract void setTimelineServiceAddress(String address); public abstract void setTimelineServiceAddress(String address);
} }

25
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
View File

@ -119,12 +119,11 @@ public class TimelineClientImpl extends TimelineClient {
private TimelineWriter timelineWriter; private TimelineWriter timelineWriter;
private volatile String timelineServiceAddress; private volatile String timelineServiceAddress;
// Retry parameters for identifying new timeline service // Retry parameters for identifying new timeline service
// TODO consider to merge with connection retry // TODO consider to merge with connection retry
private int maxServiceRetries; private int maxServiceRetries;
private long serviceRetryInterval; private long serviceRetryInterval;
private boolean timelineServiceV2 = false; private boolean timelineServiceV2 = false;
@Private @Private
@ -330,7 +329,7 @@ protected void serviceInit(Configuration conf) throws Exception {
conf.getFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, conf.getFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_VERSION); YarnConfiguration.DEFAULT_TIMELINE_SERVICE_VERSION);
LOG.info("Timeline service address: " + getTimelineServiceAddress()); LOG.info("Timeline service address: " + getTimelineServiceAddress());
} }
super.serviceInit(conf); super.serviceInit(conf);
} }
@ -410,16 +409,16 @@ public void putDomain(TimelineDomain domain) throws IOException,
YarnException { YarnException {
timelineWriter.putDomain(domain); timelineWriter.putDomain(domain);
} }
// Used for new timeline service only // Used for new timeline service only
@Private @Private
public void putObjects(String path, MultivaluedMap<String, String> params, public void putObjects(String path, MultivaluedMap<String, String> params,
Object obj) throws IOException, YarnException { Object obj) throws IOException, YarnException {
// timelineServiceAddress could haven't be initialized yet // timelineServiceAddress could haven't be initialized yet
// or stale (only for new timeline service) // or stale (only for new timeline service)
int retries = pollTimelineServiceAddress(this.maxServiceRetries); int retries = pollTimelineServiceAddress(this.maxServiceRetries);
// timelineServiceAddress could be stale, add retry logic here. // timelineServiceAddress could be stale, add retry logic here.
boolean needRetry = true; boolean needRetry = true;
while (needRetry) { while (needRetry) {
@ -436,13 +435,13 @@ public void putObjects(String path, MultivaluedMap<String, String> params,
} }
} }
} }
/** /**
* Check if reaching to maximum of retries. * Check if reaching to maximum of retries.
* @param retries * @param retries
* @param e * @param e
*/ */
private void checkRetryWithSleep(int retries, Exception e) throws private void checkRetryWithSleep(int retries, Exception e) throws
YarnException, IOException { YarnException, IOException {
if (retries > 0) { if (retries > 0) {
try { try {
@ -452,8 +451,8 @@ private void checkRetryWithSleep(int retries, Exception e) throws
} }
} else { } else {
LOG.error( LOG.error(
"TimelineClient has reached to max retry times :" + "TimelineClient has reached to max retry times :" +
this.maxServiceRetries + " for service address: " + this.maxServiceRetries + " for service address: " +
timelineServiceAddress); timelineServiceAddress);
if (e instanceof YarnException) { if (e instanceof YarnException) {
throw (YarnException)e; throw (YarnException)e;
@ -499,7 +498,7 @@ private void putObjects(
public void setTimelineServiceAddress(String address) { public void setTimelineServiceAddress(String address) {
this.timelineServiceAddress = address; this.timelineServiceAddress = address;
} }
private String getTimelineServiceAddress() { private String getTimelineServiceAddress() {
return this.timelineServiceAddress; return this.timelineServiceAddress;
} }

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -965,10 +965,10 @@
<name>yarn.nodemanager.container-manager.thread-count</name> <name>yarn.nodemanager.container-manager.thread-count</name>
<value>20</value> <value>20</value>
</property> </property>
<property> <property>
<description>Number of threads aggregator service uses.</description> <description>Number of threads collector service uses.</description>
<name>yarn.nodemanager.aggregator-service.thread-count</name> <name>yarn.nodemanager.collector-service.thread-count</name>
<value>5</value> <value>5</value>
</property> </property>
@ -1046,11 +1046,11 @@
<name>yarn.nodemanager.localizer.address</name> <name>yarn.nodemanager.localizer.address</name>
<value>${yarn.nodemanager.hostname}:8040</value> <value>${yarn.nodemanager.hostname}:8040</value>
</property> </property>
<property> <property>
<description>Address where the aggregator service IPC is.</description> <description>Address where the collector service IPC is.</description>
<name>yarn.nodemanager.aggregator-service.address</name> <name>yarn.nodemanager.collector-service.address</name>
<value>${yarn.nodemanager.hostname}:8048</value> <value>${yarn.nodemanager.hostname}:8048</value>
</property> </property>

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
View File

@ -135,7 +135,7 @@ private void testRPCTimeout(String rpcClass) throws Exception {
Assert.fail("timeout exception should have occurred!"); Assert.fail("timeout exception should have occurred!");
} }
public static Token newContainerToken(NodeId nodeId, byte[] password, public static Token newContainerToken(NodeId nodeId, byte[] password,
ContainerTokenIdentifier tokenIdentifier) { ContainerTokenIdentifier tokenIdentifier) {
// RPC layer client expects ip:port as service for tokens // RPC layer client expects ip:port as service for tokens

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
View File

@ -150,7 +150,7 @@
<include>yarn_server_common_service_protos.proto</include> <include>yarn_server_common_service_protos.proto</include>
<include>ResourceTracker.proto</include> <include>ResourceTracker.proto</include>
<include>SCMUploader.proto</include> <include>SCMUploader.proto</include>
<include>aggregatornodemanager_protocol.proto</include> <include>collectornodemanager_protocol.proto</include>
</includes> </includes>
</source> </source>
<output>${project.build.directory}/generated-sources/java</output> <output>${project.build.directory}/generated-sources/java</output>

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocol.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocol.java
View File

@ -21,36 +21,37 @@
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
/** /**
* <p>The protocol between an <code>TimelineAggregatorsCollection</code> and a * <p>The protocol between an <code>TimelineCollectorManager</code> and a
* <code>NodeManager</code> to report a new application aggregator get launched. * <code>NodeManager</code> to report a new application collector get launched.
* </p> * </p>
* *
*/ */
@Private @Private
public interface AggregatorNodemanagerProtocol { public interface CollectorNodemanagerProtocol {
/** /**
* *
* <p> * <p>
* The <code>TimelineAggregatorsCollection</code> provides a list of mapping * The <code>TimelineCollectorManager</code> provides a list of mapping
* between application and aggregator's address in * between application and collector's address in
* {@link ReportNewAggregatorsInfoRequest} to a <code>NodeManager</code> to * {@link ReportNewCollectorInfoRequest} to a <code>NodeManager</code> to
* <em>register</em> aggregator's info, include: applicationId and REST URI to * <em>register</em> collector's info, include: applicationId and REST URI to
* access aggregator. NodeManager will add them into registered aggregators * access collector. NodeManager will add them into registered collectors
* and register them into <code>ResourceManager</code> afterwards. * and register them into <code>ResourceManager</code> afterwards.
* </p> * </p>
* *
* @param request the request of registering a new aggregator or a list of aggregators * @param request the request of registering a new collector or a list of
* @return * collectors
* @return
* @throws YarnException * @throws YarnException
* @throws IOException * @throws IOException
*/ */
ReportNewAggregatorsInfoResponse reportNewAggregatorInfo( ReportNewCollectorInfoResponse reportNewCollectorInfo(
ReportNewAggregatorsInfoRequest request) ReportNewCollectorInfoRequest request)
throws YarnException, IOException; throws YarnException, IOException;
} }

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocolPB.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocolPB.java
View File

@ -20,14 +20,14 @@
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.ipc.ProtocolInfo; import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.yarn.proto.AggregatorNodemanagerProtocol.AggregatorNodemanagerProtocolService; import org.apache.hadoop.yarn.proto.CollectorNodemanagerProtocol.CollectorNodemanagerProtocolService;
@Private @Private
@Unstable @Unstable
@ProtocolInfo( @ProtocolInfo(
protocolName = "org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocolPB", protocolName = "org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB",
protocolVersion = 1) protocolVersion = 1)
public interface AggregatorNodemanagerProtocolPB extends public interface CollectorNodemanagerProtocolPB extends
AggregatorNodemanagerProtocolService.BlockingInterface { CollectorNodemanagerProtocolService.BlockingInterface {
} }

50
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/AggregatorNodemanagerProtocolPBClientImpl.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/CollectorNodemanagerProtocolPBClientImpl.java
View File

@ -30,18 +30,18 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProto;
import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol; import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocolPB; import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewAggregatorsInfoRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewAggregatorsInfoResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoResponsePBImpl;
import com.google.protobuf.ServiceException; import com.google.protobuf.ServiceException;
public class AggregatorNodemanagerProtocolPBClientImpl implements public class CollectorNodemanagerProtocolPBClientImpl implements
AggregatorNodemanagerProtocol, Closeable { CollectorNodemanagerProtocol, Closeable {
// Not a documented config. Only used for tests internally // Not a documented config. Only used for tests internally
static final String NM_COMMAND_TIMEOUT = YarnConfiguration.YARN_PREFIX static final String NM_COMMAND_TIMEOUT = YarnConfiguration.YARN_PREFIX
@ -51,39 +51,39 @@ public class AggregatorNodemanagerProtocolPBClientImpl implements
* Maximum of 1 minute timeout for a Node to react to the command * Maximum of 1 minute timeout for a Node to react to the command
*/ */
static final int DEFAULT_COMMAND_TIMEOUT = 60000; static final int DEFAULT_COMMAND_TIMEOUT = 60000;
private AggregatorNodemanagerProtocolPB proxy; private CollectorNodemanagerProtocolPB proxy;
@Private @Private
public AggregatorNodemanagerProtocolPBClientImpl(long clientVersion, public CollectorNodemanagerProtocolPBClientImpl(long clientVersion,
InetSocketAddress addr, Configuration conf) throws IOException { InetSocketAddress addr, Configuration conf) throws IOException {
RPC.setProtocolEngine(conf, AggregatorNodemanagerProtocolPB.class, RPC.setProtocolEngine(conf, CollectorNodemanagerProtocolPB.class,
ProtobufRpcEngine.class); ProtobufRpcEngine.class);
UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
int expireIntvl = conf.getInt(NM_COMMAND_TIMEOUT, DEFAULT_COMMAND_TIMEOUT); int expireIntvl = conf.getInt(NM_COMMAND_TIMEOUT, DEFAULT_COMMAND_TIMEOUT);
proxy = proxy =
(AggregatorNodemanagerProtocolPB) RPC.getProxy( (CollectorNodemanagerProtocolPB) RPC.getProxy(
AggregatorNodemanagerProtocolPB.class, CollectorNodemanagerProtocolPB.class,
clientVersion, addr, ugi, conf, clientVersion, addr, ugi, conf,
NetUtils.getDefaultSocketFactory(conf), expireIntvl); NetUtils.getDefaultSocketFactory(conf), expireIntvl);
} }
@Override @Override
public ReportNewAggregatorsInfoResponse reportNewAggregatorInfo( public ReportNewCollectorInfoResponse reportNewCollectorInfo(
ReportNewAggregatorsInfoRequest request) throws YarnException, IOException { ReportNewCollectorInfoRequest request) throws YarnException, IOException {
ReportNewAggregatorsInfoRequestProto requestProto = ReportNewCollectorInfoRequestProto requestProto =
((ReportNewAggregatorsInfoRequestPBImpl) request).getProto(); ((ReportNewCollectorInfoRequestPBImpl) request).getProto();
try { try {
return new ReportNewAggregatorsInfoResponsePBImpl( return new ReportNewCollectorInfoResponsePBImpl(
proxy.reportNewAggregatorInfo(null, requestProto)); proxy.reportNewCollectorInfo(null, requestProto));
} catch (ServiceException e) { } catch (ServiceException e) {
RPCUtil.unwrapAndThrowException(e); RPCUtil.unwrapAndThrowException(e);
return null; return null;
} }
} }
@Override @Override
public void close() { public void close() {
if (this.proxy != null) { if (this.proxy != null) {

38
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/AggregatorNodemanagerProtocolPBServiceImpl.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/CollectorNodemanagerProtocolPBServiceImpl.java
View File

@ -19,38 +19,36 @@
import java.io.IOException; import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoResponseProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoResponseProto;
import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol; import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocolPB; import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewAggregatorsInfoRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewAggregatorsInfoResponsePBImpl;
import com.google.protobuf.RpcController; import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException; import com.google.protobuf.ServiceException;
public class AggregatorNodemanagerProtocolPBServiceImpl implements public class CollectorNodemanagerProtocolPBServiceImpl implements
AggregatorNodemanagerProtocolPB { CollectorNodemanagerProtocolPB {
private AggregatorNodemanagerProtocol real; private CollectorNodemanagerProtocol real;
public AggregatorNodemanagerProtocolPBServiceImpl(AggregatorNodemanagerProtocol impl) { public CollectorNodemanagerProtocolPBServiceImpl(CollectorNodemanagerProtocol impl) {
this.real = impl; this.real = impl;
} }
@Override @Override
public ReportNewAggregatorsInfoResponseProto reportNewAggregatorInfo( public ReportNewCollectorInfoResponseProto reportNewCollectorInfo(
RpcController arg0, ReportNewAggregatorsInfoRequestProto proto) RpcController arg0, ReportNewCollectorInfoRequestProto proto)
throws ServiceException { throws ServiceException {
ReportNewAggregatorsInfoRequestPBImpl request = ReportNewCollectorInfoRequestPBImpl request =
new ReportNewAggregatorsInfoRequestPBImpl(proto); new ReportNewCollectorInfoRequestPBImpl(proto);
try { try {
ReportNewAggregatorsInfoResponse response = real.reportNewAggregatorInfo(request); ReportNewCollectorInfoResponse response = real.reportNewCollectorInfo(request);
return ((ReportNewAggregatorsInfoResponsePBImpl)response).getProto(); return ((ReportNewCollectorInfoResponsePBImpl)response).getProto();
} catch (YarnException e) { } catch (YarnException e) {
throw new ServiceException(e); throw new ServiceException(e);
} catch (IOException e) { } catch (IOException e) {

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
View File

@ -43,11 +43,11 @@ public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus,
nodeHeartbeatRequest.setNodeLabels(nodeLabels); nodeHeartbeatRequest.setNodeLabels(nodeLabels);
return nodeHeartbeatRequest; return nodeHeartbeatRequest;
} }
public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus, public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus,
MasterKey lastKnownContainerTokenMasterKey, MasterKey lastKnownContainerTokenMasterKey,
MasterKey lastKnownNMTokenMasterKey, Set<NodeLabel> nodeLabels, MasterKey lastKnownNMTokenMasterKey, Set<NodeLabel> nodeLabels,
Map<ApplicationId, String> registeredAggregators) { Map<ApplicationId, String> registeredCollectors) {
NodeHeartbeatRequest nodeHeartbeatRequest = NodeHeartbeatRequest nodeHeartbeatRequest =
Records.newRecord(NodeHeartbeatRequest.class); Records.newRecord(NodeHeartbeatRequest.class);
nodeHeartbeatRequest.setNodeStatus(nodeStatus); nodeHeartbeatRequest.setNodeStatus(nodeStatus);
@ -56,7 +56,7 @@ public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus,
nodeHeartbeatRequest nodeHeartbeatRequest
.setLastKnownNMTokenMasterKey(lastKnownNMTokenMasterKey); .setLastKnownNMTokenMasterKey(lastKnownNMTokenMasterKey);
nodeHeartbeatRequest.setNodeLabels(nodeLabels); nodeHeartbeatRequest.setNodeLabels(nodeLabels);
nodeHeartbeatRequest.setRegisteredAggregators(registeredAggregators); nodeHeartbeatRequest.setRegisteredCollectors(registeredCollectors);
return nodeHeartbeatRequest; return nodeHeartbeatRequest;
} }
@ -78,7 +78,8 @@ public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus,
public abstract void setLogAggregationReportsForApps( public abstract void setLogAggregationReportsForApps(
List<LogAggregationReport> logAggregationReportsForApps); List<LogAggregationReport> logAggregationReportsForApps);
// This tells RM registered aggregators' address info on this node // This tells RM registered collectors' address info on this node
public abstract Map<ApplicationId, String> getRegisteredAggregators(); public abstract Map<ApplicationId, String> getRegisteredCollectors();
public abstract void setRegisteredAggregators(Map<ApplicationId, String> appAggregatorsMap); public abstract void setRegisteredCollectors(Map<ApplicationId,
String> appCollectorsMap);
} }

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
View File

@ -40,10 +40,10 @@ public interface NodeHeartbeatResponse {
List<ContainerId> getContainersToBeRemovedFromNM(); List<ContainerId> getContainersToBeRemovedFromNM();
List<ApplicationId> getApplicationsToCleanup(); List<ApplicationId> getApplicationsToCleanup();
// This tells NM the aggregators' address info of related Apps // This tells NM the collectors' address info of related apps
Map<ApplicationId, String> getAppAggregatorsMap(); Map<ApplicationId, String> getAppCollectorsMap();
void setAppAggregatorsMap(Map<ApplicationId, String> appAggregatorsMap); void setAppCollectorsMap(Map<ApplicationId, String> appCollectorsMap);
void setResponseId(int responseId); void setResponseId(int responseId);
void setNodeAction(NodeAction action); void setNodeAction(NodeAction action);

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoRequest.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoRequest.java
View File

@ -22,32 +22,32 @@
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap; import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap;
import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Records;
@Private @Private
public abstract class ReportNewAggregatorsInfoRequest { public abstract class ReportNewCollectorInfoRequest {
public static ReportNewAggregatorsInfoRequest newInstance( public static ReportNewCollectorInfoRequest newInstance(
List<AppAggregatorsMap> appAggregatorsList) { List<AppCollectorsMap> appCollectorsList) {
ReportNewAggregatorsInfoRequest request = ReportNewCollectorInfoRequest request =
Records.newRecord(ReportNewAggregatorsInfoRequest.class); Records.newRecord(ReportNewCollectorInfoRequest.class);
request.setAppAggregatorsList(appAggregatorsList); request.setAppCollectorsList(appCollectorsList);
return request; return request;
} }
public static ReportNewAggregatorsInfoRequest newInstance( public static ReportNewCollectorInfoRequest newInstance(
ApplicationId id, String aggregatorAddr) { ApplicationId id, String collectorAddr) {
ReportNewAggregatorsInfoRequest request = ReportNewCollectorInfoRequest request =
Records.newRecord(ReportNewAggregatorsInfoRequest.class); Records.newRecord(ReportNewCollectorInfoRequest.class);
request.setAppAggregatorsList( request.setAppCollectorsList(
Arrays.asList(AppAggregatorsMap.newInstance(id, aggregatorAddr))); Arrays.asList(AppCollectorsMap.newInstance(id, collectorAddr)));
return request; return request;
} }
public abstract List<AppAggregatorsMap> getAppAggregatorsList(); public abstract List<AppCollectorsMap> getAppCollectorsList();
public abstract void setAppAggregatorsList( public abstract void setAppCollectorsList(
List<AppAggregatorsMap> appAggregatorsList); List<AppCollectorsMap> appCollectorsList);
} }

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoResponse.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoResponse.java
View File

@ -20,12 +20,12 @@
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Records;
public abstract class ReportNewAggregatorsInfoResponse { public abstract class ReportNewCollectorInfoResponse {
@Private @Private
public static ReportNewAggregatorsInfoResponse newInstance() { public static ReportNewCollectorInfoResponse newInstance() {
ReportNewAggregatorsInfoResponse response = ReportNewCollectorInfoResponse response =
Records.newRecord(ReportNewAggregatorsInfoResponse.class); Records.newRecord(ReportNewCollectorInfoResponse.class);
return response; return response;
} }

58
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
View File

@ -34,8 +34,8 @@
import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.LogAggregationReportProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.LogAggregationReportProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatRequestProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatRequestProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatRequestProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto;
@ -58,7 +58,7 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
private Set<NodeLabel> labels = null; private Set<NodeLabel> labels = null;
private List<LogAggregationReport> logAggregationReportsForApps = null; private List<LogAggregationReport> logAggregationReportsForApps = null;
Map<ApplicationId, String> registeredAggregators = null; Map<ApplicationId, String> registeredCollectors = null;
public NodeHeartbeatRequestPBImpl() { public NodeHeartbeatRequestPBImpl() {
builder = NodeHeartbeatRequestProto.newBuilder(); builder = NodeHeartbeatRequestProto.newBuilder();
@ -114,8 +114,8 @@ private void mergeLocalToBuilder() {
if (this.logAggregationReportsForApps != null) { if (this.logAggregationReportsForApps != null) {
addLogAggregationStatusForAppsToProto(); addLogAggregationStatusForAppsToProto();
} }
if (this.registeredAggregators != null) { if (this.registeredCollectors != null) {
addRegisteredAggregatorsToProto(); addRegisteredCollectorsToProto();
} }
} }
@ -158,13 +158,13 @@ private LogAggregationReportProto convertToProtoFormat(
return ((LogAggregationReportPBImpl) value).getProto(); return ((LogAggregationReportPBImpl) value).getProto();
} }
private void addRegisteredAggregatorsToProto() { private void addRegisteredCollectorsToProto() {
maybeInitBuilder(); maybeInitBuilder();
builder.clearRegisteredAggregators(); builder.clearRegisteredCollectors();
for (Map.Entry<ApplicationId, String> entry : registeredAggregators.entrySet()) { for (Map.Entry<ApplicationId, String> entry : registeredCollectors.entrySet()) {
builder.addRegisteredAggregators(AppAggregatorsMapProto.newBuilder() builder.addRegisteredCollectors(AppCollectorsMapProto.newBuilder()
.setAppId(convertToProtoFormat(entry.getKey())) .setAppId(convertToProtoFormat(entry.getKey()))
.setAppAggregatorAddr(entry.getValue())); .setAppCollectorAddr(entry.getValue()));
} }
} }
@ -248,35 +248,35 @@ public void setLastKnownNMTokenMasterKey(MasterKey masterKey) {
builder.clearLastKnownNmTokenMasterKey(); builder.clearLastKnownNmTokenMasterKey();
this.lastKnownNMTokenMasterKey = masterKey; this.lastKnownNMTokenMasterKey = masterKey;
} }
@Override @Override
public Map<ApplicationId, String> getRegisteredAggregators() { public Map<ApplicationId, String> getRegisteredCollectors() {
if (this.registeredAggregators != null) { if (this.registeredCollectors != null) {
return this.registeredAggregators; return this.registeredCollectors;
} }
initRegisteredAggregators(); initRegisteredCollectors();
return registeredAggregators; return registeredCollectors;
} }
private void initRegisteredAggregators() { private void initRegisteredCollectors() {
NodeHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder; NodeHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder;
List<AppAggregatorsMapProto> list = p.getRegisteredAggregatorsList(); List<AppCollectorsMapProto> list = p.getRegisteredCollectorsList();
this.registeredAggregators = new HashMap<ApplicationId, String> (); this.registeredCollectors = new HashMap<ApplicationId, String> ();
for (AppAggregatorsMapProto c : list) { for (AppCollectorsMapProto c : list) {
ApplicationId appId = convertFromProtoFormat(c.getAppId()); ApplicationId appId = convertFromProtoFormat(c.getAppId());
this.registeredAggregators.put(appId, c.getAppAggregatorAddr()); this.registeredCollectors.put(appId, c.getAppCollectorAddr());
} }
} }
@Override @Override
public void setRegisteredAggregators( public void setRegisteredCollectors(
Map<ApplicationId, String> registeredAggregators) { Map<ApplicationId, String> registeredCollectors) {
if (registeredAggregators == null || registeredAggregators.isEmpty()) { if (registeredCollectors == null || registeredCollectors.isEmpty()) {
return; return;
} }
maybeInitBuilder(); maybeInitBuilder();
this.registeredAggregators = new HashMap<ApplicationId, String>(); this.registeredCollectors = new HashMap<ApplicationId, String>();
this.registeredAggregators.putAll(registeredAggregators); this.registeredCollectors.putAll(registeredCollectors);
} }
private NodeStatusPBImpl convertFromProtoFormat(NodeStatusProto p) { private NodeStatusPBImpl convertFromProtoFormat(NodeStatusProto p) {
@ -286,11 +286,11 @@ private NodeStatusPBImpl convertFromProtoFormat(NodeStatusProto p) {
private NodeStatusProto convertToProtoFormat(NodeStatus t) { private NodeStatusProto convertToProtoFormat(NodeStatus t) {
return ((NodeStatusPBImpl)t).getProto(); return ((NodeStatusPBImpl)t).getProto();
} }
private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
return new ApplicationIdPBImpl(p); return new ApplicationIdPBImpl(p);
} }
private ApplicationIdProto convertToProtoFormat(ApplicationId t) { private ApplicationIdProto convertToProtoFormat(ApplicationId t) {
return ((ApplicationIdPBImpl) t).getProto(); return ((ApplicationIdPBImpl) t).getProto();
} }

56
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
View File

@ -46,7 +46,7 @@
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SignalContainerRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SignalContainerRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.SystemCredentialsForAppsProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.SystemCredentialsForAppsProto;
@ -69,7 +69,7 @@ public class NodeHeartbeatResponsePBImpl extends
private List<ApplicationId> applicationsToCleanup = null; private List<ApplicationId> applicationsToCleanup = null;
private Map<ApplicationId, ByteBuffer> systemCredentials = null; private Map<ApplicationId, ByteBuffer> systemCredentials = null;
private Resource resource = null; private Resource resource = null;
Map<ApplicationId, String> appAggregatorsMap = null; Map<ApplicationId, String> appCollectorsMap = null;
private MasterKey containerTokenMasterKey = null; private MasterKey containerTokenMasterKey = null;
private MasterKey nmTokenMasterKey = null; private MasterKey nmTokenMasterKey = null;
@ -127,8 +127,8 @@ private void mergeLocalToBuilder() {
if (this.resource != null) { if (this.resource != null) {
builder.setResource(convertToProtoFormat(this.resource)); builder.setResource(convertToProtoFormat(this.resource));
} }
if (this.appAggregatorsMap != null) { if (this.appCollectorsMap != null) {
addAppAggregatorsMapToProto(); addAppCollectorsMapToProto();
} }
} }
@ -142,14 +142,14 @@ private void addSystemCredentialsToProto() {
entry.getValue().duplicate()))); entry.getValue().duplicate())));
} }
} }
private void addAppAggregatorsMapToProto() { private void addAppCollectorsMapToProto() {
maybeInitBuilder(); maybeInitBuilder();
builder.clearAppAggregatorsMap(); builder.clearAppCollectorsMap();
for (Map.Entry<ApplicationId, String> entry : appAggregatorsMap.entrySet()) { for (Map.Entry<ApplicationId, String> entry : appCollectorsMap.entrySet()) {
builder.addAppAggregatorsMap(AppAggregatorsMapProto.newBuilder() builder.addAppCollectorsMap(AppCollectorsMapProto.newBuilder()
.setAppId(convertToProtoFormat(entry.getKey())) .setAppId(convertToProtoFormat(entry.getKey()))
.setAppAggregatorAddr(entry.getValue())); .setAppCollectorAddr(entry.getValue()));
} }
} }
@ -565,14 +565,14 @@ public Map<ApplicationId, ByteBuffer> getSystemCredentialsForApps() {
initSystemCredentials(); initSystemCredentials();
return systemCredentials; return systemCredentials;
} }
@Override @Override
public Map<ApplicationId, String> getAppAggregatorsMap() { public Map<ApplicationId, String> getAppCollectorsMap() {
if (this.appAggregatorsMap != null) { if (this.appCollectorsMap != null) {
return this.appAggregatorsMap; return this.appCollectorsMap;
} }
initAppAggregatorsMap(); initAppCollectorsMap();
return appAggregatorsMap; return appCollectorsMap;
} }
private void initSystemCredentials() { private void initSystemCredentials() {
@ -585,14 +585,14 @@ private void initSystemCredentials() {
this.systemCredentials.put(appId, byteBuffer); this.systemCredentials.put(appId, byteBuffer);
} }
} }
private void initAppAggregatorsMap() { private void initAppCollectorsMap() {
NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder; NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
List<AppAggregatorsMapProto> list = p.getAppAggregatorsMapList(); List<AppCollectorsMapProto> list = p.getAppCollectorsMapList();
this.appAggregatorsMap = new HashMap<ApplicationId, String> (); this.appCollectorsMap = new HashMap<ApplicationId, String> ();
for (AppAggregatorsMapProto c : list) { for (AppCollectorsMapProto c : list) {
ApplicationId appId = convertFromProtoFormat(c.getAppId()); ApplicationId appId = convertFromProtoFormat(c.getAppId());
this.appAggregatorsMap.put(appId, c.getAppAggregatorAddr()); this.appCollectorsMap.put(appId, c.getAppCollectorAddr());
} }
} }
@ -606,16 +606,16 @@ public void setSystemCredentialsForApps(
this.systemCredentials = new HashMap<ApplicationId, ByteBuffer>(); this.systemCredentials = new HashMap<ApplicationId, ByteBuffer>();
this.systemCredentials.putAll(systemCredentials); this.systemCredentials.putAll(systemCredentials);
} }
@Override @Override
public void setAppAggregatorsMap( public void setAppCollectorsMap(
Map<ApplicationId, String> appAggregatorsMap) { Map<ApplicationId, String> appCollectorsMap) {
if (appAggregatorsMap == null || appAggregatorsMap.isEmpty()) { if (appCollectorsMap == null || appCollectorsMap.isEmpty()) {
return; return;
} }
maybeInitBuilder(); maybeInitBuilder();
this.appAggregatorsMap = new HashMap<ApplicationId, String>(); this.appCollectorsMap = new HashMap<ApplicationId, String>();
this.appAggregatorsMap.putAll(appAggregatorsMap); this.appCollectorsMap.putAll(appCollectorsMap);
} }
@Override @Override

142
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoRequestPBImpl.java
View File

@ -1,142 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoRequestProtoOrBuilder;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest;
import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap;
import org.apache.hadoop.yarn.server.api.records.impl.pb.AppAggregatorsMapPBImpl;
public class ReportNewAggregatorsInfoRequestPBImpl extends
ReportNewAggregatorsInfoRequest {
ReportNewAggregatorsInfoRequestProto proto =
ReportNewAggregatorsInfoRequestProto.getDefaultInstance();
ReportNewAggregatorsInfoRequestProto.Builder builder = null;
boolean viaProto = false;
private List<AppAggregatorsMap> aggregatorsList = null;
public ReportNewAggregatorsInfoRequestPBImpl() {
builder = ReportNewAggregatorsInfoRequestProto.newBuilder();
}
public ReportNewAggregatorsInfoRequestPBImpl(
ReportNewAggregatorsInfoRequestProto proto) {
this.proto = proto;
viaProto = true;
}
public ReportNewAggregatorsInfoRequestProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
@Override
public int hashCode() {
return getProto().hashCode();
}
@Override
public boolean equals(Object other) {
if (other == null)
return false;
if (other.getClass().isAssignableFrom(this.getClass())) {
return this.getProto().equals(this.getClass().cast(other).getProto());
}
return false;
}
private void mergeLocalToProto() {
if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
proto = builder.build();
viaProto = true;
}
private void mergeLocalToBuilder() {
if (aggregatorsList != null) {
addLocalAggregatorsToProto();
}
}
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ReportNewAggregatorsInfoRequestProto.newBuilder(proto);
}
viaProto = false;
}
private void addLocalAggregatorsToProto() {
maybeInitBuilder();
builder.clearAppAggregators();
List<AppAggregatorsMapProto> protoList =
new ArrayList<AppAggregatorsMapProto>();
for (AppAggregatorsMap m : this.aggregatorsList) {
protoList.add(convertToProtoFormat(m));
}
builder.addAllAppAggregators(protoList);
}
private void initLocalAggregatorsList() {
ReportNewAggregatorsInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
List<AppAggregatorsMapProto> aggregatorsList =
p.getAppAggregatorsList();
this.aggregatorsList = new ArrayList<AppAggregatorsMap>();
for (AppAggregatorsMapProto m : aggregatorsList) {
this.aggregatorsList.add(convertFromProtoFormat(m));
}
}
@Override
public List<AppAggregatorsMap> getAppAggregatorsList() {
if (this.aggregatorsList == null) {
initLocalAggregatorsList();
}
return this.aggregatorsList;
}
@Override
public void setAppAggregatorsList(List<AppAggregatorsMap> appAggregatorsList) {
maybeInitBuilder();
if (appAggregatorsList == null) {
builder.clearAppAggregators();
}
this.aggregatorsList = appAggregatorsList;
}
private AppAggregatorsMapPBImpl convertFromProtoFormat(
AppAggregatorsMapProto p) {
return new AppAggregatorsMapPBImpl(p);
}
private AppAggregatorsMapProto convertToProtoFormat(
AppAggregatorsMap m) {
return ((AppAggregatorsMapPBImpl) m).getProto();
}
}

142
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoRequestPBImpl.java Normal file
View File

@ -0,0 +1,142 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProtoOrBuilder;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap;
import org.apache.hadoop.yarn.server.api.records.impl.pb.AppCollectorsMapPBImpl;
public class ReportNewCollectorInfoRequestPBImpl extends
ReportNewCollectorInfoRequest {
ReportNewCollectorInfoRequestProto proto =
ReportNewCollectorInfoRequestProto.getDefaultInstance();
ReportNewCollectorInfoRequestProto.Builder builder = null;
boolean viaProto = false;
private List<AppCollectorsMap> collectorsList = null;
public ReportNewCollectorInfoRequestPBImpl() {
builder = ReportNewCollectorInfoRequestProto.newBuilder();
}
public ReportNewCollectorInfoRequestPBImpl(
ReportNewCollectorInfoRequestProto proto) {
this.proto = proto;
viaProto = true;
}
public ReportNewCollectorInfoRequestProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
@Override
public int hashCode() {
return getProto().hashCode();
}
@Override
public boolean equals(Object other) {
if (other == null)
return false;
if (other.getClass().isAssignableFrom(this.getClass())) {
return this.getProto().equals(this.getClass().cast(other).getProto());
}
return false;
}
private void mergeLocalToProto() {
if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
proto = builder.build();
viaProto = true;
}
private void mergeLocalToBuilder() {
if (collectorsList != null) {
addLocalCollectorsToProto();
}
}
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ReportNewCollectorInfoRequestProto.newBuilder(proto);
}
viaProto = false;
}
private void addLocalCollectorsToProto() {
maybeInitBuilder();
builder.clearAppCollectors();
List<AppCollectorsMapProto> protoList =
new ArrayList<AppCollectorsMapProto>();
for (AppCollectorsMap m : this.collectorsList) {
protoList.add(convertToProtoFormat(m));
}
builder.addAllAppCollectors(protoList);
}
private void initLocalCollectorsList() {
ReportNewCollectorInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
List<AppCollectorsMapProto> collectorsList =
p.getAppCollectorsList();
this.collectorsList = new ArrayList<AppCollectorsMap>();
for (AppCollectorsMapProto m : collectorsList) {
this.collectorsList.add(convertFromProtoFormat(m));
}
}
@Override
public List<AppCollectorsMap> getAppCollectorsList() {
if (this.collectorsList == null) {
initLocalCollectorsList();
}
return this.collectorsList;
}
@Override
public void setAppCollectorsList(List<AppCollectorsMap> appCollectorsList) {
maybeInitBuilder();
if (appCollectorsList == null) {
builder.clearAppCollectors();
}
this.collectorsList = appCollectorsList;
}
private AppCollectorsMapPBImpl convertFromProtoFormat(
AppCollectorsMapProto p) {
return new AppCollectorsMapPBImpl(p);
}
private AppCollectorsMapProto convertToProtoFormat(
AppCollectorsMap m) {
return ((AppCollectorsMapPBImpl) m).getProto();
}
}

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoResponsePBImpl.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoResponsePBImpl.java
View File

@ -19,33 +19,33 @@
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoResponseProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoResponseProto;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
import com.google.protobuf.TextFormat; import com.google.protobuf.TextFormat;
@Private @Private
@Unstable @Unstable
public class ReportNewAggregatorsInfoResponsePBImpl extends public class ReportNewCollectorInfoResponsePBImpl extends
ReportNewAggregatorsInfoResponse { ReportNewCollectorInfoResponse {
ReportNewCollectorInfoResponseProto proto =
ReportNewCollectorInfoResponseProto.getDefaultInstance();
ReportNewCollectorInfoResponseProto.Builder builder = null;
ReportNewAggregatorsInfoResponseProto proto =
ReportNewAggregatorsInfoResponseProto.getDefaultInstance();
ReportNewAggregatorsInfoResponseProto.Builder builder = null;
boolean viaProto = false; boolean viaProto = false;
public ReportNewAggregatorsInfoResponsePBImpl() { public ReportNewCollectorInfoResponsePBImpl() {
builder = ReportNewAggregatorsInfoResponseProto.newBuilder(); builder = ReportNewCollectorInfoResponseProto.newBuilder();
} }
public ReportNewAggregatorsInfoResponsePBImpl(ReportNewAggregatorsInfoResponseProto proto) { public ReportNewCollectorInfoResponsePBImpl(ReportNewCollectorInfoResponseProto proto) {
this.proto = proto; this.proto = proto;
viaProto = true; viaProto = true;
} }
public ReportNewAggregatorsInfoResponseProto getProto() { public ReportNewCollectorInfoResponseProto getProto() {
proto = viaProto ? proto : builder.build(); proto = viaProto ? proto : builder.build();
viaProto = true; viaProto = true;
return proto; return proto;

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppAggregatorsMap.java
View File

@ -1,33 +0,0 @@
package org.apache.hadoop.yarn.server.api.records;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.Records;
@Private
public abstract class AppAggregatorsMap {
public static AppAggregatorsMap newInstance(
ApplicationId id, String aggregatorAddr) {
AppAggregatorsMap appAggregatorMap =
Records.newRecord(AppAggregatorsMap.class);
appAggregatorMap.setApplicationId(id);
appAggregatorMap.setAggregatorAddr(aggregatorAddr);
return appAggregatorMap;
}
public abstract ApplicationId getApplicationId();
public abstract void setApplicationId(
ApplicationId id);
public abstract String getAggregatorAddr();
public abstract void setAggregatorAddr(
String addr);
}

46
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorsMap.java Normal file
View File

@ -0,0 +1,46 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.api.records;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.Records;
@Private
public abstract class AppCollectorsMap {
public static AppCollectorsMap newInstance(
ApplicationId id, String collectorAddr) {
AppCollectorsMap appCollectorsMap =
Records.newRecord(AppCollectorsMap.class);
appCollectorsMap.setApplicationId(id);
appCollectorsMap.setCollectorAddr(collectorAddr);
return appCollectorsMap;
}
public abstract ApplicationId getApplicationId();
public abstract void setApplicationId(ApplicationId id);
public abstract String getCollectorAddr();
public abstract void setCollectorAddr(String addr);
}

76
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppAggregatorsMapPBImpl.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorsMapPBImpl.java
View File

@ -21,37 +21,37 @@
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap; import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProtoOrBuilder;
import com.google.protobuf.TextFormat; import com.google.protobuf.TextFormat;
@Private @Private
@Unstable @Unstable
public class AppAggregatorsMapPBImpl extends AppAggregatorsMap { public class AppCollectorsMapPBImpl extends AppCollectorsMap {
AppAggregatorsMapProto proto = AppCollectorsMapProto proto =
AppAggregatorsMapProto.getDefaultInstance(); AppCollectorsMapProto.getDefaultInstance();
AppAggregatorsMapProto.Builder builder = null; AppCollectorsMapProto.Builder builder = null;
boolean viaProto = false; boolean viaProto = false;
private ApplicationId appId = null; private ApplicationId appId = null;
private String aggregatorAddr = null; private String collectorAddr = null;
public AppAggregatorsMapPBImpl() { public AppCollectorsMapPBImpl() {
builder = AppAggregatorsMapProto.newBuilder(); builder = AppCollectorsMapProto.newBuilder();
} }
public AppAggregatorsMapPBImpl(AppAggregatorsMapProto proto) { public AppCollectorsMapPBImpl(AppCollectorsMapProto proto) {
this.proto = proto; this.proto = proto;
viaProto = true; viaProto = true;
} }
public AppAggregatorsMapProto getProto() { public AppCollectorsMapProto getProto() {
mergeLocalToProto(); mergeLocalToProto();
proto = viaProto ? proto : builder.build(); proto = viaProto ? proto : builder.build();
viaProto = true; viaProto = true;
@ -77,24 +77,24 @@ public boolean equals(Object other) {
public String toString() { public String toString() {
return TextFormat.shortDebugString(getProto()); return TextFormat.shortDebugString(getProto());
} }
@Override @Override
public ApplicationId getApplicationId() { public ApplicationId getApplicationId() {
AppAggregatorsMapProtoOrBuilder p = viaProto ? proto : builder; AppCollectorsMapProtoOrBuilder p = viaProto ? proto : builder;
if (this.appId == null && p.hasAppId()) { if (this.appId == null && p.hasAppId()) {
this.appId = convertFromProtoFormat(p.getAppId()); this.appId = convertFromProtoFormat(p.getAppId());
} }
return this.appId; return this.appId;
} }
@Override @Override
public String getAggregatorAddr() { public String getCollectorAddr() {
AppAggregatorsMapProtoOrBuilder p = viaProto ? proto : builder; AppCollectorsMapProtoOrBuilder p = viaProto ? proto : builder;
if (this.aggregatorAddr == null if (this.collectorAddr == null
&& p.hasAppAggregatorAddr()) { && p.hasAppCollectorAddr()) {
this.aggregatorAddr = p.getAppAggregatorAddr(); this.collectorAddr = p.getAppCollectorAddr();
} }
return this.aggregatorAddr; return this.collectorAddr;
} }
@Override @Override
@ -105,31 +105,31 @@ public void setApplicationId(ApplicationId appId) {
} }
this.appId = appId; this.appId = appId;
} }
@Override @Override
public void setAggregatorAddr(String aggregatorAddr) { public void setCollectorAddr(String collectorAddr) {
maybeInitBuilder(); maybeInitBuilder();
if (aggregatorAddr == null) { if (collectorAddr == null) {
builder.clearAppAggregatorAddr(); builder.clearAppCollectorAddr();
} }
this.aggregatorAddr = aggregatorAddr; this.collectorAddr = collectorAddr;
} }
private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
return new ApplicationIdPBImpl(p); return new ApplicationIdPBImpl(p);
} }
private ApplicationIdProto convertToProtoFormat(ApplicationId t) { private ApplicationIdProto convertToProtoFormat(ApplicationId t) {
return ((ApplicationIdPBImpl) t).getProto(); return ((ApplicationIdPBImpl) t).getProto();
} }
private void maybeInitBuilder() { private void maybeInitBuilder() {
if (viaProto || builder == null) { if (viaProto || builder == null) {
builder = AppAggregatorsMapProto.newBuilder(proto); builder = AppCollectorsMapProto.newBuilder(proto);
} }
viaProto = false; viaProto = false;
} }
private void mergeLocalToProto() { private void mergeLocalToProto() {
if (viaProto) { if (viaProto) {
maybeInitBuilder(); maybeInitBuilder();
@ -138,13 +138,13 @@ private void mergeLocalToProto() {
proto = builder.build(); proto = builder.build();
viaProto = true; viaProto = true;
} }
private void mergeLocalToBuilder() { private void mergeLocalToBuilder() {
if (this.appId != null) { if (this.appId != null) {
builder.setAppId(convertToProtoFormat(this.appId)); builder.setAppId(convertToProtoFormat(this.appId));
} }
if (this.aggregatorAddr != null) { if (this.collectorAddr != null) {
builder.setAppAggregatorAddr(this.aggregatorAddr); builder.setAppCollectorAddr(this.collectorAddr);
} }
} }

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/aggregatornodemanager_protocol.proto → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/collectornodemanager_protocol.proto
View File

@ -17,13 +17,13 @@
*/ */
option java_package = "org.apache.hadoop.yarn.proto"; option java_package = "org.apache.hadoop.yarn.proto";
option java_outer_classname = "AggregatorNodemanagerProtocol"; option java_outer_classname = "CollectorNodemanagerProtocol";
option java_generic_services = true; option java_generic_services = true;
option java_generate_equals_and_hash = true; option java_generate_equals_and_hash = true;
package hadoop.yarn; package hadoop.yarn;
import "yarn_server_common_service_protos.proto"; import "yarn_server_common_service_protos.proto";
service AggregatorNodemanagerProtocolService { service CollectorNodemanagerProtocolService {
rpc reportNewAggregatorInfo (ReportNewAggregatorsInfoRequestProto) returns (ReportNewAggregatorsInfoResponseProto); rpc reportNewCollectorInfo (ReportNewCollectorInfoRequestProto) returns (ReportNewCollectorInfoResponseProto);
} }

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
View File

@ -84,7 +84,7 @@ message NodeHeartbeatRequestProto {
optional MasterKeyProto last_known_nm_token_master_key = 3; optional MasterKeyProto last_known_nm_token_master_key = 3;
optional NodeLabelsProto nodeLabels = 4; optional NodeLabelsProto nodeLabels = 4;
repeated LogAggregationReportProto log_aggregation_reports_for_apps = 5; repeated LogAggregationReportProto log_aggregation_reports_for_apps = 5;
repeated AppAggregatorsMapProto registered_aggregators = 6; repeated AppCollectorsMapProto registered_collectors = 6;
} }
message LogAggregationReportProto { message LogAggregationReportProto {
@ -109,7 +109,7 @@ message NodeHeartbeatResponseProto {
repeated SignalContainerRequestProto containers_to_signal = 13; repeated SignalContainerRequestProto containers_to_signal = 13;
optional ResourceProto resource = 14; optional ResourceProto resource = 14;
optional ContainerQueuingLimitProto container_queuing_limit = 15; optional ContainerQueuingLimitProto container_queuing_limit = 15;
repeated AppAggregatorsMapProto app_aggregators_map = 16; repeated AppCollectorsMapProto app_collectors_map = 16;
} }
message ContainerQueuingLimitProto { message ContainerQueuingLimitProto {
@ -123,21 +123,21 @@ message SystemCredentialsForAppsProto {
} }
//////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////
////// From aggregator_nodemanager_protocol //////////////////////////// ////// From collector_nodemanager_protocol ////////////////////////////
//////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////
message AppAggregatorsMapProto { message AppCollectorsMapProto {
optional ApplicationIdProto appId = 1; optional ApplicationIdProto appId = 1;
optional string appAggregatorAddr = 2; optional string appCollectorAddr = 2;
} }
////////////////////////////////////////////////////// //////////////////////////////////////////////////////
/////// aggregator_nodemanager_protocol ////////////// /////// collector_nodemanager_protocol //////////////
////////////////////////////////////////////////////// //////////////////////////////////////////////////////
message ReportNewAggregatorsInfoRequestProto { message ReportNewCollectorInfoRequestProto {
repeated AppAggregatorsMapProto app_aggregators = 1; repeated AppCollectorsMapProto app_collectors = 1;
} }
message ReportNewAggregatorsInfoResponseProto { message ReportNewCollectorInfoResponseProto {
} }

116
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
View File

@ -63,10 +63,10 @@
import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol; import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap; import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap;
import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@ -76,15 +76,15 @@ public class TestRPC {
private static final String EXCEPTION_MSG = "test error"; private static final String EXCEPTION_MSG = "test error";
private static final String EXCEPTION_CAUSE = "exception cause"; private static final String EXCEPTION_CAUSE = "exception cause";
private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
public static final String ILLEGAL_NUMBER_MESSAGE = public static final String ILLEGAL_NUMBER_MESSAGE =
"aggregators' number in ReportNewAggregatorsInfoRequest is not ONE."; "collectors' number in ReportNewCollectorInfoRequest is not ONE.";
public static final String DEFAULT_AGGREGATOR_ADDR = "localhost:0"; public static final String DEFAULT_COLLECTOR_ADDR = "localhost:0";
public static final ApplicationId DEFAULT_APP_ID = public static final ApplicationId DEFAULT_APP_ID =
ApplicationId.newInstance(0, 0); ApplicationId.newInstance(0, 0);
@Test @Test
public void testUnknownCall() { public void testUnknownCall() {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
@ -116,17 +116,17 @@ public void testUnknownCall() {
server.stop(); server.stop();
} }
} }
@Test @Test
public void testRPCOnAggregatorNodeManagerProtocol() throws IOException { public void testRPCOnCollectorNodeManagerProtocol() throws IOException {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class
.getName()); .getName());
YarnRPC rpc = YarnRPC.create(conf); YarnRPC rpc = YarnRPC.create(conf);
String bindAddr = "localhost:0"; String bindAddr = "localhost:0";
InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr); InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
Server server = rpc.getServer(AggregatorNodemanagerProtocol.class, Server server = rpc.getServer(CollectorNodemanagerProtocol.class,
new DummyNMAggregatorService(), addr, conf, null, 1); new DummyNMCollectorService(), addr, conf, null, 1);
server.start(); server.start();
// Test unrelated protocol wouldn't get response // Test unrelated protocol wouldn't get response
@ -145,31 +145,31 @@ public void testRPCOnAggregatorNodeManagerProtocol() throws IOException {
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
// Test AggregatorNodemanagerProtocol get proper response // Test CollectorNodemanagerProtocol get proper response
AggregatorNodemanagerProtocol proxy = (AggregatorNodemanagerProtocol)rpc.getProxy( CollectorNodemanagerProtocol proxy = (CollectorNodemanagerProtocol)rpc.getProxy(
AggregatorNodemanagerProtocol.class, NetUtils.getConnectAddress(server), conf); CollectorNodemanagerProtocol.class, NetUtils.getConnectAddress(server), conf);
// Verify request with DEFAULT_APP_ID and DEFAULT_AGGREGATOR_ADDR get // Verify request with DEFAULT_APP_ID and DEFAULT_COLLECTOR_ADDR get
// normally response. // normally response.
try { try {
ReportNewAggregatorsInfoRequest request = ReportNewCollectorInfoRequest request =
ReportNewAggregatorsInfoRequest.newInstance( ReportNewCollectorInfoRequest.newInstance(
DEFAULT_APP_ID, DEFAULT_AGGREGATOR_ADDR); DEFAULT_APP_ID, DEFAULT_COLLECTOR_ADDR);
proxy.reportNewAggregatorInfo(request); proxy.reportNewCollectorInfo(request);
} catch (YarnException e) { } catch (YarnException e) {
Assert.fail("RPC call failured is not expected here."); Assert.fail("RPC call failured is not expected here.");
} }
// Verify empty request get YarnException back (by design in // Verify empty request get YarnException back (by design in
// DummyNMAggregatorService) // DummyNMCollectorService)
try { try {
proxy.reportNewAggregatorInfo(Records proxy.reportNewCollectorInfo(Records
.newRecord(ReportNewAggregatorsInfoRequest.class)); .newRecord(ReportNewCollectorInfoRequest.class));
Assert.fail("Excepted RPC call to fail with YarnException."); Assert.fail("Excepted RPC call to fail with YarnException.");
} catch (YarnException e) { } catch (YarnException e) {
Assert.assertTrue(e.getMessage().contains(ILLEGAL_NUMBER_MESSAGE)); Assert.assertTrue(e.getMessage().contains(ILLEGAL_NUMBER_MESSAGE));
} }
server.stop(); server.stop();
} }
@ -177,21 +177,21 @@ public void testRPCOnAggregatorNodeManagerProtocol() throws IOException {
public void testHadoopProtoRPC() throws Exception { public void testHadoopProtoRPC() throws Exception {
test(HadoopYarnProtoRPC.class.getName()); test(HadoopYarnProtoRPC.class.getName());
} }
private void test(String rpcClass) throws Exception { private void test(String rpcClass) throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass); conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
YarnRPC rpc = YarnRPC.create(conf); YarnRPC rpc = YarnRPC.create(conf);
String bindAddr = "localhost:0"; String bindAddr = "localhost:0";
InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr); InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
Server server = rpc.getServer(ContainerManagementProtocol.class, Server server = rpc.getServer(ContainerManagementProtocol.class,
new DummyContainerManager(), addr, conf, null, 1); new DummyContainerManager(), addr, conf, null, 1);
server.start(); server.start();
RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class); RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
ContainerManagementProtocol proxy = (ContainerManagementProtocol) ContainerManagementProtocol proxy = (ContainerManagementProtocol)
rpc.getProxy(ContainerManagementProtocol.class, rpc.getProxy(ContainerManagementProtocol.class,
NetUtils.getConnectAddress(server), conf); NetUtils.getConnectAddress(server), conf);
ContainerLaunchContext containerLaunchContext = ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class); recordFactory.newRecordInstance(ContainerLaunchContext.class);
ApplicationId applicationId = ApplicationId.newInstance(0, 0); ApplicationId applicationId = ApplicationId.newInstance(0, 0);
@ -255,7 +255,7 @@ public class DummyContainerManager implements ContainerManagementProtocol {
public GetContainerStatusesResponse getContainerStatuses( public GetContainerStatusesResponse getContainerStatuses(
GetContainerStatusesRequest request) GetContainerStatusesRequest request)
throws YarnException { throws YarnException {
GetContainerStatusesResponse response = GetContainerStatusesResponse response =
recordFactory.newRecordInstance(GetContainerStatusesResponse.class); recordFactory.newRecordInstance(GetContainerStatusesResponse.class);
response.setContainerStatuses(statuses); response.setContainerStatuses(statuses);
return response; return response;
@ -287,9 +287,9 @@ public StartContainersResponse startContainers(
} }
@Override @Override
public StopContainersResponse stopContainers(StopContainersRequest request) public StopContainersResponse stopContainers(StopContainersRequest request)
throws YarnException { throws YarnException {
Exception e = new Exception(EXCEPTION_MSG, Exception e = new Exception(EXCEPTION_MSG,
new Exception(EXCEPTION_CAUSE)); new Exception(EXCEPTION_CAUSE));
throw new YarnException(e); throw new YarnException(e);
} }
@ -332,32 +332,32 @@ public static Token newContainerToken(NodeId nodeId, byte[] password,
.buildTokenService(addr).toString()); .buildTokenService(addr).toString());
return containerToken; return containerToken;
} }
// A dummy implementation for AggregatorNodemanagerProtocol for test purpose, // A dummy implementation for CollectorNodemanagerProtocol for test purpose,
// it only can accept one appID, aggregatorAddr pair or throw exceptions // it only can accept one appID, collectorAddr pair or throw exceptions
public class DummyNMAggregatorService public class DummyNMCollectorService
implements AggregatorNodemanagerProtocol { implements CollectorNodemanagerProtocol {
@Override @Override
public ReportNewAggregatorsInfoResponse reportNewAggregatorInfo( public ReportNewCollectorInfoResponse reportNewCollectorInfo(
ReportNewAggregatorsInfoRequest request) ReportNewCollectorInfoRequest request)
throws YarnException, IOException { throws YarnException, IOException {
List<AppAggregatorsMap> appAggregators = request.getAppAggregatorsList(); List<AppCollectorsMap> appCollectors = request.getAppCollectorsList();
if (appAggregators.size() == 1) { if (appCollectors.size() == 1) {
// check default appID and aggregatorAddr // check default appID and collectorAddr
AppAggregatorsMap appAggregator = appAggregators.get(0); AppCollectorsMap appCollector = appCollectors.get(0);
Assert.assertEquals(appAggregator.getApplicationId(), Assert.assertEquals(appCollector.getApplicationId(),
DEFAULT_APP_ID); DEFAULT_APP_ID);
Assert.assertEquals(appAggregator.getAggregatorAddr(), Assert.assertEquals(appCollector.getCollectorAddr(),
DEFAULT_AGGREGATOR_ADDR); DEFAULT_COLLECTOR_ADDR);
} else { } else {
throw new YarnException(ILLEGAL_NUMBER_MESSAGE); throw new YarnException(ILLEGAL_NUMBER_MESSAGE);
} }
ReportNewAggregatorsInfoResponse response = ReportNewCollectorInfoResponse response =
recordFactory.newRecordInstance(ReportNewAggregatorsInfoResponse.class); recordFactory.newRecordInstance(ReportNewCollectorInfoResponse.class);
return response; return response;
} }
} }
} }

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
View File

@ -109,14 +109,14 @@ public void testNodeHeartbeatRequestPBImpl() {
original.setLastKnownNMTokenMasterKey(getMasterKey()); original.setLastKnownNMTokenMasterKey(getMasterKey());
original.setNodeStatus(getNodeStatus()); original.setNodeStatus(getNodeStatus());
original.setNodeLabels(getValidNodeLabels()); original.setNodeLabels(getValidNodeLabels());
Map<ApplicationId, String> aggregators = getAggregators(); Map<ApplicationId, String> collectors = getCollectors();
original.setRegisteredAggregators(aggregators); original.setRegisteredCollectors(collectors);
NodeHeartbeatRequestPBImpl copy = new NodeHeartbeatRequestPBImpl( NodeHeartbeatRequestPBImpl copy = new NodeHeartbeatRequestPBImpl(
original.getProto()); original.getProto());
assertEquals(1, copy.getLastKnownContainerTokenMasterKey().getKeyId()); assertEquals(1, copy.getLastKnownContainerTokenMasterKey().getKeyId());
assertEquals(1, copy.getLastKnownNMTokenMasterKey().getKeyId()); assertEquals(1, copy.getLastKnownNMTokenMasterKey().getKeyId());
assertEquals("localhost", copy.getNodeStatus().getNodeId().getHost()); assertEquals("localhost", copy.getNodeStatus().getNodeId().getHost());
assertEquals(aggregators, copy.getRegisteredAggregators()); assertEquals(collectors, copy.getRegisteredCollectors());
// check labels are coming with valid values // check labels are coming with valid values
Assert.assertTrue(original.getNodeLabels() Assert.assertTrue(original.getNodeLabels()
.containsAll(copy.getNodeLabels())); .containsAll(copy.getNodeLabels()));
@ -153,8 +153,8 @@ public void testNodeHeartbeatResponsePBImpl() {
original.setNextHeartBeatInterval(1000); original.setNextHeartBeatInterval(1000);
original.setNodeAction(NodeAction.NORMAL); original.setNodeAction(NodeAction.NORMAL);
original.setResponseId(100); original.setResponseId(100);
Map<ApplicationId, String> aggregators = getAggregators(); Map<ApplicationId, String> collectors = getCollectors();
original.setAppAggregatorsMap(aggregators); original.setAppCollectorsMap(collectors);
NodeHeartbeatResponsePBImpl copy = new NodeHeartbeatResponsePBImpl( NodeHeartbeatResponsePBImpl copy = new NodeHeartbeatResponsePBImpl(
original.getProto()); original.getProto());
@ -164,7 +164,7 @@ public void testNodeHeartbeatResponsePBImpl() {
assertEquals(1, copy.getContainerTokenMasterKey().getKeyId()); assertEquals(1, copy.getContainerTokenMasterKey().getKeyId());
assertEquals(1, copy.getNMTokenMasterKey().getKeyId()); assertEquals(1, copy.getNMTokenMasterKey().getKeyId());
assertEquals("testDiagnosticMessage", copy.getDiagnosticsMessage()); assertEquals("testDiagnosticMessage", copy.getDiagnosticsMessage());
assertEquals(aggregators, copy.getAppAggregatorsMap()); assertEquals(collectors, copy.getAppCollectorsMap());
assertEquals(false, copy.getAreNodeLabelsAcceptedByRM()); assertEquals(false, copy.getAreNodeLabelsAcceptedByRM());
} }
@ -344,15 +344,15 @@ private HashSet<NodeLabel> getValidNodeLabels() {
return nodeLabels; return nodeLabels;
} }
private Map<ApplicationId, String> getAggregators() { private Map<ApplicationId, String> getCollectors() {
ApplicationId appID = ApplicationId.newInstance(1L, 1); ApplicationId appID = ApplicationId.newInstance(1L, 1);
String aggregatorAddr = "localhost:0"; String collectorAddr = "localhost:0";
Map<ApplicationId, String> aggregatorMap = Map<ApplicationId, String> collectorMap =
new HashMap<ApplicationId, String>(); new HashMap<ApplicationId, String>();
aggregatorMap.put(appID, aggregatorAddr); collectorMap.put(appID, collectorAddr);
return aggregatorMap; return collectorMap;
} }
private ContainerStatus getContainerStatus(int applicationId, private ContainerStatus getContainerStatus(int applicationId,
int containerID, int appAttemptId) { int containerID, int appAttemptId) {
ContainerStatus status = recordFactory ContainerStatus status = recordFactory

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
View File

@ -70,19 +70,19 @@ interface QueuingContext {
ConcurrentMap<ApplicationId, Application> getApplications(); ConcurrentMap<ApplicationId, Application> getApplications();
Map<ApplicationId, Credentials> getSystemCredentialsForApps(); Map<ApplicationId, Credentials> getSystemCredentialsForApps();
/** /**
* Get the registered aggregators that located on this NM. * Get the registered collectors that located on this NM.
* @return registered * @return registered
*/ */
Map<ApplicationId, String> getRegisteredAggregators(); Map<ApplicationId, String> getRegisteredCollectors();
/** /**
* Return the known aggregators which get from RM for all active applications * Return the known collectors which get from RM for all active applications
* running on this NM. * running on this NM.
* @return known aggregators. * @return known collectors.
*/ */
Map<ApplicationId, String> getKnownAggregators(); Map<ApplicationId, String> getKnownCollectors();
ConcurrentMap<ContainerId, Container> getContainers(); ConcurrentMap<ContainerId, Container> getContainers();

60
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
View File

@ -60,7 +60,7 @@
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport; import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.aggregatormanager.NMAggregatorService; import org.apache.hadoop.yarn.server.nodemanager.collectormanager.NMCollectorService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@ -99,7 +99,7 @@ public class NodeManager extends CompositeService
private Context context; private Context context;
private AsyncDispatcher dispatcher; private AsyncDispatcher dispatcher;
private ContainerManagerImpl containerManager; private ContainerManagerImpl containerManager;
private NMAggregatorService nmAggregatorService; private NMCollectorService nmCollectorService;
private NodeStatusUpdater nodeStatusUpdater; private NodeStatusUpdater nodeStatusUpdater;
private NodeResourceMonitor nodeResourceMonitor; private NodeResourceMonitor nodeResourceMonitor;
private static CompositeServiceShutdownHook nodeManagerShutdownHook; private static CompositeServiceShutdownHook nodeManagerShutdownHook;
@ -184,9 +184,9 @@ protected ContainerManagerImpl createContainerManager(Context context,
metrics, dirsHandler); metrics, dirsHandler);
} }
} }
protected NMAggregatorService createNMAggregatorService(Context context) { protected NMCollectorService createNMCollectorService(Context context) {
return new NMAggregatorService(context); return new NMCollectorService(context);
} }
protected WebServer createWebServer(Context nmContext, protected WebServer createWebServer(Context nmContext,
@ -379,9 +379,9 @@ protected void serviceInit(Configuration conf) throws Exception {
metrics.getJvmMetrics().setPauseMonitor(pauseMonitor); metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
DefaultMetricsSystem.initialize("NodeManager"); DefaultMetricsSystem.initialize("NodeManager");
this.nmAggregatorService = createNMAggregatorService(context); this.nmCollectorService = createNMCollectorService(context);
addService(nmAggregatorService); addService(nmCollectorService);
// StatusUpdater should be added last so that it get started last // StatusUpdater should be added last so that it get started last
// so that we make sure everything is up before registering with RM. // so that we make sure everything is up before registering with RM.
@ -474,11 +474,11 @@ public static class NMContext implements Context {
protected final ConcurrentMap<ContainerId, Container> containers = protected final ConcurrentMap<ContainerId, Container> containers =
new ConcurrentSkipListMap<ContainerId, Container>(); new ConcurrentSkipListMap<ContainerId, Container>();
protected Map<ApplicationId, String> registeredAggregators = protected Map<ApplicationId, String> registeredCollectors =
new ConcurrentHashMap<ApplicationId, String>(); new ConcurrentHashMap<ApplicationId, String>();
protected Map<ApplicationId, String> knownAggregators = protected Map<ApplicationId, String> knownCollectors =
new ConcurrentHashMap<ApplicationId, String>(); new ConcurrentHashMap<ApplicationId, String>();
protected final ConcurrentMap<ContainerId, protected final ConcurrentMap<ContainerId,
@ -662,26 +662,26 @@ public OpportunisticContainerAllocator getContainerAllocator() {
} }
@Override @Override
public Map<ApplicationId, String> getRegisteredAggregators() { public Map<ApplicationId, String> getRegisteredCollectors() {
return this.registeredAggregators; return this.registeredCollectors;
} }
public void addRegisteredAggregators( public void addRegisteredCollectors(
Map<ApplicationId, String> newRegisteredAggregators) { Map<ApplicationId, String> newRegisteredCollectors) {
this.registeredAggregators.putAll(newRegisteredAggregators); this.registeredCollectors.putAll(newRegisteredCollectors);
// Update to knownAggregators as well so it can immediately be consumed by // Update to knownCollectors as well so it can immediately be consumed by
// this NM's TimelineClient. // this NM's TimelineClient.
this.knownAggregators.putAll(newRegisteredAggregators); this.knownCollectors.putAll(newRegisteredCollectors);
}
@Override
public Map<ApplicationId, String> getKnownAggregators() {
return this.knownAggregators;
} }
public void addKnownAggregators( @Override
Map<ApplicationId, String> knownAggregators) { public Map<ApplicationId, String> getKnownCollectors() {
this.knownAggregators.putAll(knownAggregators); return this.knownCollectors;
}
public void addKnownCollectors(
Map<ApplicationId, String> knownCollectors) {
this.knownCollectors.putAll(knownCollectors);
} }
} }
@ -781,10 +781,10 @@ Dispatcher getNMDispatcher(){
public Context getNMContext() { public Context getNMContext() {
return this.context; return this.context;
} }
// For testing // For testing
NMAggregatorService getNMAggregatorService() { NMCollectorService getNMCollectorService() {
return this.nmAggregatorService; return this.nmCollectorService;
} }
public static void main(String[] args) throws IOException { public static void main(String[] args) throws IOException {

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
View File

@ -814,7 +814,7 @@ public void run() {
NodeStatusUpdaterImpl.this.context NodeStatusUpdaterImpl.this.context
.getNMTokenSecretManager().getCurrentKey(), .getNMTokenSecretManager().getCurrentKey(),
nodeLabelsForHeartbeat, nodeLabelsForHeartbeat,
NodeStatusUpdaterImpl.this.context.getRegisteredAggregators()); NodeStatusUpdaterImpl.this.context.getRegisteredCollectors());
if (logAggregationEnabled) { if (logAggregationEnabled) {
// pull log aggregation status for application running in this NM // pull log aggregation status for application running in this NM
@ -907,8 +907,9 @@ public void run() {
} }
} }
Map<ApplicationId, String> knownAggregators = response.getAppAggregatorsMap(); Map<ApplicationId, String> knownCollectors =
((NodeManager.NMContext)context).addKnownAggregators(knownAggregators); response.getAppCollectorsMap();
((NodeManager.NMContext)context).addKnownCollectors(knownCollectors);
} catch (ConnectException e) { } catch (ConnectException e) {
//catch and throw the exception if tried MAX wait time to connect RM //catch and throw the exception if tried MAX wait time to connect RM

73
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/aggregatormanager/NMAggregatorService.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.yarn.server.nodemanager.aggregatormanager; package org.apache.hadoop.yarn.server.nodemanager.collectormanager;
import java.io.IOException; import java.io.IOException;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
@ -25,46 +25,43 @@
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.Server; import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol; import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse; import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap;
import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
public class NMAggregatorService extends CompositeService implements public class NMCollectorService extends CompositeService implements
AggregatorNodemanagerProtocol { CollectorNodemanagerProtocol {
private static final Log LOG = LogFactory.getLog(NMAggregatorService.class); private static final Log LOG = LogFactory.getLog(NMCollectorService.class);
final Context context; final Context context;
private Server server; private Server server;
public NMAggregatorService(Context context) { public NMCollectorService(Context context) {
super(NMAggregatorService.class.getName()); super(NMCollectorService.class.getName());
this.context = context; this.context = context;
} }
@Override @Override
protected void serviceStart() throws Exception { protected void serviceStart() throws Exception {
Configuration conf = getConfig(); Configuration conf = getConfig();
InetSocketAddress aggregatorServerAddress = conf.getSocketAddr( InetSocketAddress collectorServerAddress = conf.getSocketAddr(
YarnConfiguration.NM_BIND_HOST, YarnConfiguration.NM_BIND_HOST,
YarnConfiguration.NM_AGGREGATOR_SERVICE_ADDRESS, YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_ADDRESS, YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS,
YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_PORT); YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT);
Configuration serverConf = new Configuration(conf); Configuration serverConf = new Configuration(conf);
@ -72,42 +69,42 @@ protected void serviceStart() throws Exception {
YarnRPC rpc = YarnRPC.create(conf); YarnRPC rpc = YarnRPC.create(conf);
server = server =
rpc.getServer(AggregatorNodemanagerProtocol.class, this, rpc.getServer(CollectorNodemanagerProtocol.class, this,
aggregatorServerAddress, serverConf, collectorServerAddress, serverConf,
this.context.getNMTokenSecretManager(), this.context.getNMTokenSecretManager(),
conf.getInt(YarnConfiguration.NM_AGGREGATOR_SERVICE_THREAD_COUNT, conf.getInt(YarnConfiguration.NM_COLLECTOR_SERVICE_THREAD_COUNT,
YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_THREAD_COUNT)); YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT));
server.start(); server.start();
// start remaining services // start remaining services
super.serviceStart(); super.serviceStart();
LOG.info("NMAggregatorService started at " + aggregatorServerAddress); LOG.info("NMCollectorService started at " + collectorServerAddress);
} }
@Override @Override
public void serviceStop() throws Exception { public void serviceStop() throws Exception {
if (server != null) { if (server != null) {
server.stop(); server.stop();
} }
// TODO may cleanup app aggregators running on this NM in future. // TODO may cleanup app collectors running on this NM in future.
super.serviceStop(); super.serviceStop();
} }
@Override @Override
public ReportNewAggregatorsInfoResponse reportNewAggregatorInfo( public ReportNewCollectorInfoResponse reportNewCollectorInfo(
ReportNewAggregatorsInfoRequest request) throws IOException { ReportNewCollectorInfoRequest request) throws IOException {
List<AppAggregatorsMap> newAggregatorsList = request.getAppAggregatorsList(); List<AppCollectorsMap> newCollectorsList = request.getAppCollectorsList();
if (newAggregatorsList != null && !newAggregatorsList.isEmpty()) { if (newCollectorsList != null && !newCollectorsList.isEmpty()) {
Map<ApplicationId, String> newAggregatorsMap = Map<ApplicationId, String> newCollectorsMap =
new HashMap<ApplicationId, String>(); new HashMap<ApplicationId, String>();
for (AppAggregatorsMap aggregator : newAggregatorsList) { for (AppCollectorsMap collector : newCollectorsList) {
newAggregatorsMap.put(aggregator.getApplicationId(), aggregator.getAggregatorAddr()); newCollectorsMap.put(collector.getApplicationId(), collector.getCollectorAddr());
} }
((NodeManager.NMContext)context).addRegisteredAggregators(newAggregatorsMap); ((NodeManager.NMContext)context).addRegisteredCollectors(newCollectorsMap);
} }
return ReportNewAggregatorsInfoResponse.newInstance(); return ReportNewCollectorInfoResponse.newInstance();
} }
} }

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
View File

@ -496,10 +496,11 @@ public void transition(ApplicationImpl app, ApplicationEvent event) {
new LogHandlerAppFinishedEvent(app.appId)); new LogHandlerAppFinishedEvent(app.appId));
app.context.getNMTokenSecretManager().appFinished(app.getAppId()); app.context.getNMTokenSecretManager().appFinished(app.getAppId());
// Remove aggregator info for finished apps. // Remove collectors info for finished apps.
// TODO check we remove related aggregators info in failure cases (YARN-3038) // TODO check we remove related collectors info in failure cases
app.context.getRegisteredAggregators().remove(app.getAppId()); // (YARN-3038)
app.context.getKnownAggregators().remove(app.getAppId()); app.context.getRegisteredCollectors().remove(app.getAppId());
app.context.getKnownCollectors().remove(app.getAppId());
} }
} }

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
View File

@ -618,12 +618,12 @@ public Map<ApplicationId, Credentials> getSystemCredentialsForApps() {
} }
@Override @Override
public Map<ApplicationId, String> getRegisteredAggregators() { public Map<ApplicationId, String> getRegisteredCollectors() {
return null; return null;
} }
@Override @Override
public Map<ApplicationId, String> getKnownAggregators() { public Map<ApplicationId, String> getKnownCollectors() {
return null; return null;
} }

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
View File

@ -311,8 +311,8 @@ public FinishApplicationMasterResponse finishApplicationMaster(
RMApp rmApp = RMApp rmApp =
rmContext.getRMApps().get(applicationAttemptId.getApplicationId()); rmContext.getRMApps().get(applicationAttemptId.getApplicationId());
// Remove aggregator address when app get finished. // Remove collector address when app get finished.
rmApp.removeAggregatorAddr(); rmApp.removeCollectorAddr();
// checking whether the app exits in RMStateStore at first not to throw // checking whether the app exits in RMStateStore at first not to throw
// ApplicationDoesNotExistInCacheException before and after // ApplicationDoesNotExistInCacheException before and after
// RM work-preserving restart. // RM work-preserving restart.
@ -573,10 +573,10 @@ public AllocateResponse allocate(AllocateRequest request)
allocateResponse.setDecreasedContainers(allocation.getDecreasedContainers()); allocateResponse.setDecreasedContainers(allocation.getDecreasedContainers());
allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes()); allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes());
// add aggregator address for this application // add collector address for this application
allocateResponse.setAggregatorAddr( allocateResponse.setCollectorAddr(
this.rmContext.getRMApps().get(applicationId).getAggregatorAddr()); this.rmContext.getRMApps().get(applicationId).getCollectorAddr());
// add preemption to the allocateResponse message (if any) // add preemption to the allocateResponse message (if any)
allocateResponse allocateResponse

72
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
View File

@ -70,7 +70,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils;
import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppAggregatorUpdateEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppCollectorUpdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@ -516,11 +516,11 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
return YarnServerBuilderUtils.newNodeHeartbeatResponse(NodeAction.RESYNC, return YarnServerBuilderUtils.newNodeHeartbeatResponse(NodeAction.RESYNC,
message); message);
} }
// Check & update aggregators info from request. // Check & update collectors info from request.
// TODO make sure it won't have race condition issue for AM failed over case // TODO make sure it won't have race condition issue for AM failed over case
// that the older registration could possible override the newer one. // that the older registration could possible override the newer one.
updateAppAggregatorsMap(request); updateAppCollectorsMap(request);
// Heartbeat response // Heartbeat response
NodeHeartbeatResponse nodeHeartBeatResponse = YarnServerBuilderUtils NodeHeartbeatResponse nodeHeartBeatResponse = YarnServerBuilderUtils
@ -538,13 +538,14 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
if (!systemCredentials.isEmpty()) { if (!systemCredentials.isEmpty()) {
nodeHeartBeatResponse.setSystemCredentialsForApps(systemCredentials); nodeHeartBeatResponse.setSystemCredentialsForApps(systemCredentials);
} }
// Return aggregators' map that NM needs to know // Return collectors' map that NM needs to know
// TODO we should optimize this to only include aggreator info that NM // TODO we should optimize this to only include collector info that NM
// doesn't know yet. // doesn't know yet.
List<ApplicationId> keepAliveApps = remoteNodeStatus.getKeepAliveApplications(); List<ApplicationId> keepAliveApps =
remoteNodeStatus.getKeepAliveApplications();
if (keepAliveApps != null) { if (keepAliveApps != null) {
setAppAggregatorsMapToResponse(keepAliveApps, nodeHeartBeatResponse); setAppCollectorsMapToResponse(keepAliveApps, nodeHeartBeatResponse);
} }
// 4. Send status to RMNode, saving the latest response. // 4. Send status to RMNode, saving the latest response.
@ -589,48 +590,49 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
} }
return nodeHeartBeatResponse; return nodeHeartBeatResponse;
} }
private void setAppAggregatorsMapToResponse( private void setAppCollectorsMapToResponse(
List<ApplicationId> liveApps, NodeHeartbeatResponse response) { List<ApplicationId> liveApps, NodeHeartbeatResponse response) {
Map<ApplicationId, String> liveAppAggregatorsMap = new Map<ApplicationId, String> liveAppCollectorsMap = new
ConcurrentHashMap<ApplicationId, String>(); ConcurrentHashMap<ApplicationId, String>();
Map<ApplicationId, RMApp> rmApps = rmContext.getRMApps(); Map<ApplicationId, RMApp> rmApps = rmContext.getRMApps();
for (ApplicationId appId : liveApps) { for (ApplicationId appId : liveApps) {
String appAggregatorAddr = rmApps.get(appId).getAggregatorAddr(); String appCollectorAddr = rmApps.get(appId).getCollectorAddr();
if (appAggregatorAddr != null) { if (appCollectorAddr != null) {
liveAppAggregatorsMap.put(appId, appAggregatorAddr); liveAppCollectorsMap.put(appId, appCollectorAddr);
} else { } else {
// Log a debug info if aggregator address is not found. // Log a debug info if collector address is not found.
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
LOG.debug("Aggregator for applicaton: " + appId + " hasn't registered yet!"); LOG.debug("Collector for applicaton: " + appId +
" hasn't registered yet!");
} }
} }
} }
response.setAppAggregatorsMap(liveAppAggregatorsMap); response.setAppCollectorsMap(liveAppCollectorsMap);
} }
private void updateAppAggregatorsMap(NodeHeartbeatRequest request) { private void updateAppCollectorsMap(NodeHeartbeatRequest request) {
Map<ApplicationId, String> registeredAggregatorsMap = Map<ApplicationId, String> registeredCollectorsMap =
request.getRegisteredAggregators(); request.getRegisteredCollectors();
if (registeredAggregatorsMap != null if (registeredCollectorsMap != null
&& !registeredAggregatorsMap.isEmpty()) { && !registeredCollectorsMap.isEmpty()) {
Map<ApplicationId, RMApp> rmApps = rmContext.getRMApps(); Map<ApplicationId, RMApp> rmApps = rmContext.getRMApps();
for (Map.Entry<ApplicationId, String> entry: for (Map.Entry<ApplicationId, String> entry:
registeredAggregatorsMap.entrySet()) { registeredCollectorsMap.entrySet()) {
ApplicationId appId = entry.getKey(); ApplicationId appId = entry.getKey();
String aggregatorAddr = entry.getValue(); String collectorAddr = entry.getValue();
if (aggregatorAddr != null && !aggregatorAddr.isEmpty()) { if (collectorAddr != null && !collectorAddr.isEmpty()) {
RMApp rmApp = rmApps.get(appId); RMApp rmApp = rmApps.get(appId);
if (rmApp == null) { if (rmApp == null) {
LOG.warn("Cannot update aggregator info because application ID: " + LOG.warn("Cannot update collector info because application ID: " +
appId + " is not found in RMContext!"); appId + " is not found in RMContext!");
} else { } else {
String previousAggregatorAddr = rmApp.getAggregatorAddr(); String previousCollectorAddr = rmApp.getCollectorAddr();
if (previousAggregatorAddr == null || if (previousCollectorAddr == null ||
previousAggregatorAddr != aggregatorAddr) { previousCollectorAddr != collectorAddr) {
// sending aggregator update event. // sending collector update event.
RMAppAggregatorUpdateEvent event = RMAppCollectorUpdateEvent event =
new RMAppAggregatorUpdateEvent(appId, aggregatorAddr); new RMAppCollectorUpdateEvent(appId, collectorAddr);
rmContext.getDispatcher().getEventHandler().handle(event); rmContext.getDispatcher().getEventHandler().handle(event);
} }
} }

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
View File

@ -175,23 +175,23 @@ ApplicationReport createAndGetApplicationReport(String clientUserName,
* @return the tracking url for the application master. * @return the tracking url for the application master.
*/ */
String getTrackingUrl(); String getTrackingUrl();
/** /**
* The aggregator address for the application. * The collector address for the application.
* @return the address for the application's aggregator. * @return the address for the application's collector.
*/ */
String getAggregatorAddr(); String getCollectorAddr();
/** /**
* Set aggregator address for the application * Set collector address for the application
* @param aggregatorAddr the address of aggregator * @param collectorAddr the address of collector
*/ */
void setAggregatorAddr(String aggregatorAddr); void setCollectorAddr(String collectorAddr);
/** /**
* Remove aggregator address when application is finished or killed. * Remove collector address when application is finished or killed.
*/ */
void removeAggregatorAddr(); void removeCollectorAddr();
/** /**
* The original tracking url for the application master. * The original tracking url for the application master.

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppAggregatorUpdateEvent.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java
View File

@ -20,17 +20,18 @@
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
public class RMAppAggregatorUpdateEvent extends RMAppEvent { public class RMAppCollectorUpdateEvent extends RMAppEvent {
private final String appAggregatorAddr; private final String appCollectorAddr;
public RMAppAggregatorUpdateEvent(ApplicationId appId, String appAggregatorAddr) { public RMAppCollectorUpdateEvent(ApplicationId appId,
super(appId, RMAppEventType.AGGREGATOR_UPDATE); String appCollectorAddr) {
this.appAggregatorAddr = appAggregatorAddr; super(appId, RMAppEventType.COLLECTOR_UPDATE);
this.appCollectorAddr = appCollectorAddr;
} }
public String getAppAggregatorAddr(){ public String getAppCollectorAddr(){
return this.appAggregatorAddr; return this.appCollectorAddr;
} }
} }

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
View File

@ -30,9 +30,9 @@ public enum RMAppEventType {
// Source: Scheduler // Source: Scheduler
APP_ACCEPTED, APP_ACCEPTED,
// TODO add source later // TODO add source later
AGGREGATOR_UPDATE, COLLECTOR_UPDATE,
// Source: RMAppAttempt // Source: RMAppAttempt
ATTEMPT_REGISTERED, ATTEMPT_REGISTERED,

60
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
View File

@ -152,7 +152,7 @@ public class RMAppImpl implements RMApp, Recoverable {
private long storedFinishTime = 0; private long storedFinishTime = 0;
private int firstAttemptIdInStateStore = 1; private int firstAttemptIdInStateStore = 1;
private int nextAttemptId = 1; private int nextAttemptId = 1;
private String aggregatorAddr; private String collectorAddr;
// This field isn't protected by readlock now. // This field isn't protected by readlock now.
private volatile RMAppAttempt currentAttempt; private volatile RMAppAttempt currentAttempt;
private String queue; private String queue;
@ -201,7 +201,7 @@ public class RMAppImpl implements RMApp, Recoverable {
.addTransition(RMAppState.NEW, RMAppState.NEW, .addTransition(RMAppState.NEW, RMAppState.NEW,
RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition()) RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
.addTransition(RMAppState.NEW, RMAppState.NEW, .addTransition(RMAppState.NEW, RMAppState.NEW,
RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
.addTransition(RMAppState.NEW, RMAppState.NEW_SAVING, .addTransition(RMAppState.NEW, RMAppState.NEW_SAVING,
RMAppEventType.START, new RMAppNewlySavingTransition()) RMAppEventType.START, new RMAppNewlySavingTransition())
.addTransition(RMAppState.NEW, EnumSet.of(RMAppState.SUBMITTED, .addTransition(RMAppState.NEW, EnumSet.of(RMAppState.SUBMITTED,
@ -219,7 +219,7 @@ RMAppEventType.RECOVER, new RMAppRecoveredTransition())
.addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING, .addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING,
RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition()) RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
.addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING, .addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING,
RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
.addTransition(RMAppState.NEW_SAVING, RMAppState.SUBMITTED, .addTransition(RMAppState.NEW_SAVING, RMAppState.SUBMITTED,
RMAppEventType.APP_NEW_SAVED, new AddApplicationToSchedulerTransition()) RMAppEventType.APP_NEW_SAVED, new AddApplicationToSchedulerTransition())
.addTransition(RMAppState.NEW_SAVING, RMAppState.FINAL_SAVING, .addTransition(RMAppState.NEW_SAVING, RMAppState.FINAL_SAVING,
@ -239,7 +239,7 @@ RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
.addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED, .addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED,
RMAppEventType.MOVE, new RMAppMoveTransition()) RMAppEventType.MOVE, new RMAppMoveTransition())
.addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED, .addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED,
RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
.addTransition(RMAppState.SUBMITTED, RMAppState.FINAL_SAVING, .addTransition(RMAppState.SUBMITTED, RMAppState.FINAL_SAVING,
RMAppEventType.APP_REJECTED, RMAppEventType.APP_REJECTED,
new FinalSavingTransition( new FinalSavingTransition(
@ -257,7 +257,7 @@ RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
.addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED,
RMAppEventType.MOVE, new RMAppMoveTransition()) RMAppEventType.MOVE, new RMAppMoveTransition())
.addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED, .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED,
RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
.addTransition(RMAppState.ACCEPTED, RMAppState.RUNNING, .addTransition(RMAppState.ACCEPTED, RMAppState.RUNNING,
RMAppEventType.ATTEMPT_REGISTERED, new RMAppStateUpdateTransition( RMAppEventType.ATTEMPT_REGISTERED, new RMAppStateUpdateTransition(
YarnApplicationState.RUNNING)) YarnApplicationState.RUNNING))
@ -286,7 +286,7 @@ RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
.addTransition(RMAppState.RUNNING, RMAppState.RUNNING, .addTransition(RMAppState.RUNNING, RMAppState.RUNNING,
RMAppEventType.MOVE, new RMAppMoveTransition()) RMAppEventType.MOVE, new RMAppMoveTransition())
.addTransition(RMAppState.RUNNING, RMAppState.RUNNING, .addTransition(RMAppState.RUNNING, RMAppState.RUNNING,
RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
.addTransition(RMAppState.RUNNING, RMAppState.FINAL_SAVING, .addTransition(RMAppState.RUNNING, RMAppState.FINAL_SAVING,
RMAppEventType.ATTEMPT_UNREGISTERED, RMAppEventType.ATTEMPT_UNREGISTERED,
new FinalSavingTransition( new FinalSavingTransition(
@ -317,7 +317,7 @@ RMAppEventType.KILL, new KillAttemptTransition())
RMAppEventType.APP_RUNNING_ON_NODE, RMAppEventType.APP_RUNNING_ON_NODE,
new AppRunningOnNodeTransition()) new AppRunningOnNodeTransition())
.addTransition(RMAppState.FINAL_SAVING, RMAppState.FINAL_SAVING, .addTransition(RMAppState.FINAL_SAVING, RMAppState.FINAL_SAVING,
RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
// ignorable transitions // ignorable transitions
.addTransition(RMAppState.FINAL_SAVING, RMAppState.FINAL_SAVING, .addTransition(RMAppState.FINAL_SAVING, RMAppState.FINAL_SAVING,
EnumSet.of(RMAppEventType.NODE_UPDATE, RMAppEventType.KILL, EnumSet.of(RMAppEventType.NODE_UPDATE, RMAppEventType.KILL,
@ -330,7 +330,7 @@ RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition())
RMAppEventType.APP_RUNNING_ON_NODE, RMAppEventType.APP_RUNNING_ON_NODE,
new AppRunningOnNodeTransition()) new AppRunningOnNodeTransition())
.addTransition(RMAppState.FINISHING, RMAppState.FINISHING, .addTransition(RMAppState.FINISHING, RMAppState.FINISHING,
RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
// ignorable transitions // ignorable transitions
.addTransition(RMAppState.FINISHING, RMAppState.FINISHING, .addTransition(RMAppState.FINISHING, RMAppState.FINISHING,
EnumSet.of(RMAppEventType.NODE_UPDATE, EnumSet.of(RMAppEventType.NODE_UPDATE,
@ -343,7 +343,7 @@ RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition())
RMAppEventType.APP_RUNNING_ON_NODE, RMAppEventType.APP_RUNNING_ON_NODE,
new AppRunningOnNodeTransition()) new AppRunningOnNodeTransition())
.addTransition(RMAppState.KILLING, RMAppState.KILLING, .addTransition(RMAppState.KILLING, RMAppState.KILLING,
RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition()) RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
.addTransition(RMAppState.KILLING, RMAppState.FINAL_SAVING, .addTransition(RMAppState.KILLING, RMAppState.FINAL_SAVING,
RMAppEventType.ATTEMPT_KILLED, RMAppEventType.ATTEMPT_KILLED,
new FinalSavingTransition( new FinalSavingTransition(
@ -577,20 +577,20 @@ public String getQueue() {
public void setQueue(String queue) { public void setQueue(String queue) {
this.queue = queue; this.queue = queue;
} }
@Override @Override
public String getAggregatorAddr() { public String getCollectorAddr() {
return this.aggregatorAddr; return this.collectorAddr;
} }
@Override @Override
public void setAggregatorAddr(String aggregatorAddr) { public void setCollectorAddr(String collectorAddr) {
this.aggregatorAddr = aggregatorAddr; this.collectorAddr = collectorAddr;
} }
@Override @Override
public void removeAggregatorAddr() { public void removeCollectorAddr() {
this.aggregatorAddr = null; this.collectorAddr = null;
} }
@Override @Override
@ -864,8 +864,8 @@ public void recover(RMState state) {
// send the ATS create Event // send the ATS create Event
sendATSCreateEvent(this, this.startTime); sendATSCreateEvent(this, this.startTime);
//TODO recover aggregator address. //TODO recover collector address.
//this.aggregatorAddr = appState.getAggregatorAddr(); //this.collectorAddr = appState.getCollectorAddr();
RMAppAttemptImpl preAttempt = null; RMAppAttemptImpl preAttempt = null;
for (ApplicationAttemptId attemptId : for (ApplicationAttemptId attemptId :
@ -938,22 +938,22 @@ public void transition(RMAppImpl app, RMAppEvent event) {
}; };
} }
private static final class RMAppAggregatorUpdateTransition private static final class RMAppCollectorUpdateTransition
extends RMAppTransition { extends RMAppTransition {
public void transition(RMAppImpl app, RMAppEvent event) { public void transition(RMAppImpl app, RMAppEvent event) {
LOG.info("Updating aggregator info for app: " + app.getApplicationId()); LOG.info("Updating collector info for app: " + app.getApplicationId());
RMAppAggregatorUpdateEvent appAggregatorUpdateEvent = RMAppCollectorUpdateEvent appCollectorUpdateEvent =
(RMAppAggregatorUpdateEvent) event; (RMAppCollectorUpdateEvent) event;
// Update aggregator address // Update collector address
app.setAggregatorAddr(appAggregatorUpdateEvent.getAppAggregatorAddr()); app.setCollectorAddr(appCollectorUpdateEvent.getAppCollectorAddr());
// TODO persistent to RMStateStore for recover // TODO persistent to RMStateStore for recover
// Save to RMStateStore // Save to RMStateStore
}; };
} }
private static final class RMAppNodeUpdateTransition extends RMAppTransition { private static final class RMAppNodeUpdateTransition extends RMAppTransition {
public void transition(RMAppImpl app, RMAppEvent event) { public void transition(RMAppImpl app, RMAppEvent event) {
RMAppNodeUpdateEvent nodeUpdateEvent = (RMAppNodeUpdateEvent) event; RMAppNodeUpdateEvent nodeUpdateEvent = (RMAppNodeUpdateEvent) event;

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
View File

@ -95,15 +95,15 @@ public StringBuilder getDiagnostics() {
throw new UnsupportedOperationException("Not supported yet."); throw new UnsupportedOperationException("Not supported yet.");
} }
@Override @Override
public String getAggregatorAddr() { public String getCollectorAddr() {
throw new UnsupportedOperationException("Not supported yet."); throw new UnsupportedOperationException("Not supported yet.");
} }
@Override @Override
public void setAggregatorAddr(String aggregatorAddr) { public void setCollectorAddr(String collectorAddr) {
throw new UnsupportedOperationException("Not supported yet."); throw new UnsupportedOperationException("Not supported yet.");
} }
@Override @Override
public void removeAggregatorAddr() { public void removeCollectorAddr() {
throw new UnsupportedOperationException("Not supported yet."); throw new UnsupportedOperationException("Not supported yet.");
} }
@Override @Override

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
View File

@ -302,17 +302,17 @@ public CallerContext getCallerContext() {
throw new UnsupportedOperationException("Not supported yet."); throw new UnsupportedOperationException("Not supported yet.");
} }
public String getAggregatorAddr() { public String getCollectorAddr() {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public void removeAggregatorAddr() {
throw new UnsupportedOperationException("Not supported yet."); throw new UnsupportedOperationException("Not supported yet.");
} }
@Override @Override
public void setAggregatorAddr(String aggregatorAddr) { public void removeCollectorAddr() {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public void setCollectorAddr(String collectorAddr) {
throw new UnsupportedOperationException("Not supported yet."); throw new UnsupportedOperationException("Not supported yet.");
} }
} }

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
View File

@ -56,6 +56,11 @@
<artifactId>junit</artifactId> <artifactId>junit</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency --> <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
View File

@ -1,25 +1,50 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.timelineservice; package org.apache.hadoop.yarn.server.timelineservice;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.aggregator.PerNodeTimelineAggregatorsAuxService; import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import static org.junit.Assert.fail;
public class TestTimelineServiceClientIntegration { public class TestTimelineServiceClientIntegration {
private static PerNodeTimelineAggregatorsAuxService auxService; private static TimelineCollectorManager collectorManager;
private static PerNodeTimelineCollectorsAuxService auxService;
@BeforeClass @BeforeClass
public static void setupClass() throws Exception { public static void setupClass() throws Exception {
try { try {
auxService = PerNodeTimelineAggregatorsAuxService.launchServer(new String[0]); collectorManager = new MyTimelineCollectorManager();
auxService =
PerNodeTimelineCollectorsAuxService.launchServer(new String[0],
collectorManager);
auxService.addApplication(ApplicationId.newInstance(0, 1)); auxService.addApplication(ApplicationId.newInstance(0, 1));
} catch (ExitUtil.ExitException e) { } catch (ExitUtil.ExitException e) {
fail(); fail();
@ -38,6 +63,9 @@ public void testPutEntities() throws Exception {
TimelineClient client = TimelineClient client =
TimelineClient.createTimelineClient(ApplicationId.newInstance(0, 1)); TimelineClient.createTimelineClient(ApplicationId.newInstance(0, 1));
try { try {
// set the timeline service address manually
client.setTimelineServiceAddress(
collectorManager.getRestServerBindAddress());
client.init(new YarnConfiguration()); client.init(new YarnConfiguration());
client.start(); client.start();
TimelineEntity entity = new TimelineEntity(); TimelineEntity entity = new TimelineEntity();
@ -45,10 +73,20 @@ public void testPutEntities() throws Exception {
entity.setId("test entity id"); entity.setId("test entity id");
client.putEntities(entity); client.putEntities(entity);
client.putEntitiesAsync(entity); client.putEntitiesAsync(entity);
} catch(Exception e) {
fail();
} finally { } finally {
client.stop(); client.stop();
} }
} }
private static class MyTimelineCollectorManager extends
TimelineCollectorManager {
public MyTimelineCollectorManager() {
super();
}
@Override
protected CollectorNodemanagerProtocol getNMCollectorService() {
return mock(CollectorNodemanagerProtocol.class);
}
}
} }

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
View File

@ -56,6 +56,11 @@
<artifactId>hadoop-yarn-common</artifactId> <artifactId>hadoop-yarn-common</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId> <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
@ -71,6 +76,11 @@
<artifactId>guice</artifactId> <artifactId>guice</artifactId>
</dependency> </dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency> <dependency>
<groupId>javax.servlet</groupId> <groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId> <artifactId>servlet-api</artifactId>

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/AppLevelTimelineAggregator.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
View File

@ -16,7 +16,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.yarn.server.timelineservice.aggregator; package org.apache.hadoop.yarn.server.timelineservice.collector;
import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
@ -30,12 +30,12 @@
*/ */
@Private @Private
@Unstable @Unstable
public class AppLevelTimelineAggregator extends TimelineAggregator { public class AppLevelTimelineCollector extends TimelineCollector {
private final String applicationId; private final String applicationId;
// TODO define key metadata such as flow metadata, user, and queue // TODO define key metadata such as flow metadata, user, and queue
public AppLevelTimelineAggregator(String applicationId) { public AppLevelTimelineCollector(String applicationId) {
super(AppLevelTimelineAggregator.class.getName() + " - " + applicationId); super(AppLevelTimelineCollector.class.getName() + " - " + applicationId);
this.applicationId = applicationId; this.applicationId = applicationId;
} }

87
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/PerNodeTimelineAggregatorsAuxService.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
View File

@ -16,7 +16,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.yarn.server.timelineservice.aggregator; package org.apache.hadoop.yarn.server.timelineservice.collector;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
@ -42,85 +42,85 @@
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
/** /**
* The top-level server for the per-node timeline aggregator collection. Currently * The top-level server for the per-node timeline collector manager. Currently
* it is defined as an auxiliary service to accommodate running within another * it is defined as an auxiliary service to accommodate running within another
* daemon (e.g. node manager). * daemon (e.g. node manager).
*/ */
@Private @Private
@Unstable @Unstable
public class PerNodeTimelineAggregatorsAuxService extends AuxiliaryService { public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
private static final Log LOG = private static final Log LOG =
LogFactory.getLog(PerNodeTimelineAggregatorsAuxService.class); LogFactory.getLog(PerNodeTimelineCollectorsAuxService.class);
private static final int SHUTDOWN_HOOK_PRIORITY = 30; private static final int SHUTDOWN_HOOK_PRIORITY = 30;
private final TimelineAggregatorsCollection aggregatorCollection; private final TimelineCollectorManager collectorManager;
public PerNodeTimelineAggregatorsAuxService() { public PerNodeTimelineCollectorsAuxService() {
// use the same singleton // use the same singleton
this(TimelineAggregatorsCollection.getInstance()); this(TimelineCollectorManager.getInstance());
} }
@VisibleForTesting PerNodeTimelineAggregatorsAuxService( @VisibleForTesting PerNodeTimelineCollectorsAuxService(
TimelineAggregatorsCollection aggregatorCollection) { TimelineCollectorManager collectorsManager) {
super("timeline_aggregator"); super("timeline_collector");
this.aggregatorCollection = aggregatorCollection; this.collectorManager = collectorsManager;
} }
@Override @Override
protected void serviceInit(Configuration conf) throws Exception { protected void serviceInit(Configuration conf) throws Exception {
aggregatorCollection.init(conf); collectorManager.init(conf);
super.serviceInit(conf); super.serviceInit(conf);
} }
@Override @Override
protected void serviceStart() throws Exception { protected void serviceStart() throws Exception {
aggregatorCollection.start(); collectorManager.start();
super.serviceStart(); super.serviceStart();
} }
@Override @Override
protected void serviceStop() throws Exception { protected void serviceStop() throws Exception {
aggregatorCollection.stop(); collectorManager.stop();
super.serviceStop(); super.serviceStop();
} }
// these methods can be used as the basis for future service methods if the // these methods can be used as the basis for future service methods if the
// per-node aggregator runs separate from the node manager // per-node collector runs separate from the node manager
/** /**
* Creates and adds an app level aggregator for the specified application id. * Creates and adds an app level collector for the specified application id.
* The aggregator is also initialized and started. If the service already * The collector is also initialized and started. If the service already
* exists, no new service is created. * exists, no new service is created.
* *
* @return whether it was added successfully * @return whether it was added successfully
*/ */
public boolean addApplication(ApplicationId appId) { public boolean addApplication(ApplicationId appId) {
AppLevelTimelineAggregator aggregator = AppLevelTimelineCollector collector =
new AppLevelTimelineAggregator(appId.toString()); new AppLevelTimelineCollector(appId.toString());
return (aggregatorCollection.putIfAbsent(appId, aggregator) return (collectorManager.putIfAbsent(appId, collector)
== aggregator); == collector);
} }
/** /**
* Removes the app level aggregator for the specified application id. The * Removes the app level collector for the specified application id. The
* aggregator is also stopped as a result. If the aggregator does not exist, no * collector is also stopped as a result. If the collector does not exist, no
* change is made. * change is made.
* *
* @return whether it was removed successfully * @return whether it was removed successfully
*/ */
public boolean removeApplication(ApplicationId appId) { public boolean removeApplication(ApplicationId appId) {
String appIdString = appId.toString(); String appIdString = appId.toString();
return aggregatorCollection.remove(appIdString); return collectorManager.remove(appIdString);
} }
/** /**
* Creates and adds an app level aggregator for the specified application id. * Creates and adds an app level collector for the specified application id.
* The aggregator is also initialized and started. If the aggregator already * The collector is also initialized and started. If the collector already
* exists, no new aggregator is created. * exists, no new collector is created.
*/ */
@Override @Override
public void initializeContainer(ContainerInitializationContext context) { public void initializeContainer(ContainerInitializationContext context) {
// intercept the event of the AM container being created and initialize the // intercept the event of the AM container being created and initialize the
// app level aggregator service // app level collector service
if (isApplicationMaster(context)) { if (isApplicationMaster(context)) {
ApplicationId appId = context.getContainerId(). ApplicationId appId = context.getContainerId().
getApplicationAttemptId().getApplicationId(); getApplicationAttemptId().getApplicationId();
@ -129,14 +129,14 @@ public void initializeContainer(ContainerInitializationContext context) {
} }
/** /**
* Removes the app level aggregator for the specified application id. The * Removes the app level collector for the specified application id. The
* aggregator is also stopped as a result. If the aggregator does not exist, no * collector is also stopped as a result. If the collector does not exist, no
* change is made. * change is made.
*/ */
@Override @Override
public void stopContainer(ContainerTerminationContext context) { public void stopContainer(ContainerTerminationContext context) {
// intercept the event of the AM container being stopped and remove the app // intercept the event of the AM container being stopped and remove the app
// level aggregator service // level collector service
if (isApplicationMaster(context)) { if (isApplicationMaster(context)) {
ApplicationId appId = context.getContainerId(). ApplicationId appId = context.getContainerId().
getApplicationAttemptId().getApplicationId(); getApplicationAttemptId().getApplicationId();
@ -154,7 +154,7 @@ private boolean isApplicationMaster(ContainerContext context) {
@VisibleForTesting @VisibleForTesting
boolean hasApplication(String appId) { boolean hasApplication(String appId) {
return aggregatorCollection.containsKey(appId); return collectorManager.containsKey(appId);
} }
@Override @Override
@ -173,30 +173,33 @@ public ByteBuffer getMetaData() {
} }
@VisibleForTesting @VisibleForTesting
public static PerNodeTimelineAggregatorsAuxService launchServer(String[] args) { public static PerNodeTimelineCollectorsAuxService
launchServer(String[] args, TimelineCollectorManager collectorManager) {
Thread Thread
.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); .setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(PerNodeTimelineAggregatorsAuxService.class, args, StringUtils.startupShutdownMessage(
LOG); PerNodeTimelineCollectorsAuxService.class, args, LOG);
PerNodeTimelineAggregatorsAuxService auxService = null; PerNodeTimelineCollectorsAuxService auxService = null;
try { try {
auxService = new PerNodeTimelineAggregatorsAuxService(); auxService = collectorManager == null ?
new PerNodeTimelineCollectorsAuxService() :
new PerNodeTimelineCollectorsAuxService(collectorManager);
ShutdownHookManager.get().addShutdownHook(new ShutdownHook(auxService), ShutdownHookManager.get().addShutdownHook(new ShutdownHook(auxService),
SHUTDOWN_HOOK_PRIORITY); SHUTDOWN_HOOK_PRIORITY);
YarnConfiguration conf = new YarnConfiguration(); YarnConfiguration conf = new YarnConfiguration();
auxService.init(conf); auxService.init(conf);
auxService.start(); auxService.start();
} catch (Throwable t) { } catch (Throwable t) {
LOG.fatal("Error starting PerNodeAggregatorServer", t); LOG.fatal("Error starting PerNodeTimelineCollectorServer", t);
ExitUtil.terminate(-1, "Error starting PerNodeAggregatorServer"); ExitUtil.terminate(-1, "Error starting PerNodeTimelineCollectorServer");
} }
return auxService; return auxService;
} }
private static class ShutdownHook implements Runnable { private static class ShutdownHook implements Runnable {
private final PerNodeTimelineAggregatorsAuxService auxService; private final PerNodeTimelineCollectorsAuxService auxService;
public ShutdownHook(PerNodeTimelineAggregatorsAuxService auxService) { public ShutdownHook(PerNodeTimelineCollectorsAuxService auxService) {
this.auxService = auxService; this.auxService = auxService;
} }
@ -206,6 +209,6 @@ public void run() {
} }
public static void main(String[] args) { public static void main(String[] args) {
launchServer(args); launchServer(args, null);
} }
} }

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregator.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
View File

@ -16,7 +16,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.yarn.server.timelineservice.aggregator; package org.apache.hadoop.yarn.server.timelineservice.collector;
import java.io.IOException; import java.io.IOException;
@ -27,12 +27,12 @@
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl; import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
/** /**
* Service that handles writes to the timeline service and writes them to the * Service that handles writes to the timeline service and writes them to the
* backing storage. * backing storage.
@ -42,12 +42,12 @@
*/ */
@Private @Private
@Unstable @Unstable
public abstract class TimelineAggregator extends CompositeService { public abstract class TimelineCollector extends CompositeService {
private static final Log LOG = LogFactory.getLog(TimelineAggregator.class); private static final Log LOG = LogFactory.getLog(TimelineCollector.class);
private TimelineWriter writer; private TimelineWriter writer;
public TimelineAggregator(String name) { public TimelineCollector(String name) {
super(name); super(name);
} }

187
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorsCollection.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
View File

@ -16,11 +16,14 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.yarn.server.timelineservice.aggregator; package org.apache.hadoop.yarn.server.timelineservice.collector;
import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
import java.io.IOException; import java.io.IOException;
import java.net.URI;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
import java.net.URI;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@ -39,64 +42,64 @@
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol; import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER; import com.google.common.annotations.VisibleForTesting;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
/** /**
* Class that manages adding and removing aggregators and their lifecycle. It * Class that manages adding and removing collectors and their lifecycle. It
* provides thread safety access to the aggregators inside. * provides thread safety access to the collectors inside.
* *
* It is a singleton, and instances should be obtained via * It is a singleton, and instances should be obtained via
* {@link #getInstance()}. * {@link #getInstance()}.
*/ */
@Private @Private
@Unstable @Unstable
public class TimelineAggregatorsCollection extends CompositeService { public class TimelineCollectorManager extends CompositeService {
private static final Log LOG = private static final Log LOG =
LogFactory.getLog(TimelineAggregatorsCollection.class); LogFactory.getLog(TimelineCollectorManager.class);
private static final TimelineAggregatorsCollection INSTANCE = private static final TimelineCollectorManager INSTANCE =
new TimelineAggregatorsCollection(); new TimelineCollectorManager();
// access to this map is synchronized with the map itself // access to this map is synchronized with the map itself
private final Map<String, TimelineAggregator> aggregators = private final Map<String, TimelineCollector> collectors =
Collections.synchronizedMap( Collections.synchronizedMap(
new HashMap<String, TimelineAggregator>()); new HashMap<String, TimelineCollector>());
// REST server for this aggregator collection // REST server for this collector manager
private HttpServer2 timelineRestServer; private HttpServer2 timelineRestServer;
private String timelineRestServerBindAddress; private String timelineRestServerBindAddress;
private AggregatorNodemanagerProtocol nmAggregatorService;
private InetSocketAddress nmAggregatorServiceAddress;
static final String AGGREGATOR_COLLECTION_ATTR_KEY = "aggregator.collection"; private CollectorNodemanagerProtocol nmCollectorService;
static TimelineAggregatorsCollection getInstance() { private InetSocketAddress nmCollectorServiceAddress;
static final String COLLECTOR_MANAGER_ATTR_KEY = "collector.manager";
static TimelineCollectorManager getInstance() {
return INSTANCE; return INSTANCE;
} }
TimelineAggregatorsCollection() { @VisibleForTesting
super(TimelineAggregatorsCollection.class.getName()); protected TimelineCollectorManager() {
super(TimelineCollectorManager.class.getName());
} }
@Override @Override
public void serviceInit(Configuration conf) throws Exception { public void serviceInit(Configuration conf) throws Exception {
this.nmAggregatorServiceAddress = conf.getSocketAddr( this.nmCollectorServiceAddress = conf.getSocketAddr(
YarnConfiguration.NM_BIND_HOST, YarnConfiguration.NM_BIND_HOST,
YarnConfiguration.NM_AGGREGATOR_SERVICE_ADDRESS, YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_ADDRESS, YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS,
YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_PORT); YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT);
} }
@Override @Override
protected void serviceStart() throws Exception { protected void serviceStart() throws Exception {
startWebApp(); startWebApp();
@ -112,95 +115,95 @@ protected void serviceStop() throws Exception {
} }
/** /**
* Put the aggregator into the collection if an aggregator mapped by id does * Put the collector into the collection if an collector mapped by id does
* not exist. * not exist.
* *
* @throws YarnRuntimeException if there was any exception in initializing and * @throws YarnRuntimeException if there was any exception in initializing and
* starting the app level service * starting the app level service
* @return the aggregator associated with id after the potential put. * @return the collector associated with id after the potential put.
*/ */
public TimelineAggregator putIfAbsent(ApplicationId appId, public TimelineCollector putIfAbsent(ApplicationId appId,
TimelineAggregator aggregator) { TimelineCollector collector) {
String id = appId.toString(); String id = appId.toString();
TimelineAggregator aggregatorInTable; TimelineCollector collectorInTable;
boolean aggregatorIsNew = false; boolean collectorIsNew = false;
synchronized (aggregators) { synchronized (collectors) {
aggregatorInTable = aggregators.get(id); collectorInTable = collectors.get(id);
if (aggregatorInTable == null) { if (collectorInTable == null) {
try { try {
// initialize, start, and add it to the collection so it can be // initialize, start, and add it to the collection so it can be
// cleaned up when the parent shuts down // cleaned up when the parent shuts down
aggregator.init(getConfig()); collector.init(getConfig());
aggregator.start(); collector.start();
aggregators.put(id, aggregator); collectors.put(id, collector);
LOG.info("the aggregator for " + id + " was added"); LOG.info("the collector for " + id + " was added");
aggregatorInTable = aggregator; collectorInTable = collector;
aggregatorIsNew = true; collectorIsNew = true;
} catch (Exception e) { } catch (Exception e) {
throw new YarnRuntimeException(e); throw new YarnRuntimeException(e);
} }
} else { } else {
String msg = "the aggregator for " + id + " already exists!"; String msg = "the collector for " + id + " already exists!";
LOG.error(msg); LOG.error(msg);
} }
} }
// Report to NM if a new aggregator is added. // Report to NM if a new collector is added.
if (aggregatorIsNew) { if (collectorIsNew) {
try { try {
reportNewAggregatorToNM(appId); reportNewCollectorToNM(appId);
} catch (Exception e) { } catch (Exception e) {
// throw exception here as it cannot be used if failed report to NM // throw exception here as it cannot be used if failed report to NM
LOG.error("Failed to report a new aggregator for application: " + appId + LOG.error("Failed to report a new collector for application: " + appId +
" to NM Aggregator Services."); " to the NM Collector Service.");
throw new YarnRuntimeException(e); throw new YarnRuntimeException(e);
} }
} }
return aggregatorInTable; return collectorInTable;
} }
/** /**
* Removes the aggregator for the specified id. The aggregator is also stopped * Removes the collector for the specified id. The collector is also stopped
* as a result. If the aggregator does not exist, no change is made. * as a result. If the collector does not exist, no change is made.
* *
* @return whether it was removed successfully * @return whether it was removed successfully
*/ */
public boolean remove(String id) { public boolean remove(String id) {
synchronized (aggregators) { synchronized (collectors) {
TimelineAggregator aggregator = aggregators.remove(id); TimelineCollector collector = collectors.remove(id);
if (aggregator == null) { if (collector == null) {
String msg = "the aggregator for " + id + " does not exist!"; String msg = "the collector for " + id + " does not exist!";
LOG.error(msg); LOG.error(msg);
return false; return false;
} else { } else {
// stop the service to do clean up // stop the service to do clean up
aggregator.stop(); collector.stop();
LOG.info("the aggregator service for " + id + " was removed"); LOG.info("the collector service for " + id + " was removed");
return true; return true;
} }
} }
} }
/** /**
* Returns the aggregator for the specified id. * Returns the collector for the specified id.
* *
* @return the aggregator or null if it does not exist * @return the collector or null if it does not exist
*/ */
public TimelineAggregator get(String id) { public TimelineCollector get(String id) {
return aggregators.get(id); return collectors.get(id);
} }
/** /**
* Returns whether the aggregator for the specified id exists in this * Returns whether the collector for the specified id exists in this
* collection. * collection.
*/ */
public boolean containsKey(String id) { public boolean containsKey(String id) {
return aggregators.containsKey(id); return collectors.containsKey(id);
} }
/** /**
* Launch the REST web server for this aggregator collection * Launch the REST web server for this collector manager
*/ */
private void startWebApp() { private void startWebApp() {
Configuration conf = getConfig(); Configuration conf = getConfig();
@ -211,7 +214,7 @@ private void startWebApp() {
WebAppUtils.getAHSWebAppURLWithoutScheme(conf)); WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress( this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
NetUtils.createSocketAddr(bindAddress)); NetUtils.createSocketAddr(bindAddress));
LOG.info("Instantiating the per-node aggregator webapp at " + LOG.info("Instantiating the per-node collector webapp at " +
timelineRestServerBindAddress); timelineRestServerBindAddress);
try { try {
Configuration confForInfoServer = new Configuration(conf); Configuration confForInfoServer = new Configuration(conf);
@ -232,40 +235,44 @@ private void startWebApp() {
options, new String[] {"/*"}); options, new String[] {"/*"});
timelineRestServer.addJerseyResourcePackage( timelineRestServer.addJerseyResourcePackage(
TimelineAggregatorWebService.class.getPackage().getName() + ";" TimelineCollectorWebService.class.getPackage().getName() + ";"
+ GenericExceptionHandler.class.getPackage().getName() + ";" + GenericExceptionHandler.class.getPackage().getName() + ";"
+ YarnJacksonJaxbJsonProvider.class.getPackage().getName(), + YarnJacksonJaxbJsonProvider.class.getPackage().getName(),
"/*"); "/*");
timelineRestServer.setAttribute(AGGREGATOR_COLLECTION_ATTR_KEY, timelineRestServer.setAttribute(COLLECTOR_MANAGER_ATTR_KEY,
TimelineAggregatorsCollection.getInstance()); TimelineCollectorManager.getInstance());
timelineRestServer.start(); timelineRestServer.start();
} catch (Exception e) { } catch (Exception e) {
String msg = "The per-node aggregator webapp failed to start."; String msg = "The per-node collector webapp failed to start.";
LOG.error(msg, e); LOG.error(msg, e);
throw new YarnRuntimeException(msg, e); throw new YarnRuntimeException(msg, e);
} }
} }
private void reportNewAggregatorToNM(ApplicationId appId) private void reportNewCollectorToNM(ApplicationId appId)
throws YarnException, IOException { throws YarnException, IOException {
this.nmAggregatorService = getNMAggregatorService(); this.nmCollectorService = getNMCollectorService();
ReportNewAggregatorsInfoRequest request = ReportNewCollectorInfoRequest request =
ReportNewAggregatorsInfoRequest.newInstance(appId, ReportNewCollectorInfoRequest.newInstance(appId,
this.timelineRestServerBindAddress); this.timelineRestServerBindAddress);
LOG.info("Report a new aggregator for application: " + appId + LOG.info("Report a new collector for application: " + appId +
" to NM Aggregator Services."); " to the NM Collector Service.");
nmAggregatorService.reportNewAggregatorInfo(request); nmCollectorService.reportNewCollectorInfo(request);
} }
// protected for test @VisibleForTesting
protected AggregatorNodemanagerProtocol getNMAggregatorService(){ protected CollectorNodemanagerProtocol getNMCollectorService() {
Configuration conf = getConfig(); Configuration conf = getConfig();
final YarnRPC rpc = YarnRPC.create(conf); final YarnRPC rpc = YarnRPC.create(conf);
// TODO Security settings. // TODO Security settings.
return (AggregatorNodemanagerProtocol) rpc.getProxy( return (CollectorNodemanagerProtocol) rpc.getProxy(
AggregatorNodemanagerProtocol.class, CollectorNodemanagerProtocol.class,
nmAggregatorServiceAddress, conf); nmCollectorServiceAddress, conf);
}
@VisibleForTesting
public String getRestServerBindAddress() {
return timelineRestServerBindAddress;
} }
} }

41
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorWebService.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
View File

@ -16,12 +16,18 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.yarn.server.timelineservice.aggregator; package org.apache.hadoop.yarn.server.timelineservice.collector;
import javax.servlet.ServletContext; import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.*; import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context; import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response; import javax.ws.rs.core.Response;
@ -36,7 +42,6 @@
import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.ForbiddenException; import org.apache.hadoop.yarn.webapp.ForbiddenException;
@ -53,9 +58,9 @@
@Unstable @Unstable
@Singleton @Singleton
@Path("/ws/v2/timeline") @Path("/ws/v2/timeline")
public class TimelineAggregatorWebService { public class TimelineCollectorWebService {
private static final Log LOG = private static final Log LOG =
LogFactory.getLog(TimelineAggregatorWebService.class); LogFactory.getLog(TimelineCollectorWebService.class);
private @Context ServletContext context; private @Context ServletContext context;
@ -95,12 +100,12 @@ public AboutInfo about(
@Context HttpServletRequest req, @Context HttpServletRequest req,
@Context HttpServletResponse res) { @Context HttpServletResponse res) {
init(res); init(res);
return new AboutInfo("Timeline API"); return new AboutInfo("Timeline Collector API");
} }
/** /**
* Accepts writes to the aggregator, and returns a response. It simply routes * Accepts writes to the collector, and returns a response. It simply routes
* the request to the app level aggregator. It expects an application as a * the request to the app level collector. It expects an application as a
* context. * context.
*/ */
@PUT @PUT
@ -128,12 +133,12 @@ public Response putEntities(
if (appId == null) { if (appId == null) {
return Response.status(Response.Status.BAD_REQUEST).build(); return Response.status(Response.Status.BAD_REQUEST).build();
} }
TimelineAggregator service = getAggregatorService(req, appId); TimelineCollector collector = getCollector(req, appId);
if (service == null) { if (collector == null) {
LOG.error("Application not found"); LOG.error("Application not found");
throw new NotFoundException(); // different exception? throw new NotFoundException(); // different exception?
} }
service.postEntities(entities, callerUgi); collector.postEntities(entities, callerUgi);
return Response.ok().build(); return Response.ok().build();
} catch (Exception e) { } catch (Exception e) {
LOG.error("Error putting entities", e); LOG.error("Error putting entities", e);
@ -143,8 +148,6 @@ public Response putEntities(
} }
private String parseApplicationId(String appId) { private String parseApplicationId(String appId) {
// Make sure the appId is not null and is valid
ApplicationId appID;
try { try {
if (appId != null) { if (appId != null) {
return ConverterUtils.toApplicationId(appId.trim()).toString(); return ConverterUtils.toApplicationId(appId.trim()).toString();
@ -156,13 +159,13 @@ private String parseApplicationId(String appId) {
} }
} }
private TimelineAggregator private TimelineCollector
getAggregatorService(HttpServletRequest req, String appIdToParse) { getCollector(HttpServletRequest req, String appIdToParse) {
String appIdString = parseApplicationId(appIdToParse); String appIdString = parseApplicationId(appIdToParse);
final TimelineAggregatorsCollection aggregatorCollection = final TimelineCollectorManager collectorManager =
(TimelineAggregatorsCollection) context.getAttribute( (TimelineCollectorManager) context.getAttribute(
TimelineAggregatorsCollection.AGGREGATOR_COLLECTION_ATTR_KEY); TimelineCollectorManager.COLLECTOR_MANAGER_ATTR_KEY);
return aggregatorCollection.get(appIdString); return collectorManager.get(appIdString);
} }
private void init(HttpServletResponse response) { private void init(HttpServletResponse response) {

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
View File

@ -65,7 +65,7 @@ public class FileSystemTimelineWriterImpl extends AbstractService
* Stores the entire information in {@link TimelineEntity} to the * Stores the entire information in {@link TimelineEntity} to the
* timeline store. Any errors occurring for individual write request objects * timeline store. Any errors occurring for individual write request objects
* will be reported in the response. * will be reported in the response.
* *
* @param data * @param data
* a {@link TimelineEntity} object * a {@link TimelineEntity} object
* @return {@link TimelineWriteResponse} object. * @return {@link TimelineWriteResponse} object.
@ -116,10 +116,10 @@ private void write(TimelineEntity entity,
* Aggregates the entity information to the timeline store based on which * Aggregates the entity information to the timeline store based on which
* track this entity is to be rolled up to The tracks along which aggregations * track this entity is to be rolled up to The tracks along which aggregations
* are to be done are given by {@link TimelineAggregationTrack} * are to be done are given by {@link TimelineAggregationTrack}
* *
* Any errors occurring for individual write request objects will be reported * Any errors occurring for individual write request objects will be reported
* in the response. * in the response.
* *
* @param data * @param data
* a {@link TimelineEntity} object * a {@link TimelineEntity} object
* a {@link TimelineAggregationTrack} enum value * a {@link TimelineAggregationTrack} enum value

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestAppLevelTimelineAggregator.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestAppLevelTimelineCollector.java
View File

@ -16,8 +16,8 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.yarn.server.timelineservice.aggregator; package org.apache.hadoop.yarn.server.timelineservice.collector;
public class TestAppLevelTimelineAggregator { public class TestAppLevelTimelineCollector {
} }

48
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestPerNodeTimelineAggregatorsAuxService.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
View File

@ -16,7 +16,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.yarn.server.timelineservice.aggregator; package org.apache.hadoop.yarn.server.timelineservice.collector;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
@ -32,14 +32,15 @@
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.ContainerInitializationContext; import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
import org.apache.hadoop.yarn.server.api.ContainerTerminationContext; import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
import org.junit.Test; import org.junit.Test;
public class TestPerNodeTimelineAggregatorsAuxService { public class TestPerNodeTimelineCollectorsAuxService {
private ApplicationAttemptId appAttemptId; private ApplicationAttemptId appAttemptId;
public TestPerNodeTimelineAggregatorsAuxService() { public TestPerNodeTimelineCollectorsAuxService() {
ApplicationId appId = ApplicationId appId =
ApplicationId.newInstance(System.currentTimeMillis(), 1); ApplicationId.newInstance(System.currentTimeMillis(), 1);
appAttemptId = ApplicationAttemptId.newInstance(appId, 1); appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
@ -47,7 +48,8 @@ public TestPerNodeTimelineAggregatorsAuxService() {
@Test @Test
public void testAddApplication() throws Exception { public void testAddApplication() throws Exception {
PerNodeTimelineAggregatorsAuxService auxService = createAggregatorAndAddApplication(); PerNodeTimelineCollectorsAuxService auxService =
createCollectorAndAddApplication();
// auxService should have a single app // auxService should have a single app
assertTrue(auxService.hasApplication( assertTrue(auxService.hasApplication(
appAttemptId.getApplicationId().toString())); appAttemptId.getApplicationId().toString()));
@ -56,7 +58,7 @@ public void testAddApplication() throws Exception {
@Test @Test
public void testAddApplicationNonAMContainer() throws Exception { public void testAddApplicationNonAMContainer() throws Exception {
PerNodeTimelineAggregatorsAuxService auxService = createAggregator(); PerNodeTimelineCollectorsAuxService auxService = createCollector();
ContainerId containerId = getContainerId(2L); // not an AM ContainerId containerId = getContainerId(2L); // not an AM
ContainerInitializationContext context = ContainerInitializationContext context =
@ -70,7 +72,8 @@ public void testAddApplicationNonAMContainer() throws Exception {
@Test @Test
public void testRemoveApplication() throws Exception { public void testRemoveApplication() throws Exception {
PerNodeTimelineAggregatorsAuxService auxService = createAggregatorAndAddApplication(); PerNodeTimelineCollectorsAuxService auxService =
createCollectorAndAddApplication();
// auxService should have a single app // auxService should have a single app
String appIdStr = appAttemptId.getApplicationId().toString(); String appIdStr = appAttemptId.getApplicationId().toString();
assertTrue(auxService.hasApplication(appIdStr)); assertTrue(auxService.hasApplication(appIdStr));
@ -87,7 +90,8 @@ public void testRemoveApplication() throws Exception {
@Test @Test
public void testRemoveApplicationNonAMContainer() throws Exception { public void testRemoveApplicationNonAMContainer() throws Exception {
PerNodeTimelineAggregatorsAuxService auxService = createAggregatorAndAddApplication(); PerNodeTimelineCollectorsAuxService auxService =
createCollectorAndAddApplication();
// auxService should have a single app // auxService should have a single app
String appIdStr = appAttemptId.getApplicationId().toString(); String appIdStr = appAttemptId.getApplicationId().toString();
assertTrue(auxService.hasApplication(appIdStr)); assertTrue(auxService.hasApplication(appIdStr));
@ -105,10 +109,11 @@ public void testRemoveApplicationNonAMContainer() throws Exception {
@Test(timeout = 60000) @Test(timeout = 60000)
public void testLaunch() throws Exception { public void testLaunch() throws Exception {
ExitUtil.disableSystemExit(); ExitUtil.disableSystemExit();
PerNodeTimelineAggregatorsAuxService auxService = null; PerNodeTimelineCollectorsAuxService auxService = null;
try { try {
auxService = auxService =
PerNodeTimelineAggregatorsAuxService.launchServer(new String[0]); PerNodeTimelineCollectorsAuxService.launchServer(new String[0],
createCollectorManager());
} catch (ExitUtil.ExitException e) { } catch (ExitUtil.ExitException e) {
assertEquals(0, e.status); assertEquals(0, e.status);
ExitUtil.resetFirstExitException(); ExitUtil.resetFirstExitException();
@ -120,8 +125,9 @@ public void testLaunch() throws Exception {
} }
} }
private PerNodeTimelineAggregatorsAuxService createAggregatorAndAddApplication() { private PerNodeTimelineCollectorsAuxService
PerNodeTimelineAggregatorsAuxService auxService = createAggregator(); createCollectorAndAddApplication() {
PerNodeTimelineCollectorsAuxService auxService = createCollector();
// create an AM container // create an AM container
ContainerId containerId = getAMContainerId(); ContainerId containerId = getAMContainerId();
ContainerInitializationContext context = ContainerInitializationContext context =
@ -131,15 +137,23 @@ private PerNodeTimelineAggregatorsAuxService createAggregatorAndAddApplication()
return auxService; return auxService;
} }
private PerNodeTimelineAggregatorsAuxService createAggregator() { private PerNodeTimelineCollectorsAuxService createCollector() {
TimelineAggregatorsCollection TimelineCollectorManager collectorManager = createCollectorManager();
aggregatorsCollection = spy(new TimelineAggregatorsCollection()); PerNodeTimelineCollectorsAuxService auxService =
doReturn(new Configuration()).when(aggregatorsCollection).getConfig(); spy(new PerNodeTimelineCollectorsAuxService(collectorManager));
PerNodeTimelineAggregatorsAuxService auxService =
spy(new PerNodeTimelineAggregatorsAuxService(aggregatorsCollection));
return auxService; return auxService;
} }
private TimelineCollectorManager createCollectorManager() {
TimelineCollectorManager collectorManager =
spy(new TimelineCollectorManager());
doReturn(new Configuration()).when(collectorManager).getConfig();
CollectorNodemanagerProtocol nmCollectorService =
mock(CollectorNodemanagerProtocol.class);
doReturn(nmCollectorService).when(collectorManager).getNMCollectorService();
return collectorManager;
}
private ContainerId getAMContainerId() { private ContainerId getAMContainerId() {
return getContainerId(1L); return getContainerId(1L);
} }

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestTimelineAggregatorsCollection.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
View File

@ -16,11 +16,12 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.yarn.server.timelineservice.aggregator; package org.apache.hadoop.yarn.server.timelineservice.collector;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy; import static org.mockito.Mockito.spy;
import java.util.ArrayList; import java.util.ArrayList;
@ -30,18 +31,16 @@
import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.junit.Test; import org.junit.Test;
public class TestTimelineAggregatorsCollection { public class TestTimelineCollectorManager {
@Test(timeout=60000) @Test(timeout=60000)
public void testMultithreadedAdd() throws Exception { public void testMultithreadedAdd() throws Exception {
final TimelineAggregatorsCollection aggregatorCollection = final TimelineCollectorManager collectorManager = createCollectorManager();
spy(new TimelineAggregatorsCollection());
doReturn(new Configuration()).when(aggregatorCollection).getConfig();
final int NUM_APPS = 5; final int NUM_APPS = 5;
List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>(); List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
@ -49,9 +48,9 @@ public void testMultithreadedAdd() throws Exception {
final ApplicationId appId = ApplicationId.newInstance(0L, i); final ApplicationId appId = ApplicationId.newInstance(0L, i);
Callable<Boolean> task = new Callable<Boolean>() { Callable<Boolean> task = new Callable<Boolean>() {
public Boolean call() { public Boolean call() {
AppLevelTimelineAggregator aggregator = AppLevelTimelineCollector collector =
new AppLevelTimelineAggregator(appId.toString()); new AppLevelTimelineCollector(appId.toString());
return (aggregatorCollection.putIfAbsent(appId, aggregator) == aggregator); return (collectorManager.putIfAbsent(appId, collector) == collector);
} }
}; };
tasks.add(task); tasks.add(task);
@ -67,15 +66,14 @@ public Boolean call() {
} }
// check the keys // check the keys
for (int i = 0; i < NUM_APPS; i++) { for (int i = 0; i < NUM_APPS; i++) {
assertTrue(aggregatorCollection.containsKey(String.valueOf(i))); final ApplicationId appId = ApplicationId.newInstance(0L, i);
assertTrue(collectorManager.containsKey(appId.toString()));
} }
} }
@Test @Test
public void testMultithreadedAddAndRemove() throws Exception { public void testMultithreadedAddAndRemove() throws Exception {
final TimelineAggregatorsCollection aggregatorCollection = final TimelineCollectorManager collectorManager = createCollectorManager();
spy(new TimelineAggregatorsCollection());
doReturn(new Configuration()).when(aggregatorCollection).getConfig();
final int NUM_APPS = 5; final int NUM_APPS = 5;
List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>(); List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
@ -83,11 +81,11 @@ public void testMultithreadedAddAndRemove() throws Exception {
final ApplicationId appId = ApplicationId.newInstance(0L, i); final ApplicationId appId = ApplicationId.newInstance(0L, i);
Callable<Boolean> task = new Callable<Boolean>() { Callable<Boolean> task = new Callable<Boolean>() {
public Boolean call() { public Boolean call() {
AppLevelTimelineAggregator aggregator = AppLevelTimelineCollector collector =
new AppLevelTimelineAggregator(appId.toString()); new AppLevelTimelineCollector(appId.toString());
boolean successPut = boolean successPut =
(aggregatorCollection.putIfAbsent(appId, aggregator) == aggregator); (collectorManager.putIfAbsent(appId, collector) == collector);
return successPut && aggregatorCollection.remove(appId.toString()); return successPut && collectorManager.remove(appId.toString());
} }
}; };
tasks.add(task); tasks.add(task);
@ -103,7 +101,18 @@ public Boolean call() {
} }
// check the keys // check the keys
for (int i = 0; i < NUM_APPS; i++) { for (int i = 0; i < NUM_APPS; i++) {
assertFalse(aggregatorCollection.containsKey(String.valueOf(i))); final ApplicationId appId = ApplicationId.newInstance(0L, i);
assertFalse(collectorManager.containsKey(appId.toString()));
} }
} }
private TimelineCollectorManager createCollectorManager() {
final TimelineCollectorManager collectorManager =
spy(new TimelineCollectorManager());
doReturn(new Configuration()).when(collectorManager).getConfig();
CollectorNodemanagerProtocol nmCollectorService =
mock(CollectorNodemanagerProtocol.class);
doReturn(nmCollectorService).when(collectorManager).getNMCollectorService();
return collectorManager;
}
} }

43
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
View File

@ -27,12 +27,12 @@
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.List; import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.Test; import org.junit.Test;
import org.apache.commons.io.FileUtils;
public class TestFileSystemTimelineWriterImpl { public class TestFileSystemTimelineWriterImpl {
@ -42,9 +42,6 @@ public class TestFileSystemTimelineWriterImpl {
*/ */
@Test @Test
public void testWriteEntityToFile() throws Exception { public void testWriteEntityToFile() throws Exception {
String name = "unit_test_BaseAggregator_testWriteEntityToFile_"
+ Long.toString(System.currentTimeMillis());
TimelineEntities te = new TimelineEntities(); TimelineEntities te = new TimelineEntities();
TimelineEntity entity = new TimelineEntity(); TimelineEntity entity = new TimelineEntity();
String id = "hello"; String id = "hello";
@ -55,25 +52,27 @@ public void testWriteEntityToFile() throws Exception {
entity.setModifiedTime(1425016502000L); entity.setModifiedTime(1425016502000L);
te.addEntity(entity); te.addEntity(entity);
FileSystemTimelineWriterImpl fsi = new FileSystemTimelineWriterImpl(); try (FileSystemTimelineWriterImpl fsi =
fsi.serviceInit(new Configuration()); new FileSystemTimelineWriterImpl()) {
fsi.write(te); fsi.serviceInit(new Configuration());
fsi.write(te);
String fileName = fsi.getOutputRoot() + "/" + type + "/" + id String fileName = fsi.getOutputRoot() + "/" + type + "/" + id
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION; + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
Path path = Paths.get(fileName); Path path = Paths.get(fileName);
File f = new File(fileName); File f = new File(fileName);
assertTrue(f.exists() && !f.isDirectory()); assertTrue(f.exists() && !f.isDirectory());
List<String> data = Files.readAllLines(path, StandardCharsets.UTF_8); List<String> data = Files.readAllLines(path, StandardCharsets.UTF_8);
// ensure there's only one entity + 1 new line // ensure there's only one entity + 1 new line
assertTrue(data.size() == 2); assertTrue(data.size() == 2);
String d = data.get(0); String d = data.get(0);
// confirm the contents same as what was written // confirm the contents same as what was written
assertEquals(d, TimelineUtils.dumpTimelineRecordtoJSON(entity)); assertEquals(d, TimelineUtils.dumpTimelineRecordtoJSON(entity));
// delete the directory // delete the directory
File outputDir = new File(fsi.getOutputRoot()); File outputDir = new File(fsi.getOutputRoot());
FileUtils.deleteDirectory(outputDir); FileUtils.deleteDirectory(outputDir);
assertTrue(!(f.exists())); assertTrue(!(f.exists()));
}
} }
} }