big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-1633. Defined user-facing entity, entity-info and event objects related to Application Timeline feature. Contributed by Zhijie Shen.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1563356 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2014-02-01 00:09:53 +00:00
parent f6bfe98639
commit 6e8c1bf200
7 changed files with 862 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -77,6 +77,9 @@ Release 2.4.0 - UNRELEASED
YARN-1413. Implemented serving of aggregated-logs in the ApplicationHistory YARN-1413. Implemented serving of aggregated-logs in the ApplicationHistory
server. (Mayank Bansal via vinodkv) server. (Mayank Bansal via vinodkv)
YARN-1633. Defined user-facing entity, entity-info and event objects related
to Application Timeline feature. (Zhijie Shen via vinodkv)
IMPROVEMENTS IMPROVEMENTS
YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via

88
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntities.java Normal file
View File

@ -0,0 +1,88 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.api.records.apptimeline;
import java.util.ArrayList;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
/**
* The class that hosts a list of application timeline entities.
*/
@XmlRootElement(name = "entities")
@XmlAccessorType(XmlAccessType.NONE)
@Public
@Unstable
public class ATSEntities {
private List<ATSEntity> entities =
new ArrayList<ATSEntity>();
public ATSEntities() {
}
/**
* Get a list of entities
*
* @return a list of entities
*/
@XmlElement(name = "entities")
public List<ATSEntity> getEntities() {
return entities;
}
/**
* Add a single entity into the existing entity list
*
* @param entity
* a single entity
*/
public void addEntity(ATSEntity entity) {
entities.add(entity);
}
/**
* All a list of entities into the existing entity list
*
* @param entities
* a list of entities
*/
public void addEntities(List<ATSEntity> entities) {
this.entities.addAll(entities);
}
/**
* Set the entity list to the given list of entities
*
* @param entities
* a list of entities
*/
public void setEntities(List<ATSEntity> entities) {
this.entities = entities;
}
}

314
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEntity.java Normal file
View File

@ -0,0 +1,314 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.api.records.apptimeline;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
/**
* <p>
* The class that contains the the meta information of some conceptual entity of
* an application and its related events. The entity can be an application, an
* application attempt, a container or whatever the user-defined object.
* </p>
*
* <p>
* Primary filters will be used to index the entities in
* <code>ApplicationTimelineStore</code>, such that users should carefully
* choose the information they want to store as the primary filters. The
* remaining can be stored as other information.
* </p>
*/
@XmlRootElement(name = "entity")
@XmlAccessorType(XmlAccessType.NONE)
@Public
@Unstable
public class ATSEntity {
private String entityType;
private String entityId;
private long startTime;
private List<ATSEvent> events = new ArrayList<ATSEvent>();
private Map<String, List<Object>> relatedEntities =
new HashMap<String, List<Object>>();
private Map<String, Object> primaryFilters =
new HashMap<String, Object>();
private Map<String, Object> otherInfo =
new HashMap<String, Object>();
public ATSEntity() {
}
/**
* Get the entity type
*
* @return the entity type
*/
@XmlElement(name = "entitytype")
public String getEntityType() {
return entityType;
}
/**
* Set the entity type
*
* @param entityType
* the entity type
*/
public void setEntityType(String entityType) {
this.entityType = entityType;
}
/**
* Get the entity Id
*
* @return the entity Id
*/
@XmlElement(name = "entity")
public String getEntityId() {
return entityId;
}
/**
* Set the entity Id
*
* @param entityId
* the entity Id
*/
public void setEntityId(String entityId) {
this.entityId = entityId;
}
/**
* Get the start time of the entity
*
* @return the start time of the entity
*/
@XmlElement(name = "starttime")
public long getStartTime() {
return startTime;
}
/**
* Set the start time of the entity
*
* @param startTime
* the start time of the entity
*/
public void setStartTime(long startTime) {
this.startTime = startTime;
}
/**
* Get a list of events related to the entity
*
* @return a list of events related to the entity
*/
@XmlElement(name = "events")
public List<ATSEvent> getEvents() {
return events;
}
/**
* Add a single event related to the entity to the existing event list
*
* @param event
* a single event related to the entity
*/
public void addEvent(ATSEvent event) {
events.add(event);
}
/**
* Add a list of events related to the entity to the existing event list
*
* @param events
* a list of events related to the entity
*/
public void addEvents(List<ATSEvent> events) {
this.events.addAll(events);
}
/**
* Set the event list to the given list of events related to the entity
*
* @param events
* events a list of events related to the entity
*/
public void setEvents(List<ATSEvent> events) {
this.events = events;
}
/**
* Get the related entities
*
* @return the related entities
*/
@XmlElement(name = "relatedentities")
public Map<String, List<Object>> getRelatedEntities() {
return relatedEntities;
}
/**
* Add a list of entity of the same type to the existing related entity map
*
* @param entityType
* the entity type
* @param entityIds
* a list of entity Ids
*/
public void addRelatedEntity(String entityType, List<Object> entityIds) {
List<Object> thisRelatedEntity = relatedEntities.get(entityType);
relatedEntities.put(entityType, entityIds);
if (thisRelatedEntity == null) {
relatedEntities.put(entityType, entityIds);
} else {
thisRelatedEntity.addAll(entityIds);
}
}
/**
* Add a map of related entities to the existing related entity map
*
* @param relatedEntities
* a map of related entities
*/
public void addRelatedEntities(
Map<String, List<Object>> relatedEntities) {
for (Map.Entry<String, List<Object>> relatedEntity : relatedEntities
.entrySet()) {
List<Object> thisRelatedEntity =
this.relatedEntities.get(relatedEntity.getKey());
if (thisRelatedEntity == null) {
this.relatedEntities.put(
relatedEntity.getKey(), relatedEntity.getValue());
} else {
thisRelatedEntity.addAll(relatedEntity.getValue());
}
}
}
/**
* Set the related entity map to the given map of related entities
*
* @param relatedEntities
* a map of related entities
*/
public void setRelatedEntities(
Map<String, List<Object>> relatedEntities) {
this.relatedEntities = relatedEntities;
}
/**
* Get the primary filters
*
* @return the primary filters
*/
@XmlElement(name = "primaryfilters")
public Map<String, Object> getPrimaryFilters() {
return primaryFilters;
}
/**
* Add a single piece of primary filter to the existing primary filter map
*
* @param key
* the primary filter key
* @param value
* the primary filter value
*/
public void addPrimaryFilter(String key, Object value) {
primaryFilters.put(key, value);
}
/**
* Add a map of primary filters to the existing primary filter map
*
* @param primaryFilters
* a map of primary filters
*/
public void addPrimaryFilters(Map<String, Object> primaryFilters) {
this.primaryFilters.putAll(primaryFilters);
}
/**
* Set the primary filter map to the given map of primary filters
*
* @param primaryFilters
* a map of primary filters
*/
public void setPrimaryFilters(Map<String, Object> primaryFilters) {
this.primaryFilters = primaryFilters;
}
/**
* Get the other information of the entity
*
* @return the other information of the entity
*/
@XmlElement(name = "otherinfo")
public Map<String, Object> getOtherInfo() {
return otherInfo;
}
/**
* Add one piece of other information of the entity to the existing other info
* map
*
* @param key
* the other information key
* @param value
* the other information value
*/
public void addOtherInfo(String key, Object value) {
this.otherInfo.put(key, value);
}
/**
* Add a map of other information of the entity to the existing other info map
*
* @param otherInfo
* a map of other information
*/
public void addOtherInfo(Map<String, Object> otherInfo) {
this.otherInfo.putAll(otherInfo);
}
/**
* Set the other info map to the given map of other information
*
* @param otherInfo
* a map of other information
*/
public void setOtherInfo(Map<String, Object> otherInfo) {
this.otherInfo = otherInfo;
}
}

134
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvent.java Normal file
View File

@ -0,0 +1,134 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.api.records.apptimeline;
import java.util.HashMap;
import java.util.Map;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
/**
* The class that contains the information of an event that is related to some
* conceptual entity of an application. Users are free to define what the event
* means, such as starting an application, getting allocated a container and
* etc.
*/
@XmlRootElement(name = "event")
@XmlAccessorType(XmlAccessType.NONE)
@Public
@Unstable
public class ATSEvent {
private long timestamp;
private String eventType;
private Map<String, Object> eventInfo = new HashMap<String, Object>();
public ATSEvent() {
}
/**
* Get the timestamp of the event
*
* @return the timestamp of the event
*/
@XmlElement(name = "timestamp")
public long getTimestamp() {
return timestamp;
}
/**
* Set the timestamp of the event
*
* @param timestamp
* the timestamp of the event
*/
public void setTimestamp(long timestamp) {
this.timestamp = timestamp;
}
/**
* Get the event type
*
* @return the event type
*/
@XmlElement(name = "eventtype")
public String getEventType() {
return eventType;
}
/**
* Set the event type
*
* @param eventType
* the event type
*/
public void setEventType(String eventType) {
this.eventType = eventType;
}
/**
* Set the information of the event
*
* @return the information of the event
*/
@XmlElement(name = "eventinfo")
public Map<String, Object> getEventInfo() {
return eventInfo;
}
/**
* Add one piece of the information of the event to the existing information
* map
*
* @param key
* the information key
* @param value
* the information value
*/
public void addEventInfo(String key, Object value) {
this.eventInfo.put(key, value);
}
/**
* Add a map of the information of the event to the existing information map
*
* @param eventInfo
* a map of of the information of the event
*/
public void addEventInfo(Map<String, Object> eventInfo) {
this.eventInfo.putAll(eventInfo);
}
/**
* Set the information map to the given map of the information of the event
*
* @param eventInfo
* a map of of the information of the event
*/
public void setEventInfo(Map<String, Object> eventInfo) {
this.eventInfo = eventInfo;
}
}

189
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/ATSEvents.java Normal file
View File

@ -0,0 +1,189 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.api.records.apptimeline;
import java.util.ArrayList;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
/**
* The class that hosts a list of events, which are categorized according to
* their related entities.
*/
@XmlRootElement(name = "events")
@XmlAccessorType(XmlAccessType.NONE)
@Public
@Unstable
public class ATSEvents {
private List<ATSEventsOfOneEntity> allEvents =
new ArrayList<ATSEventsOfOneEntity>();
public ATSEvents() {
}
/**
* Get a list of {@link ATSEventsOfOneEntity} instances
*
* @return a list of {@link ATSEventsOfOneEntity} instances
*/
@XmlElement(name = "events")
public List<ATSEventsOfOneEntity> getAllEvents() {
return allEvents;
}
/**
* Add a single {@link ATSEventsOfOneEntity} instance into the existing list
*
* @param eventsOfOneEntity
* a single {@link ATSEventsOfOneEntity} instance
*/
public void addEvent(ATSEventsOfOneEntity eventsOfOneEntity) {
allEvents.add(eventsOfOneEntity);
}
/**
* Add a list of {@link ATSEventsOfOneEntity} instances into the existing list
*
* @param allEvents
* a list of {@link ATSEventsOfOneEntity} instances
*/
public void addEvents(List<ATSEventsOfOneEntity> allEvents) {
this.allEvents.addAll(allEvents);
}
/**
* Set the list to the given list of {@link ATSEventsOfOneEntity} instances
*
* @param allEvents
* a list of {@link ATSEventsOfOneEntity} instances
*/
public void setEvents(List<ATSEventsOfOneEntity> allEvents) {
this.allEvents.clear();
this.allEvents.addAll(allEvents);
}
/**
* The class that hosts a list of events that are only related to one entity.
*/
@XmlRootElement(name = "events")
@XmlAccessorType(XmlAccessType.NONE)
@Public
@Unstable
public static class ATSEventsOfOneEntity {
private String entityId;
private String entityType;
private List<ATSEvent> events = new ArrayList<ATSEvent>();
public ATSEventsOfOneEntity() {
}
/**
* Get the entity Id
*
* @return the entity Id
*/
@XmlElement(name = "entity")
public String getEntityId() {
return entityId;
}
/**
* Set the entity Id
*
* @param entityId
* the entity Id
*/
public void setEntityId(String entityId) {
this.entityId = entityId;
}
/**
* Get the entity type
*
* @return the entity type
*/
@XmlElement(name = "entitytype")
public String getEntityType() {
return entityType;
}
/**
* Set the entity type
*
* @param entityType
* the entity type
*/
public void setEntityType(String entityType) {
this.entityType = entityType;
}
/**
* Get a list of events
*
* @return a list of events
*/
@XmlElement(name = "events")
public List<ATSEvent> getEvents() {
return events;
}
/**
* Add a single event to the existing event list
*
* @param event
* a single event
*/
public void addEntity(ATSEvent event) {
events.add(event);
}
/**
* Add a list of event to the existing event list
*
* @param events
* a list of events
*/
public void addEvents(List<ATSEvent> events) {
this.events.addAll(events);
}
/**
* Set the event list to the given list of events
*
* @param events
* a list of events
*/
public void setEvents(List<ATSEvent> events) {
this.events = events;
}
}
}

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/package-info.java Normal file
View File

@ -0,0 +1,21 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@InterfaceAudience.Public
package org.apache.hadoop.yarn.api.records.apptimeline;
import org.apache.hadoop.classification.InterfaceAudience;

113
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java Normal file
View File

@ -0,0 +1,113 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.api.records.apptimeline;
import java.util.Arrays;
import junit.framework.Assert;
import org.junit.Test;
public class TestApplicationTimelineRecords {
@Test
public void testATSEntities() {
ATSEntities entities = new ATSEntities();
for (int j = 0; j < 2; ++j) {
ATSEntity entity = new ATSEntity();
entity.setEntityId("entity id " + j);
entity.setEntityType("entity type " + j);
entity.setStartTime(System.currentTimeMillis());
for (int i = 0; i < 2; ++i) {
ATSEvent event = new ATSEvent();
event.setTimestamp(System.currentTimeMillis());
event.setEventType("event type " + i);
event.addEventInfo("key1", "val1");
event.addEventInfo("key2", "val2");
entity.addEvent(event);
}
entity.addRelatedEntity(
"test ref type 1", Arrays.asList((Object) "test ref id 1"));
entity.addRelatedEntity(
"test ref type 2", Arrays.asList((Object) "test ref id 2"));
entity.addPrimaryFilter("pkey1", "pval1");
entity.addPrimaryFilter("pkey2", "pval2");
entity.addOtherInfo("okey1", "oval1");
entity.addOtherInfo("okey2", "oval2");
entities.addEntity(entity);
}
Assert.assertEquals(2, entities.getEntities().size());
ATSEntity entity1 = entities.getEntities().get(0);
Assert.assertEquals("entity id 0", entity1.getEntityId());
Assert.assertEquals("entity type 0", entity1.getEntityType());
Assert.assertEquals(2, entity1.getRelatedEntities().size());
Assert.assertEquals(2, entity1.getEvents().size());
Assert.assertEquals(2, entity1.getPrimaryFilters().size());
Assert.assertEquals(2, entity1.getOtherInfo().size());
ATSEntity entity2 = entities.getEntities().get(1);
Assert.assertEquals("entity id 1", entity2.getEntityId());
Assert.assertEquals("entity type 1", entity2.getEntityType());
Assert.assertEquals(2, entity2.getRelatedEntities().size());
Assert.assertEquals(2, entity2.getEvents().size());
Assert.assertEquals(2, entity2.getPrimaryFilters().size());
Assert.assertEquals(2, entity2.getOtherInfo().size());
}
@Test
public void testATSEvents() {
ATSEvents events = new ATSEvents();
for (int j = 0; j < 2; ++j) {
ATSEvents.ATSEventsOfOneEntity partEvents =
new ATSEvents.ATSEventsOfOneEntity();
partEvents.setEntityId("entity id " + j);
partEvents.setEntityType("entity type " + j);
for (int i = 0; i < 2; ++i) {
ATSEvent event = new ATSEvent();
event.setTimestamp(System.currentTimeMillis());
event.setEventType("event type " + i);
event.addEventInfo("key1", "val1");
event.addEventInfo("key2", "val2");
partEvents.addEntity(event);
}
events.addEvent(partEvents);
}
Assert.assertEquals(2, events.getAllEvents().size());
ATSEvents.ATSEventsOfOneEntity partEvents1 = events.getAllEvents().get(0);
Assert.assertEquals("entity id 0", partEvents1.getEntityId());
Assert.assertEquals("entity type 0", partEvents1.getEntityType());
Assert.assertEquals(2, partEvents1.getEvents().size());
ATSEvent event11 = partEvents1.getEvents().get(0);
Assert.assertEquals("event type 0", event11.getEventType());
Assert.assertEquals(2, event11.getEventInfo().size());
ATSEvent event12 = partEvents1.getEvents().get(1);
Assert.assertEquals("event type 1", event12.getEventType());
Assert.assertEquals(2, event12.getEventInfo().size());
ATSEvents.ATSEventsOfOneEntity partEvents2 = events.getAllEvents().get(1);
Assert.assertEquals("entity id 1", partEvents2.getEntityId());
Assert.assertEquals("entity type 1", partEvents2.getEntityType());
Assert.assertEquals(2, partEvents2.getEvents().size());
ATSEvent event21 = partEvents2.getEvents().get(0);
Assert.assertEquals("event type 0", event21.getEventType());
Assert.assertEquals(2, event21.getEventInfo().size());
ATSEvent event22 = partEvents2.getEvents().get(1);
Assert.assertEquals("event type 1", event22.getEventType());
Assert.assertEquals(2, event22.getEventInfo().size());
}
}