Made a number of miscellaneous fixes for javac, javadoc, and checstyle warnings.
This commit is contained in:
parent
c5dbde0cc4
commit
6cf6ab7b78
@ -46,7 +46,6 @@
|
|||||||
import org.apache.hadoop.mapred.JobConf;
|
import org.apache.hadoop.mapred.JobConf;
|
||||||
import org.apache.hadoop.mapred.TaskStatus;
|
import org.apache.hadoop.mapred.TaskStatus;
|
||||||
import org.apache.hadoop.mapreduce.Counter;
|
import org.apache.hadoop.mapreduce.Counter;
|
||||||
import org.apache.hadoop.mapreduce.CounterGroup;
|
|
||||||
import org.apache.hadoop.mapreduce.Counters;
|
import org.apache.hadoop.mapreduce.Counters;
|
||||||
import org.apache.hadoop.mapreduce.JobCounter;
|
import org.apache.hadoop.mapreduce.JobCounter;
|
||||||
import org.apache.hadoop.mapreduce.MRJobConfig;
|
import org.apache.hadoop.mapreduce.MRJobConfig;
|
||||||
@ -76,11 +75,8 @@
|
|||||||
import org.apache.hadoop.yarn.event.EventHandler;
|
import org.apache.hadoop.yarn.event.EventHandler;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||||
import org.codehaus.jackson.JsonNode;
|
|
||||||
import org.codehaus.jackson.map.ObjectMapper;
|
import org.codehaus.jackson.map.ObjectMapper;
|
||||||
import org.codehaus.jackson.node.ArrayNode;
|
|
||||||
import org.codehaus.jackson.node.JsonNodeFactory;
|
import org.codehaus.jackson.node.JsonNodeFactory;
|
||||||
import org.codehaus.jackson.node.ObjectNode;
|
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import com.sun.jersey.api.client.ClientHandlerException;
|
import com.sun.jersey.api.client.ClientHandlerException;
|
||||||
|
@ -829,7 +829,7 @@ private List<Container> getResources() throws Exception {
|
|||||||
if (collectorAddr != null && !collectorAddr.isEmpty()
|
if (collectorAddr != null && !collectorAddr.isEmpty()
|
||||||
&& appContext.getTimelineClient() != null) {
|
&& appContext.getTimelineClient() != null) {
|
||||||
appContext.getTimelineClient().setTimelineServiceAddress(
|
appContext.getTimelineClient().setTimelineServiceAddress(
|
||||||
response.getCollectorAddr());
|
response.getCollectorAddr());
|
||||||
}
|
}
|
||||||
|
|
||||||
for (ContainerStatus cont : finishedContainers) {
|
for (ContainerStatus cont : finishedContainers) {
|
||||||
|
@ -786,7 +786,8 @@ public TestParams() {
|
|||||||
public TestParams(boolean isLastAMRetry) {
|
public TestParams(boolean isLastAMRetry) {
|
||||||
this(AppContext.class, isLastAMRetry);
|
this(AppContext.class, isLastAMRetry);
|
||||||
}
|
}
|
||||||
public TestParams(Class<? extends AppContext> contextClass, boolean isLastAMRetry) {
|
public TestParams(Class<? extends AppContext> contextClass,
|
||||||
|
boolean isLastAMRetry) {
|
||||||
this.isLastAMRetry = isLastAMRetry;
|
this.isLastAMRetry = isLastAMRetry;
|
||||||
mockAppContext = mockAppContext(contextClass, appId, this.isLastAMRetry);
|
mockAppContext = mockAppContext(contextClass, appId, this.isLastAMRetry);
|
||||||
}
|
}
|
||||||
|
@ -42,9 +42,17 @@ public interface HistoryEvent {
|
|||||||
/** Set the Avro datum wrapped by this. */
|
/** Set the Avro datum wrapped by this. */
|
||||||
void setDatum(Object datum);
|
void setDatum(Object datum);
|
||||||
|
|
||||||
/** Map HistoryEvent to TimelineEvent */
|
/**
|
||||||
|
* Map HistoryEvent to TimelineEvent.
|
||||||
|
*
|
||||||
|
* @return the timeline event
|
||||||
|
*/
|
||||||
TimelineEvent toTimelineEvent();
|
TimelineEvent toTimelineEvent();
|
||||||
|
|
||||||
/** Counters or Metrics if any else return null. */
|
/**
|
||||||
|
* Counters or Metrics if any else return null.
|
||||||
|
*
|
||||||
|
* @return the set of timeline metrics
|
||||||
|
*/
|
||||||
Set<TimelineMetric> getTimelineMetrics();
|
Set<TimelineMetric> getTimelineMetrics();
|
||||||
}
|
}
|
||||||
|
@ -64,7 +64,7 @@ public class TestMRTimelineEventHandling {
|
|||||||
|
|
||||||
private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";
|
private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";
|
||||||
private static final Log LOG =
|
private static final Log LOG =
|
||||||
LogFactory.getLog(TestMRTimelineEventHandling.class);
|
LogFactory.getLog(TestMRTimelineEventHandling.class);
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testTimelineServiceStartInMiniCluster() throws Exception {
|
public void testTimelineServiceStartInMiniCluster() throws Exception {
|
||||||
@ -169,6 +169,7 @@ public void testMRTimelineEventHandling() throws Exception {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("deprecation")
|
||||||
@Test
|
@Test
|
||||||
public void testMRNewTimelineServiceEventHandling() throws Exception {
|
public void testMRNewTimelineServiceEventHandling() throws Exception {
|
||||||
LOG.info("testMRNewTimelineServiceEventHandling start.");
|
LOG.info("testMRNewTimelineServiceEventHandling start.");
|
||||||
@ -183,7 +184,7 @@ public void testMRNewTimelineServiceEventHandling() throws Exception {
|
|||||||
// enable aux-service based timeline collectors
|
// enable aux-service based timeline collectors
|
||||||
conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
|
conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
|
||||||
conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME
|
conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME
|
||||||
+ ".class", PerNodeTimelineCollectorsAuxService.class.getName());
|
+ ".class", PerNodeTimelineCollectorsAuxService.class.getName());
|
||||||
|
|
||||||
conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
|
conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
|
||||||
|
|
||||||
@ -245,7 +246,8 @@ public void testMRNewTimelineServiceEventHandling() throws Exception {
|
|||||||
}
|
}
|
||||||
// Cleanup test file
|
// Cleanup test file
|
||||||
String testRoot =
|
String testRoot =
|
||||||
FileSystemTimelineWriterImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT;
|
FileSystemTimelineWriterImpl.
|
||||||
|
DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT;
|
||||||
File testRootFolder = new File(testRoot);
|
File testRootFolder = new File(testRoot);
|
||||||
if(testRootFolder.isDirectory()) {
|
if(testRootFolder.isDirectory()) {
|
||||||
FileUtils.deleteDirectory(testRootFolder);
|
FileUtils.deleteDirectory(testRootFolder);
|
||||||
@ -320,8 +322,10 @@ private void checkNewTimelineEvent(ApplicationId appId,
|
|||||||
" does not exist.",
|
" does not exist.",
|
||||||
taskFolder.isDirectory());
|
taskFolder.isDirectory());
|
||||||
|
|
||||||
String taskEventFileName = appId.toString().replaceAll("application", "task")
|
String taskEventFileName =
|
||||||
+ "_m_000000" + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
|
appId.toString().replaceAll("application", "task") +
|
||||||
|
"_m_000000" +
|
||||||
|
FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
|
||||||
|
|
||||||
String taskEventFilePath = outputDirTask + taskEventFileName;
|
String taskEventFilePath = outputDirTask + taskEventFileName;
|
||||||
File taskEventFile = new File(taskEventFilePath);
|
File taskEventFile = new File(taskEventFilePath);
|
||||||
@ -372,10 +376,12 @@ private void verifyEntity(File entityFile, String eventId,
|
|||||||
reader = new BufferedReader(new FileReader(entityFile));
|
reader = new BufferedReader(new FileReader(entityFile));
|
||||||
while ((strLine = reader.readLine()) != null) {
|
while ((strLine = reader.readLine()) != null) {
|
||||||
if (strLine.trim().length() > 0) {
|
if (strLine.trim().length() > 0) {
|
||||||
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
|
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
|
||||||
FileSystemTimelineReaderImpl.getTimelineRecordFromJSON(
|
entity =
|
||||||
strLine.trim(),
|
FileSystemTimelineReaderImpl.getTimelineRecordFromJSON(
|
||||||
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.class);
|
strLine.trim(),
|
||||||
|
org.apache.hadoop.yarn.api.records.timelineservice.
|
||||||
|
TimelineEntity.class);
|
||||||
if (eventId == null) {
|
if (eventId == null) {
|
||||||
// Job metrics are published without any events for
|
// Job metrics are published without any events for
|
||||||
// ApplicationEntity. There is also possibility that some other
|
// ApplicationEntity. There is also possibility that some other
|
||||||
|
@ -621,8 +621,8 @@ static RunningJob runJob(JobConf conf, Path inDir, Path outDir, int numMaps,
|
|||||||
return job;
|
return job;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void waitForAppFinished(RunningJob job, MiniMRYarnCluster cluster)
|
public static void waitForAppFinished(RunningJob job,
|
||||||
throws IOException {
|
MiniMRYarnCluster cluster) throws IOException {
|
||||||
ApplicationId appId = ApplicationId.newInstance(
|
ApplicationId appId = ApplicationId.newInstance(
|
||||||
Long.parseLong(job.getID().getJtIdentifier()), job.getID().getId());
|
Long.parseLong(job.getID().getJtIdentifier()), job.getID().getId());
|
||||||
ConcurrentMap<ApplicationId, RMApp> rmApps =
|
ConcurrentMap<ApplicationId, RMApp> rmApps =
|
||||||
|
@ -33,7 +33,8 @@
|
|||||||
* to the timeline service.
|
* to the timeline service.
|
||||||
*/
|
*/
|
||||||
abstract class EntityWriterV2
|
abstract class EntityWriterV2
|
||||||
extends org.apache.hadoop.mapreduce.Mapper<IntWritable,IntWritable,Writable,Writable> {
|
extends org.apache.hadoop.mapreduce.Mapper
|
||||||
|
<IntWritable, IntWritable, Writable, Writable> {
|
||||||
@Override
|
@Override
|
||||||
public void map(IntWritable key, IntWritable val, Context context)
|
public void map(IntWritable key, IntWritable val, Context context)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
@ -89,8 +89,8 @@ protected void writeEntities(Configuration tlConf,
|
|||||||
parser.parseHistoryFile(job.getJobHistoryFilePath());
|
parser.parseHistoryFile(job.getJobHistoryFilePath());
|
||||||
Configuration jobConf =
|
Configuration jobConf =
|
||||||
parser.parseConfiguration(job.getJobConfFilePath());
|
parser.parseConfiguration(job.getJobConfFilePath());
|
||||||
LOG.info("parsed the job history file and the configuration file for job"
|
LOG.info("parsed the job history file and the configuration file " +
|
||||||
+ jobIdStr);
|
"for job " + jobIdStr);
|
||||||
|
|
||||||
// set the context
|
// set the context
|
||||||
// flow id: job name, flow run id: timestamp, user id
|
// flow id: job name, flow run id: timestamp, user id
|
||||||
|
@ -32,12 +32,12 @@ interface SimpleEntityWriterConstants {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* To ensure that the compression really gets exercised, generate a
|
* To ensure that the compression really gets exercised, generate a
|
||||||
* random alphanumeric fixed length payload
|
* random alphanumeric fixed length payload.
|
||||||
*/
|
*/
|
||||||
char[] ALPHA_NUMS = new char[] { 'a', 'b', 'c', 'd', 'e', 'f',
|
char[] ALPHA_NUMS = new char[] {'a', 'b', 'c', 'd', 'e', 'f',
|
||||||
'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
|
'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
|
||||||
's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D',
|
's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D',
|
||||||
'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
|
'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
|
||||||
'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2',
|
'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2',
|
||||||
'3', '4', '5', '6', '7', '8', '9', '0', ' ' };
|
'3', '4', '5', '6', '7', '8', '9', '0', ' '};
|
||||||
}
|
}
|
||||||
|
@ -39,11 +39,13 @@
|
|||||||
* configuration.
|
* configuration.
|
||||||
*/
|
*/
|
||||||
class SimpleEntityWriterV1
|
class SimpleEntityWriterV1
|
||||||
extends org.apache.hadoop.mapreduce.Mapper<IntWritable,IntWritable,Writable,Writable>
|
extends org.apache.hadoop.mapreduce.Mapper
|
||||||
|
<IntWritable, IntWritable, Writable, Writable>
|
||||||
implements SimpleEntityWriterConstants {
|
implements SimpleEntityWriterConstants {
|
||||||
private static final Log LOG = LogFactory.getLog(SimpleEntityWriterV1.class);
|
private static final Log LOG = LogFactory.getLog(SimpleEntityWriterV1.class);
|
||||||
|
|
||||||
public void map(IntWritable key, IntWritable val, Context context) throws IOException {
|
public void map(IntWritable key, IntWritable val, Context context)
|
||||||
|
throws IOException {
|
||||||
TimelineClient tlc = new TimelineClientImpl();
|
TimelineClient tlc = new TimelineClientImpl();
|
||||||
Configuration conf = context.getConfiguration();
|
Configuration conf = context.getConfiguration();
|
||||||
|
|
||||||
|
@ -90,9 +90,10 @@ private TimelineEntity createJobEntity(JobInfo jobInfo, Configuration conf) {
|
|||||||
return job;
|
return job;
|
||||||
}
|
}
|
||||||
|
|
||||||
private Set<TimelineEntity> createTaskAndTaskAttemptEntities(JobInfo jobInfo) {
|
private Set<TimelineEntity>
|
||||||
|
createTaskAndTaskAttemptEntities(JobInfo jobInfo) {
|
||||||
Set<TimelineEntity> entities = new HashSet<>();
|
Set<TimelineEntity> entities = new HashSet<>();
|
||||||
Map<TaskID,TaskInfo> taskInfoMap = jobInfo.getAllTasks();
|
Map<TaskID, TaskInfo> taskInfoMap = jobInfo.getAllTasks();
|
||||||
LOG.info("job " + jobInfo.getJobId()+ " has " + taskInfoMap.size() +
|
LOG.info("job " + jobInfo.getJobId()+ " has " + taskInfoMap.size() +
|
||||||
" tasks");
|
" tasks");
|
||||||
for (TaskInfo taskInfo: taskInfoMap.values()) {
|
for (TaskInfo taskInfo: taskInfoMap.values()) {
|
||||||
@ -124,7 +125,7 @@ private TimelineEntity createTaskEntity(TaskInfo taskInfo) {
|
|||||||
|
|
||||||
private Set<TimelineEntity> createTaskAttemptEntities(TaskInfo taskInfo) {
|
private Set<TimelineEntity> createTaskAttemptEntities(TaskInfo taskInfo) {
|
||||||
Set<TimelineEntity> taskAttempts = new HashSet<TimelineEntity>();
|
Set<TimelineEntity> taskAttempts = new HashSet<TimelineEntity>();
|
||||||
Map<TaskAttemptID,TaskAttemptInfo> taskAttemptInfoMap =
|
Map<TaskAttemptID, TaskAttemptInfo> taskAttemptInfoMap =
|
||||||
taskInfo.getAllTaskAttempts();
|
taskInfo.getAllTaskAttempts();
|
||||||
LOG.info("task " + taskInfo.getTaskId() + " has " +
|
LOG.info("task " + taskInfo.getTaskId() + " has " +
|
||||||
taskAttemptInfoMap.size() + " task attempts");
|
taskAttemptInfoMap.size() + " task attempts");
|
||||||
@ -135,7 +136,8 @@ private Set<TimelineEntity> createTaskAttemptEntities(TaskInfo taskInfo) {
|
|||||||
return taskAttempts;
|
return taskAttempts;
|
||||||
}
|
}
|
||||||
|
|
||||||
private TimelineEntity createTaskAttemptEntity(TaskAttemptInfo taskAttemptInfo) {
|
private TimelineEntity
|
||||||
|
createTaskAttemptEntity(TaskAttemptInfo taskAttemptInfo) {
|
||||||
TimelineEntity taskAttempt = new TimelineEntity();
|
TimelineEntity taskAttempt = new TimelineEntity();
|
||||||
taskAttempt.setEntityType(TASK_ATTEMPT);
|
taskAttempt.setEntityType(TASK_ATTEMPT);
|
||||||
taskAttempt.setEntityId(taskAttemptInfo.getAttemptId().toString());
|
taskAttempt.setEntityId(taskAttemptInfo.getAttemptId().toString());
|
||||||
|
@ -27,11 +27,6 @@
|
|||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.mapreduce.Counter;
|
|
||||||
import org.apache.hadoop.mapreduce.CounterGroup;
|
|
||||||
import org.apache.hadoop.mapreduce.Counters;
|
|
||||||
import org.apache.hadoop.mapreduce.TaskAttemptID;
|
|
||||||
import org.apache.hadoop.mapreduce.TaskID;
|
|
||||||
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
|
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
|
||||||
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
|
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
|
||||||
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
|
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
|
||||||
@ -109,7 +104,7 @@ private TimelineEntity createJobEntity(JobInfo jobInfo, Configuration conf) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void addConfiguration(TimelineEntity job, Configuration conf) {
|
private void addConfiguration(TimelineEntity job, Configuration conf) {
|
||||||
for (Map.Entry<String,String> e: conf) {
|
for (Map.Entry<String, String> e: conf) {
|
||||||
job.addConfig(e.getKey(), e.getValue());
|
job.addConfig(e.getKey(), e.getValue());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -130,7 +125,7 @@ private void addMetrics(TimelineEntity entity, Counters counters) {
|
|||||||
private List<TimelineEntity> createTaskAndTaskAttemptEntities(
|
private List<TimelineEntity> createTaskAndTaskAttemptEntities(
|
||||||
JobInfo jobInfo) {
|
JobInfo jobInfo) {
|
||||||
List<TimelineEntity> entities = new ArrayList<>();
|
List<TimelineEntity> entities = new ArrayList<>();
|
||||||
Map<TaskID,TaskInfo> taskInfoMap = jobInfo.getAllTasks();
|
Map<TaskID, TaskInfo> taskInfoMap = jobInfo.getAllTasks();
|
||||||
LOG.info("job " + jobInfo.getJobId()+ " has " + taskInfoMap.size() +
|
LOG.info("job " + jobInfo.getJobId()+ " has " + taskInfoMap.size() +
|
||||||
" tasks");
|
" tasks");
|
||||||
for (TaskInfo taskInfo: taskInfoMap.values()) {
|
for (TaskInfo taskInfo: taskInfoMap.values()) {
|
||||||
@ -167,7 +162,7 @@ private TimelineEntity createTaskEntity(TaskInfo taskInfo) {
|
|||||||
|
|
||||||
private Set<TimelineEntity> createTaskAttemptEntities(TaskInfo taskInfo) {
|
private Set<TimelineEntity> createTaskAttemptEntities(TaskInfo taskInfo) {
|
||||||
Set<TimelineEntity> taskAttempts = new HashSet<TimelineEntity>();
|
Set<TimelineEntity> taskAttempts = new HashSet<TimelineEntity>();
|
||||||
Map<TaskAttemptID,TaskAttemptInfo> taskAttemptInfoMap =
|
Map<TaskAttemptID, TaskAttemptInfo> taskAttemptInfoMap =
|
||||||
taskInfo.getAllTaskAttempts();
|
taskInfo.getAllTaskAttempts();
|
||||||
LOG.info("task " + taskInfo.getTaskId() + " has " +
|
LOG.info("task " + taskInfo.getTaskId() + " has " +
|
||||||
taskAttemptInfoMap.size() + " task attempts");
|
taskAttemptInfoMap.size() + " task attempts");
|
||||||
|
@ -137,7 +137,8 @@ public static boolean parseArgs(String[] args, Job job) throws IOException {
|
|||||||
default:
|
default:
|
||||||
// use the current timestamp as the "run id" of the test: this will
|
// use the current timestamp as the "run id" of the test: this will
|
||||||
// be used as simulating the cluster timestamp for apps
|
// be used as simulating the cluster timestamp for apps
|
||||||
conf.setLong(SimpleEntityWriterConstants.TIMELINE_SERVICE_PERFORMANCE_RUN_ID,
|
conf.setLong(
|
||||||
|
SimpleEntityWriterConstants.TIMELINE_SERVICE_PERFORMANCE_RUN_ID,
|
||||||
System.currentTimeMillis());
|
System.currentTimeMillis());
|
||||||
switch (timeline_service_version) {
|
switch (timeline_service_version) {
|
||||||
case TIMELINE_SERVICE_VERSION_2:
|
case TIMELINE_SERVICE_VERSION_2:
|
||||||
|
@ -181,10 +181,11 @@ public void serviceInit(Configuration conf) throws Exception {
|
|||||||
}
|
}
|
||||||
if (enableTimelineAuxService) {
|
if (enableTimelineAuxService) {
|
||||||
conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
|
conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
|
||||||
new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, TIMELINE_AUX_SERVICE_NAME });
|
new String[] {ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID,
|
||||||
|
TIMELINE_AUX_SERVICE_NAME});
|
||||||
} else {
|
} else {
|
||||||
conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
|
conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
|
||||||
new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
|
new String[] {ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID});
|
||||||
}
|
}
|
||||||
conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT,
|
conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT,
|
||||||
ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class,
|
ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class,
|
||||||
|
@ -24,4 +24,3 @@
|
|||||||
package org.apache.hadoop.yarn.api.records.timelineservice;
|
package org.apache.hadoop.yarn.api.records.timelineservice;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
|
||||||
|
@ -500,7 +500,8 @@ public static boolean isAclEnabled(Configuration conf) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* The setting that controls whether yarn system metrics is published on the
|
* The setting that controls whether yarn system metrics is published on the
|
||||||
* timeline server or not by RM and NM. This configuration setting is for ATS V2
|
* timeline server or not by RM and NM. This configuration setting is for
|
||||||
|
* ATS v2.
|
||||||
*/
|
*/
|
||||||
public static final String SYSTEM_METRICS_PUBLISHER_ENABLED = YARN_PREFIX
|
public static final String SYSTEM_METRICS_PUBLISHER_ENABLED = YARN_PREFIX
|
||||||
+ "system-metrics-publisher.enabled";
|
+ "system-metrics-publisher.enabled";
|
||||||
@ -840,7 +841,7 @@ public static boolean isAclEnabled(Configuration conf) {
|
|||||||
|
|
||||||
/** Number of threads container manager uses.*/
|
/** Number of threads container manager uses.*/
|
||||||
public static final String NM_COLLECTOR_SERVICE_THREAD_COUNT =
|
public static final String NM_COLLECTOR_SERVICE_THREAD_COUNT =
|
||||||
NM_PREFIX + "collector-service.thread-count";
|
NM_PREFIX + "collector-service.thread-count";
|
||||||
public static final int DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT = 5;
|
public static final int DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT = 5;
|
||||||
|
|
||||||
/** Number of threads used in cleanup.*/
|
/** Number of threads used in cleanup.*/
|
||||||
@ -872,7 +873,7 @@ public static boolean isAclEnabled(Configuration conf) {
|
|||||||
|
|
||||||
/** Address where the collector service IPC is.*/
|
/** Address where the collector service IPC is.*/
|
||||||
public static final String NM_COLLECTOR_SERVICE_ADDRESS =
|
public static final String NM_COLLECTOR_SERVICE_ADDRESS =
|
||||||
NM_PREFIX + "collector-service.address";
|
NM_PREFIX + "collector-service.address";
|
||||||
public static final int DEFAULT_NM_COLLECTOR_SERVICE_PORT = 8048;
|
public static final int DEFAULT_NM_COLLECTOR_SERVICE_PORT = 8048;
|
||||||
public static final String DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS =
|
public static final String DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS =
|
||||||
"0.0.0.0:" + DEFAULT_NM_LOCALIZER_PORT;
|
"0.0.0.0:" + DEFAULT_NM_LOCALIZER_PORT;
|
||||||
|
@ -36,6 +36,8 @@ private TimelineServiceHelper() {
|
|||||||
/**
|
/**
|
||||||
* Cast map to HashMap for generic type.
|
* Cast map to HashMap for generic type.
|
||||||
* @param originalMap the map need to be casted
|
* @param originalMap the map need to be casted
|
||||||
|
* @param <E> key type
|
||||||
|
* @param <V> value type
|
||||||
* @return casted HashMap object
|
* @return casted HashMap object
|
||||||
*/
|
*/
|
||||||
public static <E, V> HashMap<E, V> mapCastToHashMap(
|
public static <E, V> HashMap<E, V> mapCastToHashMap(
|
||||||
|
@ -102,7 +102,6 @@
|
|||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
|
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
|
||||||
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
|
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
|
||||||
import org.apache.log4j.LogManager;
|
import org.apache.log4j.LogManager;
|
||||||
|
|
||||||
@ -747,7 +746,7 @@ protected boolean finish() {
|
|||||||
DSEvent.DS_APP_ATTEMPT_END);
|
DSEvent.DS_APP_ATTEMPT_END);
|
||||||
} else {
|
} else {
|
||||||
publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
|
publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
|
||||||
DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi);
|
DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -858,7 +857,7 @@ public void onContainersCompleted(List<ContainerStatus> completedContainers) {
|
|||||||
publishContainerEndEventOnTimelineServiceV2(containerStatus);
|
publishContainerEndEventOnTimelineServiceV2(containerStatus);
|
||||||
} else {
|
} else {
|
||||||
publishContainerEndEvent(
|
publishContainerEndEvent(
|
||||||
timelineClient, containerStatus, domainId, appSubmitterUgi);
|
timelineClient, containerStatus, domainId, appSubmitterUgi);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -988,8 +987,8 @@ public void onContainerStarted(ContainerId containerId,
|
|||||||
container);
|
container);
|
||||||
} else {
|
} else {
|
||||||
applicationMaster.publishContainerStartEvent(
|
applicationMaster.publishContainerStartEvent(
|
||||||
applicationMaster.timelineClient, container,
|
applicationMaster.timelineClient, container,
|
||||||
applicationMaster.domainId, applicationMaster.appSubmitterUgi);
|
applicationMaster.domainId, applicationMaster.appSubmitterUgi);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1348,8 +1347,10 @@ Thread createLaunchContainerThread(Container allocatedContainer,
|
|||||||
|
|
||||||
private void publishContainerStartEventOnTimelineServiceV2(
|
private void publishContainerStartEventOnTimelineServiceV2(
|
||||||
Container container) {
|
Container container) {
|
||||||
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
|
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
|
||||||
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
|
entity =
|
||||||
|
new org.apache.hadoop.yarn.api.records.timelineservice.
|
||||||
|
TimelineEntity();
|
||||||
entity.setId(container.getId().toString());
|
entity.setId(container.getId().toString());
|
||||||
entity.setType(DSEntity.DS_CONTAINER.toString());
|
entity.setType(DSEntity.DS_CONTAINER.toString());
|
||||||
long ts = System.currentTimeMillis();
|
long ts = System.currentTimeMillis();
|
||||||
@ -1381,8 +1382,10 @@ public TimelinePutResponse run() throws Exception {
|
|||||||
|
|
||||||
private void publishContainerEndEventOnTimelineServiceV2(
|
private void publishContainerEndEventOnTimelineServiceV2(
|
||||||
final ContainerStatus container) {
|
final ContainerStatus container) {
|
||||||
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
|
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
|
||||||
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
|
entity =
|
||||||
|
new org.apache.hadoop.yarn.api.records.timelineservice.
|
||||||
|
TimelineEntity();
|
||||||
entity.setId(container.getContainerId().toString());
|
entity.setId(container.getContainerId().toString());
|
||||||
entity.setType(DSEntity.DS_CONTAINER.toString());
|
entity.setType(DSEntity.DS_CONTAINER.toString());
|
||||||
//entity.setDomainId(domainId);
|
//entity.setDomainId(domainId);
|
||||||
@ -1412,8 +1415,10 @@ public TimelinePutResponse run() throws Exception {
|
|||||||
|
|
||||||
private void publishApplicationAttemptEventOnTimelineServiceV2(
|
private void publishApplicationAttemptEventOnTimelineServiceV2(
|
||||||
DSEvent appEvent) {
|
DSEvent appEvent) {
|
||||||
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
|
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
|
||||||
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
|
entity =
|
||||||
|
new org.apache.hadoop.yarn.api.records.timelineservice.
|
||||||
|
TimelineEntity();
|
||||||
entity.setId(appAttemptID.toString());
|
entity.setId(appAttemptID.toString());
|
||||||
entity.setType(DSEntity.DS_APP_ATTEMPT.toString());
|
entity.setType(DSEntity.DS_APP_ATTEMPT.toString());
|
||||||
long ts = System.currentTimeMillis();
|
long ts = System.currentTimeMillis();
|
||||||
|
@ -153,7 +153,8 @@ private void setupInternal(int numNodeManager, float timelineVersion)
|
|||||||
ProcfsBasedProcessTree.class.getName());
|
ProcfsBasedProcessTree.class.getName());
|
||||||
conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
|
conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
|
||||||
conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
|
conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
|
||||||
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
|
conf.setBoolean(
|
||||||
|
YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
|
||||||
true);
|
true);
|
||||||
conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
|
conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
|
||||||
true);
|
true);
|
||||||
@ -180,11 +181,13 @@ private void setupInternal(int numNodeManager, float timelineVersion)
|
|||||||
// disable v1 timeline server since we no longer have a server here
|
// disable v1 timeline server since we no longer have a server here
|
||||||
// enable aux-service based timeline aggregators
|
// enable aux-service based timeline aggregators
|
||||||
conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
|
conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
|
||||||
conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME
|
conf.set(YarnConfiguration.NM_AUX_SERVICES + "." +
|
||||||
+ ".class", PerNodeTimelineCollectorsAuxService.class.getName());
|
TIMELINE_AUX_SERVICE_NAME + ".class",
|
||||||
|
PerNodeTimelineCollectorsAuxService.class.getName());
|
||||||
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
|
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
|
||||||
FileSystemTimelineWriterImpl.class,
|
FileSystemTimelineWriterImpl.class,
|
||||||
org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter.class);
|
org.apache.hadoop.yarn.server.timelineservice.storage.
|
||||||
|
TimelineWriter.class);
|
||||||
} else {
|
} else {
|
||||||
Assert.fail("Wrong timeline version number: " + timelineVersion);
|
Assert.fail("Wrong timeline version number: " + timelineVersion);
|
||||||
}
|
}
|
||||||
@ -395,7 +398,8 @@ public void run() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
|
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
|
||||||
&& appReport.getFinalApplicationStatus() != FinalApplicationStatus.UNDEFINED) {
|
&& appReport.getFinalApplicationStatus() !=
|
||||||
|
FinalApplicationStatus.UNDEFINED) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -431,7 +435,7 @@ public void run() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void checkTimelineV1(boolean haveDomain) throws Exception {
|
private void checkTimelineV1(boolean haveDomain) throws Exception {
|
||||||
TimelineDomain domain = null;
|
TimelineDomain domain = null;
|
||||||
if (haveDomain) {
|
if (haveDomain) {
|
||||||
domain = yarnCluster.getApplicationHistoryServer()
|
domain = yarnCluster.getApplicationHistoryServer()
|
||||||
.getTimelineStore().getDomain("TEST_DOMAIN");
|
.getTimelineStore().getDomain("TEST_DOMAIN");
|
||||||
@ -545,7 +549,7 @@ private void checkTimelineV2(boolean haveDomain, ApplicationId appId,
|
|||||||
if (numOfContainerFinishedOccurences > 0) {
|
if (numOfContainerFinishedOccurences > 0) {
|
||||||
break;
|
break;
|
||||||
} else {
|
} else {
|
||||||
Thread.sleep(500l);
|
Thread.sleep(500L);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Assert.assertEquals(
|
Assert.assertEquals(
|
||||||
@ -577,7 +581,7 @@ private void checkTimelineV2(boolean haveDomain, ApplicationId appId,
|
|||||||
if (numOfStringOccurences > 0) {
|
if (numOfStringOccurences > 0) {
|
||||||
break;
|
break;
|
||||||
} else {
|
} else {
|
||||||
Thread.sleep(500l);
|
Thread.sleep(500L);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Assert.assertEquals(
|
Assert.assertEquals(
|
||||||
@ -631,8 +635,9 @@ private long getNumOfStringOccurences(File entityFile, String searchString)
|
|||||||
try {
|
try {
|
||||||
reader = new BufferedReader(new FileReader(entityFile));
|
reader = new BufferedReader(new FileReader(entityFile));
|
||||||
while ((strLine = reader.readLine()) != null) {
|
while ((strLine = reader.readLine()) != null) {
|
||||||
if (strLine.trim().contains(searchString))
|
if (strLine.trim().contains(searchString)) {
|
||||||
actualCount++;
|
actualCount++;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
reader.close();
|
reader.close();
|
||||||
|
@ -30,9 +30,7 @@
|
|||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.rules.TestName;
|
|
||||||
|
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
|
|
||||||
|
@ -463,15 +463,15 @@ public NMTokenCache getNMTokenCache() {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Register TimelineClient to AMRMClient.
|
* Register TimelineClient to AMRMClient.
|
||||||
* @param timelineClient
|
* @param client the timeline client to register
|
||||||
*/
|
*/
|
||||||
public void registerTimelineClient(TimelineClient timelineClient) {
|
public void registerTimelineClient(TimelineClient client) {
|
||||||
this.timelineClient = timelineClient;
|
this.timelineClient = client;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get registered timeline client.
|
* Get registered timeline client.
|
||||||
* @return
|
* @return the registered timeline client
|
||||||
*/
|
*/
|
||||||
public TimelineClient getRegisteredTimeineClient() {
|
public TimelineClient getRegisteredTimeineClient() {
|
||||||
return this.timelineClient;
|
return this.timelineClient;
|
||||||
@ -481,7 +481,7 @@ public TimelineClient getRegisteredTimeineClient() {
|
|||||||
* Wait for <code>check</code> to return true for each 1000 ms.
|
* Wait for <code>check</code> to return true for each 1000 ms.
|
||||||
* See also {@link #waitFor(com.google.common.base.Supplier, int)}
|
* See also {@link #waitFor(com.google.common.base.Supplier, int)}
|
||||||
* and {@link #waitFor(com.google.common.base.Supplier, int, int)}
|
* and {@link #waitFor(com.google.common.base.Supplier, int, int)}
|
||||||
* @param check
|
* @param check the condition for which it should wait
|
||||||
*/
|
*/
|
||||||
public void waitFor(Supplier<Boolean> check) throws InterruptedException {
|
public void waitFor(Supplier<Boolean> check) throws InterruptedException {
|
||||||
waitFor(check, 1000);
|
waitFor(check, 1000);
|
||||||
|
@ -304,7 +304,7 @@ public void registerTimelineClient(TimelineClient timelineClient) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Get registered timeline client.
|
* Get registered timeline client.
|
||||||
* @return
|
* @return the registered timeline client
|
||||||
*/
|
*/
|
||||||
public TimelineClient getRegisteredTimeineClient() {
|
public TimelineClient getRegisteredTimeineClient() {
|
||||||
return client.getRegisteredTimeineClient();
|
return client.getRegisteredTimeineClient();
|
||||||
@ -325,7 +325,7 @@ public abstract void updateBlacklist(List<String> blacklistAdditions,
|
|||||||
* Wait for <code>check</code> to return true for each 1000 ms.
|
* Wait for <code>check</code> to return true for each 1000 ms.
|
||||||
* See also {@link #waitFor(com.google.common.base.Supplier, int)}
|
* See also {@link #waitFor(com.google.common.base.Supplier, int)}
|
||||||
* and {@link #waitFor(com.google.common.base.Supplier, int, int)}
|
* and {@link #waitFor(com.google.common.base.Supplier, int, int)}
|
||||||
* @param check
|
* @param check the condition for which it should wait
|
||||||
*/
|
*/
|
||||||
public void waitFor(Supplier<Boolean> check) throws InterruptedException {
|
public void waitFor(Supplier<Boolean> check) throws InterruptedException {
|
||||||
waitFor(check, 1000);
|
waitFor(check, 1000);
|
||||||
|
@ -30,8 +30,6 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
|
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
|
||||||
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
|
|
||||||
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
|
|
||||||
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
|
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
|
||||||
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
|
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
|
||||||
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
|
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
|
||||||
@ -55,10 +53,12 @@ public abstract class TimelineClient extends AbstractService implements
|
|||||||
* construct and initialize a timeline client if the following operations are
|
* construct and initialize a timeline client if the following operations are
|
||||||
* supposed to be conducted by that user.
|
* supposed to be conducted by that user.
|
||||||
*/
|
*/
|
||||||
protected ApplicationId contextAppId;
|
private ApplicationId contextAppId;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates an instance of the timeline v.1.x client.
|
* Creates an instance of the timeline v.1.x client.
|
||||||
|
*
|
||||||
|
* @return the created timeline client instance
|
||||||
*/
|
*/
|
||||||
@Public
|
@Public
|
||||||
public static TimelineClient createTimelineClient() {
|
public static TimelineClient createTimelineClient() {
|
||||||
@ -68,6 +68,10 @@ public static TimelineClient createTimelineClient() {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates an instance of the timeline v.2 client.
|
* Creates an instance of the timeline v.2 client.
|
||||||
|
*
|
||||||
|
* @param appId the application id with which the timeline client is
|
||||||
|
* associated
|
||||||
|
* @return the created timeline client instance
|
||||||
*/
|
*/
|
||||||
@Public
|
@Public
|
||||||
public static TimelineClient createTimelineClient(ApplicationId appId) {
|
public static TimelineClient createTimelineClient(ApplicationId appId) {
|
||||||
@ -91,8 +95,8 @@ protected TimelineClient(String name, ApplicationId appId) {
|
|||||||
* @param entities
|
* @param entities
|
||||||
* the collection of {@link TimelineEntity}
|
* the collection of {@link TimelineEntity}
|
||||||
* @return the error information if the sent entities are not correctly stored
|
* @return the error information if the sent entities are not correctly stored
|
||||||
* @throws IOException
|
* @throws IOException if there are I/O errors
|
||||||
* @throws YarnException
|
* @throws YarnException if entities are incomplete/invalid
|
||||||
*/
|
*/
|
||||||
@Public
|
@Public
|
||||||
public abstract TimelinePutResponse putEntities(
|
public abstract TimelinePutResponse putEntities(
|
||||||
@ -112,8 +116,8 @@ public abstract TimelinePutResponse putEntities(
|
|||||||
* @param entities
|
* @param entities
|
||||||
* the collection of {@link TimelineEntity}
|
* the collection of {@link TimelineEntity}
|
||||||
* @return the error information if the sent entities are not correctly stored
|
* @return the error information if the sent entities are not correctly stored
|
||||||
* @throws IOException
|
* @throws IOException if there are I/O errors
|
||||||
* @throws YarnException
|
* @throws YarnException if entities are incomplete/invalid
|
||||||
*/
|
*/
|
||||||
@Public
|
@Public
|
||||||
public abstract TimelinePutResponse putEntities(
|
public abstract TimelinePutResponse putEntities(
|
||||||
@ -212,15 +216,15 @@ public abstract void cancelDelegationToken(
|
|||||||
* for a non-v.2 timeline client instance, a YarnException is thrown.
|
* for a non-v.2 timeline client instance, a YarnException is thrown.
|
||||||
* </p>
|
* </p>
|
||||||
*
|
*
|
||||||
* @param entities
|
* @param entities the collection of {@link
|
||||||
* the collection of {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
|
* org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
* @throws YarnException
|
* @throws YarnException
|
||||||
*/
|
*/
|
||||||
@Public
|
@Public
|
||||||
public abstract void putEntities(
|
public abstract void putEntities(
|
||||||
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity... entities)
|
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity...
|
||||||
throws IOException, YarnException;
|
entities) throws IOException, YarnException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <p>
|
* <p>
|
||||||
@ -230,15 +234,15 @@ public abstract void putEntities(
|
|||||||
* non-v.2 timeline client instance, a YarnException is thrown.
|
* non-v.2 timeline client instance, a YarnException is thrown.
|
||||||
* </p>
|
* </p>
|
||||||
*
|
*
|
||||||
* @param entities
|
* @param entities the collection of {@link
|
||||||
* the collection of {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
|
* org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
* @throws YarnException
|
* @throws YarnException
|
||||||
*/
|
*/
|
||||||
@Public
|
@Public
|
||||||
public abstract void putEntitiesAsync(
|
public abstract void putEntitiesAsync(
|
||||||
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity... entities)
|
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity...
|
||||||
throws IOException, YarnException;
|
entities) throws IOException, YarnException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <p>
|
* <p>
|
||||||
|
@ -391,8 +391,8 @@ public TimelinePutResponse putEntities(
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void putEntities(
|
public void putEntities(
|
||||||
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity... entities)
|
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity...
|
||||||
throws IOException, YarnException {
|
entities) throws IOException, YarnException {
|
||||||
if (!timelineServiceV2) {
|
if (!timelineServiceV2) {
|
||||||
throw new YarnException("v.2 method is invoked on a v.1.x client");
|
throw new YarnException("v.2 method is invoked on a v.1.x client");
|
||||||
}
|
}
|
||||||
@ -401,8 +401,8 @@ public void putEntities(
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void putEntitiesAsync(
|
public void putEntitiesAsync(
|
||||||
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity... entities)
|
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity...
|
||||||
throws IOException, YarnException {
|
entities) throws IOException, YarnException {
|
||||||
if (!timelineServiceV2) {
|
if (!timelineServiceV2) {
|
||||||
throw new YarnException("v.2 method is invoked on a v.1.x client");
|
throw new YarnException("v.2 method is invoked on a v.1.x client");
|
||||||
}
|
}
|
||||||
@ -494,7 +494,8 @@ protected void putObjects(
|
|||||||
throw new IOException(re);
|
throw new IOException(re);
|
||||||
}
|
}
|
||||||
if (resp == null ||
|
if (resp == null ||
|
||||||
resp.getClientResponseStatus() != ClientResponse.Status.OK) {
|
resp.getStatusInfo().getStatusCode() !=
|
||||||
|
ClientResponse.Status.OK.getStatusCode()) {
|
||||||
String msg = "Response from the timeline server is " +
|
String msg = "Response from the timeline server is " +
|
||||||
((resp == null) ? "null":
|
((resp == null) ? "null":
|
||||||
"not successful," + " HTTP error code: " + resp.getStatus()
|
"not successful," + " HTTP error code: " + resp.getStatus()
|
||||||
@ -530,7 +531,8 @@ public Token<TimelineDelegationTokenIdentifier> run()
|
|||||||
// TODO we should add retry logic here if timelineServiceAddress is
|
// TODO we should add retry logic here if timelineServiceAddress is
|
||||||
// not available immediately.
|
// not available immediately.
|
||||||
return (Token) authUrl.getDelegationToken(
|
return (Token) authUrl.getDelegationToken(
|
||||||
constructResURI(getConfig(), getTimelineServiceAddress(), false).toURL(),
|
constructResURI(getConfig(),
|
||||||
|
getTimelineServiceAddress(), false).toURL(),
|
||||||
token, renewer, doAsUser);
|
token, renewer, doAsUser);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@ -911,17 +913,21 @@ public boolean shouldRetryOn(Exception e) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private final class EntitiesHolder extends FutureTask<Void> {
|
private final class EntitiesHolder extends FutureTask<Void> {
|
||||||
private final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities entities;
|
private final
|
||||||
|
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities
|
||||||
|
entities;
|
||||||
private final boolean isSync;
|
private final boolean isSync;
|
||||||
|
|
||||||
EntitiesHolder(
|
EntitiesHolder(
|
||||||
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities entities,
|
final
|
||||||
|
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities
|
||||||
|
entities,
|
||||||
final boolean isSync) {
|
final boolean isSync) {
|
||||||
super(new Callable<Void>() {
|
super(new Callable<Void>() {
|
||||||
// publishEntities()
|
// publishEntities()
|
||||||
public Void call() throws Exception {
|
public Void call() throws Exception {
|
||||||
MultivaluedMap<String, String> params = new MultivaluedMapImpl();
|
MultivaluedMap<String, String> params = new MultivaluedMapImpl();
|
||||||
params.add("appid", contextAppId.toString());
|
params.add("appid", getContextAppId().toString());
|
||||||
params.add("async", Boolean.toString(!isSync));
|
params.add("async", Boolean.toString(!isSync));
|
||||||
putObjects("entities", params, entities);
|
putObjects("entities", params, entities);
|
||||||
return null;
|
return null;
|
||||||
@ -935,7 +941,8 @@ public boolean isSync() {
|
|||||||
return isSync;
|
return isSync;
|
||||||
}
|
}
|
||||||
|
|
||||||
public org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities getEntities() {
|
public org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities
|
||||||
|
getEntities() {
|
||||||
return entities;
|
return entities;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -947,7 +954,7 @@ public org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities getEn
|
|||||||
private class TimelineEntityDispatcher {
|
private class TimelineEntityDispatcher {
|
||||||
/**
|
/**
|
||||||
* Time period for which the timelineclient will wait for draining after
|
* Time period for which the timelineclient will wait for draining after
|
||||||
* stop
|
* stop.
|
||||||
*/
|
*/
|
||||||
private static final long DRAIN_TIME_PERIOD = 2000L;
|
private static final long DRAIN_TIME_PERIOD = 2000L;
|
||||||
|
|
||||||
@ -1063,17 +1070,20 @@ private void publishWithoutBlockingOnQueue(
|
|||||||
}
|
}
|
||||||
|
|
||||||
public void dispatchEntities(boolean sync,
|
public void dispatchEntities(boolean sync,
|
||||||
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity[] entitiesTobePublished)
|
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity[]
|
||||||
throws YarnException {
|
entitiesTobePublished) throws YarnException {
|
||||||
if (executor.isShutdown()) {
|
if (executor.isShutdown()) {
|
||||||
throw new YarnException("Timeline client is in the process of stopping,"
|
throw new YarnException("Timeline client is in the process of stopping,"
|
||||||
+ " not accepting any more TimelineEntities");
|
+ " not accepting any more TimelineEntities");
|
||||||
}
|
}
|
||||||
|
|
||||||
// wrap all TimelineEntity into TimelineEntities object
|
// wrap all TimelineEntity into TimelineEntities object
|
||||||
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities entities =
|
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities
|
||||||
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities();
|
entities =
|
||||||
for (org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity : entitiesTobePublished) {
|
new org.apache.hadoop.yarn.api.records.timelineservice.
|
||||||
|
TimelineEntities();
|
||||||
|
for (org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
|
||||||
|
entity : entitiesTobePublished) {
|
||||||
entities.addEntity(entity);
|
entities.addEntity(entity);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -100,7 +100,8 @@ public void testTimelineEntities() throws Exception {
|
|||||||
}
|
}
|
||||||
entity.addMetric(metric2);
|
entity.addMetric(metric2);
|
||||||
|
|
||||||
TimelineMetric metric3 = new TimelineMetric(TimelineMetric.Type.SINGLE_VALUE);
|
TimelineMetric metric3 =
|
||||||
|
new TimelineMetric(TimelineMetric.Type.SINGLE_VALUE);
|
||||||
metric3.setId("test metric id 1");
|
metric3.setId("test metric id 1");
|
||||||
metric3.addValue(4L, (short) 4);
|
metric3.addValue(4L, (short) 4);
|
||||||
Assert.assertEquals("metric3 should equal to metric2! ", metric3, metric2);
|
Assert.assertEquals("metric3 should equal to metric2! ", metric3, metric2);
|
||||||
@ -212,18 +213,22 @@ public void testFirstClassCitizenEntities() throws Exception {
|
|||||||
ApplicationAttemptId.newInstance(
|
ApplicationAttemptId.newInstance(
|
||||||
ApplicationId.newInstance(0, 1), 1), 1).toString());
|
ApplicationId.newInstance(0, 1), 1), 1).toString());
|
||||||
|
|
||||||
cluster.addChild(TimelineEntityType.YARN_FLOW_RUN.toString(), flow1.getId());
|
cluster.addChild(TimelineEntityType.YARN_FLOW_RUN.toString(),
|
||||||
|
flow1.getId());
|
||||||
flow1
|
flow1
|
||||||
.setParent(TimelineEntityType.YARN_CLUSTER.toString(), cluster.getId());
|
.setParent(TimelineEntityType.YARN_CLUSTER.toString(), cluster.getId());
|
||||||
flow1.addChild(TimelineEntityType.YARN_FLOW_RUN.toString(), flow2.getId());
|
flow1.addChild(TimelineEntityType.YARN_FLOW_RUN.toString(), flow2.getId());
|
||||||
flow2.setParent(TimelineEntityType.YARN_FLOW_RUN.toString(), flow1.getId());
|
flow2.setParent(TimelineEntityType.YARN_FLOW_RUN.toString(), flow1.getId());
|
||||||
flow2.addChild(TimelineEntityType.YARN_APPLICATION.toString(), app1.getId());
|
flow2.addChild(TimelineEntityType.YARN_APPLICATION.toString(),
|
||||||
flow2.addChild(TimelineEntityType.YARN_APPLICATION.toString(), app2.getId());
|
app1.getId());
|
||||||
|
flow2.addChild(TimelineEntityType.YARN_APPLICATION.toString(),
|
||||||
|
app2.getId());
|
||||||
app1.setParent(TimelineEntityType.YARN_FLOW_RUN.toString(), flow2.getId());
|
app1.setParent(TimelineEntityType.YARN_FLOW_RUN.toString(), flow2.getId());
|
||||||
app1.addChild(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
|
app1.addChild(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
|
||||||
appAttempt.getId());
|
appAttempt.getId());
|
||||||
appAttempt
|
appAttempt
|
||||||
.setParent(TimelineEntityType.YARN_APPLICATION.toString(), app1.getId());
|
.setParent(TimelineEntityType.YARN_APPLICATION.toString(),
|
||||||
|
app1.getId());
|
||||||
app2.setParent(TimelineEntityType.YARN_FLOW_RUN.toString(), flow2.getId());
|
app2.setParent(TimelineEntityType.YARN_FLOW_RUN.toString(), flow2.getId());
|
||||||
appAttempt.addChild(TimelineEntityType.YARN_CONTAINER.toString(),
|
appAttempt.addChild(TimelineEntityType.YARN_CONTAINER.toString(),
|
||||||
container.getId());
|
container.getId());
|
||||||
|
@ -43,7 +43,7 @@ public class TestTimelineClientV2Impl {
|
|||||||
private static final Log LOG =
|
private static final Log LOG =
|
||||||
LogFactory.getLog(TestTimelineClientV2Impl.class);
|
LogFactory.getLog(TestTimelineClientV2Impl.class);
|
||||||
private TestV2TimelineClient client;
|
private TestV2TimelineClient client;
|
||||||
private static long TIME_TO_SLEEP = 150;
|
private static final long TIME_TO_SLEEP = 150L;
|
||||||
private static final String EXCEPTION_MSG = "Exception in the content";
|
private static final String EXCEPTION_MSG = "Exception in the content";
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
@ -62,12 +62,12 @@ public void setup() {
|
|||||||
public TestName currTestName = new TestName();
|
public TestName currTestName = new TestName();
|
||||||
private YarnConfiguration conf;
|
private YarnConfiguration conf;
|
||||||
|
|
||||||
private TestV2TimelineClient createTimelineClient(YarnConfiguration conf) {
|
private TestV2TimelineClient createTimelineClient(YarnConfiguration config) {
|
||||||
ApplicationId id = ApplicationId.newInstance(0, 0);
|
ApplicationId id = ApplicationId.newInstance(0, 0);
|
||||||
TestV2TimelineClient client = new TestV2TimelineClient(id);
|
TestV2TimelineClient tc = new TestV2TimelineClient(id);
|
||||||
client.init(conf);
|
tc.init(config);
|
||||||
client.start();
|
tc.start();
|
||||||
return client;
|
return tc;
|
||||||
}
|
}
|
||||||
|
|
||||||
private class TestV2TimelineClientForExceptionHandling
|
private class TestV2TimelineClientForExceptionHandling
|
||||||
@ -76,12 +76,16 @@ public TestV2TimelineClientForExceptionHandling(ApplicationId id) {
|
|||||||
super(id);
|
super(id);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected boolean throwYarnException;
|
private boolean throwYarnException;
|
||||||
|
|
||||||
public void setThrowYarnException(boolean throwYarnException) {
|
public void setThrowYarnException(boolean throwYarnException) {
|
||||||
this.throwYarnException = throwYarnException;
|
this.throwYarnException = throwYarnException;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public boolean isThrowYarnException() {
|
||||||
|
return throwYarnException;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void putObjects(URI base, String path,
|
protected void putObjects(URI base, String path,
|
||||||
MultivaluedMap<String, String> params, Object obj)
|
MultivaluedMap<String, String> params, Object obj)
|
||||||
@ -123,7 +127,7 @@ public TestV2TimelineClient(ApplicationId id) {
|
|||||||
protected void putObjects(String path,
|
protected void putObjects(String path,
|
||||||
MultivaluedMap<String, String> params, Object obj)
|
MultivaluedMap<String, String> params, Object obj)
|
||||||
throws IOException, YarnException {
|
throws IOException, YarnException {
|
||||||
if (throwYarnException) {
|
if (isThrowYarnException()) {
|
||||||
throw new YarnException("ActualException");
|
throw new YarnException("ActualException");
|
||||||
}
|
}
|
||||||
publishedEntities.add((TimelineEntities) obj);
|
publishedEntities.add((TimelineEntities) obj);
|
||||||
@ -139,17 +143,17 @@ protected void putObjects(String path,
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExceptionMultipleRetry() {
|
public void testExceptionMultipleRetry() {
|
||||||
TestV2TimelineClientForExceptionHandling client =
|
TestV2TimelineClientForExceptionHandling c =
|
||||||
new TestV2TimelineClientForExceptionHandling(
|
new TestV2TimelineClientForExceptionHandling(
|
||||||
ApplicationId.newInstance(0, 0));
|
ApplicationId.newInstance(0, 0));
|
||||||
int maxRetries = 2;
|
int maxRetries = 2;
|
||||||
conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
|
conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
|
||||||
maxRetries);
|
maxRetries);
|
||||||
client.init(conf);
|
c.init(conf);
|
||||||
client.start();
|
c.start();
|
||||||
client.setTimelineServiceAddress("localhost:12345");
|
c.setTimelineServiceAddress("localhost:12345");
|
||||||
try {
|
try {
|
||||||
client.putEntities(new TimelineEntity());
|
c.putEntities(new TimelineEntity());
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
Assert.fail("YARN exception is expected");
|
Assert.fail("YARN exception is expected");
|
||||||
} catch (YarnException e) {
|
} catch (YarnException e) {
|
||||||
@ -161,9 +165,9 @@ public void testExceptionMultipleRetry() {
|
|||||||
"TimelineClient has reached to max retry times : " + maxRetries));
|
"TimelineClient has reached to max retry times : " + maxRetries));
|
||||||
}
|
}
|
||||||
|
|
||||||
client.setThrowYarnException(true);
|
c.setThrowYarnException(true);
|
||||||
try {
|
try {
|
||||||
client.putEntities(new TimelineEntity());
|
c.putEntities(new TimelineEntity());
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
Assert.fail("YARN exception is expected");
|
Assert.fail("YARN exception is expected");
|
||||||
} catch (YarnException e) {
|
} catch (YarnException e) {
|
||||||
@ -173,7 +177,7 @@ public void testExceptionMultipleRetry() {
|
|||||||
Assert.assertTrue("YARN exception is expected",
|
Assert.assertTrue("YARN exception is expected",
|
||||||
cause.getMessage().contains(EXCEPTION_MSG));
|
cause.getMessage().contains(EXCEPTION_MSG));
|
||||||
}
|
}
|
||||||
client.stop();
|
c.stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@ -348,7 +352,6 @@ private void printReceivedEntities() {
|
|||||||
for (int i = 0; i < client.getNumOfTimelineEntitiesPublished(); i++) {
|
for (int i = 0; i < client.getNumOfTimelineEntitiesPublished(); i++) {
|
||||||
TimelineEntities publishedEntities = client.getPublishedEntities(i);
|
TimelineEntities publishedEntities = client.getPublishedEntities(i);
|
||||||
StringBuilder entitiesPerPublish = new StringBuilder();
|
StringBuilder entitiesPerPublish = new StringBuilder();
|
||||||
;
|
|
||||||
for (TimelineEntity entity : publishedEntities.getEntities()) {
|
for (TimelineEntity entity : publishedEntities.getEntities()) {
|
||||||
entitiesPerPublish.append(entity.getId());
|
entitiesPerPublish.append(entity.getId());
|
||||||
entitiesPerPublish.append(",");
|
entitiesPerPublish.append(",");
|
||||||
|
@ -37,18 +37,21 @@ public void testMapCastToHashMap() {
|
|||||||
|
|
||||||
// Test empty hashmap be casted to a empty hashmap
|
// Test empty hashmap be casted to a empty hashmap
|
||||||
Map<String, String> emptyHashMap = new HashMap<String, String>();
|
Map<String, String> emptyHashMap = new HashMap<String, String>();
|
||||||
Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(emptyHashMap).size(), 0);
|
Assert.assertEquals(
|
||||||
|
TimelineServiceHelper.mapCastToHashMap(emptyHashMap).size(), 0);
|
||||||
|
|
||||||
// Test empty non-hashmap be casted to a empty hashmap
|
// Test empty non-hashmap be casted to a empty hashmap
|
||||||
Map<String, String> emptyTreeMap = new TreeMap<String, String>();
|
Map<String, String> emptyTreeMap = new TreeMap<String, String>();
|
||||||
Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(emptyTreeMap).size(), 0);
|
Assert.assertEquals(
|
||||||
|
TimelineServiceHelper.mapCastToHashMap(emptyTreeMap).size(), 0);
|
||||||
|
|
||||||
// Test non-empty hashmap be casted to hashmap correctly
|
// Test non-empty hashmap be casted to hashmap correctly
|
||||||
Map<String, String> firstHashMap = new HashMap<String, String>();
|
Map<String, String> firstHashMap = new HashMap<String, String>();
|
||||||
String key = "KEY";
|
String key = "KEY";
|
||||||
String value = "VALUE";
|
String value = "VALUE";
|
||||||
firstHashMap.put(key, value);
|
firstHashMap.put(key, value);
|
||||||
Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(firstHashMap), firstHashMap);
|
Assert.assertEquals(
|
||||||
|
TimelineServiceHelper.mapCastToHashMap(firstHashMap), firstHashMap);
|
||||||
|
|
||||||
// Test non-empty non-hashmap is casted correctly.
|
// Test non-empty non-hashmap is casted correctly.
|
||||||
Map<String, String> firstTreeMap = new TreeMap<String, String>();
|
Map<String, String> firstTreeMap = new TreeMap<String, String>();
|
||||||
@ -59,17 +62,21 @@ public void testMapCastToHashMap() {
|
|||||||
Assert.assertEquals(alternateHashMap.get(key), value);
|
Assert.assertEquals(alternateHashMap.get(key), value);
|
||||||
|
|
||||||
// Test complicated hashmap be casted correctly
|
// Test complicated hashmap be casted correctly
|
||||||
Map<String, Set<String>> complicatedHashMap = new HashMap<String, Set<String>>();
|
Map<String, Set<String>> complicatedHashMap =
|
||||||
|
new HashMap<String, Set<String>>();
|
||||||
Set<String> hashSet = new HashSet<String>();
|
Set<String> hashSet = new HashSet<String>();
|
||||||
hashSet.add(value);
|
hashSet.add(value);
|
||||||
complicatedHashMap.put(key, hashSet);
|
complicatedHashMap.put(key, hashSet);
|
||||||
Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedHashMap),
|
Assert.assertEquals(
|
||||||
|
TimelineServiceHelper.mapCastToHashMap(complicatedHashMap),
|
||||||
complicatedHashMap);
|
complicatedHashMap);
|
||||||
|
|
||||||
// Test complicated non-hashmap get casted correctly
|
// Test complicated non-hashmap get casted correctly
|
||||||
Map<String, Set<String>> complicatedTreeMap = new TreeMap<String, Set<String>>();
|
Map<String, Set<String>> complicatedTreeMap =
|
||||||
|
new TreeMap<String, Set<String>>();
|
||||||
complicatedTreeMap.put(key, hashSet);
|
complicatedTreeMap.put(key, hashSet);
|
||||||
Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedTreeMap).get(key),
|
Assert.assertEquals(
|
||||||
|
TimelineServiceHelper.mapCastToHashMap(complicatedTreeMap).get(key),
|
||||||
hashSet);
|
hashSet);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -48,9 +48,9 @@ public interface CollectorNodemanagerProtocol {
|
|||||||
*
|
*
|
||||||
* @param request the request of registering a new collector or a list of
|
* @param request the request of registering a new collector or a list of
|
||||||
* collectors
|
* collectors
|
||||||
* @return
|
* @return the response for registering the new collector
|
||||||
* @throws YarnException
|
* @throws YarnException if the request is invalid
|
||||||
* @throws IOException
|
* @throws IOException if there are I/O errors
|
||||||
*/
|
*/
|
||||||
ReportNewCollectorInfoResponse reportNewCollectorInfo(
|
ReportNewCollectorInfoResponse reportNewCollectorInfo(
|
||||||
ReportNewCollectorInfoRequest request)
|
ReportNewCollectorInfoRequest request)
|
||||||
@ -63,9 +63,9 @@ ReportNewCollectorInfoResponse reportNewCollectorInfo(
|
|||||||
* </p>
|
* </p>
|
||||||
* @param request the request of getting the aggregator context information of
|
* @param request the request of getting the aggregator context information of
|
||||||
* the given application
|
* the given application
|
||||||
* @return
|
* @return the response for registering the new collector
|
||||||
* @throws YarnException
|
* @throws YarnException if the request is invalid
|
||||||
* @throws IOException
|
* @throws IOException if there are I/O errors
|
||||||
*/
|
*/
|
||||||
GetTimelineCollectorContextResponse getTimelineCollectorContext(
|
GetTimelineCollectorContextResponse getTimelineCollectorContext(
|
||||||
GetTimelineCollectorContextRequest request)
|
GetTimelineCollectorContextRequest request)
|
||||||
|
@ -25,7 +25,8 @@
|
|||||||
@Private
|
@Private
|
||||||
@Unstable
|
@Unstable
|
||||||
@ProtocolInfo(
|
@ProtocolInfo(
|
||||||
protocolName = "org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB",
|
protocolName =
|
||||||
|
"org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB",
|
||||||
protocolVersion = 1)
|
protocolVersion = 1)
|
||||||
public interface CollectorNodemanagerProtocolPB extends
|
public interface CollectorNodemanagerProtocolPB extends
|
||||||
CollectorNodemanagerProtocolService.BlockingInterface {
|
CollectorNodemanagerProtocolService.BlockingInterface {
|
||||||
|
@ -53,7 +53,7 @@ public class CollectorNodemanagerProtocolPBClientImpl implements
|
|||||||
+ "rpc.nm-command-timeout";
|
+ "rpc.nm-command-timeout";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Maximum of 1 minute timeout for a Node to react to the command
|
* Maximum of 1 minute timeout for a Node to react to the command.
|
||||||
*/
|
*/
|
||||||
static final int DEFAULT_COMMAND_TIMEOUT = 60000;
|
static final int DEFAULT_COMMAND_TIMEOUT = 60000;
|
||||||
|
|
||||||
@ -63,7 +63,7 @@ public class CollectorNodemanagerProtocolPBClientImpl implements
|
|||||||
public CollectorNodemanagerProtocolPBClientImpl(long clientVersion,
|
public CollectorNodemanagerProtocolPBClientImpl(long clientVersion,
|
||||||
InetSocketAddress addr, Configuration conf) throws IOException {
|
InetSocketAddress addr, Configuration conf) throws IOException {
|
||||||
RPC.setProtocolEngine(conf, CollectorNodemanagerProtocolPB.class,
|
RPC.setProtocolEngine(conf, CollectorNodemanagerProtocolPB.class,
|
||||||
ProtobufRpcEngine.class);
|
ProtobufRpcEngine.class);
|
||||||
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
||||||
|
|
||||||
int expireIntvl = conf.getInt(NM_COMMAND_TIMEOUT, DEFAULT_COMMAND_TIMEOUT);
|
int expireIntvl = conf.getInt(NM_COMMAND_TIMEOUT, DEFAULT_COMMAND_TIMEOUT);
|
||||||
|
@ -41,7 +41,8 @@ public class CollectorNodemanagerProtocolPBServiceImpl implements
|
|||||||
|
|
||||||
private CollectorNodemanagerProtocol real;
|
private CollectorNodemanagerProtocol real;
|
||||||
|
|
||||||
public CollectorNodemanagerProtocolPBServiceImpl(CollectorNodemanagerProtocol impl) {
|
public CollectorNodemanagerProtocolPBServiceImpl(
|
||||||
|
CollectorNodemanagerProtocol impl) {
|
||||||
this.real = impl;
|
this.real = impl;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -52,7 +53,8 @@ public ReportNewCollectorInfoResponseProto reportNewCollectorInfo(
|
|||||||
ReportNewCollectorInfoRequestPBImpl request =
|
ReportNewCollectorInfoRequestPBImpl request =
|
||||||
new ReportNewCollectorInfoRequestPBImpl(proto);
|
new ReportNewCollectorInfoRequestPBImpl(proto);
|
||||||
try {
|
try {
|
||||||
ReportNewCollectorInfoResponse response = real.reportNewCollectorInfo(request);
|
ReportNewCollectorInfoResponse response =
|
||||||
|
real.reportNewCollectorInfo(request);
|
||||||
return ((ReportNewCollectorInfoResponsePBImpl)response).getProto();
|
return ((ReportNewCollectorInfoResponsePBImpl)response).getProto();
|
||||||
} catch (YarnException e) {
|
} catch (YarnException e) {
|
||||||
throw new ServiceException(e);
|
throw new ServiceException(e);
|
||||||
|
@ -29,10 +29,10 @@
|
|||||||
public class GetTimelineCollectorContextRequestPBImpl extends
|
public class GetTimelineCollectorContextRequestPBImpl extends
|
||||||
GetTimelineCollectorContextRequest {
|
GetTimelineCollectorContextRequest {
|
||||||
|
|
||||||
GetTimelineCollectorContextRequestProto
|
private GetTimelineCollectorContextRequestProto
|
||||||
proto = GetTimelineCollectorContextRequestProto.getDefaultInstance();
|
proto = GetTimelineCollectorContextRequestProto.getDefaultInstance();
|
||||||
GetTimelineCollectorContextRequestProto.Builder builder = null;
|
private GetTimelineCollectorContextRequestProto.Builder builder = null;
|
||||||
boolean viaProto = false;
|
private boolean viaProto = false;
|
||||||
|
|
||||||
private ApplicationId appId = null;
|
private ApplicationId appId = null;
|
||||||
|
|
||||||
@ -60,8 +60,9 @@ public int hashCode() {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean equals(Object other) {
|
public boolean equals(Object other) {
|
||||||
if (other == null)
|
if (other == null) {
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
if (other.getClass().isAssignableFrom(this.getClass())) {
|
if (other.getClass().isAssignableFrom(this.getClass())) {
|
||||||
return this.getProto().equals(this.getClass().cast(other).getProto());
|
return this.getProto().equals(this.getClass().cast(other).getProto());
|
||||||
}
|
}
|
||||||
@ -80,8 +81,9 @@ private void mergeLocalToBuilder() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void mergeLocalToProto() {
|
private void mergeLocalToProto() {
|
||||||
if (viaProto)
|
if (viaProto) {
|
||||||
maybeInitBuilder();
|
maybeInitBuilder();
|
||||||
|
}
|
||||||
mergeLocalToBuilder();
|
mergeLocalToBuilder();
|
||||||
proto = builder.build();
|
proto = builder.build();
|
||||||
viaProto = true;
|
viaProto = true;
|
||||||
@ -100,7 +102,8 @@ public ApplicationId getApplicationId() {
|
|||||||
return this.appId;
|
return this.appId;
|
||||||
}
|
}
|
||||||
|
|
||||||
GetTimelineCollectorContextRequestProtoOrBuilder p = viaProto ? proto : builder;
|
GetTimelineCollectorContextRequestProtoOrBuilder p =
|
||||||
|
viaProto ? proto : builder;
|
||||||
if (!p.hasAppId()) {
|
if (!p.hasAppId()) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@ -110,14 +113,16 @@ public ApplicationId getApplicationId() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setApplicationId(ApplicationId appId) {
|
public void setApplicationId(ApplicationId id) {
|
||||||
maybeInitBuilder();
|
maybeInitBuilder();
|
||||||
if (appId == null)
|
if (id == null) {
|
||||||
builder.clearAppId();
|
builder.clearAppId();
|
||||||
this.appId = appId;
|
}
|
||||||
|
this.appId = id;
|
||||||
}
|
}
|
||||||
|
|
||||||
private ApplicationIdPBImpl convertFromProtoFormat(YarnProtos.ApplicationIdProto p) {
|
private ApplicationIdPBImpl convertFromProtoFormat(
|
||||||
|
YarnProtos.ApplicationIdProto p) {
|
||||||
return new ApplicationIdPBImpl(p);
|
return new ApplicationIdPBImpl(p);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -26,10 +26,10 @@
|
|||||||
public class GetTimelineCollectorContextResponsePBImpl extends
|
public class GetTimelineCollectorContextResponsePBImpl extends
|
||||||
GetTimelineCollectorContextResponse {
|
GetTimelineCollectorContextResponse {
|
||||||
|
|
||||||
GetTimelineCollectorContextResponseProto proto =
|
private GetTimelineCollectorContextResponseProto proto =
|
||||||
GetTimelineCollectorContextResponseProto.getDefaultInstance();
|
GetTimelineCollectorContextResponseProto.getDefaultInstance();
|
||||||
GetTimelineCollectorContextResponseProto.Builder builder = null;
|
private GetTimelineCollectorContextResponseProto.Builder builder = null;
|
||||||
boolean viaProto = false;
|
private boolean viaProto = false;
|
||||||
|
|
||||||
public GetTimelineCollectorContextResponsePBImpl() {
|
public GetTimelineCollectorContextResponsePBImpl() {
|
||||||
builder = GetTimelineCollectorContextResponseProto.newBuilder();
|
builder = GetTimelineCollectorContextResponseProto.newBuilder();
|
||||||
@ -55,8 +55,9 @@ public int hashCode() {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean equals(Object other) {
|
public boolean equals(Object other) {
|
||||||
if (other == null)
|
if (other == null) {
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
if (other.getClass().isAssignableFrom(this.getClass())) {
|
if (other.getClass().isAssignableFrom(this.getClass())) {
|
||||||
return this.getProto().equals(this.getClass().cast(other).getProto());
|
return this.getProto().equals(this.getClass().cast(other).getProto());
|
||||||
}
|
}
|
||||||
@ -69,8 +70,9 @@ public String toString() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void mergeLocalToProto() {
|
private void mergeLocalToProto() {
|
||||||
if (viaProto)
|
if (viaProto) {
|
||||||
maybeInitBuilder();
|
maybeInitBuilder();
|
||||||
|
}
|
||||||
proto = builder.build();
|
proto = builder.build();
|
||||||
viaProto = true;
|
viaProto = true;
|
||||||
}
|
}
|
||||||
@ -84,7 +86,8 @@ private void maybeInitBuilder() {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getUserId() {
|
public String getUserId() {
|
||||||
GetTimelineCollectorContextResponseProtoOrBuilder p = viaProto ? proto : builder;
|
GetTimelineCollectorContextResponseProtoOrBuilder p =
|
||||||
|
viaProto ? proto : builder;
|
||||||
if (!p.hasUserId()) {
|
if (!p.hasUserId()) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@ -103,7 +106,8 @@ public void setUserId(String userId) {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getFlowName() {
|
public String getFlowName() {
|
||||||
GetTimelineCollectorContextResponseProtoOrBuilder p = viaProto ? proto : builder;
|
GetTimelineCollectorContextResponseProtoOrBuilder p =
|
||||||
|
viaProto ? proto : builder;
|
||||||
if (!p.hasFlowName()) {
|
if (!p.hasFlowName()) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@ -122,7 +126,8 @@ public void setFlowName(String flowName) {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getFlowVersion() {
|
public String getFlowVersion() {
|
||||||
GetTimelineCollectorContextResponseProtoOrBuilder p = viaProto ? proto : builder;
|
GetTimelineCollectorContextResponseProtoOrBuilder p =
|
||||||
|
viaProto ? proto : builder;
|
||||||
if (!p.hasFlowVersion()) {
|
if (!p.hasFlowVersion()) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@ -141,7 +146,8 @@ public void setFlowVersion(String flowVersion) {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public long getFlowRunId() {
|
public long getFlowRunId() {
|
||||||
GetTimelineCollectorContextResponseProtoOrBuilder p = viaProto ? proto : builder;
|
GetTimelineCollectorContextResponseProtoOrBuilder p =
|
||||||
|
viaProto ? proto : builder;
|
||||||
return p.getFlowRunId();
|
return p.getFlowRunId();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -58,7 +58,7 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
|
|||||||
private Set<NodeLabel> labels = null;
|
private Set<NodeLabel> labels = null;
|
||||||
private List<LogAggregationReport> logAggregationReportsForApps = null;
|
private List<LogAggregationReport> logAggregationReportsForApps = null;
|
||||||
|
|
||||||
Map<ApplicationId, String> registeredCollectors = null;
|
private Map<ApplicationId, String> registeredCollectors = null;
|
||||||
|
|
||||||
public NodeHeartbeatRequestPBImpl() {
|
public NodeHeartbeatRequestPBImpl() {
|
||||||
builder = NodeHeartbeatRequestProto.newBuilder();
|
builder = NodeHeartbeatRequestProto.newBuilder();
|
||||||
@ -161,10 +161,11 @@ private LogAggregationReportProto convertToProtoFormat(
|
|||||||
private void addRegisteredCollectorsToProto() {
|
private void addRegisteredCollectorsToProto() {
|
||||||
maybeInitBuilder();
|
maybeInitBuilder();
|
||||||
builder.clearRegisteredCollectors();
|
builder.clearRegisteredCollectors();
|
||||||
for (Map.Entry<ApplicationId, String> entry : registeredCollectors.entrySet()) {
|
for (Map.Entry<ApplicationId, String> entry :
|
||||||
|
registeredCollectors.entrySet()) {
|
||||||
builder.addRegisteredCollectors(AppCollectorsMapProto.newBuilder()
|
builder.addRegisteredCollectors(AppCollectorsMapProto.newBuilder()
|
||||||
.setAppId(convertToProtoFormat(entry.getKey()))
|
.setAppId(convertToProtoFormat(entry.getKey()))
|
||||||
.setAppCollectorAddr(entry.getValue()));
|
.setAppCollectorAddr(entry.getValue()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -69,7 +69,7 @@ public class NodeHeartbeatResponsePBImpl extends
|
|||||||
private List<ApplicationId> applicationsToCleanup = null;
|
private List<ApplicationId> applicationsToCleanup = null;
|
||||||
private Map<ApplicationId, ByteBuffer> systemCredentials = null;
|
private Map<ApplicationId, ByteBuffer> systemCredentials = null;
|
||||||
private Resource resource = null;
|
private Resource resource = null;
|
||||||
Map<ApplicationId, String> appCollectorsMap = null;
|
private Map<ApplicationId, String> appCollectorsMap = null;
|
||||||
|
|
||||||
private MasterKey containerTokenMasterKey = null;
|
private MasterKey containerTokenMasterKey = null;
|
||||||
private MasterKey nmTokenMasterKey = null;
|
private MasterKey nmTokenMasterKey = null;
|
||||||
@ -148,8 +148,8 @@ private void addAppCollectorsMapToProto() {
|
|||||||
builder.clearAppCollectorsMap();
|
builder.clearAppCollectorsMap();
|
||||||
for (Map.Entry<ApplicationId, String> entry : appCollectorsMap.entrySet()) {
|
for (Map.Entry<ApplicationId, String> entry : appCollectorsMap.entrySet()) {
|
||||||
builder.addAppCollectorsMap(AppCollectorsMapProto.newBuilder()
|
builder.addAppCollectorsMap(AppCollectorsMapProto.newBuilder()
|
||||||
.setAppId(convertToProtoFormat(entry.getKey()))
|
.setAppId(convertToProtoFormat(entry.getKey()))
|
||||||
.setAppCollectorAddr(entry.getValue()));
|
.setAppCollectorAddr(entry.getValue()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -30,11 +30,11 @@
|
|||||||
public class ReportNewCollectorInfoRequestPBImpl extends
|
public class ReportNewCollectorInfoRequestPBImpl extends
|
||||||
ReportNewCollectorInfoRequest {
|
ReportNewCollectorInfoRequest {
|
||||||
|
|
||||||
ReportNewCollectorInfoRequestProto proto =
|
private ReportNewCollectorInfoRequestProto proto =
|
||||||
ReportNewCollectorInfoRequestProto.getDefaultInstance();
|
ReportNewCollectorInfoRequestProto.getDefaultInstance();
|
||||||
|
|
||||||
ReportNewCollectorInfoRequestProto.Builder builder = null;
|
private ReportNewCollectorInfoRequestProto.Builder builder = null;
|
||||||
boolean viaProto = false;
|
private boolean viaProto = false;
|
||||||
|
|
||||||
private List<AppCollectorsMap> collectorsList = null;
|
private List<AppCollectorsMap> collectorsList = null;
|
||||||
|
|
||||||
@ -62,8 +62,9 @@ public int hashCode() {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean equals(Object other) {
|
public boolean equals(Object other) {
|
||||||
if (other == null)
|
if (other == null) {
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
if (other.getClass().isAssignableFrom(this.getClass())) {
|
if (other.getClass().isAssignableFrom(this.getClass())) {
|
||||||
return this.getProto().equals(this.getClass().cast(other).getProto());
|
return this.getProto().equals(this.getClass().cast(other).getProto());
|
||||||
}
|
}
|
||||||
@ -71,8 +72,9 @@ public boolean equals(Object other) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private void mergeLocalToProto() {
|
private void mergeLocalToProto() {
|
||||||
if (viaProto)
|
if (viaProto) {
|
||||||
maybeInitBuilder();
|
maybeInitBuilder();
|
||||||
|
}
|
||||||
mergeLocalToBuilder();
|
mergeLocalToBuilder();
|
||||||
proto = builder.build();
|
proto = builder.build();
|
||||||
viaProto = true;
|
viaProto = true;
|
||||||
@ -104,10 +106,10 @@ private void addLocalCollectorsToProto() {
|
|||||||
|
|
||||||
private void initLocalCollectorsList() {
|
private void initLocalCollectorsList() {
|
||||||
ReportNewCollectorInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
|
ReportNewCollectorInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
|
||||||
List<AppCollectorsMapProto> collectorsList =
|
List<AppCollectorsMapProto> list =
|
||||||
p.getAppCollectorsList();
|
p.getAppCollectorsList();
|
||||||
this.collectorsList = new ArrayList<AppCollectorsMap>();
|
this.collectorsList = new ArrayList<AppCollectorsMap>();
|
||||||
for (AppCollectorsMapProto m : collectorsList) {
|
for (AppCollectorsMapProto m : list) {
|
||||||
this.collectorsList.add(convertFromProtoFormat(m));
|
this.collectorsList.add(convertFromProtoFormat(m));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -29,18 +29,19 @@
|
|||||||
public class ReportNewCollectorInfoResponsePBImpl extends
|
public class ReportNewCollectorInfoResponsePBImpl extends
|
||||||
ReportNewCollectorInfoResponse {
|
ReportNewCollectorInfoResponse {
|
||||||
|
|
||||||
ReportNewCollectorInfoResponseProto proto =
|
private ReportNewCollectorInfoResponseProto proto =
|
||||||
ReportNewCollectorInfoResponseProto.getDefaultInstance();
|
ReportNewCollectorInfoResponseProto.getDefaultInstance();
|
||||||
|
|
||||||
ReportNewCollectorInfoResponseProto.Builder builder = null;
|
private ReportNewCollectorInfoResponseProto.Builder builder = null;
|
||||||
|
|
||||||
boolean viaProto = false;
|
private boolean viaProto = false;
|
||||||
|
|
||||||
public ReportNewCollectorInfoResponsePBImpl() {
|
public ReportNewCollectorInfoResponsePBImpl() {
|
||||||
builder = ReportNewCollectorInfoResponseProto.newBuilder();
|
builder = ReportNewCollectorInfoResponseProto.newBuilder();
|
||||||
}
|
}
|
||||||
|
|
||||||
public ReportNewCollectorInfoResponsePBImpl(ReportNewCollectorInfoResponseProto proto) {
|
public ReportNewCollectorInfoResponsePBImpl(
|
||||||
|
ReportNewCollectorInfoResponseProto proto) {
|
||||||
this.proto = proto;
|
this.proto = proto;
|
||||||
viaProto = true;
|
viaProto = true;
|
||||||
}
|
}
|
||||||
@ -58,8 +59,9 @@ public int hashCode() {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean equals(Object other) {
|
public boolean equals(Object other) {
|
||||||
if (other == null)
|
if (other == null) {
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
if (other.getClass().isAssignableFrom(this.getClass())) {
|
if (other.getClass().isAssignableFrom(this.getClass())) {
|
||||||
return this.getProto().equals(this.getClass().cast(other).getProto());
|
return this.getProto().equals(this.getClass().cast(other).getProto());
|
||||||
}
|
}
|
||||||
|
@ -33,11 +33,11 @@
|
|||||||
@Unstable
|
@Unstable
|
||||||
public class AppCollectorsMapPBImpl extends AppCollectorsMap {
|
public class AppCollectorsMapPBImpl extends AppCollectorsMap {
|
||||||
|
|
||||||
AppCollectorsMapProto proto =
|
private AppCollectorsMapProto proto =
|
||||||
AppCollectorsMapProto.getDefaultInstance();
|
AppCollectorsMapProto.getDefaultInstance();
|
||||||
|
|
||||||
AppCollectorsMapProto.Builder builder = null;
|
private AppCollectorsMapProto.Builder builder = null;
|
||||||
boolean viaProto = false;
|
private boolean viaProto = false;
|
||||||
|
|
||||||
private ApplicationId appId = null;
|
private ApplicationId appId = null;
|
||||||
private String collectorAddr = null;
|
private String collectorAddr = null;
|
||||||
@ -52,7 +52,7 @@ public AppCollectorsMapPBImpl(AppCollectorsMapProto proto) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public AppCollectorsMapProto getProto() {
|
public AppCollectorsMapProto getProto() {
|
||||||
mergeLocalToProto();
|
mergeLocalToProto();
|
||||||
proto = viaProto ? proto : builder.build();
|
proto = viaProto ? proto : builder.build();
|
||||||
viaProto = true;
|
viaProto = true;
|
||||||
return proto;
|
return proto;
|
||||||
@ -65,8 +65,9 @@ public int hashCode() {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean equals(Object other) {
|
public boolean equals(Object other) {
|
||||||
if (other == null)
|
if (other == null) {
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
if (other.getClass().isAssignableFrom(this.getClass())) {
|
if (other.getClass().isAssignableFrom(this.getClass())) {
|
||||||
return this.getProto().equals(this.getClass().cast(other).getProto());
|
return this.getProto().equals(this.getClass().cast(other).getProto());
|
||||||
}
|
}
|
||||||
@ -98,12 +99,12 @@ public String getCollectorAddr() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setApplicationId(ApplicationId appId) {
|
public void setApplicationId(ApplicationId id) {
|
||||||
maybeInitBuilder();
|
maybeInitBuilder();
|
||||||
if (appId == null) {
|
if (id == null) {
|
||||||
builder.clearAppId();
|
builder.clearAppId();
|
||||||
}
|
}
|
||||||
this.appId = appId;
|
this.appId = id;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -77,7 +77,8 @@ public class TestRPC {
|
|||||||
|
|
||||||
private static final String EXCEPTION_MSG = "test error";
|
private static final String EXCEPTION_MSG = "test error";
|
||||||
private static final String EXCEPTION_CAUSE = "exception cause";
|
private static final String EXCEPTION_CAUSE = "exception cause";
|
||||||
private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
|
private static final RecordFactory RECORD_FACTORY =
|
||||||
|
RecordFactoryProvider.getRecordFactory(null);
|
||||||
|
|
||||||
public static final String ILLEGAL_NUMBER_MESSAGE =
|
public static final String ILLEGAL_NUMBER_MESSAGE =
|
||||||
"collectors' number in ReportNewCollectorInfoRequest is not ONE.";
|
"collectors' number in ReportNewCollectorInfoRequest is not ONE.";
|
||||||
@ -101,7 +102,8 @@ public void testUnknownCall() {
|
|||||||
|
|
||||||
// Any unrelated protocol would do
|
// Any unrelated protocol would do
|
||||||
ApplicationClientProtocol proxy = (ApplicationClientProtocol) rpc.getProxy(
|
ApplicationClientProtocol proxy = (ApplicationClientProtocol) rpc.getProxy(
|
||||||
ApplicationClientProtocol.class, NetUtils.getConnectAddress(server), conf);
|
ApplicationClientProtocol.class, NetUtils.getConnectAddress(server),
|
||||||
|
conf);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
proxy.getNewApplication(Records
|
proxy.getNewApplication(Records
|
||||||
@ -111,7 +113,8 @@ public void testUnknownCall() {
|
|||||||
Assert.assertTrue(e.getMessage().matches(
|
Assert.assertTrue(e.getMessage().matches(
|
||||||
"Unknown method getNewApplication called on.*"
|
"Unknown method getNewApplication called on.*"
|
||||||
+ "org.apache.hadoop.yarn.proto.ApplicationClientProtocol"
|
+ "org.apache.hadoop.yarn.proto.ApplicationClientProtocol"
|
||||||
+ "\\$ApplicationClientProtocolService\\$BlockingInterface protocol."));
|
+ "\\$ApplicationClientProtocolService\\$BlockingInterface "
|
||||||
|
+ "protocol."));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
} finally {
|
} finally {
|
||||||
@ -132,8 +135,10 @@ public void testRPCOnCollectorNodeManagerProtocol() throws IOException {
|
|||||||
server.start();
|
server.start();
|
||||||
|
|
||||||
// Test unrelated protocol wouldn't get response
|
// Test unrelated protocol wouldn't get response
|
||||||
ApplicationClientProtocol unknownProxy = (ApplicationClientProtocol) rpc.getProxy(
|
ApplicationClientProtocol unknownProxy =
|
||||||
ApplicationClientProtocol.class, NetUtils.getConnectAddress(server), conf);
|
(ApplicationClientProtocol) rpc.getProxy(
|
||||||
|
ApplicationClientProtocol.class, NetUtils.getConnectAddress(server),
|
||||||
|
conf);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
unknownProxy.getNewApplication(Records
|
unknownProxy.getNewApplication(Records
|
||||||
@ -143,14 +148,17 @@ public void testRPCOnCollectorNodeManagerProtocol() throws IOException {
|
|||||||
Assert.assertTrue(e.getMessage().matches(
|
Assert.assertTrue(e.getMessage().matches(
|
||||||
"Unknown method getNewApplication called on.*"
|
"Unknown method getNewApplication called on.*"
|
||||||
+ "org.apache.hadoop.yarn.proto.ApplicationClientProtocol"
|
+ "org.apache.hadoop.yarn.proto.ApplicationClientProtocol"
|
||||||
+ "\\$ApplicationClientProtocolService\\$BlockingInterface protocol."));
|
+ "\\$ApplicationClientProtocolService\\$BlockingInterface "
|
||||||
|
+ "protocol."));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test CollectorNodemanagerProtocol get proper response
|
// Test CollectorNodemanagerProtocol get proper response
|
||||||
CollectorNodemanagerProtocol proxy = (CollectorNodemanagerProtocol)rpc.getProxy(
|
CollectorNodemanagerProtocol proxy =
|
||||||
CollectorNodemanagerProtocol.class, NetUtils.getConnectAddress(server), conf);
|
(CollectorNodemanagerProtocol)rpc.getProxy(
|
||||||
|
CollectorNodemanagerProtocol.class, NetUtils.getConnectAddress(server),
|
||||||
|
conf);
|
||||||
// Verify request with DEFAULT_APP_ID and DEFAULT_COLLECTOR_ADDR get
|
// Verify request with DEFAULT_APP_ID and DEFAULT_COLLECTOR_ADDR get
|
||||||
// normally response.
|
// normally response.
|
||||||
try {
|
try {
|
||||||
@ -196,7 +204,8 @@ public void testRPCOnCollectorNodeManagerProtocol() throws IOException {
|
|||||||
Assert.fail("RPC call failured is expected here.");
|
Assert.fail("RPC call failured is expected here.");
|
||||||
} catch (YarnException | IOException e) {
|
} catch (YarnException | IOException e) {
|
||||||
Assert.assertTrue(e instanceof YarnException);
|
Assert.assertTrue(e instanceof YarnException);
|
||||||
Assert.assertTrue(e.getMessage().contains("The application is not found."));
|
Assert.assertTrue(e.getMessage().contains(
|
||||||
|
"The application is not found."));
|
||||||
}
|
}
|
||||||
server.stop();
|
server.stop();
|
||||||
}
|
}
|
||||||
@ -215,12 +224,13 @@ private void test(String rpcClass) throws Exception {
|
|||||||
Server server = rpc.getServer(ContainerManagementProtocol.class,
|
Server server = rpc.getServer(ContainerManagementProtocol.class,
|
||||||
new DummyContainerManager(), addr, conf, null, 1);
|
new DummyContainerManager(), addr, conf, null, 1);
|
||||||
server.start();
|
server.start();
|
||||||
RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
|
RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class,
|
||||||
|
ProtobufRpcEngine.class);
|
||||||
ContainerManagementProtocol proxy = (ContainerManagementProtocol)
|
ContainerManagementProtocol proxy = (ContainerManagementProtocol)
|
||||||
rpc.getProxy(ContainerManagementProtocol.class,
|
rpc.getProxy(ContainerManagementProtocol.class,
|
||||||
NetUtils.getConnectAddress(server), conf);
|
NetUtils.getConnectAddress(server), conf);
|
||||||
ContainerLaunchContext containerLaunchContext =
|
ContainerLaunchContext containerLaunchContext =
|
||||||
recordFactory.newRecordInstance(ContainerLaunchContext.class);
|
RECORD_FACTORY.newRecordInstance(ContainerLaunchContext.class);
|
||||||
|
|
||||||
ApplicationId applicationId = ApplicationId.newInstance(0, 0);
|
ApplicationId applicationId = ApplicationId.newInstance(0, 0);
|
||||||
ApplicationAttemptId applicationAttemptId =
|
ApplicationAttemptId applicationAttemptId =
|
||||||
@ -257,10 +267,10 @@ private void test(String rpcClass) throws Exception {
|
|||||||
boolean exception = false;
|
boolean exception = false;
|
||||||
try {
|
try {
|
||||||
StopContainersRequest stopRequest =
|
StopContainersRequest stopRequest =
|
||||||
recordFactory.newRecordInstance(StopContainersRequest.class);
|
RECORD_FACTORY.newRecordInstance(StopContainersRequest.class);
|
||||||
stopRequest.setContainerIds(containerIds);
|
stopRequest.setContainerIds(containerIds);
|
||||||
proxy.stopContainers(stopRequest);
|
proxy.stopContainers(stopRequest);
|
||||||
} catch (YarnException e) {
|
} catch (YarnException e) {
|
||||||
exception = true;
|
exception = true;
|
||||||
Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
|
Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
|
||||||
Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
|
Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
|
||||||
@ -284,7 +294,7 @@ public GetContainerStatusesResponse getContainerStatuses(
|
|||||||
GetContainerStatusesRequest request)
|
GetContainerStatusesRequest request)
|
||||||
throws YarnException {
|
throws YarnException {
|
||||||
GetContainerStatusesResponse response =
|
GetContainerStatusesResponse response =
|
||||||
recordFactory.newRecordInstance(GetContainerStatusesResponse.class);
|
RECORD_FACTORY.newRecordInstance(GetContainerStatusesResponse.class);
|
||||||
response.setContainerStatuses(statuses);
|
response.setContainerStatuses(statuses);
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
@ -293,8 +303,9 @@ public GetContainerStatusesResponse getContainerStatuses(
|
|||||||
public StartContainersResponse startContainers(
|
public StartContainersResponse startContainers(
|
||||||
StartContainersRequest requests) throws YarnException {
|
StartContainersRequest requests) throws YarnException {
|
||||||
StartContainersResponse response =
|
StartContainersResponse response =
|
||||||
recordFactory.newRecordInstance(StartContainersResponse.class);
|
RECORD_FACTORY.newRecordInstance(StartContainersResponse.class);
|
||||||
for (StartContainerRequest request : requests.getStartContainerRequests()) {
|
for (StartContainerRequest request :
|
||||||
|
requests.getStartContainerRequests()) {
|
||||||
Token containerToken = request.getContainerToken();
|
Token containerToken = request.getContainerToken();
|
||||||
ContainerTokenIdentifier tokenId = null;
|
ContainerTokenIdentifier tokenId = null;
|
||||||
|
|
||||||
@ -304,7 +315,7 @@ public StartContainersResponse startContainers(
|
|||||||
throw RPCUtil.getRemoteException(e);
|
throw RPCUtil.getRemoteException(e);
|
||||||
}
|
}
|
||||||
ContainerStatus status =
|
ContainerStatus status =
|
||||||
recordFactory.newRecordInstance(ContainerStatus.class);
|
RECORD_FACTORY.newRecordInstance(ContainerStatus.class);
|
||||||
status.setState(ContainerState.RUNNING);
|
status.setState(ContainerState.RUNNING);
|
||||||
status.setContainerId(tokenId.getContainerID());
|
status.setContainerId(tokenId.getContainerID());
|
||||||
status.setExitStatus(0);
|
status.setExitStatus(0);
|
||||||
@ -324,7 +335,8 @@ public StopContainersResponse stopContainers(StopContainersRequest request)
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public IncreaseContainersResourceResponse increaseContainersResource(
|
public IncreaseContainersResourceResponse increaseContainersResource(
|
||||||
IncreaseContainersResourceRequest request) throws YarnException, IOException {
|
IncreaseContainersResourceRequest request)
|
||||||
|
throws YarnException, IOException {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -383,7 +395,8 @@ public ReportNewCollectorInfoResponse reportNewCollectorInfo(
|
|||||||
}
|
}
|
||||||
|
|
||||||
ReportNewCollectorInfoResponse response =
|
ReportNewCollectorInfoResponse response =
|
||||||
recordFactory.newRecordInstance(ReportNewCollectorInfoResponse.class);
|
RECORD_FACTORY.newRecordInstance(
|
||||||
|
ReportNewCollectorInfoResponse.class);
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -392,8 +405,8 @@ public GetTimelineCollectorContextResponse getTimelineCollectorContext(
|
|||||||
GetTimelineCollectorContextRequest request)
|
GetTimelineCollectorContextRequest request)
|
||||||
throws YarnException, IOException {
|
throws YarnException, IOException {
|
||||||
if (request.getApplicationId().getId() == 1) {
|
if (request.getApplicationId().getId() == 1) {
|
||||||
return GetTimelineCollectorContextResponse.newInstance(
|
return GetTimelineCollectorContextResponse.newInstance(
|
||||||
"test_user_id", "test_flow_name", "test_flow_version", 12345678L);
|
"test_user_id", "test_flow_name", "test_flow_version", 12345678L);
|
||||||
} else {
|
} else {
|
||||||
throw new YarnException("The application is not found.");
|
throw new YarnException("The application is not found.");
|
||||||
}
|
}
|
||||||
|
@ -187,8 +187,8 @@ protected ContainerManagerImpl createContainerManager(Context context,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected NMCollectorService createNMCollectorService(Context context) {
|
protected NMCollectorService createNMCollectorService(Context ctxt) {
|
||||||
return new NMCollectorService(context);
|
return new NMCollectorService(ctxt);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected WebServer createWebServer(Context nmContext,
|
protected WebServer createWebServer(Context nmContext,
|
||||||
|
@ -816,7 +816,8 @@ public void run() {
|
|||||||
NodeStatusUpdaterImpl.this.context
|
NodeStatusUpdaterImpl.this.context
|
||||||
.getNMTokenSecretManager().getCurrentKey(),
|
.getNMTokenSecretManager().getCurrentKey(),
|
||||||
nodeLabelsForHeartbeat,
|
nodeLabelsForHeartbeat,
|
||||||
NodeStatusUpdaterImpl.this.context.getRegisteredCollectors());
|
NodeStatusUpdaterImpl.this.context
|
||||||
|
.getRegisteredCollectors());
|
||||||
|
|
||||||
if (logAggregationEnabled) {
|
if (logAggregationEnabled) {
|
||||||
// pull log aggregation status for application running in this NM
|
// pull log aggregation status for application running in this NM
|
||||||
@ -939,23 +940,6 @@ public void run() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Caller should take care of sending non null nodelabels for both
|
|
||||||
* arguments
|
|
||||||
*
|
|
||||||
* @param nodeLabelsNew
|
|
||||||
* @param nodeLabelsOld
|
|
||||||
* @return if the New node labels are diff from the older one.
|
|
||||||
*/
|
|
||||||
private boolean areNodeLabelsUpdated(Set<NodeLabel> nodeLabelsNew,
|
|
||||||
Set<NodeLabel> nodeLabelsOld) {
|
|
||||||
if (nodeLabelsNew.size() != nodeLabelsOld.size()
|
|
||||||
|| !nodeLabelsOld.containsAll(nodeLabelsNew)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void updateTimelineClientsAddress(
|
private void updateTimelineClientsAddress(
|
||||||
NodeHeartbeatResponse response) {
|
NodeHeartbeatResponse response) {
|
||||||
Map<ApplicationId, String> knownCollectorsMap =
|
Map<ApplicationId, String> knownCollectorsMap =
|
||||||
|
@ -230,8 +230,7 @@ public ContainerManagerImpl(Context context, ContainerExecutor exec,
|
|||||||
nmMetricsPublisher = createNMTimelinePublisher(context);
|
nmMetricsPublisher = createNMTimelinePublisher(context);
|
||||||
context.setNMTimelinePublisher(nmMetricsPublisher);
|
context.setNMTimelinePublisher(nmMetricsPublisher);
|
||||||
}
|
}
|
||||||
this.containersMonitor =
|
this.containersMonitor = createContainersMonitor(exec);
|
||||||
new ContainersMonitorImpl(exec, dispatcher, this.context);
|
|
||||||
addService(this.containersMonitor);
|
addService(this.containersMonitor);
|
||||||
|
|
||||||
dispatcher.register(ContainerEventType.class,
|
dispatcher.register(ContainerEventType.class,
|
||||||
@ -447,8 +446,9 @@ protected SharedCacheUploadService createSharedCacheUploaderService() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
protected NMTimelinePublisher createNMTimelinePublisher(Context context) {
|
protected NMTimelinePublisher createNMTimelinePublisher(Context ctxt) {
|
||||||
NMTimelinePublisher nmTimelinePublisherLocal = new NMTimelinePublisher(context);
|
NMTimelinePublisher nmTimelinePublisherLocal =
|
||||||
|
new NMTimelinePublisher(ctxt);
|
||||||
addIfService(nmTimelinePublisherLocal);
|
addIfService(nmTimelinePublisherLocal);
|
||||||
return nmTimelinePublisherLocal;
|
return nmTimelinePublisherLocal;
|
||||||
}
|
}
|
||||||
|
@ -25,7 +25,8 @@ public class ApplicationContainerFinishedEvent extends ApplicationEvent {
|
|||||||
private ContainerStatus containerStatus;
|
private ContainerStatus containerStatus;
|
||||||
|
|
||||||
public ApplicationContainerFinishedEvent(ContainerStatus containerStatus) {
|
public ApplicationContainerFinishedEvent(ContainerStatus containerStatus) {
|
||||||
super(containerStatus.getContainerId().getApplicationAttemptId().getApplicationId(),
|
super(containerStatus.getContainerId().getApplicationAttemptId().
|
||||||
|
getApplicationId(),
|
||||||
ApplicationEventType.APPLICATION_CONTAINER_FINISHED);
|
ApplicationEventType.APPLICATION_CONTAINER_FINISHED);
|
||||||
this.containerStatus = containerStatus;
|
this.containerStatus = containerStatus;
|
||||||
}
|
}
|
||||||
|
@ -74,7 +74,6 @@
|
|||||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerMetrics;
|
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerMetrics;
|
||||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerStartMonitoringEvent;
|
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerStartMonitoringEvent;
|
||||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerStopMonitoringEvent;
|
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerStopMonitoringEvent;
|
||||||
import org.apache.hadoop.yarn.server.nodemanager.Context;
|
|
||||||
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
|
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
|
||||||
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
|
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
|
||||||
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerState;
|
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerState;
|
||||||
@ -87,7 +86,6 @@
|
|||||||
import org.apache.hadoop.yarn.state.StateMachine;
|
import org.apache.hadoop.yarn.state.StateMachine;
|
||||||
import org.apache.hadoop.yarn.state.StateMachineFactory;
|
import org.apache.hadoop.yarn.state.StateMachineFactory;
|
||||||
import org.apache.hadoop.yarn.util.Clock;
|
import org.apache.hadoop.yarn.util.Clock;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
|
||||||
import org.apache.hadoop.yarn.util.SystemClock;
|
import org.apache.hadoop.yarn.util.SystemClock;
|
||||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||||
|
|
||||||
|
@ -435,8 +435,9 @@ public void run() {
|
|||||||
+ " for the first time");
|
+ " for the first time");
|
||||||
|
|
||||||
ResourceCalculatorProcessTree pt =
|
ResourceCalculatorProcessTree pt =
|
||||||
ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(
|
ResourceCalculatorProcessTree.
|
||||||
pId, processTreeClass, conf);
|
getResourceCalculatorProcessTree(
|
||||||
|
pId, processTreeClass, conf);
|
||||||
ptInfo.setPid(pId);
|
ptInfo.setPid(pId);
|
||||||
ptInfo.setProcessTree(pt);
|
ptInfo.setProcessTree(pt);
|
||||||
|
|
||||||
|
@ -33,12 +33,12 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.NodeId;
|
import org.apache.hadoop.yarn.api.records.NodeId;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
|
import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
|
||||||
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation;
|
|
||||||
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
|
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
|
||||||
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
|
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
|
||||||
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
|
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
|
||||||
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
|
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
|
||||||
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
|
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
|
||||||
|
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation;
|
||||||
import org.apache.hadoop.yarn.client.api.TimelineClient;
|
import org.apache.hadoop.yarn.client.api.TimelineClient;
|
||||||
import org.apache.hadoop.yarn.event.AsyncDispatcher;
|
import org.apache.hadoop.yarn.event.AsyncDispatcher;
|
||||||
import org.apache.hadoop.yarn.event.Dispatcher;
|
import org.apache.hadoop.yarn.event.Dispatcher;
|
||||||
@ -56,6 +56,8 @@
|
|||||||
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
|
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
|
||||||
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
|
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
|
||||||
|
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Metrics publisher service that publishes data to the timeline service v.2. It
|
* Metrics publisher service that publishes data to the timeline service v.2. It
|
||||||
* is used only if the timeline service v.2 is enabled and the system publishing
|
* is used only if the timeline service v.2 is enabled and the system publishing
|
||||||
@ -73,7 +75,7 @@ public class NMTimelinePublisher extends CompositeService {
|
|||||||
|
|
||||||
private String httpAddress;
|
private String httpAddress;
|
||||||
|
|
||||||
protected final Map<ApplicationId, TimelineClient> appToClientMap;
|
private final Map<ApplicationId, TimelineClient> appToClientMap;
|
||||||
|
|
||||||
public NMTimelinePublisher(Context context) {
|
public NMTimelinePublisher(Context context) {
|
||||||
super(NMTimelinePublisher.class.getName());
|
super(NMTimelinePublisher.class.getName());
|
||||||
@ -99,6 +101,11 @@ protected void serviceStart() throws Exception {
|
|||||||
this.httpAddress = nodeId.getHost() + ":" + context.getHttpPort();
|
this.httpAddress = nodeId.getHost() + ":" + context.getHttpPort();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
Map<ApplicationId, TimelineClient> getAppToClientMap() {
|
||||||
|
return appToClientMap;
|
||||||
|
}
|
||||||
|
|
||||||
protected void handleNMTimelineEvent(NMTimelineEvent event) {
|
protected void handleNMTimelineEvent(NMTimelineEvent event) {
|
||||||
switch (event.getType()) {
|
switch (event.getType()) {
|
||||||
case TIMELINE_ENTITY_PUBLISH:
|
case TIMELINE_ENTITY_PUBLISH:
|
||||||
|
@ -85,7 +85,6 @@
|
|||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto;
|
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto;
|
||||||
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
|
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
|
||||||
import org.apache.hadoop.yarn.server.api.ContainerContext;
|
|
||||||
import org.apache.hadoop.yarn.server.api.ResourceTracker;
|
import org.apache.hadoop.yarn.server.api.ResourceTracker;
|
||||||
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
|
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
|
||||||
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
|
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
|
||||||
@ -1707,9 +1706,9 @@ protected NMContext createNMContext(
|
|||||||
NMContainerTokenSecretManager containerTokenSecretManager,
|
NMContainerTokenSecretManager containerTokenSecretManager,
|
||||||
NMTokenSecretManagerInNM nmTokenSecretManager,
|
NMTokenSecretManagerInNM nmTokenSecretManager,
|
||||||
NMStateStoreService store, boolean isDistributedSchedulingEnabled,
|
NMStateStoreService store, boolean isDistributedSchedulingEnabled,
|
||||||
Configuration conf) {
|
Configuration config) {
|
||||||
return new MyNMContext(containerTokenSecretManager,
|
return new MyNMContext(containerTokenSecretManager,
|
||||||
nmTokenSecretManager, conf);
|
nmTokenSecretManager, config);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -645,8 +645,9 @@ private Container createMockedContainer(ApplicationId appId, int containerId) {
|
|||||||
when(c.getLaunchContext()).thenReturn(launchContext);
|
when(c.getLaunchContext()).thenReturn(launchContext);
|
||||||
when(launchContext.getApplicationACLs()).thenReturn(
|
when(launchContext.getApplicationACLs()).thenReturn(
|
||||||
new HashMap<ApplicationAccessType, String>());
|
new HashMap<ApplicationAccessType, String>());
|
||||||
when(c.cloneAndGetContainerStatus()).thenReturn(BuilderUtils.newContainerStatus(cId,
|
when(c.cloneAndGetContainerStatus()).thenReturn(
|
||||||
ContainerState.NEW, "", 0, Resource.newInstance(1024, 1)));
|
BuilderUtils.newContainerStatus(cId,
|
||||||
|
ContainerState.NEW, "", 0, Resource.newInstance(1024, 1)));
|
||||||
return c;
|
return c;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -99,7 +99,6 @@
|
|||||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||||
import org.apache.hadoop.yarn.util.Apps;
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
|
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
|
||||||
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
|
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
|
||||||
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
|
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
|
||||||
import org.hamcrest.CoreMatchers;
|
import org.hamcrest.CoreMatchers;
|
||||||
@ -111,16 +110,17 @@
|
|||||||
public class TestContainerLaunch extends BaseContainerManagerTest {
|
public class TestContainerLaunch extends BaseContainerManagerTest {
|
||||||
|
|
||||||
private static final String INVALID_JAVA_HOME = "/no/jvm/here";
|
private static final String INVALID_JAVA_HOME = "/no/jvm/here";
|
||||||
protected Context distContext = new NMContext(new NMContainerTokenSecretManager(
|
private Context distContext =
|
||||||
conf), new NMTokenSecretManagerInNM(), null,
|
new NMContext(new NMContainerTokenSecretManager(conf),
|
||||||
new ApplicationACLsManager(conf), new NMNullStateStoreService(), false,
|
new NMTokenSecretManagerInNM(), null,
|
||||||
conf) {
|
new ApplicationACLsManager(conf), new NMNullStateStoreService(),
|
||||||
public int getHttpPort() {
|
false, conf) {
|
||||||
return HTTP_PORT;
|
public int getHttpPort() {
|
||||||
};
|
return HTTP_PORT;
|
||||||
public NodeId getNodeId() {
|
};
|
||||||
return NodeId.newInstance("ahost", 1234);
|
public NodeId getNodeId() {
|
||||||
};
|
return NodeId.newInstance("ahost", 1234);
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
public TestContainerLaunch() throws UnsupportedFileSystemException {
|
public TestContainerLaunch() throws UnsupportedFileSystemException {
|
||||||
|
@ -55,8 +55,8 @@ public void testContainerResourceUsage() {
|
|||||||
when(context.getHttpPort()).thenReturn(0);
|
when(context.getHttpPort()).thenReturn(0);
|
||||||
NMTimelinePublisher publisher = new NMTimelinePublisher(context) {
|
NMTimelinePublisher publisher = new NMTimelinePublisher(context) {
|
||||||
public void createTimelineClient(ApplicationId appId) {
|
public void createTimelineClient(ApplicationId appId) {
|
||||||
if (!appToClientMap.containsKey(appId)) {
|
if (!getAppToClientMap().containsKey(appId)) {
|
||||||
appToClientMap.put(appId, timelineClient);
|
getAppToClientMap().put(appId, timelineClient);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -24,7 +24,6 @@
|
|||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
import org.apache.hadoop.yarn.client.api.TimelineClient;
|
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
|
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
|
||||||
@ -40,10 +39,9 @@ public class MockApp implements Application {
|
|||||||
Map<ContainerId, Container> containers = new HashMap<ContainerId, Container>();
|
Map<ContainerId, Container> containers = new HashMap<ContainerId, Container>();
|
||||||
ApplicationState appState;
|
ApplicationState appState;
|
||||||
Application app;
|
Application app;
|
||||||
String flowName;
|
private String flowName;
|
||||||
String flowVersion;
|
private String flowVersion;
|
||||||
long flowRunId;
|
private long flowRunId;
|
||||||
TimelineClient timelineClient = null;
|
|
||||||
|
|
||||||
public MockApp(int uniqId) {
|
public MockApp(int uniqId) {
|
||||||
this("mockUser", 1234, uniqId);
|
this("mockUser", 1234, uniqId);
|
||||||
|
@ -373,7 +373,8 @@ public RMApplicationHistoryWriter getRMApplicationHistoryWriter() {
|
|||||||
@Override
|
@Override
|
||||||
public void setRMTimelineCollectorManager(
|
public void setRMTimelineCollectorManager(
|
||||||
RMTimelineCollectorManager timelineCollectorManager) {
|
RMTimelineCollectorManager timelineCollectorManager) {
|
||||||
activeServiceContext.setRMTimelineCollectorManager(timelineCollectorManager);
|
activeServiceContext.setRMTimelineCollectorManager(
|
||||||
|
timelineCollectorManager);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -311,7 +311,7 @@ public void containerFinished(RMContainer container, long finishedTime) {
|
|||||||
entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO,
|
entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO,
|
||||||
container.getAllocatedNode().getHost());
|
container.getAllocatedNode().getHost());
|
||||||
entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO,
|
entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO,
|
||||||
container.getAllocatedNode().getPort());
|
container.getAllocatedNode().getPort());
|
||||||
entity.setOtherInfo(entityInfo);
|
entity.setOtherInfo(entityInfo);
|
||||||
tEvent.setEventInfo(eventInfo);
|
tEvent.setEventInfo(eventInfo);
|
||||||
|
|
||||||
|
@ -76,9 +76,9 @@
|
|||||||
public class TestSystemMetricsPublisherForV2 {
|
public class TestSystemMetricsPublisherForV2 {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* is the folder where the FileSystemTimelineWriterImpl writes the entities
|
* The folder where the FileSystemTimelineWriterImpl writes the entities.
|
||||||
*/
|
*/
|
||||||
protected static File testRootDir = new File("target",
|
private static File testRootDir = new File("target",
|
||||||
TestSystemMetricsPublisherForV2.class.getName() + "-localDir")
|
TestSystemMetricsPublisherForV2.class.getName() + "-localDir")
|
||||||
.getAbsoluteFile();
|
.getAbsoluteFile();
|
||||||
|
|
||||||
@ -151,7 +151,8 @@ private static Configuration getTimelineV2Conf() {
|
|||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
Assert
|
Assert
|
||||||
.fail("Exception while setting the TIMELINE_SERVICE_STORAGE_DIR_ROOT ");
|
.fail("Exception while setting the " +
|
||||||
|
"TIMELINE_SERVICE_STORAGE_DIR_ROOT ");
|
||||||
}
|
}
|
||||||
return conf;
|
return conf;
|
||||||
}
|
}
|
||||||
@ -159,30 +160,30 @@ private static Configuration getTimelineV2Conf() {
|
|||||||
@Test
|
@Test
|
||||||
public void testSystemMetricPublisherInitialization() {
|
public void testSystemMetricPublisherInitialization() {
|
||||||
@SuppressWarnings("resource")
|
@SuppressWarnings("resource")
|
||||||
TimelineServiceV2Publisher metricsPublisher =
|
TimelineServiceV2Publisher publisher =
|
||||||
new TimelineServiceV2Publisher(mock(RMContext.class));
|
new TimelineServiceV2Publisher(mock(RMContext.class));
|
||||||
try {
|
try {
|
||||||
Configuration conf = getTimelineV2Conf();
|
Configuration conf = getTimelineV2Conf();
|
||||||
conf.setBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_EVENTS_ENABLED,
|
conf.setBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_EVENTS_ENABLED,
|
||||||
YarnConfiguration.DEFAULT_RM_PUBLISH_CONTAINER_EVENTS_ENABLED);
|
YarnConfiguration.DEFAULT_RM_PUBLISH_CONTAINER_EVENTS_ENABLED);
|
||||||
metricsPublisher.init(conf);
|
publisher.init(conf);
|
||||||
assertFalse(
|
assertFalse(
|
||||||
"Default configuration should not publish container events from RM",
|
"Default configuration should not publish container events from RM",
|
||||||
metricsPublisher.isPublishContainerEvents());
|
publisher.isPublishContainerEvents());
|
||||||
|
|
||||||
metricsPublisher.stop();
|
publisher.stop();
|
||||||
|
|
||||||
metricsPublisher = new TimelineServiceV2Publisher(mock(RMContext.class));
|
publisher = new TimelineServiceV2Publisher(mock(RMContext.class));
|
||||||
conf = getTimelineV2Conf();
|
conf = getTimelineV2Conf();
|
||||||
metricsPublisher.init(conf);
|
publisher.init(conf);
|
||||||
assertTrue("Expected to have registered event handlers and set ready to "
|
assertTrue("Expected to have registered event handlers and set ready to "
|
||||||
+ "publish events after init",
|
+ "publish events after init",
|
||||||
metricsPublisher.isPublishContainerEvents());
|
publisher.isPublishContainerEvents());
|
||||||
metricsPublisher.start();
|
publisher.start();
|
||||||
assertTrue("Expected to publish container events from RM",
|
assertTrue("Expected to publish container events from RM",
|
||||||
metricsPublisher.isPublishContainerEvents());
|
publisher.isPublishContainerEvents());
|
||||||
} finally {
|
} finally {
|
||||||
metricsPublisher.stop();
|
publisher.stop();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -243,7 +244,7 @@ public void testPublishAppAttemptMetrics() throws Exception {
|
|||||||
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
|
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
|
||||||
File appFile = new File(outputDirApp, timelineServiceFileName);
|
File appFile = new File(outputDirApp, timelineServiceFileName);
|
||||||
Assert.assertTrue(appFile.exists());
|
Assert.assertTrue(appFile.exists());
|
||||||
verifyEntity(appFile,2, AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
|
verifyEntity(appFile, 2, AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(timeout = 10000)
|
@Test(timeout = 10000)
|
||||||
|
@ -134,7 +134,8 @@ public void testPutExtendedEntities() throws Exception {
|
|||||||
ApplicationEntity app = new ApplicationEntity();
|
ApplicationEntity app = new ApplicationEntity();
|
||||||
app.setId(appId.toString());
|
app.setId(appId.toString());
|
||||||
flow.addChild(app.getType(), app.getId());
|
flow.addChild(app.getType(), app.getId());
|
||||||
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
|
ApplicationAttemptId attemptId =
|
||||||
|
ApplicationAttemptId.newInstance(appId, 1);
|
||||||
ApplicationAttemptEntity appAttempt = new ApplicationAttemptEntity();
|
ApplicationAttemptEntity appAttempt = new ApplicationAttemptEntity();
|
||||||
appAttempt.setId(attemptId.toString());
|
appAttempt.setId(attemptId.toString());
|
||||||
ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
|
ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
|
||||||
@ -144,8 +145,10 @@ public void testPutExtendedEntities() throws Exception {
|
|||||||
user.setId(UserGroupInformation.getCurrentUser().getShortUserName());
|
user.setId(UserGroupInformation.getCurrentUser().getShortUserName());
|
||||||
QueueEntity queue = new QueueEntity();
|
QueueEntity queue = new QueueEntity();
|
||||||
queue.setId("default_queue");
|
queue.setId("default_queue");
|
||||||
client.putEntities(cluster, flow, app, appAttempt, container, user, queue);
|
client.putEntities(cluster, flow, app, appAttempt, container, user,
|
||||||
client.putEntitiesAsync(cluster, flow, app, appAttempt, container, user, queue);
|
queue);
|
||||||
|
client.putEntitiesAsync(cluster, flow, app, appAttempt, container, user,
|
||||||
|
queue);
|
||||||
} finally {
|
} finally {
|
||||||
client.stop();
|
client.stop();
|
||||||
}
|
}
|
||||||
|
@ -256,17 +256,17 @@ private static void loadData() throws Exception {
|
|||||||
entity5.addEvent(event54);
|
entity5.addEvent(event54);
|
||||||
Map<String, Set<String>> isRelatedTo1 = new HashMap<String, Set<String>>();
|
Map<String, Set<String>> isRelatedTo1 = new HashMap<String, Set<String>>();
|
||||||
isRelatedTo1.put("type2",
|
isRelatedTo1.put("type2",
|
||||||
Sets.newHashSet("entity21","entity22","entity23","entity24"));
|
Sets.newHashSet("entity21", "entity22", "entity23", "entity24"));
|
||||||
isRelatedTo1.put("type4", Sets.newHashSet("entity41","entity42"));
|
isRelatedTo1.put("type4", Sets.newHashSet("entity41", "entity42"));
|
||||||
isRelatedTo1.put("type1", Sets.newHashSet("entity14","entity15"));
|
isRelatedTo1.put("type1", Sets.newHashSet("entity14", "entity15"));
|
||||||
isRelatedTo1.put("type3",
|
isRelatedTo1.put("type3",
|
||||||
Sets.newHashSet("entity31", "entity35", "entity32", "entity33"));
|
Sets.newHashSet("entity31", "entity35", "entity32", "entity33"));
|
||||||
entity5.addIsRelatedToEntities(isRelatedTo1);
|
entity5.addIsRelatedToEntities(isRelatedTo1);
|
||||||
Map<String, Set<String>> relatesTo1 = new HashMap<String, Set<String>>();
|
Map<String, Set<String>> relatesTo1 = new HashMap<String, Set<String>>();
|
||||||
relatesTo1.put("type2",
|
relatesTo1.put("type2",
|
||||||
Sets.newHashSet("entity21","entity22","entity23","entity24"));
|
Sets.newHashSet("entity21", "entity22", "entity23", "entity24"));
|
||||||
relatesTo1.put("type4", Sets.newHashSet("entity41","entity42"));
|
relatesTo1.put("type4", Sets.newHashSet("entity41", "entity42"));
|
||||||
relatesTo1.put("type1", Sets.newHashSet("entity14","entity15"));
|
relatesTo1.put("type1", Sets.newHashSet("entity14", "entity15"));
|
||||||
relatesTo1.put("type3",
|
relatesTo1.put("type3",
|
||||||
Sets.newHashSet("entity31", "entity35", "entity32", "entity33"));
|
Sets.newHashSet("entity31", "entity35", "entity32", "entity33"));
|
||||||
entity5.addRelatesToEntities(relatesTo1);
|
entity5.addRelatesToEntities(relatesTo1);
|
||||||
@ -317,16 +317,16 @@ private static void loadData() throws Exception {
|
|||||||
entity6.addEvent(event64);
|
entity6.addEvent(event64);
|
||||||
Map<String, Set<String>> isRelatedTo2 = new HashMap<String, Set<String>>();
|
Map<String, Set<String>> isRelatedTo2 = new HashMap<String, Set<String>>();
|
||||||
isRelatedTo2.put("type2",
|
isRelatedTo2.put("type2",
|
||||||
Sets.newHashSet("entity21","entity22","entity23","entity24"));
|
Sets.newHashSet("entity21", "entity22", "entity23", "entity24"));
|
||||||
isRelatedTo2.put("type5", Sets.newHashSet("entity51","entity52"));
|
isRelatedTo2.put("type5", Sets.newHashSet("entity51", "entity52"));
|
||||||
isRelatedTo2.put("type6", Sets.newHashSet("entity61","entity66"));
|
isRelatedTo2.put("type6", Sets.newHashSet("entity61", "entity66"));
|
||||||
isRelatedTo2.put("type3", Sets.newHashSet("entity31"));
|
isRelatedTo2.put("type3", Sets.newHashSet("entity31"));
|
||||||
entity6.addIsRelatedToEntities(isRelatedTo2);
|
entity6.addIsRelatedToEntities(isRelatedTo2);
|
||||||
Map<String, Set<String>> relatesTo2 = new HashMap<String, Set<String>>();
|
Map<String, Set<String>> relatesTo2 = new HashMap<String, Set<String>>();
|
||||||
relatesTo2.put("type2",
|
relatesTo2.put("type2",
|
||||||
Sets.newHashSet("entity21","entity22","entity23","entity24"));
|
Sets.newHashSet("entity21", "entity22", "entity23", "entity24"));
|
||||||
relatesTo2.put("type5", Sets.newHashSet("entity51","entity52"));
|
relatesTo2.put("type5", Sets.newHashSet("entity51", "entity52"));
|
||||||
relatesTo2.put("type6", Sets.newHashSet("entity61","entity66"));
|
relatesTo2.put("type6", Sets.newHashSet("entity61", "entity66"));
|
||||||
relatesTo2.put("type3", Sets.newHashSet("entity31"));
|
relatesTo2.put("type3", Sets.newHashSet("entity31"));
|
||||||
entity6.addRelatesToEntities(relatesTo2);
|
entity6.addRelatesToEntities(relatesTo2);
|
||||||
te5.addEntity(entity6);
|
te5.addEntity(entity6);
|
||||||
@ -391,10 +391,11 @@ private static ClientResponse getResponse(Client client, URI uri)
|
|||||||
client.resource(uri).accept(MediaType.APPLICATION_JSON)
|
client.resource(uri).accept(MediaType.APPLICATION_JSON)
|
||||||
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
|
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
|
||||||
if (resp == null ||
|
if (resp == null ||
|
||||||
resp.getClientResponseStatus() != ClientResponse.Status.OK) {
|
resp.getStatusInfo().getStatusCode() !=
|
||||||
|
ClientResponse.Status.OK.getStatusCode()) {
|
||||||
String msg = "";
|
String msg = "";
|
||||||
if (resp != null) {
|
if (resp != null) {
|
||||||
msg = resp.getClientResponseStatus().toString();
|
msg = String.valueOf(resp.getStatusInfo().getStatusCode());
|
||||||
}
|
}
|
||||||
throw new IOException("Incorrect response from timeline reader. " +
|
throw new IOException("Incorrect response from timeline reader. " +
|
||||||
"Status=" + msg);
|
"Status=" + msg);
|
||||||
@ -406,7 +407,8 @@ private static class DummyURLConnectionFactory
|
|||||||
implements HttpURLConnectionFactory {
|
implements HttpURLConnectionFactory {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public HttpURLConnection getHttpURLConnection(final URL url) throws IOException {
|
public HttpURLConnection getHttpURLConnection(final URL url)
|
||||||
|
throws IOException {
|
||||||
try {
|
try {
|
||||||
return (HttpURLConnection)url.openConnection();
|
return (HttpURLConnection)url.openConnection();
|
||||||
} catch (UndeclaredThrowableException e) {
|
} catch (UndeclaredThrowableException e) {
|
||||||
@ -422,10 +424,10 @@ private static TimelineEntity newEntity(String type, String id) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static TimelineMetric newMetric(TimelineMetric.Type type,
|
private static TimelineMetric newMetric(TimelineMetric.Type type,
|
||||||
String id, long ts, Number value) {
|
String id, long t, Number value) {
|
||||||
TimelineMetric metric = new TimelineMetric(type);
|
TimelineMetric metric = new TimelineMetric(type);
|
||||||
metric.setId(id);
|
metric.setId(id);
|
||||||
metric.addValue(ts, value);
|
metric.addValue(t, value);
|
||||||
return metric;
|
return metric;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -463,7 +465,7 @@ private static void verifyHttpResponse(Client client, URI uri,
|
|||||||
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
|
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
|
||||||
assertNotNull(resp);
|
assertNotNull(resp);
|
||||||
assertTrue("Response from server should have been " + status,
|
assertTrue("Response from server should have been " + status,
|
||||||
resp.getClientResponseStatus().equals(status));
|
resp.getStatusInfo().getStatusCode() == status.getStatusCode());
|
||||||
System.out.println("Response is: " + resp.getEntity(String.class));
|
System.out.println("Response is: " + resp.getEntity(String.class));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -866,7 +868,7 @@ public void testUIDQueryWithAndWithoutFlowContextInfo() throws Exception {
|
|||||||
String appUIDWithoutFlowInfo = "cluster1!application_1111111111_1111";
|
String appUIDWithoutFlowInfo = "cluster1!application_1111111111_1111";
|
||||||
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/"+
|
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/"+
|
||||||
"app-uid/" + appUIDWithoutFlowInfo);
|
"app-uid/" + appUIDWithoutFlowInfo);
|
||||||
resp = getResponse(client, uri);;
|
resp = getResponse(client, uri);
|
||||||
TimelineEntity appEntity2 = resp.getEntity(TimelineEntity.class);
|
TimelineEntity appEntity2 = resp.getEntity(TimelineEntity.class);
|
||||||
assertNotNull(appEntity2);
|
assertNotNull(appEntity2);
|
||||||
assertEquals(
|
assertEquals(
|
||||||
@ -893,7 +895,7 @@ public void testUIDQueryWithAndWithoutFlowContextInfo() throws Exception {
|
|||||||
String entityUIDWithFlowInfo = appUIDWithFlowInfo + "!type1!entity1";
|
String entityUIDWithFlowInfo = appUIDWithFlowInfo + "!type1!entity1";
|
||||||
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/"+
|
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/"+
|
||||||
"entity-uid/" + entityUIDWithFlowInfo);
|
"entity-uid/" + entityUIDWithFlowInfo);
|
||||||
resp = getResponse(client, uri);;
|
resp = getResponse(client, uri);
|
||||||
TimelineEntity singleEntity1 = resp.getEntity(TimelineEntity.class);
|
TimelineEntity singleEntity1 = resp.getEntity(TimelineEntity.class);
|
||||||
assertNotNull(singleEntity1);
|
assertNotNull(singleEntity1);
|
||||||
assertEquals("type1", singleEntity1.getType());
|
assertEquals("type1", singleEntity1.getType());
|
||||||
@ -903,7 +905,7 @@ public void testUIDQueryWithAndWithoutFlowContextInfo() throws Exception {
|
|||||||
appUIDWithoutFlowInfo + "!type1!entity1";
|
appUIDWithoutFlowInfo + "!type1!entity1";
|
||||||
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/"+
|
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/"+
|
||||||
"entity-uid/" + entityUIDWithoutFlowInfo);
|
"entity-uid/" + entityUIDWithoutFlowInfo);
|
||||||
resp = getResponse(client, uri);;
|
resp = getResponse(client, uri);
|
||||||
TimelineEntity singleEntity2 = resp.getEntity(TimelineEntity.class);
|
TimelineEntity singleEntity2 = resp.getEntity(TimelineEntity.class);
|
||||||
assertNotNull(singleEntity2);
|
assertNotNull(singleEntity2);
|
||||||
assertEquals("type1", singleEntity2.getType());
|
assertEquals("type1", singleEntity2.getType());
|
||||||
@ -1054,12 +1056,12 @@ public void testGetApp() throws Exception {
|
|||||||
assertEquals("application_1111111111_2222", entity.getId());
|
assertEquals("application_1111111111_2222", entity.getId());
|
||||||
assertEquals(1, entity.getMetrics().size());
|
assertEquals(1, entity.getMetrics().size());
|
||||||
TimelineMetric m4 = newMetric(TimelineMetric.Type.SINGLE_VALUE,
|
TimelineMetric m4 = newMetric(TimelineMetric.Type.SINGLE_VALUE,
|
||||||
"MAP_SLOT_MILLIS", ts - 80000, 101L);
|
"MAP_SLOT_MILLIS", ts - 80000, 101L);
|
||||||
for (TimelineMetric metric : entity.getMetrics()) {
|
for (TimelineMetric metric : entity.getMetrics()) {
|
||||||
assertTrue(verifyMetrics(metric, m4));
|
assertTrue(verifyMetrics(metric, m4));
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
client.destroy();
|
client.destroy();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1425,7 +1427,8 @@ public void testGetEntitiesInfoFilters() throws Exception {
|
|||||||
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
|
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
|
||||||
"timeline/clusters/cluster1/apps/application_1111111111_1111/" +
|
"timeline/clusters/cluster1/apps/application_1111111111_1111/" +
|
||||||
"entities/type1?infofilters=(info1%20eq%20cluster1%20AND%20info4%20" +
|
"entities/type1?infofilters=(info1%20eq%20cluster1%20AND%20info4%20" +
|
||||||
"eq%2035000)%20OR%20(info1%20eq%20cluster2%20AND%20info2%20eq%202.0)");
|
"eq%2035000)%20OR%20(info1%20eq%20cluster2%20AND%20info2%20eq%202.0" +
|
||||||
|
")");
|
||||||
resp = getResponse(client, uri);
|
resp = getResponse(client, uri);
|
||||||
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
|
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
|
||||||
assertNotNull(entities);
|
assertNotNull(entities);
|
||||||
|
@ -444,17 +444,17 @@ private static void loadEntities() throws IOException {
|
|||||||
te.addEntity(entity2);
|
te.addEntity(entity2);
|
||||||
HBaseTimelineWriterImpl hbi = null;
|
HBaseTimelineWriterImpl hbi = null;
|
||||||
try {
|
try {
|
||||||
hbi = new HBaseTimelineWriterImpl(util.getConfiguration());
|
hbi = new HBaseTimelineWriterImpl(util.getConfiguration());
|
||||||
hbi.init(util.getConfiguration());
|
hbi.init(util.getConfiguration());
|
||||||
hbi.start();
|
hbi.start();
|
||||||
String cluster = "cluster1";
|
String cluster = "cluster1";
|
||||||
String user = "user1";
|
String user = "user1";
|
||||||
String flow = "some_flow_name";
|
String flow = "some_flow_name";
|
||||||
String flowVersion = "AB7822C10F1111";
|
String flowVersion = "AB7822C10F1111";
|
||||||
long runid = 1002345678919L;
|
long runid = 1002345678919L;
|
||||||
String appName = "application_1231111111_1111";
|
String appName = "application_1231111111_1111";
|
||||||
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
|
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
|
||||||
hbi.stop();
|
hbi.stop();
|
||||||
} finally {
|
} finally {
|
||||||
if (hbi != null) {
|
if (hbi != null) {
|
||||||
hbi.stop();
|
hbi.stop();
|
||||||
@ -531,7 +531,7 @@ public void testWriteNullApplicationToHBase() throws Exception {
|
|||||||
int count = 0;
|
int count = 0;
|
||||||
for (Result rr = resultScanner.next(); rr != null;
|
for (Result rr = resultScanner.next(); rr != null;
|
||||||
rr = resultScanner.next()) {
|
rr = resultScanner.next()) {
|
||||||
count++;
|
count++;
|
||||||
}
|
}
|
||||||
// there should be no rows written
|
// there should be no rows written
|
||||||
// no exceptions thrown during write
|
// no exceptions thrown during write
|
||||||
@ -1173,7 +1173,7 @@ public void testEvents() throws IOException {
|
|||||||
for (TimelineEvent e : events) {
|
for (TimelineEvent e : events) {
|
||||||
assertEquals(eventId, e.getId());
|
assertEquals(eventId, e.getId());
|
||||||
assertEquals(expTs, Long.valueOf(e.getTimestamp()));
|
assertEquals(expTs, Long.valueOf(e.getTimestamp()));
|
||||||
Map<String,Object> info = e.getInfo();
|
Map<String, Object> info = e.getInfo();
|
||||||
assertEquals(1, info.size());
|
assertEquals(1, info.size());
|
||||||
for (Map.Entry<String, Object> infoEntry : info.entrySet()) {
|
for (Map.Entry<String, Object> infoEntry : info.entrySet()) {
|
||||||
assertEquals(expKey, infoEntry.getKey());
|
assertEquals(expKey, infoEntry.getKey());
|
||||||
@ -1249,7 +1249,7 @@ public void testEventsWithEmptyInfo() throws IOException {
|
|||||||
// the qualifier is a compound key
|
// the qualifier is a compound key
|
||||||
// hence match individual values
|
// hence match individual values
|
||||||
assertEquals(eventId, eventColumnName.getId());
|
assertEquals(eventId, eventColumnName.getId());
|
||||||
assertEquals(expTs,eventColumnName.getTimestamp());
|
assertEquals(expTs, eventColumnName.getTimestamp());
|
||||||
// key must be empty
|
// key must be empty
|
||||||
assertNull(eventColumnName.getInfoKey());
|
assertNull(eventColumnName.getInfoKey());
|
||||||
Object value = e.getValue();
|
Object value = e.getValue();
|
||||||
@ -1280,7 +1280,7 @@ public void testEventsWithEmptyInfo() throws IOException {
|
|||||||
for (TimelineEvent e : events) {
|
for (TimelineEvent e : events) {
|
||||||
assertEquals(eventId, e.getId());
|
assertEquals(eventId, e.getId());
|
||||||
assertEquals(expTs, Long.valueOf(e.getTimestamp()));
|
assertEquals(expTs, Long.valueOf(e.getTimestamp()));
|
||||||
Map<String,Object> info = e.getInfo();
|
Map<String, Object> info = e.getInfo();
|
||||||
assertTrue(info == null || info.isEmpty());
|
assertTrue(info == null || info.isEmpty());
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
@ -1337,7 +1337,7 @@ public void testEventsEscapeTs() throws IOException {
|
|||||||
for (TimelineEvent e : events) {
|
for (TimelineEvent e : events) {
|
||||||
assertEquals(eventId, e.getId());
|
assertEquals(eventId, e.getId());
|
||||||
assertEquals(expTs, e.getTimestamp());
|
assertEquals(expTs, e.getTimestamp());
|
||||||
Map<String,Object> info = e.getInfo();
|
Map<String, Object> info = e.getInfo();
|
||||||
assertEquals(1, info.size());
|
assertEquals(1, info.size());
|
||||||
for (Map.Entry<String, Object> infoEntry : info.entrySet()) {
|
for (Map.Entry<String, Object> infoEntry : info.entrySet()) {
|
||||||
assertEquals(expKey, infoEntry.getKey());
|
assertEquals(expKey, infoEntry.getKey());
|
||||||
@ -1417,14 +1417,14 @@ public void testNonIntegralMetricValues() throws IOException {
|
|||||||
public void testReadEntities() throws Exception {
|
public void testReadEntities() throws Exception {
|
||||||
TimelineEntity entity = reader.getEntity(
|
TimelineEntity entity = reader.getEntity(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", "hello"),
|
1002345678919L, "application_1231111111_1111", "world", "hello"),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
||||||
assertNotNull(entity);
|
assertNotNull(entity);
|
||||||
assertEquals(3, entity.getConfigs().size());
|
assertEquals(3, entity.getConfigs().size());
|
||||||
assertEquals(1, entity.getIsRelatedToEntities().size());
|
assertEquals(1, entity.getIsRelatedToEntities().size());
|
||||||
Set<TimelineEntity> entities = reader.getEntities(
|
Set<TimelineEntity> entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world",
|
1002345678919L, "application_1231111111_1111", "world",
|
||||||
null), new TimelineEntityFilters(),
|
null), new TimelineEntityFilters(),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
||||||
assertEquals(3, entities.size());
|
assertEquals(3, entities.size());
|
||||||
@ -1460,7 +1460,7 @@ public void testReadEntities() throws Exception {
|
|||||||
public void testFilterEntitiesByCreatedTime() throws Exception {
|
public void testFilterEntitiesByCreatedTime() throws Exception {
|
||||||
Set<TimelineEntity> entities = reader.getEntities(
|
Set<TimelineEntity> entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, 1425016502000L, 1425016502040L, null,
|
new TimelineEntityFilters(null, 1425016502000L, 1425016502040L, null,
|
||||||
null, null, null, null, null), new TimelineDataToRetrieve());
|
null, null, null, null, null), new TimelineDataToRetrieve());
|
||||||
assertEquals(3, entities.size());
|
assertEquals(3, entities.size());
|
||||||
@ -1468,12 +1468,12 @@ public void testFilterEntitiesByCreatedTime() throws Exception {
|
|||||||
if (!entity.getId().equals("hello") && !entity.getId().equals("hello1") &&
|
if (!entity.getId().equals("hello") && !entity.getId().equals("hello1") &&
|
||||||
!entity.getId().equals("hello2")) {
|
!entity.getId().equals("hello2")) {
|
||||||
Assert.fail("Entities with ids' hello, hello1 and hello2 should be" +
|
Assert.fail("Entities with ids' hello, hello1 and hello2 should be" +
|
||||||
" present");
|
" present");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, 1425016502015L, null, null, null, null,
|
new TimelineEntityFilters(null, 1425016502015L, null, null, null, null,
|
||||||
null, null, null), new TimelineDataToRetrieve());
|
null, null, null), new TimelineDataToRetrieve());
|
||||||
assertEquals(2, entities.size());
|
assertEquals(2, entities.size());
|
||||||
@ -1485,15 +1485,15 @@ public void testFilterEntitiesByCreatedTime() throws Exception {
|
|||||||
}
|
}
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, 1425016502015L, null, null, null,
|
new TimelineEntityFilters(null, null, 1425016502015L, null, null, null,
|
||||||
null, null, null), new TimelineDataToRetrieve());
|
null, null, null), new TimelineDataToRetrieve());
|
||||||
assertEquals(1, entities.size());
|
assertEquals(1, entities.size());
|
||||||
for (TimelineEntity entity : entities) {
|
for (TimelineEntity entity : entities) {
|
||||||
if (!entity.getId().equals("hello")) {
|
if (!entity.getId().equals("hello")) {
|
||||||
Assert.fail("Entity with id hello should be present");
|
Assert.fail("Entity with id hello should be present");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@ -1518,7 +1518,7 @@ public void testReadEntitiesRelationsAndEventFiltersDefaultView()
|
|||||||
new HashSet<Object>(Arrays.asList("relatedto5"))));
|
new HashSet<Object>(Arrays.asList("relatedto5"))));
|
||||||
Set<TimelineEntity> entities = reader.getEntities(
|
Set<TimelineEntity> entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, relatesTo, isRelatedTo,
|
new TimelineEntityFilters(null, null, null, relatesTo, isRelatedTo,
|
||||||
null, null, null, eventFilter), new TimelineDataToRetrieve());
|
null, null, null, eventFilter), new TimelineDataToRetrieve());
|
||||||
assertEquals(1, entities.size());
|
assertEquals(1, entities.size());
|
||||||
@ -1547,7 +1547,7 @@ public void testReadEntitiesEventFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "end_event"));
|
TimelineCompareOp.NOT_EQUAL, "end_event"));
|
||||||
Set<TimelineEntity> entities = reader.getEntities(
|
Set<TimelineEntity> entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
null, ef),
|
null, ef),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
||||||
@ -1568,7 +1568,7 @@ public void testReadEntitiesEventFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "end_event"));
|
TimelineCompareOp.NOT_EQUAL, "end_event"));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
null, ef1),
|
null, ef1),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
@ -1587,7 +1587,7 @@ public void testReadEntitiesEventFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "end_event"));
|
TimelineCompareOp.NOT_EQUAL, "end_event"));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
null, ef2),
|
null, ef2),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
@ -1609,7 +1609,7 @@ public void testReadEntitiesEventFilters() throws Exception {
|
|||||||
TimelineCompareOp.EQUAL, "dummy_event"));
|
TimelineCompareOp.EQUAL, "dummy_event"));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
null, ef3),
|
null, ef3),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
@ -1626,7 +1626,7 @@ public void testReadEntitiesEventFilters() throws Exception {
|
|||||||
TimelineFilterList ef4 = new TimelineFilterList(Operator.OR, list1, list2);
|
TimelineFilterList ef4 = new TimelineFilterList(Operator.OR, list1, list2);
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
null, ef4),
|
null, ef4),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
@ -1647,7 +1647,7 @@ public void testReadEntitiesEventFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "end_event"));
|
TimelineCompareOp.NOT_EQUAL, "end_event"));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
null, ef5),
|
null, ef5),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
@ -1656,8 +1656,8 @@ public void testReadEntitiesEventFilters() throws Exception {
|
|||||||
for (TimelineEntity timelineEntity : entities) {
|
for (TimelineEntity timelineEntity : entities) {
|
||||||
eventCnt += timelineEntity.getEvents().size();
|
eventCnt += timelineEntity.getEvents().size();
|
||||||
if (!timelineEntity.getId().equals("hello")) {
|
if (!timelineEntity.getId().equals("hello")) {
|
||||||
Assert.fail("Entity id should have been hello");
|
Assert.fail("Entity id should have been hello");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
assertEquals(0, eventCnt);
|
assertEquals(0, eventCnt);
|
||||||
}
|
}
|
||||||
@ -1673,7 +1673,7 @@ public void testReadEntitiesIsRelatedTo() throws Exception {
|
|||||||
new HashSet<Object>(Arrays.asList("relatedto4"))));
|
new HashSet<Object>(Arrays.asList("relatedto4"))));
|
||||||
Set<TimelineEntity> entities = reader.getEntities(
|
Set<TimelineEntity> entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, irt, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, irt, null, null, null,
|
||||||
null),
|
null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
||||||
@ -1822,7 +1822,7 @@ public void testReadEntitiesRelatesTo() throws Exception {
|
|||||||
new HashSet<Object>(Arrays.asList("relatesto4"))));
|
new HashSet<Object>(Arrays.asList("relatesto4"))));
|
||||||
Set<TimelineEntity> entities = reader.getEntities(
|
Set<TimelineEntity> entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, rt, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, rt, null, null, null, null,
|
||||||
null),
|
null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
||||||
@ -2000,7 +2000,7 @@ public void testReadEntitiesRelatesTo() throws Exception {
|
|||||||
public void testReadEntitiesDefaultView() throws Exception {
|
public void testReadEntitiesDefaultView() throws Exception {
|
||||||
TimelineEntity e1 = reader.getEntity(
|
TimelineEntity e1 = reader.getEntity(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", "hello"),
|
1002345678919L, "application_1231111111_1111", "world", "hello"),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
assertNotNull(e1);
|
assertNotNull(e1);
|
||||||
assertTrue(e1.getInfo().isEmpty() && e1.getConfigs().isEmpty() &&
|
assertTrue(e1.getInfo().isEmpty() && e1.getConfigs().isEmpty() &&
|
||||||
@ -2008,7 +2008,7 @@ public void testReadEntitiesDefaultView() throws Exception {
|
|||||||
e1.getRelatesToEntities().isEmpty());
|
e1.getRelatesToEntities().isEmpty());
|
||||||
Set<TimelineEntity> es1 = reader.getEntities(
|
Set<TimelineEntity> es1 = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(),
|
new TimelineEntityFilters(),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
assertEquals(3, es1.size());
|
assertEquals(3, es1.size());
|
||||||
@ -2023,7 +2023,7 @@ public void testReadEntitiesDefaultView() throws Exception {
|
|||||||
public void testReadEntitiesByFields() throws Exception {
|
public void testReadEntitiesByFields() throws Exception {
|
||||||
TimelineEntity e1 = reader.getEntity(
|
TimelineEntity e1 = reader.getEntity(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", "hello"),
|
1002345678919L, "application_1231111111_1111", "world", "hello"),
|
||||||
new TimelineDataToRetrieve(
|
new TimelineDataToRetrieve(
|
||||||
null, null, EnumSet.of(Field.INFO, Field.CONFIGS), null));
|
null, null, EnumSet.of(Field.INFO, Field.CONFIGS), null));
|
||||||
assertNotNull(e1);
|
assertNotNull(e1);
|
||||||
@ -2031,7 +2031,7 @@ public void testReadEntitiesByFields() throws Exception {
|
|||||||
assertEquals(0, e1.getIsRelatedToEntities().size());
|
assertEquals(0, e1.getIsRelatedToEntities().size());
|
||||||
Set<TimelineEntity> es1 = reader.getEntities(
|
Set<TimelineEntity> es1 = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(),
|
new TimelineEntityFilters(),
|
||||||
new TimelineDataToRetrieve(
|
new TimelineDataToRetrieve(
|
||||||
null, null, EnumSet.of(Field.IS_RELATED_TO, Field.METRICS), null));
|
null, null, EnumSet.of(Field.IS_RELATED_TO, Field.METRICS), null));
|
||||||
@ -2056,13 +2056,13 @@ public void testReadEntitiesConfigPrefix() throws Exception {
|
|||||||
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "cfg_"));
|
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "cfg_"));
|
||||||
TimelineEntity e1 = reader.getEntity(
|
TimelineEntity e1 = reader.getEntity(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", "hello"),
|
1002345678919L, "application_1231111111_1111", "world", "hello"),
|
||||||
new TimelineDataToRetrieve(list, null, null, null));
|
new TimelineDataToRetrieve(list, null, null, null));
|
||||||
assertNotNull(e1);
|
assertNotNull(e1);
|
||||||
assertEquals(1, e1.getConfigs().size());
|
assertEquals(1, e1.getConfigs().size());
|
||||||
Set<TimelineEntity> es1 = reader.getEntities(
|
Set<TimelineEntity> es1 = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(),
|
new TimelineEntityFilters(),
|
||||||
new TimelineDataToRetrieve(list, null, null, null));
|
new TimelineDataToRetrieve(list, null, null, null));
|
||||||
int cfgCnt = 0;
|
int cfgCnt = 0;
|
||||||
@ -2092,7 +2092,7 @@ public void testReadEntitiesConfigFilters() throws Exception {
|
|||||||
new TimelineFilterList(Operator.OR, list1, list2);
|
new TimelineFilterList(Operator.OR, list1, list2);
|
||||||
Set<TimelineEntity> entities = reader.getEntities(
|
Set<TimelineEntity> entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null,
|
||||||
confFilterList, null, null),
|
confFilterList, null, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
|
||||||
@ -2106,7 +2106,7 @@ public void testReadEntitiesConfigFilters() throws Exception {
|
|||||||
|
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null,
|
||||||
confFilterList, null, null),
|
confFilterList, null, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
||||||
@ -2122,7 +2122,7 @@ public void testReadEntitiesConfigFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "cfg_param1", "value1"));
|
TimelineCompareOp.NOT_EQUAL, "cfg_param1", "value1"));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null,
|
||||||
confFilterList1, null, null),
|
confFilterList1, null, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
|
||||||
@ -2141,7 +2141,7 @@ public void testReadEntitiesConfigFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "config_param2", "value2"));
|
TimelineCompareOp.NOT_EQUAL, "config_param2", "value2"));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null,
|
||||||
confFilterList2, null, null),
|
confFilterList2, null, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
|
||||||
@ -2153,7 +2153,7 @@ public void testReadEntitiesConfigFilters() throws Exception {
|
|||||||
TimelineCompareOp.EQUAL, "dummy_config", "value1"));
|
TimelineCompareOp.EQUAL, "dummy_config", "value1"));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null,
|
||||||
confFilterList3, null, null),
|
confFilterList3, null, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
|
||||||
@ -2165,7 +2165,7 @@ public void testReadEntitiesConfigFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "dummy_config", "value1"));
|
TimelineCompareOp.NOT_EQUAL, "dummy_config", "value1"));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null,
|
||||||
confFilterList4, null, null),
|
confFilterList4, null, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
|
||||||
@ -2177,7 +2177,7 @@ public void testReadEntitiesConfigFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "dummy_config", "value1", false));
|
TimelineCompareOp.NOT_EQUAL, "dummy_config", "value1", false));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null,
|
||||||
confFilterList5, null, null),
|
confFilterList5, null, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS),
|
||||||
@ -2195,7 +2195,7 @@ public void testReadEntitiesConfigFilterPrefix() throws Exception {
|
|||||||
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "cfg_"));
|
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "cfg_"));
|
||||||
Set<TimelineEntity> entities = reader.getEntities(
|
Set<TimelineEntity> entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null,
|
||||||
confFilterList, null, null),
|
confFilterList, null, null),
|
||||||
new TimelineDataToRetrieve(list, null, null, null));
|
new TimelineDataToRetrieve(list, null, null, null));
|
||||||
@ -2226,7 +2226,7 @@ public void testReadEntitiesConfigFilterPrefix() throws Exception {
|
|||||||
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "config_"));
|
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "config_"));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null,
|
||||||
confFilterList1, null, null),
|
confFilterList1, null, null),
|
||||||
new TimelineDataToRetrieve(confsToRetrieve, null, null, null));
|
new TimelineDataToRetrieve(confsToRetrieve, null, null, null));
|
||||||
@ -2237,7 +2237,7 @@ public void testReadEntitiesConfigFilterPrefix() throws Exception {
|
|||||||
for (String confKey : entity.getConfigs().keySet()) {
|
for (String confKey : entity.getConfigs().keySet()) {
|
||||||
assertTrue("Config key returned should start with config_",
|
assertTrue("Config key returned should start with config_",
|
||||||
confKey.startsWith("config_"));
|
confKey.startsWith("config_"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
assertEquals(2, cfgCnt);
|
assertEquals(2, cfgCnt);
|
||||||
}
|
}
|
||||||
@ -2249,13 +2249,13 @@ public void testReadEntitiesMetricPrefix() throws Exception {
|
|||||||
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
|
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
|
||||||
TimelineEntity e1 = reader.getEntity(
|
TimelineEntity e1 = reader.getEntity(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", "hello"),
|
1002345678919L, "application_1231111111_1111", "world", "hello"),
|
||||||
new TimelineDataToRetrieve(null, list, null, null));
|
new TimelineDataToRetrieve(null, list, null, null));
|
||||||
assertNotNull(e1);
|
assertNotNull(e1);
|
||||||
assertEquals(1, e1.getMetrics().size());
|
assertEquals(1, e1.getMetrics().size());
|
||||||
Set<TimelineEntity> es1 = reader.getEntities(
|
Set<TimelineEntity> es1 = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(),
|
new TimelineEntityFilters(),
|
||||||
new TimelineDataToRetrieve(null, list, null, null));
|
new TimelineDataToRetrieve(null, list, null, null));
|
||||||
int metricCnt = 0;
|
int metricCnt = 0;
|
||||||
@ -2283,7 +2283,7 @@ public void testReadEntitiesMetricFilters() throws Exception {
|
|||||||
new TimelineFilterList(Operator.OR, list1, list2);
|
new TimelineFilterList(Operator.OR, list1, list2);
|
||||||
Set<TimelineEntity> entities = reader.getEntities(
|
Set<TimelineEntity> entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
metricFilterList, null),
|
metricFilterList, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
|
||||||
@ -2297,7 +2297,7 @@ public void testReadEntitiesMetricFilters() throws Exception {
|
|||||||
|
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
metricFilterList, null),
|
metricFilterList, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
||||||
@ -2315,7 +2315,7 @@ public void testReadEntitiesMetricFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "MAP1_BYTES", 30));
|
TimelineCompareOp.NOT_EQUAL, "MAP1_BYTES", 30));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
metricFilterList1, null),
|
metricFilterList1, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
|
||||||
@ -2334,7 +2334,7 @@ public void testReadEntitiesMetricFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "MAP1_BYTES", 30));
|
TimelineCompareOp.NOT_EQUAL, "MAP1_BYTES", 30));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
metricFilterList2, null),
|
metricFilterList2, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
|
||||||
@ -2346,7 +2346,7 @@ public void testReadEntitiesMetricFilters() throws Exception {
|
|||||||
TimelineCompareOp.EQUAL, "dummy_metric", 5));
|
TimelineCompareOp.EQUAL, "dummy_metric", 5));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
metricFilterList3, null),
|
metricFilterList3, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
|
||||||
@ -2358,7 +2358,7 @@ public void testReadEntitiesMetricFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "dummy_metric", 5));
|
TimelineCompareOp.NOT_EQUAL, "dummy_metric", 5));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
metricFilterList4, null),
|
metricFilterList4, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
|
||||||
@ -2370,7 +2370,7 @@ public void testReadEntitiesMetricFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "dummy_metric", 5, false));
|
TimelineCompareOp.NOT_EQUAL, "dummy_metric", 5, false));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
metricFilterList5, null),
|
metricFilterList5, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS),
|
||||||
@ -2388,7 +2388,7 @@ public void testReadEntitiesMetricFilterPrefix() throws Exception {
|
|||||||
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
|
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
|
||||||
Set<TimelineEntity> entities = reader.getEntities(
|
Set<TimelineEntity> entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
metricFilterList, null),
|
metricFilterList, null),
|
||||||
new TimelineDataToRetrieve(null, list, null, null));
|
new TimelineDataToRetrieve(null, list, null, null));
|
||||||
@ -2417,7 +2417,7 @@ public void testReadEntitiesMetricFilterPrefix() throws Exception {
|
|||||||
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
|
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
metricFilterList1, null),
|
metricFilterList1, null),
|
||||||
new TimelineDataToRetrieve(
|
new TimelineDataToRetrieve(
|
||||||
@ -2436,9 +2436,9 @@ public void testReadEntitiesMetricFilterPrefix() throws Exception {
|
|||||||
assertEquals(2, metricCnt);
|
assertEquals(2, metricCnt);
|
||||||
|
|
||||||
entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1",
|
entities = reader.getEntities(new TimelineReaderContext("cluster1", "user1",
|
||||||
"some_flow_name", 1002345678919L, "application_1231111111_1111","world",
|
"some_flow_name", 1002345678919L, "application_1231111111_1111",
|
||||||
null), new TimelineEntityFilters(null, null, null, null, null, null,
|
"world", null), new TimelineEntityFilters(null, null, null, null, null,
|
||||||
null, metricFilterList1, null), new TimelineDataToRetrieve(null,
|
null, null, metricFilterList1, null), new TimelineDataToRetrieve(null,
|
||||||
metricsToRetrieve, EnumSet.of(Field.METRICS), Integer.MAX_VALUE));
|
metricsToRetrieve, EnumSet.of(Field.METRICS), Integer.MAX_VALUE));
|
||||||
assertEquals(2, entities.size());
|
assertEquals(2, entities.size());
|
||||||
metricCnt = 0;
|
metricCnt = 0;
|
||||||
@ -2471,7 +2471,7 @@ public void testReadEntitiesInfoFilters() throws Exception {
|
|||||||
new TimelineFilterList(Operator.OR, list1, list2);
|
new TimelineFilterList(Operator.OR, list1, list2);
|
||||||
Set<TimelineEntity> entities = reader.getEntities(
|
Set<TimelineEntity> entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, infoFilterList,
|
new TimelineEntityFilters(null, null, null, null, null, infoFilterList,
|
||||||
null, null, null),
|
null, null, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
|
||||||
@ -2487,7 +2487,7 @@ public void testReadEntitiesInfoFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "infoMapKey1", "infoMapValue1"));
|
TimelineCompareOp.NOT_EQUAL, "infoMapKey1", "infoMapValue1"));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, infoFilterList1,
|
new TimelineEntityFilters(null, null, null, null, null, infoFilterList1,
|
||||||
null, null, null),
|
null, null, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
|
||||||
@ -2505,7 +2505,7 @@ public void testReadEntitiesInfoFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "infoMapKey3", 71.4));
|
TimelineCompareOp.NOT_EQUAL, "infoMapKey3", 71.4));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, infoFilterList2,
|
new TimelineEntityFilters(null, null, null, null, null, infoFilterList2,
|
||||||
null, null, null),
|
null, null, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
|
||||||
@ -2516,7 +2516,7 @@ public void testReadEntitiesInfoFilters() throws Exception {
|
|||||||
TimelineCompareOp.EQUAL, "dummy_info", "some_value"));
|
TimelineCompareOp.EQUAL, "dummy_info", "some_value"));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, infoFilterList3,
|
new TimelineEntityFilters(null, null, null, null, null, infoFilterList3,
|
||||||
null, null, null),
|
null, null, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
|
||||||
@ -2527,7 +2527,7 @@ public void testReadEntitiesInfoFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "dummy_info", "some_value"));
|
TimelineCompareOp.NOT_EQUAL, "dummy_info", "some_value"));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, infoFilterList4,
|
new TimelineEntityFilters(null, null, null, null, null, infoFilterList4,
|
||||||
null, null, null),
|
null, null, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
|
||||||
@ -2538,7 +2538,7 @@ public void testReadEntitiesInfoFilters() throws Exception {
|
|||||||
TimelineCompareOp.NOT_EQUAL, "dummy_info", "some_value", false));
|
TimelineCompareOp.NOT_EQUAL, "dummy_info", "some_value", false));
|
||||||
entities = reader.getEntities(
|
entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, "application_1231111111_1111","world", null),
|
1002345678919L, "application_1231111111_1111", "world", null),
|
||||||
new TimelineEntityFilters(null, null, null, null, null, infoFilterList5,
|
new TimelineEntityFilters(null, null, null, null, null, infoFilterList5,
|
||||||
null, null, null),
|
null, null, null),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO), null));
|
||||||
@ -2593,19 +2593,19 @@ public void testReadApps() throws Exception {
|
|||||||
@Test
|
@Test
|
||||||
public void testFilterAppsByCreatedTime() throws Exception {
|
public void testFilterAppsByCreatedTime() throws Exception {
|
||||||
Set<TimelineEntity> entities = reader.getEntities(
|
Set<TimelineEntity> entities = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
||||||
null),
|
null),
|
||||||
new TimelineEntityFilters(null, 1425016502000L, 1425016502040L, null,
|
new TimelineEntityFilters(null, 1425016502000L, 1425016502040L, null,
|
||||||
null, null, null, null, null),
|
null, null, null, null, null),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
assertEquals(3, entities.size());
|
assertEquals(3, entities.size());
|
||||||
for (TimelineEntity entity : entities) {
|
for (TimelineEntity entity : entities) {
|
||||||
if (!entity.getId().equals("application_1111111111_2222") &&
|
if (!entity.getId().equals("application_1111111111_2222") &&
|
||||||
!entity.getId().equals("application_1111111111_3333") &&
|
!entity.getId().equals("application_1111111111_3333") &&
|
||||||
!entity.getId().equals("application_1111111111_4444")) {
|
!entity.getId().equals("application_1111111111_4444")) {
|
||||||
Assert.fail("Entities with ids' application_1111111111_2222, " +
|
Assert.fail("Entities with ids' application_1111111111_2222, " +
|
||||||
"application_1111111111_3333 and application_1111111111_4444" +
|
"application_1111111111_3333 and application_1111111111_4444" +
|
||||||
" should be present");
|
" should be present");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -2736,8 +2736,8 @@ public void testReadAppsIsRelatedTo() throws Exception {
|
|||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
||||||
null),
|
null),
|
||||||
new TimelineEntityFilters(null, null, null, null, irt1, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, irt1, null, null,
|
||||||
null),
|
null, null),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
assertEquals(1, entities.size());
|
assertEquals(1, entities.size());
|
||||||
isRelatedToCnt = 0;
|
isRelatedToCnt = 0;
|
||||||
@ -2760,8 +2760,8 @@ public void testReadAppsIsRelatedTo() throws Exception {
|
|||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
||||||
null),
|
null),
|
||||||
new TimelineEntityFilters(null, null, null, null, irt2, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, irt2, null, null,
|
||||||
null),
|
null, null),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
assertEquals(2, entities.size());
|
assertEquals(2, entities.size());
|
||||||
isRelatedToCnt = 0;
|
isRelatedToCnt = 0;
|
||||||
@ -2783,8 +2783,8 @@ public void testReadAppsIsRelatedTo() throws Exception {
|
|||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
||||||
null),
|
null),
|
||||||
new TimelineEntityFilters(null, null, null, null, irt3, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, irt3, null, null,
|
||||||
null),
|
null, null),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
assertEquals(1, entities.size());
|
assertEquals(1, entities.size());
|
||||||
isRelatedToCnt = 0;
|
isRelatedToCnt = 0;
|
||||||
@ -2807,8 +2807,8 @@ public void testReadAppsIsRelatedTo() throws Exception {
|
|||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
||||||
null),
|
null),
|
||||||
new TimelineEntityFilters(null, null, null, null, irt4, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, irt4, null, null,
|
||||||
null),
|
null, null),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
assertEquals(0, entities.size());
|
assertEquals(0, entities.size());
|
||||||
|
|
||||||
@ -2820,8 +2820,8 @@ public void testReadAppsIsRelatedTo() throws Exception {
|
|||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
||||||
null),
|
null),
|
||||||
new TimelineEntityFilters(null, null, null, null, irt5, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, irt5, null, null,
|
||||||
null),
|
null, null),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
assertEquals(0, entities.size());
|
assertEquals(0, entities.size());
|
||||||
|
|
||||||
@ -2841,8 +2841,8 @@ public void testReadAppsIsRelatedTo() throws Exception {
|
|||||||
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
|
||||||
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
|
||||||
null),
|
null),
|
||||||
new TimelineEntityFilters(null, null, null, null, irt6, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, irt6, null, null,
|
||||||
null),
|
null, null),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
assertEquals(1, entities.size());
|
assertEquals(1, entities.size());
|
||||||
isRelatedToCnt = 0;
|
isRelatedToCnt = 0;
|
||||||
@ -3335,8 +3335,8 @@ public void testReadAppsEventFilters() throws Exception {
|
|||||||
for (TimelineEntity timelineEntity : entities) {
|
for (TimelineEntity timelineEntity : entities) {
|
||||||
eventCnt += timelineEntity.getEvents().size();
|
eventCnt += timelineEntity.getEvents().size();
|
||||||
if (!timelineEntity.getId().equals("application_1111111111_2222")) {
|
if (!timelineEntity.getId().equals("application_1111111111_2222")) {
|
||||||
Assert.fail("Entity id should have been application_1111111111_2222");
|
Assert.fail("Entity id should have been application_1111111111_2222");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
assertEquals(0, eventCnt);
|
assertEquals(0, eventCnt);
|
||||||
}
|
}
|
||||||
|
@ -74,8 +74,8 @@ public static void cleanup() throws Exception {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static PhoenixOfflineAggregationWriterImpl
|
private static PhoenixOfflineAggregationWriterImpl
|
||||||
setupPhoenixClusterAndWriterForTest(YarnConfiguration conf)
|
setupPhoenixClusterAndWriterForTest(YarnConfiguration conf)
|
||||||
throws Exception{
|
throws Exception {
|
||||||
Map<String, String> props = new HashMap<>();
|
Map<String, String> props = new HashMap<>();
|
||||||
// Must update config before starting server
|
// Must update config before starting server
|
||||||
props.put(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB,
|
props.put(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB,
|
||||||
|
@ -31,12 +31,14 @@
|
|||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generates the data/entities for the FlowRun and FlowActivity Tables
|
* Generates the data/entities for the FlowRun and FlowActivity Tables.
|
||||||
*/
|
*/
|
||||||
class TestFlowDataGenerator {
|
final class TestFlowDataGenerator {
|
||||||
|
private TestFlowDataGenerator() {
|
||||||
|
}
|
||||||
|
|
||||||
private static final String metric1 = "MAP_SLOT_MILLIS";
|
private static final String METRIC_1 = "MAP_SLOT_MILLIS";
|
||||||
private static final String metric2 = "HDFS_BYTES_READ";
|
private static final String METRIC_2 = "HDFS_BYTES_READ";
|
||||||
public static final long END_TS_INCR = 10000L;
|
public static final long END_TS_INCR = 10000L;
|
||||||
|
|
||||||
static TimelineEntity getEntityMetricsApp1(long insertTs, Configuration c1) {
|
static TimelineEntity getEntityMetricsApp1(long insertTs, Configuration c1) {
|
||||||
@ -51,7 +53,7 @@ static TimelineEntity getEntityMetricsApp1(long insertTs, Configuration c1) {
|
|||||||
// add metrics
|
// add metrics
|
||||||
Set<TimelineMetric> metrics = new HashSet<>();
|
Set<TimelineMetric> metrics = new HashSet<>();
|
||||||
TimelineMetric m1 = new TimelineMetric();
|
TimelineMetric m1 = new TimelineMetric();
|
||||||
m1.setId(metric1);
|
m1.setId(METRIC_1);
|
||||||
Map<Long, Number> metricValues = new HashMap<Long, Number>();
|
Map<Long, Number> metricValues = new HashMap<Long, Number>();
|
||||||
long ts = insertTs;
|
long ts = insertTs;
|
||||||
|
|
||||||
@ -64,10 +66,10 @@ static TimelineEntity getEntityMetricsApp1(long insertTs, Configuration c1) {
|
|||||||
metrics.add(m1);
|
metrics.add(m1);
|
||||||
|
|
||||||
TimelineMetric m2 = new TimelineMetric();
|
TimelineMetric m2 = new TimelineMetric();
|
||||||
m2.setId(metric2);
|
m2.setId(METRIC_2);
|
||||||
metricValues = new HashMap<Long, Number>();
|
metricValues = new HashMap<Long, Number>();
|
||||||
ts = System.currentTimeMillis();
|
ts = System.currentTimeMillis();
|
||||||
for (int k=1; k< 100 ; k++) {
|
for (int k = 1; k < 100; k++) {
|
||||||
metricValues.put(ts - k*100000L, 31L);
|
metricValues.put(ts - k*100000L, 31L);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -81,7 +83,8 @@ static TimelineEntity getEntityMetricsApp1(long insertTs, Configuration c1) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static TimelineEntity getEntityMetricsApp1Complete(long insertTs, Configuration c1) {
|
static TimelineEntity getEntityMetricsApp1Complete(long insertTs,
|
||||||
|
Configuration c1) {
|
||||||
TimelineEntity entity = new TimelineEntity();
|
TimelineEntity entity = new TimelineEntity();
|
||||||
String id = "flowRunMetrics_test";
|
String id = "flowRunMetrics_test";
|
||||||
String type = TimelineEntityType.YARN_APPLICATION.toString();
|
String type = TimelineEntityType.YARN_APPLICATION.toString();
|
||||||
@ -93,7 +96,7 @@ static TimelineEntity getEntityMetricsApp1Complete(long insertTs, Configuration
|
|||||||
// add metrics
|
// add metrics
|
||||||
Set<TimelineMetric> metrics = new HashSet<>();
|
Set<TimelineMetric> metrics = new HashSet<>();
|
||||||
TimelineMetric m1 = new TimelineMetric();
|
TimelineMetric m1 = new TimelineMetric();
|
||||||
m1.setId(metric1);
|
m1.setId(METRIC_1);
|
||||||
Map<Long, Number> metricValues = new HashMap<Long, Number>();
|
Map<Long, Number> metricValues = new HashMap<Long, Number>();
|
||||||
long ts = insertTs;
|
long ts = insertTs;
|
||||||
|
|
||||||
@ -103,7 +106,7 @@ static TimelineEntity getEntityMetricsApp1Complete(long insertTs, Configuration
|
|||||||
metrics.add(m1);
|
metrics.add(m1);
|
||||||
|
|
||||||
TimelineMetric m2 = new TimelineMetric();
|
TimelineMetric m2 = new TimelineMetric();
|
||||||
m2.setId(metric2);
|
m2.setId(METRIC_2);
|
||||||
metricValues = new HashMap<Long, Number>();
|
metricValues = new HashMap<Long, Number>();
|
||||||
ts = insertTs;
|
ts = insertTs;
|
||||||
metricValues.put(ts - 80000, 57L);
|
metricValues.put(ts - 80000, 57L);
|
||||||
@ -134,7 +137,7 @@ static TimelineEntity getEntityMetricsApp1(long insertTs) {
|
|||||||
// add metrics
|
// add metrics
|
||||||
Set<TimelineMetric> metrics = new HashSet<>();
|
Set<TimelineMetric> metrics = new HashSet<>();
|
||||||
TimelineMetric m1 = new TimelineMetric();
|
TimelineMetric m1 = new TimelineMetric();
|
||||||
m1.setId(metric1);
|
m1.setId(METRIC_1);
|
||||||
Map<Long, Number> metricValues = new HashMap<Long, Number>();
|
Map<Long, Number> metricValues = new HashMap<Long, Number>();
|
||||||
long ts = insertTs;
|
long ts = insertTs;
|
||||||
metricValues.put(ts - 100000, 2L);
|
metricValues.put(ts - 100000, 2L);
|
||||||
@ -144,7 +147,7 @@ static TimelineEntity getEntityMetricsApp1(long insertTs) {
|
|||||||
metrics.add(m1);
|
metrics.add(m1);
|
||||||
|
|
||||||
TimelineMetric m2 = new TimelineMetric();
|
TimelineMetric m2 = new TimelineMetric();
|
||||||
m2.setId(metric2);
|
m2.setId(METRIC_2);
|
||||||
metricValues = new HashMap<Long, Number>();
|
metricValues = new HashMap<Long, Number>();
|
||||||
ts = insertTs;
|
ts = insertTs;
|
||||||
metricValues.put(ts - 100000, 31L);
|
metricValues.put(ts - 100000, 31L);
|
||||||
@ -177,7 +180,7 @@ static TimelineEntity getEntityMetricsApp2(long insertTs) {
|
|||||||
// add metrics
|
// add metrics
|
||||||
Set<TimelineMetric> metrics = new HashSet<>();
|
Set<TimelineMetric> metrics = new HashSet<>();
|
||||||
TimelineMetric m1 = new TimelineMetric();
|
TimelineMetric m1 = new TimelineMetric();
|
||||||
m1.setId(metric1);
|
m1.setId(METRIC_1);
|
||||||
Map<Long, Number> metricValues = new HashMap<Long, Number>();
|
Map<Long, Number> metricValues = new HashMap<Long, Number>();
|
||||||
long ts = insertTs;
|
long ts = insertTs;
|
||||||
metricValues.put(ts - 100000, 5L);
|
metricValues.put(ts - 100000, 5L);
|
||||||
@ -208,7 +211,7 @@ static TimelineEntity getEntity1() {
|
|||||||
// add metrics
|
// add metrics
|
||||||
Set<TimelineMetric> metrics = new HashSet<>();
|
Set<TimelineMetric> metrics = new HashSet<>();
|
||||||
TimelineMetric m1 = new TimelineMetric();
|
TimelineMetric m1 = new TimelineMetric();
|
||||||
m1.setId(metric1);
|
m1.setId(METRIC_1);
|
||||||
Map<Long, Number> metricValues = new HashMap<Long, Number>();
|
Map<Long, Number> metricValues = new HashMap<Long, Number>();
|
||||||
long ts = System.currentTimeMillis();
|
long ts = System.currentTimeMillis();
|
||||||
metricValues.put(ts - 120000, 100000000L);
|
metricValues.put(ts - 120000, 100000000L);
|
||||||
@ -232,7 +235,7 @@ static TimelineEntity getEntity1() {
|
|||||||
|
|
||||||
event = new TimelineEvent();
|
event = new TimelineEvent();
|
||||||
event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
|
event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
|
||||||
long expTs = cTime + 21600000;// start time + 6hrs
|
long expTs = cTime + 21600000; // start time + 6hrs
|
||||||
event.setTimestamp(expTs);
|
event.setTimestamp(expTs);
|
||||||
event.addInfo(expKey, expVal);
|
event.addInfo(expKey, expVal);
|
||||||
entity.addEvent(event);
|
entity.addEvent(event);
|
||||||
@ -250,7 +253,7 @@ static TimelineEntity getAFullEntity(long ts, long endTs) {
|
|||||||
// add metrics
|
// add metrics
|
||||||
Set<TimelineMetric> metrics = new HashSet<>();
|
Set<TimelineMetric> metrics = new HashSet<>();
|
||||||
TimelineMetric m1 = new TimelineMetric();
|
TimelineMetric m1 = new TimelineMetric();
|
||||||
m1.setId(metric1);
|
m1.setId(METRIC_1);
|
||||||
Map<Long, Number> metricValues = new HashMap<Long, Number>();
|
Map<Long, Number> metricValues = new HashMap<Long, Number>();
|
||||||
metricValues.put(ts - 120000, 100000000L);
|
metricValues.put(ts - 120000, 100000000L);
|
||||||
metricValues.put(ts - 100000, 200000000L);
|
metricValues.put(ts - 100000, 200000000L);
|
||||||
@ -262,7 +265,7 @@ static TimelineEntity getAFullEntity(long ts, long endTs) {
|
|||||||
m1.setValues(metricValues);
|
m1.setValues(metricValues);
|
||||||
metrics.add(m1);
|
metrics.add(m1);
|
||||||
TimelineMetric m2 = new TimelineMetric();
|
TimelineMetric m2 = new TimelineMetric();
|
||||||
m2.setId(metric2);
|
m2.setId(METRIC_2);
|
||||||
metricValues = new HashMap<Long, Number>();
|
metricValues = new HashMap<Long, Number>();
|
||||||
metricValues.put(ts - 900000, 31L);
|
metricValues.put(ts - 900000, 31L);
|
||||||
metricValues.put(ts - 30000, 57L);
|
metricValues.put(ts - 30000, 57L);
|
||||||
@ -281,7 +284,7 @@ static TimelineEntity getAFullEntity(long ts, long endTs) {
|
|||||||
|
|
||||||
event = new TimelineEvent();
|
event = new TimelineEvent();
|
||||||
event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
|
event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
|
||||||
long expTs = ts + 21600000;// start time + 6hrs
|
long expTs = ts + 21600000; // start time + 6hrs
|
||||||
event.setTimestamp(expTs);
|
event.setTimestamp(expTs);
|
||||||
event.addInfo(expKey, expVal);
|
event.addInfo(expKey, expVal);
|
||||||
entity.addEvent(event);
|
entity.addEvent(event);
|
||||||
|
@ -58,7 +58,7 @@
|
|||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tests the FlowRun and FlowActivity Tables
|
* Tests the FlowRun and FlowActivity Tables.
|
||||||
*/
|
*/
|
||||||
public class TestHBaseStorageFlowActivity {
|
public class TestHBaseStorageFlowActivity {
|
||||||
|
|
||||||
@ -114,7 +114,7 @@ public void testWriteFlowRunMinMax() throws Exception {
|
|||||||
String appName = "application_100000000000_1111";
|
String appName = "application_100000000000_1111";
|
||||||
long minStartTs = 1424995200300L;
|
long minStartTs = 1424995200300L;
|
||||||
long greaterStartTs = 1424995200300L + 864000L;
|
long greaterStartTs = 1424995200300L + 864000L;
|
||||||
long endTs = 1424995200300L + 86000000L;;
|
long endTs = 1424995200300L + 86000000L;
|
||||||
TimelineEntity entityMinStartTime = TestFlowDataGenerator
|
TimelineEntity entityMinStartTime = TestFlowDataGenerator
|
||||||
.getEntityMinStartTime(minStartTs);
|
.getEntityMinStartTime(minStartTs);
|
||||||
|
|
||||||
@ -209,7 +209,7 @@ public void testWriteFlowRunMinMax() throws Exception {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Write 1 application entity and checks the record for today in the flow
|
* Write 1 application entity and checks the record for today in the flow
|
||||||
* activity table
|
* activity table.
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void testWriteFlowActivityOneFlow() throws Exception {
|
public void testWriteFlowActivityOneFlow() throws Exception {
|
||||||
@ -313,10 +313,10 @@ private void checkFlowActivityTable(String cluster, String user, String flow,
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Writes 3 applications each with a different run id and version for the same
|
* Writes 3 applications each with a different run id and version for the same
|
||||||
* {cluster, user, flow}
|
* {cluster, user, flow}.
|
||||||
*
|
*
|
||||||
* They should be getting inserted into one record in the flow activity table
|
* They should be getting inserted into one record in the flow activity table
|
||||||
* with 3 columns, one per run id
|
* with 3 columns, one per run id.
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void testFlowActivityTableOneFlowMultipleRunIds() throws IOException {
|
public void testFlowActivityTableOneFlowMultipleRunIds() throws IOException {
|
||||||
@ -425,7 +425,8 @@ private void checkFlowActivityTableSeveralRuns(String cluster, String user,
|
|||||||
s.setStartRow(startRow);
|
s.setStartRow(startRow);
|
||||||
String clusterStop = cluster + "1";
|
String clusterStop = cluster + "1";
|
||||||
byte[] stopRow =
|
byte[] stopRow =
|
||||||
new FlowActivityRowKey(clusterStop, appCreatedTime, user, flow).getRowKey();
|
new FlowActivityRowKey(clusterStop, appCreatedTime, user, flow)
|
||||||
|
.getRowKey();
|
||||||
s.setStopRow(stopRow);
|
s.setStopRow(stopRow);
|
||||||
Connection conn = ConnectionFactory.createConnection(c1);
|
Connection conn = ConnectionFactory.createConnection(c1);
|
||||||
Table table1 = conn.getTable(TableName
|
Table table1 = conn.getTable(TableName
|
||||||
|
@ -69,7 +69,7 @@
|
|||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tests the FlowRun and FlowActivity Tables
|
* Tests the FlowRun and FlowActivity Tables.
|
||||||
*/
|
*/
|
||||||
public class TestHBaseStorageFlowRun {
|
public class TestHBaseStorageFlowRun {
|
||||||
|
|
||||||
@ -356,18 +356,20 @@ public void testWriteFlowRunMetricsOneFlow() throws Exception {
|
|||||||
/*
|
/*
|
||||||
* checks the batch limits on a scan
|
* checks the batch limits on a scan
|
||||||
*/
|
*/
|
||||||
void checkFlowRunTableBatchLimit(String cluster, String user,
|
void checkFlowRunTableBatchLimit(String cluster, String user,
|
||||||
String flow, long runid, Configuration c1) throws IOException {
|
String flow, long runid, Configuration c1) throws IOException {
|
||||||
|
|
||||||
Scan s = new Scan();
|
Scan s = new Scan();
|
||||||
s.addFamily(FlowRunColumnFamily.INFO.getBytes());
|
s.addFamily(FlowRunColumnFamily.INFO.getBytes());
|
||||||
byte[] startRow = new FlowRunRowKey(cluster, user, flow, runid).getRowKey();
|
byte[] startRow =
|
||||||
|
new FlowRunRowKey(cluster, user, flow, runid).getRowKey();
|
||||||
s.setStartRow(startRow);
|
s.setStartRow(startRow);
|
||||||
// set a batch limit
|
// set a batch limit
|
||||||
int batchLimit = 2;
|
int batchLimit = 2;
|
||||||
s.setBatch(batchLimit);
|
s.setBatch(batchLimit);
|
||||||
String clusterStop = cluster + "1";
|
String clusterStop = cluster + "1";
|
||||||
byte[] stopRow = new FlowRunRowKey(clusterStop, user, flow, runid).getRowKey();
|
byte[] stopRow =
|
||||||
|
new FlowRunRowKey(clusterStop, user, flow, runid).getRowKey();
|
||||||
s.setStopRow(stopRow);
|
s.setStopRow(stopRow);
|
||||||
Connection conn = ConnectionFactory.createConnection(c1);
|
Connection conn = ConnectionFactory.createConnection(c1);
|
||||||
Table table1 = conn
|
Table table1 = conn
|
||||||
|
@ -63,14 +63,14 @@
|
|||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tests the FlowRun and FlowActivity Tables
|
* Tests the FlowRun and FlowActivity Tables.
|
||||||
*/
|
*/
|
||||||
public class TestHBaseStorageFlowRunCompaction {
|
public class TestHBaseStorageFlowRunCompaction {
|
||||||
|
|
||||||
private static HBaseTestingUtility util;
|
private static HBaseTestingUtility util;
|
||||||
|
|
||||||
private static final String metric1 = "MAP_SLOT_MILLIS";
|
private static final String METRIC_1 = "MAP_SLOT_MILLIS";
|
||||||
private static final String metric2 = "HDFS_BYTES_READ";
|
private static final String METRIC_2 = "HDFS_BYTES_READ";
|
||||||
|
|
||||||
private final byte[] aRowKey = Bytes.toBytes("a");
|
private final byte[] aRowKey = Bytes.toBytes("a");
|
||||||
private final byte[] aFamily = Bytes.toBytes("family");
|
private final byte[] aFamily = Bytes.toBytes("family");
|
||||||
@ -89,8 +89,8 @@ private static void createSchema() throws IOException {
|
|||||||
TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
|
TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** writes non numeric data into flow run table
|
/** Writes non numeric data into flow run table
|
||||||
* reads it back
|
* reads it back.
|
||||||
*
|
*
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
@ -262,7 +262,7 @@ public void testWriteScanBatchLimit() throws Exception {
|
|||||||
.getFamilyMap(FlowRunColumnFamily.INFO.getBytes());
|
.getFamilyMap(FlowRunColumnFamily.INFO.getBytes());
|
||||||
// we expect all back in one next call
|
// we expect all back in one next call
|
||||||
assertEquals(4, values.size());
|
assertEquals(4, values.size());
|
||||||
System.out.println(" values size " + values.size() + " " + batchLimit );
|
System.out.println(" values size " + values.size() + " " + batchLimit);
|
||||||
rowCount++;
|
rowCount++;
|
||||||
}
|
}
|
||||||
// should get back 1 row with each invocation
|
// should get back 1 row with each invocation
|
||||||
@ -325,11 +325,12 @@ public void testWriteFlowRunCompaction() throws Exception {
|
|||||||
.valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
|
.valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
|
||||||
List<Region> regions = server.getOnlineRegions(TableName
|
List<Region> regions = server.getOnlineRegions(TableName
|
||||||
.valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
|
.valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
|
||||||
assertTrue("Didn't find any regions for primary table!", regions.size() > 0);
|
assertTrue("Didn't find any regions for primary table!",
|
||||||
|
regions.size() > 0);
|
||||||
// flush and compact all the regions of the primary table
|
// flush and compact all the regions of the primary table
|
||||||
for (Region region : regions) {
|
for (Region region : regions) {
|
||||||
region.flush(true);
|
region.flush(true);
|
||||||
region.compact(true);
|
region.compact(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
// check flow run for one flow many apps
|
// check flow run for one flow many apps
|
||||||
@ -363,13 +364,13 @@ private void checkFlowRunTable(String cluster, String user, String flow,
|
|||||||
rowCount++;
|
rowCount++;
|
||||||
// check metric1
|
// check metric1
|
||||||
byte[] q = ColumnHelper.getColumnQualifier(
|
byte[] q = ColumnHelper.getColumnQualifier(
|
||||||
FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), metric1);
|
FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), METRIC_1);
|
||||||
assertTrue(values.containsKey(q));
|
assertTrue(values.containsKey(q));
|
||||||
assertEquals(141, Bytes.toLong(values.get(q)));
|
assertEquals(141, Bytes.toLong(values.get(q)));
|
||||||
|
|
||||||
// check metric2
|
// check metric2
|
||||||
q = ColumnHelper.getColumnQualifier(
|
q = ColumnHelper.getColumnQualifier(
|
||||||
FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), metric2);
|
FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), METRIC_2);
|
||||||
assertTrue(values.containsKey(q));
|
assertTrue(values.containsKey(q));
|
||||||
assertEquals(57, Bytes.toLong(values.get(q)));
|
assertEquals(57, Bytes.toLong(values.get(q)));
|
||||||
}
|
}
|
||||||
@ -385,7 +386,7 @@ private FlowScanner getFlowScannerForTestingCompaction() {
|
|||||||
// okay to pass in nulls for the constructor arguments
|
// okay to pass in nulls for the constructor arguments
|
||||||
// because all we want to do is invoke the process summation
|
// because all we want to do is invoke the process summation
|
||||||
FlowScanner fs = new FlowScanner(null, null,
|
FlowScanner fs = new FlowScanner(null, null,
|
||||||
(request.isMajor() == true ? FlowScannerOperation.MAJOR_COMPACTION
|
(request.isMajor() ? FlowScannerOperation.MAJOR_COMPACTION
|
||||||
: FlowScannerOperation.MINOR_COMPACTION));
|
: FlowScannerOperation.MINOR_COMPACTION));
|
||||||
assertNotNull(fs);
|
assertNotNull(fs);
|
||||||
return fs;
|
return fs;
|
||||||
@ -404,7 +405,7 @@ public void checkProcessSummationMoreCellsSumFinal2()
|
|||||||
long currentTimestamp = System.currentTimeMillis();
|
long currentTimestamp = System.currentTimeMillis();
|
||||||
long cell1Ts = 1200120L;
|
long cell1Ts = 1200120L;
|
||||||
long cell2Ts = TimestampGenerator.getSupplementedTimestamp(
|
long cell2Ts = TimestampGenerator.getSupplementedTimestamp(
|
||||||
System.currentTimeMillis(),"application_123746661110_11202");
|
System.currentTimeMillis(), "application_123746661110_11202");
|
||||||
long cell3Ts = 1277719L;
|
long cell3Ts = 1277719L;
|
||||||
long cell4Ts = currentTimestamp - 10;
|
long cell4Ts = currentTimestamp - 10;
|
||||||
|
|
||||||
@ -571,7 +572,8 @@ public void checkProcessSummationMoreCellsSumFinalMany() throws IOException {
|
|||||||
// of type SUM and SUM_FINAL
|
// of type SUM and SUM_FINAL
|
||||||
// NOT cells of SUM_FINAL will expire
|
// NOT cells of SUM_FINAL will expire
|
||||||
@Test
|
@Test
|
||||||
public void checkProcessSummationMoreCellsSumFinalVariedTags() throws IOException {
|
public void checkProcessSummationMoreCellsSumFinalVariedTags()
|
||||||
|
throws IOException {
|
||||||
FlowScanner fs = getFlowScannerForTestingCompaction();
|
FlowScanner fs = getFlowScannerForTestingCompaction();
|
||||||
int countFinal = 20100;
|
int countFinal = 20100;
|
||||||
int countNotFinal = 1000;
|
int countNotFinal = 1000;
|
||||||
@ -585,7 +587,8 @@ public void checkProcessSummationMoreCellsSumFinalVariedTags() throws IOExceptio
|
|||||||
long cellTsFinalStart = 10001120L;
|
long cellTsFinalStart = 10001120L;
|
||||||
long cellTsFinal = cellTsFinalStart;
|
long cellTsFinal = cellTsFinalStart;
|
||||||
|
|
||||||
long cellTsFinalStartNotExpire = TimestampGenerator.getSupplementedTimestamp(
|
long cellTsFinalStartNotExpire =
|
||||||
|
TimestampGenerator.getSupplementedTimestamp(
|
||||||
System.currentTimeMillis(), "application_10266666661166_118821");
|
System.currentTimeMillis(), "application_10266666661166_118821");
|
||||||
long cellTsFinalNotExpire = cellTsFinalStartNotExpire;
|
long cellTsFinalNotExpire = cellTsFinalStartNotExpire;
|
||||||
|
|
||||||
|
@ -53,7 +53,6 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
|
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
|
||||||
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
|
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
|
||||||
import org.apache.hadoop.yarn.api.records.timelineservice.UserEntity;
|
import org.apache.hadoop.yarn.api.records.timelineservice.UserEntity;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
|
||||||
import org.apache.hadoop.yarn.webapp.ForbiddenException;
|
import org.apache.hadoop.yarn.webapp.ForbiddenException;
|
||||||
import org.apache.hadoop.yarn.webapp.NotFoundException;
|
import org.apache.hadoop.yarn.webapp.NotFoundException;
|
||||||
|
|
||||||
@ -180,7 +179,7 @@ public Response putEntities(
|
|||||||
private static ApplicationId parseApplicationId(String appId) {
|
private static ApplicationId parseApplicationId(String appId) {
|
||||||
try {
|
try {
|
||||||
if (appId != null) {
|
if (appId != null) {
|
||||||
return ConverterUtils.toApplicationId(appId.trim());
|
return ApplicationId.fromString(appId.trim());
|
||||||
} else {
|
} else {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -26,4 +26,3 @@
|
|||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
|
||||||
|
@ -20,7 +20,6 @@
|
|||||||
|
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Encodes and decodes {@link ApplicationId} for row keys.
|
* Encodes and decodes {@link ApplicationId} for row keys.
|
||||||
@ -50,7 +49,7 @@ public AppIdKeyConverter() {
|
|||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public byte[] encode(String appIdStr) {
|
public byte[] encode(String appIdStr) {
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(appIdStr);
|
ApplicationId appId = ApplicationId.fromString(appIdStr);
|
||||||
byte[] appIdBytes = new byte[getKeySize()];
|
byte[] appIdBytes = new byte[getKeySize()];
|
||||||
byte[] clusterTs = Bytes.toBytes(
|
byte[] clusterTs = Bytes.toBytes(
|
||||||
LongConverter.invertLong(appId.getClusterTimestamp()));
|
LongConverter.invertLong(appId.getClusterTimestamp()));
|
||||||
|
@ -21,7 +21,6 @@
|
|||||||
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Utility class that allows HBase coprocessors to interact with unique
|
* Utility class that allows HBase coprocessors to interact with unique
|
||||||
@ -99,7 +98,7 @@ private static long getAppIdSuffix(String appIdStr) {
|
|||||||
if (appIdStr == null) {
|
if (appIdStr == null) {
|
||||||
return 0L;
|
return 0L;
|
||||||
}
|
}
|
||||||
ApplicationId appId = ConverterUtils.toApplicationId(appIdStr);
|
ApplicationId appId = ApplicationId.fromString(appIdStr);
|
||||||
long id = appId.getId() % TS_MULTIPLIER;
|
long id = appId.getId() % TS_MULTIPLIER;
|
||||||
return id;
|
return id;
|
||||||
}
|
}
|
||||||
|
@ -26,4 +26,3 @@
|
|||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
|
|
||||||
|
@ -88,9 +88,9 @@ public void testStartWebApp() throws Exception {
|
|||||||
|
|
||||||
@Test(timeout=60000)
|
@Test(timeout=60000)
|
||||||
public void testMultithreadedAdd() throws Exception {
|
public void testMultithreadedAdd() throws Exception {
|
||||||
final int NUM_APPS = 5;
|
final int numApps = 5;
|
||||||
List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
|
List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
|
||||||
for (int i = 0; i < NUM_APPS; i++) {
|
for (int i = 0; i < numApps; i++) {
|
||||||
final ApplicationId appId = ApplicationId.newInstance(0L, i);
|
final ApplicationId appId = ApplicationId.newInstance(0L, i);
|
||||||
Callable<Boolean> task = new Callable<Boolean>() {
|
Callable<Boolean> task = new Callable<Boolean>() {
|
||||||
public Boolean call() {
|
public Boolean call() {
|
||||||
@ -101,7 +101,7 @@ public Boolean call() {
|
|||||||
};
|
};
|
||||||
tasks.add(task);
|
tasks.add(task);
|
||||||
}
|
}
|
||||||
ExecutorService executor = Executors.newFixedThreadPool(NUM_APPS);
|
ExecutorService executor = Executors.newFixedThreadPool(numApps);
|
||||||
try {
|
try {
|
||||||
List<Future<Boolean>> futures = executor.invokeAll(tasks);
|
List<Future<Boolean>> futures = executor.invokeAll(tasks);
|
||||||
for (Future<Boolean> future: futures) {
|
for (Future<Boolean> future: futures) {
|
||||||
@ -111,7 +111,7 @@ public Boolean call() {
|
|||||||
executor.shutdownNow();
|
executor.shutdownNow();
|
||||||
}
|
}
|
||||||
// check the keys
|
// check the keys
|
||||||
for (int i = 0; i < NUM_APPS; i++) {
|
for (int i = 0; i < numApps; i++) {
|
||||||
final ApplicationId appId = ApplicationId.newInstance(0L, i);
|
final ApplicationId appId = ApplicationId.newInstance(0L, i);
|
||||||
assertTrue(collectorManager.containsTimelineCollector(appId));
|
assertTrue(collectorManager.containsTimelineCollector(appId));
|
||||||
}
|
}
|
||||||
@ -119,9 +119,9 @@ public Boolean call() {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testMultithreadedAddAndRemove() throws Exception {
|
public void testMultithreadedAddAndRemove() throws Exception {
|
||||||
final int NUM_APPS = 5;
|
final int numApps = 5;
|
||||||
List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
|
List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
|
||||||
for (int i = 0; i < NUM_APPS; i++) {
|
for (int i = 0; i < numApps; i++) {
|
||||||
final ApplicationId appId = ApplicationId.newInstance(0L, i);
|
final ApplicationId appId = ApplicationId.newInstance(0L, i);
|
||||||
Callable<Boolean> task = new Callable<Boolean>() {
|
Callable<Boolean> task = new Callable<Boolean>() {
|
||||||
public Boolean call() {
|
public Boolean call() {
|
||||||
@ -134,7 +134,7 @@ public Boolean call() {
|
|||||||
};
|
};
|
||||||
tasks.add(task);
|
tasks.add(task);
|
||||||
}
|
}
|
||||||
ExecutorService executor = Executors.newFixedThreadPool(NUM_APPS);
|
ExecutorService executor = Executors.newFixedThreadPool(numApps);
|
||||||
try {
|
try {
|
||||||
List<Future<Boolean>> futures = executor.invokeAll(tasks);
|
List<Future<Boolean>> futures = executor.invokeAll(tasks);
|
||||||
for (Future<Boolean> future: futures) {
|
for (Future<Boolean> future: futures) {
|
||||||
@ -144,16 +144,16 @@ public Boolean call() {
|
|||||||
executor.shutdownNow();
|
executor.shutdownNow();
|
||||||
}
|
}
|
||||||
// check the keys
|
// check the keys
|
||||||
for (int i = 0; i < NUM_APPS; i++) {
|
for (int i = 0; i < numApps; i++) {
|
||||||
final ApplicationId appId = ApplicationId.newInstance(0L, i);
|
final ApplicationId appId = ApplicationId.newInstance(0L, i);
|
||||||
assertFalse(collectorManager.containsTimelineCollector(appId));
|
assertFalse(collectorManager.containsTimelineCollector(appId));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private NodeTimelineCollectorManager createCollectorManager() {
|
private NodeTimelineCollectorManager createCollectorManager() {
|
||||||
final NodeTimelineCollectorManager collectorManager =
|
final NodeTimelineCollectorManager cm =
|
||||||
spy(new NodeTimelineCollectorManager());
|
spy(new NodeTimelineCollectorManager());
|
||||||
doReturn(new Configuration()).when(collectorManager).getConfig();
|
doReturn(new Configuration()).when(cm).getConfig();
|
||||||
CollectorNodemanagerProtocol nmCollectorService =
|
CollectorNodemanagerProtocol nmCollectorService =
|
||||||
mock(CollectorNodemanagerProtocol.class);
|
mock(CollectorNodemanagerProtocol.class);
|
||||||
GetTimelineCollectorContextResponse response =
|
GetTimelineCollectorContextResponse response =
|
||||||
@ -164,7 +164,7 @@ private NodeTimelineCollectorManager createCollectorManager() {
|
|||||||
} catch (YarnException | IOException e) {
|
} catch (YarnException | IOException e) {
|
||||||
fail();
|
fail();
|
||||||
}
|
}
|
||||||
doReturn(nmCollectorService).when(collectorManager).getNMCollectorService();
|
doReturn(nmCollectorService).when(cm).getNMCollectorService();
|
||||||
return collectorManager;
|
return cm;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -111,7 +111,7 @@ public void testRemoveApplication() throws Exception {
|
|||||||
// a configured period
|
// a configured period
|
||||||
assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()));
|
assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()));
|
||||||
for (int i = 0; i < 4; i++) {
|
for (int i = 0; i < 4; i++) {
|
||||||
Thread.sleep(500l);
|
Thread.sleep(500L);
|
||||||
if (!auxService.hasApplication(appAttemptId.getApplicationId())) {
|
if (!auxService.hasApplication(appAttemptId.getApplicationId())) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@ -154,7 +154,7 @@ public void testLaunch() throws Exception {
|
|||||||
|
|
||||||
private PerNodeTimelineCollectorsAuxService
|
private PerNodeTimelineCollectorsAuxService
|
||||||
createCollectorAndAddApplication() {
|
createCollectorAndAddApplication() {
|
||||||
PerNodeTimelineCollectorsAuxService auxService = createCollector();
|
PerNodeTimelineCollectorsAuxService service = createCollector();
|
||||||
// create an AM container
|
// create an AM container
|
||||||
ContainerId containerId = getAMContainerId();
|
ContainerId containerId = getAMContainerId();
|
||||||
ContainerInitializationContext context =
|
ContainerInitializationContext context =
|
||||||
@ -162,17 +162,17 @@ public void testLaunch() throws Exception {
|
|||||||
when(context.getContainerId()).thenReturn(containerId);
|
when(context.getContainerId()).thenReturn(containerId);
|
||||||
when(context.getContainerType()).thenReturn(
|
when(context.getContainerType()).thenReturn(
|
||||||
ContainerType.APPLICATION_MASTER);
|
ContainerType.APPLICATION_MASTER);
|
||||||
auxService.initializeContainer(context);
|
service.initializeContainer(context);
|
||||||
return auxService;
|
return service;
|
||||||
}
|
}
|
||||||
|
|
||||||
private PerNodeTimelineCollectorsAuxService createCollector() {
|
private PerNodeTimelineCollectorsAuxService createCollector() {
|
||||||
NodeTimelineCollectorManager collectorManager = createCollectorManager();
|
NodeTimelineCollectorManager collectorManager = createCollectorManager();
|
||||||
PerNodeTimelineCollectorsAuxService auxService =
|
PerNodeTimelineCollectorsAuxService service =
|
||||||
spy(new PerNodeTimelineCollectorsAuxService(collectorManager));
|
spy(new PerNodeTimelineCollectorsAuxService(collectorManager));
|
||||||
auxService.init(conf);
|
service.init(conf);
|
||||||
auxService.start();
|
service.start();
|
||||||
return auxService;
|
return service;
|
||||||
}
|
}
|
||||||
|
|
||||||
private NodeTimelineCollectorManager createCollectorManager() {
|
private NodeTimelineCollectorManager createCollectorManager() {
|
||||||
|
@ -45,11 +45,11 @@ public void testSplitUsingEscapeAndDelimChar() throws Exception {
|
|||||||
public void testJoinAndEscapeStrings() throws Exception {
|
public void testJoinAndEscapeStrings() throws Exception {
|
||||||
assertEquals("*!cluster!*!b**o***!xer!oozie**",
|
assertEquals("*!cluster!*!b**o***!xer!oozie**",
|
||||||
TimelineReaderUtils.joinAndEscapeStrings(
|
TimelineReaderUtils.joinAndEscapeStrings(
|
||||||
new String[] { "!cluster", "!b*o*!xer", "oozie*"}, '!', '*'));
|
new String[] {"!cluster", "!b*o*!xer", "oozie*"}, '!', '*'));
|
||||||
assertEquals("*!cluster!*!b**o***!xer!!",
|
assertEquals("*!cluster!*!b**o***!xer!!",
|
||||||
TimelineReaderUtils.joinAndEscapeStrings(
|
TimelineReaderUtils.joinAndEscapeStrings(
|
||||||
new String[] { "!cluster", "!b*o*!xer", "", ""}, '!', '*'));
|
new String[] {"!cluster", "!b*o*!xer", "", ""}, '!', '*'));
|
||||||
assertNull(TimelineReaderUtils.joinAndEscapeStrings(
|
assertNull(TimelineReaderUtils.joinAndEscapeStrings(
|
||||||
new String[] { "!cluster", "!b*o*!xer", null, ""}, '!', '*'));
|
new String[] {"!cluster", "!b*o*!xer", null, ""}, '!', '*'));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -110,7 +110,8 @@ private static void verifyHttpResponse(Client client, URI uri,
|
|||||||
client.resource(uri).accept(MediaType.APPLICATION_JSON)
|
client.resource(uri).accept(MediaType.APPLICATION_JSON)
|
||||||
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
|
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
|
||||||
assertNotNull(resp);
|
assertNotNull(resp);
|
||||||
assertEquals(resp.getClientResponseStatus(), expectedStatus);
|
assertEquals(resp.getStatusInfo().getStatusCode(),
|
||||||
|
expectedStatus.getStatusCode());
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Client createClient() {
|
private static Client createClient() {
|
||||||
@ -126,10 +127,11 @@ private static ClientResponse getResponse(Client client, URI uri)
|
|||||||
client.resource(uri).accept(MediaType.APPLICATION_JSON)
|
client.resource(uri).accept(MediaType.APPLICATION_JSON)
|
||||||
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
|
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
|
||||||
if (resp == null ||
|
if (resp == null ||
|
||||||
resp.getClientResponseStatus() != ClientResponse.Status.OK) {
|
resp.getStatusInfo().getStatusCode() !=
|
||||||
|
ClientResponse.Status.OK.getStatusCode()) {
|
||||||
String msg = new String();
|
String msg = new String();
|
||||||
if (resp != null) {
|
if (resp != null) {
|
||||||
msg = resp.getClientResponseStatus().toString();
|
msg = String.valueOf(resp.getStatusInfo().getStatusCode());
|
||||||
}
|
}
|
||||||
throw new IOException("Incorrect response from timeline reader. " +
|
throw new IOException("Incorrect response from timeline reader. " +
|
||||||
"Status=" + msg);
|
"Status=" + msg);
|
||||||
@ -141,7 +143,8 @@ private static class DummyURLConnectionFactory
|
|||||||
implements HttpURLConnectionFactory {
|
implements HttpURLConnectionFactory {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public HttpURLConnection getHttpURLConnection(final URL url) throws IOException {
|
public HttpURLConnection getHttpURLConnection(final URL url)
|
||||||
|
throws IOException {
|
||||||
try {
|
try {
|
||||||
return (HttpURLConnection)url.openConnection();
|
return (HttpURLConnection)url.openConnection();
|
||||||
} catch (UndeclaredThrowableException e) {
|
} catch (UndeclaredThrowableException e) {
|
||||||
|
@ -779,8 +779,8 @@ public void testRelationFiltersParsing() throws Exception {
|
|||||||
),
|
),
|
||||||
new TimelineFilterList(
|
new TimelineFilterList(
|
||||||
new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
|
new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
|
||||||
"type4", Sets.newHashSet((Object)"entity43","entity44",
|
"type4", Sets.newHashSet((Object)"entity43", "entity44",
|
||||||
"entity47","entity49")),
|
"entity47", "entity49")),
|
||||||
new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
|
new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
|
||||||
"type7", Sets.newHashSet((Object)"entity71"))
|
"type7", Sets.newHashSet((Object)"entity71"))
|
||||||
)
|
)
|
||||||
|
@ -60,7 +60,7 @@ public void testUIDEncodingDecoding() throws Exception {
|
|||||||
+ "YARN_CONTAINER!container_1111111111_1111_01_000001", uid);
|
+ "YARN_CONTAINER!container_1111111111_1111_01_000001", uid);
|
||||||
assertEquals(
|
assertEquals(
|
||||||
context, TimelineUIDConverter.GENERIC_ENTITY_UID.decodeUID(uid));
|
context, TimelineUIDConverter.GENERIC_ENTITY_UID.decodeUID(uid));
|
||||||
context = new TimelineReaderContext("yarn_cluster",null, null, null,
|
context = new TimelineReaderContext("yarn_cluster", null, null, null,
|
||||||
"application_1111111111_1111", "YARN_CONTAINER",
|
"application_1111111111_1111", "YARN_CONTAINER",
|
||||||
"container_1111111111_1111_01_000001");
|
"container_1111111111_1111_01_000001");
|
||||||
uid = TimelineUIDConverter.GENERIC_ENTITY_UID.encodeUID(context);
|
uid = TimelineUIDConverter.GENERIC_ENTITY_UID.encodeUID(context);
|
||||||
|
@ -58,9 +58,9 @@
|
|||||||
|
|
||||||
public class TestFileSystemTimelineReaderImpl {
|
public class TestFileSystemTimelineReaderImpl {
|
||||||
|
|
||||||
private static final String rootDir =
|
private static final String ROOT_DIR =
|
||||||
FileSystemTimelineReaderImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT;
|
FileSystemTimelineReaderImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT;
|
||||||
FileSystemTimelineReaderImpl reader;
|
private FileSystemTimelineReaderImpl reader;
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void setup() throws Exception {
|
public static void setup() throws Exception {
|
||||||
@ -68,22 +68,22 @@ public static void setup() throws Exception {
|
|||||||
// Create app flow mapping file.
|
// Create app flow mapping file.
|
||||||
CSVFormat format =
|
CSVFormat format =
|
||||||
CSVFormat.DEFAULT.withHeader("APP", "USER", "FLOW", "FLOWRUN");
|
CSVFormat.DEFAULT.withHeader("APP", "USER", "FLOW", "FLOWRUN");
|
||||||
String appFlowMappingFile = rootDir + "/entities/cluster1/" +
|
String appFlowMappingFile = ROOT_DIR + "/entities/cluster1/" +
|
||||||
FileSystemTimelineReaderImpl.APP_FLOW_MAPPING_FILE;
|
FileSystemTimelineReaderImpl.APP_FLOW_MAPPING_FILE;
|
||||||
try (PrintWriter out =
|
try (PrintWriter out =
|
||||||
new PrintWriter(new BufferedWriter(
|
new PrintWriter(new BufferedWriter(
|
||||||
new FileWriter(appFlowMappingFile, true)));
|
new FileWriter(appFlowMappingFile, true)));
|
||||||
CSVPrinter printer = new CSVPrinter(out, format)){
|
CSVPrinter printer = new CSVPrinter(out, format)){
|
||||||
printer.printRecord("app1", "user1", "flow1", 1);
|
printer.printRecord("app1", "user1", "flow1", 1);
|
||||||
printer.printRecord("app2","user1","flow1,flow",1);
|
printer.printRecord("app2", "user1", "flow1,flow", 1);
|
||||||
printer.close();
|
printer.close();
|
||||||
}
|
}
|
||||||
(new File(rootDir)).deleteOnExit();
|
(new File(ROOT_DIR)).deleteOnExit();
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterClass
|
@AfterClass
|
||||||
public static void tearDown() throws Exception {
|
public static void tearDown() throws Exception {
|
||||||
FileUtils.deleteDirectory(new File(rootDir));
|
FileUtils.deleteDirectory(new File(ROOT_DIR));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
@ -91,7 +91,7 @@ public void init() throws Exception {
|
|||||||
reader = new FileSystemTimelineReaderImpl();
|
reader = new FileSystemTimelineReaderImpl();
|
||||||
Configuration conf = new YarnConfiguration();
|
Configuration conf = new YarnConfiguration();
|
||||||
conf.set(FileSystemTimelineReaderImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
|
conf.set(FileSystemTimelineReaderImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
|
||||||
rootDir);
|
ROOT_DIR);
|
||||||
reader.init(conf);
|
reader.init(conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -112,7 +112,7 @@ private static void writeEntityFile(TimelineEntity entity, File dir)
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static void loadEntityData() throws Exception {
|
private static void loadEntityData() throws Exception {
|
||||||
File appDir = new File(rootDir +
|
File appDir = new File(ROOT_DIR +
|
||||||
"/entities/cluster1/user1/flow1/1/app1/app/");
|
"/entities/cluster1/user1/flow1/1/app1/app/");
|
||||||
TimelineEntity entity11 = new TimelineEntity();
|
TimelineEntity entity11 = new TimelineEntity();
|
||||||
entity11.setId("id_1");
|
entity11.setId("id_1");
|
||||||
@ -138,7 +138,7 @@ private static void loadEntityData() throws Exception {
|
|||||||
metric2.addValue(1425016502016L, 34);
|
metric2.addValue(1425016502016L, 34);
|
||||||
metrics.add(metric2);
|
metrics.add(metric2);
|
||||||
entity11.setMetrics(metrics);
|
entity11.setMetrics(metrics);
|
||||||
Map<String,String> configs = new HashMap<String, String>();
|
Map<String, String> configs = new HashMap<String, String>();
|
||||||
configs.put("config_1", "127");
|
configs.put("config_1", "127");
|
||||||
entity11.setConfigs(configs);
|
entity11.setConfigs(configs);
|
||||||
entity11.addRelatesToEntity("flow", "flow1");
|
entity11.addRelatesToEntity("flow", "flow1");
|
||||||
@ -179,7 +179,7 @@ private static void loadEntityData() throws Exception {
|
|||||||
Map<String, Object> info2 = new HashMap<String, Object>();
|
Map<String, Object> info2 = new HashMap<String, Object>();
|
||||||
info1.put("info2", 4);
|
info1.put("info2", 4);
|
||||||
entity2.addInfo(info2);
|
entity2.addInfo(info2);
|
||||||
Map<String,String> configs2 = new HashMap<String, String>();
|
Map<String, String> configs2 = new HashMap<String, String>();
|
||||||
configs2.put("config_1", "129");
|
configs2.put("config_1", "129");
|
||||||
configs2.put("config_3", "def");
|
configs2.put("config_3", "def");
|
||||||
entity2.setConfigs(configs2);
|
entity2.setConfigs(configs2);
|
||||||
@ -216,7 +216,7 @@ private static void loadEntityData() throws Exception {
|
|||||||
info3.put("info2", 3.5);
|
info3.put("info2", 3.5);
|
||||||
info3.put("info4", 20);
|
info3.put("info4", 20);
|
||||||
entity3.addInfo(info3);
|
entity3.addInfo(info3);
|
||||||
Map<String,String> configs3 = new HashMap<String, String>();
|
Map<String, String> configs3 = new HashMap<String, String>();
|
||||||
configs3.put("config_1", "123");
|
configs3.put("config_1", "123");
|
||||||
configs3.put("config_3", "abc");
|
configs3.put("config_3", "abc");
|
||||||
entity3.setConfigs(configs3);
|
entity3.setConfigs(configs3);
|
||||||
@ -254,7 +254,7 @@ private static void loadEntityData() throws Exception {
|
|||||||
entity4.addEvent(event44);
|
entity4.addEvent(event44);
|
||||||
writeEntityFile(entity4, appDir);
|
writeEntityFile(entity4, appDir);
|
||||||
|
|
||||||
File appDir2 = new File(rootDir +
|
File appDir2 = new File(ROOT_DIR +
|
||||||
"/entities/cluster1/user1/flow1,flow/1/app2/app/");
|
"/entities/cluster1/user1/flow1,flow/1/app2/app/");
|
||||||
TimelineEntity entity5 = new TimelineEntity();
|
TimelineEntity entity5 = new TimelineEntity();
|
||||||
entity5.setId("id_5");
|
entity5.setId("id_5");
|
||||||
@ -298,7 +298,7 @@ public void testGetEntityByClusterAndApp() throws Exception {
|
|||||||
Assert.assertEquals(0, result.getMetrics().size());
|
Assert.assertEquals(0, result.getMetrics().size());
|
||||||
}
|
}
|
||||||
|
|
||||||
/** This test checks whether we can handle commas in app flow mapping csv */
|
/** This test checks whether we can handle commas in app flow mapping csv. */
|
||||||
@Test
|
@Test
|
||||||
public void testAppFlowMappingCsv() throws Exception {
|
public void testAppFlowMappingCsv() throws Exception {
|
||||||
// Test getting an entity by cluster and app where flow entry
|
// Test getting an entity by cluster and app where flow entry
|
||||||
@ -317,7 +317,7 @@ public void testAppFlowMappingCsv() throws Exception {
|
|||||||
public void testGetEntityCustomFields() throws Exception {
|
public void testGetEntityCustomFields() throws Exception {
|
||||||
// Specified fields in addition to default view will be returned.
|
// Specified fields in addition to default view will be returned.
|
||||||
TimelineEntity result = reader.getEntity(
|
TimelineEntity result = reader.getEntity(
|
||||||
new TimelineReaderContext("cluster1","user1", "flow1", 1L, "app1",
|
new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
|
||||||
"app", "id_1"),
|
"app", "id_1"),
|
||||||
new TimelineDataToRetrieve(null, null,
|
new TimelineDataToRetrieve(null, null,
|
||||||
EnumSet.of(Field.INFO, Field.CONFIGS, Field.METRICS), null));
|
EnumSet.of(Field.INFO, Field.CONFIGS, Field.METRICS), null));
|
||||||
@ -336,7 +336,7 @@ public void testGetEntityCustomFields() throws Exception {
|
|||||||
public void testGetEntityAllFields() throws Exception {
|
public void testGetEntityAllFields() throws Exception {
|
||||||
// All fields of TimelineEntity will be returned.
|
// All fields of TimelineEntity will be returned.
|
||||||
TimelineEntity result = reader.getEntity(
|
TimelineEntity result = reader.getEntity(
|
||||||
new TimelineReaderContext("cluster1","user1", "flow1", 1L, "app1",
|
new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
|
||||||
"app", "id_1"),
|
"app", "id_1"),
|
||||||
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL), null));
|
||||||
Assert.assertEquals(
|
Assert.assertEquals(
|
||||||
@ -381,9 +381,9 @@ public void testGetEntitiesWithLimit() throws Exception {
|
|||||||
"app", null),
|
"app", null),
|
||||||
new TimelineEntityFilters(3L, null, null, null, null, null, null,
|
new TimelineEntityFilters(3L, null, null, null, null, null, null,
|
||||||
null, null), new TimelineDataToRetrieve());
|
null, null), new TimelineDataToRetrieve());
|
||||||
// Even though 2 entities out of 4 have same created time, one entity
|
// Even though 2 entities out of 4 have same created time, one entity
|
||||||
// is left out due to limit
|
// is left out due to limit
|
||||||
Assert.assertEquals(3, result.size());
|
Assert.assertEquals(3, result.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@ -474,9 +474,9 @@ public void testGetFilteredEntities() throws Exception {
|
|||||||
// Get entities based on event filters.
|
// Get entities based on event filters.
|
||||||
TimelineFilterList eventFilters = new TimelineFilterList();
|
TimelineFilterList eventFilters = new TimelineFilterList();
|
||||||
eventFilters.addFilter(
|
eventFilters.addFilter(
|
||||||
new TimelineExistsFilter(TimelineCompareOp.EQUAL,"event_2"));
|
new TimelineExistsFilter(TimelineCompareOp.EQUAL, "event_2"));
|
||||||
eventFilters.addFilter(
|
eventFilters.addFilter(
|
||||||
new TimelineExistsFilter(TimelineCompareOp.EQUAL,"event_4"));
|
new TimelineExistsFilter(TimelineCompareOp.EQUAL, "event_4"));
|
||||||
result = reader.getEntities(
|
result = reader.getEntities(
|
||||||
new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
|
new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
|
||||||
"app", null),
|
"app", null),
|
||||||
@ -642,7 +642,7 @@ public void testGetFilteredEntities() throws Exception {
|
|||||||
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
new TimelineEntityFilters(null, null, null, null, null, null, null,
|
||||||
metricFilterList2, null),
|
metricFilterList2, null),
|
||||||
new TimelineDataToRetrieve());
|
new TimelineDataToRetrieve());
|
||||||
Assert.assertEquals(1, result.size());
|
Assert.assertEquals(1, result.size());
|
||||||
for (TimelineEntity entity : result) {
|
for (TimelineEntity entity : result) {
|
||||||
if (!entity.getId().equals("id_1")) {
|
if (!entity.getId().equals("id_1")) {
|
||||||
Assert.fail("Incorrect filtering based on metric filters");
|
Assert.fail("Incorrect filtering based on metric filters");
|
||||||
@ -757,7 +757,7 @@ public void testGetFilteredEntities() throws Exception {
|
|||||||
Assert.fail("Incorrect filtering based on info filters");
|
Assert.fail("Incorrect filtering based on info filters");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testGetEntitiesByRelations() throws Exception {
|
public void testGetEntitiesByRelations() throws Exception {
|
||||||
|
@ -41,7 +41,8 @@
|
|||||||
public class TestFileSystemTimelineWriterImpl {
|
public class TestFileSystemTimelineWriterImpl {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Unit test for PoC YARN 3264
|
* Unit test for PoC YARN 3264.
|
||||||
|
*
|
||||||
* @throws Exception
|
* @throws Exception
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
@ -84,8 +85,8 @@ public void testWriteEntityToFile() throws Exception {
|
|||||||
"app_id", te);
|
"app_id", te);
|
||||||
|
|
||||||
String fileName = fsi.getOutputRoot() +
|
String fileName = fsi.getOutputRoot() +
|
||||||
"/entities/cluster_id/user_id/flow_name/flow_version/12345678/app_id/" +
|
"/entities/cluster_id/user_id/flow_name/flow_version/12345678/" +
|
||||||
type + "/" + id +
|
"app_id/" + type + "/" + id +
|
||||||
FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
|
FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
|
||||||
Path path = Paths.get(fileName);
|
Path path = Paths.get(fileName);
|
||||||
File f = new File(fileName);
|
File f = new File(fileName);
|
||||||
|
@ -60,7 +60,8 @@ public class TestRowKeys {
|
|||||||
if (sepByteLen <= byteArr.length) {
|
if (sepByteLen <= byteArr.length) {
|
||||||
for (int i = 0; i < sepByteLen; i++) {
|
for (int i = 0; i < sepByteLen; i++) {
|
||||||
byteArr[byteArr.length - sepByteLen + i] =
|
byteArr[byteArr.length - sepByteLen + i] =
|
||||||
(byte) (longMaxByteArr[byteArr.length - sepByteLen + i] - QUALIFIER_SEP_BYTES[i]);
|
(byte) (longMaxByteArr[byteArr.length - sepByteLen + i] -
|
||||||
|
QUALIFIER_SEP_BYTES[i]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
clusterTs = Bytes.toLong(byteArr);
|
clusterTs = Bytes.toLong(byteArr);
|
||||||
@ -73,7 +74,8 @@ private static void verifyRowPrefixBytes(byte[] byteRowKeyPrefix) {
|
|||||||
for (int i = 0; i < sepLen; i++) {
|
for (int i = 0; i < sepLen; i++) {
|
||||||
assertTrue(
|
assertTrue(
|
||||||
"Row key prefix not encoded properly.",
|
"Row key prefix not encoded properly.",
|
||||||
byteRowKeyPrefix[byteRowKeyPrefix.length - sepLen + i] == QUALIFIER_SEP_BYTES[i]);
|
byteRowKeyPrefix[byteRowKeyPrefix.length - sepLen + i] ==
|
||||||
|
QUALIFIER_SEP_BYTES[i]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -94,9 +96,9 @@ public void testApplicationRowKey() {
|
|||||||
.getRowKeyPrefix();
|
.getRowKeyPrefix();
|
||||||
byte[][] splits =
|
byte[][] splits =
|
||||||
Separator.QUALIFIERS.split(byteRowKeyPrefix,
|
Separator.QUALIFIERS.split(byteRowKeyPrefix,
|
||||||
new int[] { Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
|
new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
|
||||||
Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
|
Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
|
||||||
Separator.VARIABLE_SIZE });
|
Separator.VARIABLE_SIZE});
|
||||||
assertEquals(5, splits.length);
|
assertEquals(5, splits.length);
|
||||||
assertEquals(0, splits[4].length);
|
assertEquals(0, splits[4].length);
|
||||||
assertEquals(FLOW_NAME,
|
assertEquals(FLOW_NAME,
|
||||||
@ -153,10 +155,10 @@ public void testEntityRowKey() {
|
|||||||
byte[][] splits =
|
byte[][] splits =
|
||||||
Separator.QUALIFIERS.split(
|
Separator.QUALIFIERS.split(
|
||||||
byteRowKeyPrefix,
|
byteRowKeyPrefix,
|
||||||
new int[] { Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
|
new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
|
||||||
Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
|
Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
|
||||||
AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE,
|
AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE,
|
||||||
Separator.VARIABLE_SIZE });
|
Separator.VARIABLE_SIZE});
|
||||||
assertEquals(7, splits.length);
|
assertEquals(7, splits.length);
|
||||||
assertEquals(0, splits[6].length);
|
assertEquals(0, splits[6].length);
|
||||||
assertEquals(APPLICATION_ID, new AppIdKeyConverter().decode(splits[4]));
|
assertEquals(APPLICATION_ID, new AppIdKeyConverter().decode(splits[4]));
|
||||||
@ -170,9 +172,9 @@ public void testEntityRowKey() {
|
|||||||
splits =
|
splits =
|
||||||
Separator.QUALIFIERS.split(
|
Separator.QUALIFIERS.split(
|
||||||
byteRowKeyPrefix,
|
byteRowKeyPrefix,
|
||||||
new int[] { Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
|
new int[] {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
|
||||||
Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
|
Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
|
||||||
AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE });
|
AppIdKeyConverter.getKeySize(), Separator.VARIABLE_SIZE});
|
||||||
assertEquals(6, splits.length);
|
assertEquals(6, splits.length);
|
||||||
assertEquals(0, splits[5].length);
|
assertEquals(0, splits[5].length);
|
||||||
AppIdKeyConverter appIdKeyConverter = new AppIdKeyConverter();
|
AppIdKeyConverter appIdKeyConverter = new AppIdKeyConverter();
|
||||||
@ -207,8 +209,8 @@ public void testFlowActivityRowKey() {
|
|||||||
new FlowActivityRowKeyPrefix(CLUSTER, ts).getRowKeyPrefix();
|
new FlowActivityRowKeyPrefix(CLUSTER, ts).getRowKeyPrefix();
|
||||||
splits =
|
splits =
|
||||||
Separator.QUALIFIERS.split(byteRowKeyPrefix,
|
Separator.QUALIFIERS.split(byteRowKeyPrefix,
|
||||||
new int[] { Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
|
new int[] {Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
|
||||||
Separator.VARIABLE_SIZE });
|
Separator.VARIABLE_SIZE});
|
||||||
assertEquals(3, splits.length);
|
assertEquals(3, splits.length);
|
||||||
assertEquals(0, splits[2].length);
|
assertEquals(0, splits[2].length);
|
||||||
assertEquals(CLUSTER,
|
assertEquals(CLUSTER,
|
||||||
|
@ -86,7 +86,7 @@ public void testEncodedValues() {
|
|||||||
testEncodeDecode("Double-escape %2$ and %9$ or %%2$ or %%3$, nor %%%2$" +
|
testEncodeDecode("Double-escape %2$ and %9$ or %%2$ or %%3$, nor %%%2$" +
|
||||||
"= no problem!",
|
"= no problem!",
|
||||||
Separator.QUALIFIERS, Separator.VALUES, Separator.SPACE, Separator.TAB);
|
Separator.QUALIFIERS, Separator.VALUES, Separator.SPACE, Separator.TAB);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSplits() {
|
public void testSplits() {
|
||||||
@ -101,10 +101,10 @@ public void testSplits() {
|
|||||||
byte[] intVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxIntBytes,
|
byte[] intVal1Arr = Bytes.add(sepByteArr, Bytes.copy(maxIntBytes,
|
||||||
sepByteArr.length, Bytes.SIZEOF_INT - sepByteArr.length));
|
sepByteArr.length, Bytes.SIZEOF_INT - sepByteArr.length));
|
||||||
byte[] arr = separator.join(
|
byte[] arr = separator.join(
|
||||||
Bytes.toBytes(separator.encode(str1)),longVal1Arr,
|
Bytes.toBytes(separator.encode(str1)), longVal1Arr,
|
||||||
Bytes.toBytes(separator.encode(str2)), intVal1Arr);
|
Bytes.toBytes(separator.encode(str2)), intVal1Arr);
|
||||||
int[] sizes = { Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
|
int[] sizes = {Separator.VARIABLE_SIZE, Bytes.SIZEOF_LONG,
|
||||||
Separator.VARIABLE_SIZE, Bytes.SIZEOF_INT };
|
Separator.VARIABLE_SIZE, Bytes.SIZEOF_INT};
|
||||||
byte[][] splits = separator.split(arr, sizes);
|
byte[][] splits = separator.split(arr, sizes);
|
||||||
assertEquals(4, splits.length);
|
assertEquals(4, splits.length);
|
||||||
assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
|
assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
|
||||||
@ -116,7 +116,7 @@ public void testSplits() {
|
|||||||
sepByteArr.length), sepByteArr);
|
sepByteArr.length), sepByteArr);
|
||||||
intVal1Arr = Bytes.add(Bytes.copy(maxIntBytes, 0, Bytes.SIZEOF_INT -
|
intVal1Arr = Bytes.add(Bytes.copy(maxIntBytes, 0, Bytes.SIZEOF_INT -
|
||||||
sepByteArr.length), sepByteArr);
|
sepByteArr.length), sepByteArr);
|
||||||
arr = separator.join(Bytes.toBytes(separator.encode(str1)),longVal1Arr,
|
arr = separator.join(Bytes.toBytes(separator.encode(str1)), longVal1Arr,
|
||||||
Bytes.toBytes(separator.encode(str2)), intVal1Arr);
|
Bytes.toBytes(separator.encode(str2)), intVal1Arr);
|
||||||
splits = separator.split(arr, sizes);
|
splits = separator.split(arr, sizes);
|
||||||
assertEquals(4, splits.length);
|
assertEquals(4, splits.length);
|
||||||
@ -129,7 +129,7 @@ public void testSplits() {
|
|||||||
sepByteArr.length, 4 - sepByteArr.length), sepByteArr);
|
sepByteArr.length, 4 - sepByteArr.length), sepByteArr);
|
||||||
longVal1Arr = Bytes.add(longVal1Arr, Bytes.copy(maxLongBytes, 4, 3 -
|
longVal1Arr = Bytes.add(longVal1Arr, Bytes.copy(maxLongBytes, 4, 3 -
|
||||||
sepByteArr.length), sepByteArr);
|
sepByteArr.length), sepByteArr);
|
||||||
arr = separator.join(Bytes.toBytes(separator.encode(str1)),longVal1Arr,
|
arr = separator.join(Bytes.toBytes(separator.encode(str1)), longVal1Arr,
|
||||||
Bytes.toBytes(separator.encode(str2)), intVal1Arr);
|
Bytes.toBytes(separator.encode(str2)), intVal1Arr);
|
||||||
splits = separator.split(arr, sizes);
|
splits = separator.split(arr, sizes);
|
||||||
assertEquals(4, splits.length);
|
assertEquals(4, splits.length);
|
||||||
@ -140,8 +140,8 @@ public void testSplits() {
|
|||||||
|
|
||||||
arr = separator.join(Bytes.toBytes(separator.encode(str1)),
|
arr = separator.join(Bytes.toBytes(separator.encode(str1)),
|
||||||
Bytes.toBytes(separator.encode(str2)), intVal1Arr, longVal1Arr);
|
Bytes.toBytes(separator.encode(str2)), intVal1Arr, longVal1Arr);
|
||||||
int[] sizes1 = { Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
|
int[] sizes1 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
|
||||||
Bytes.SIZEOF_INT, Bytes.SIZEOF_LONG };
|
Bytes.SIZEOF_INT, Bytes.SIZEOF_LONG};
|
||||||
splits = separator.split(arr, sizes1);
|
splits = separator.split(arr, sizes1);
|
||||||
assertEquals(4, splits.length);
|
assertEquals(4, splits.length);
|
||||||
assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
|
assertEquals(str1, separator.decode(Bytes.toString(splits[0])));
|
||||||
@ -150,15 +150,15 @@ public void testSplits() {
|
|||||||
assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[3]));
|
assertEquals(Bytes.toLong(longVal1Arr), Bytes.toLong(splits[3]));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
int[] sizes2 = { Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
|
int[] sizes2 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE,
|
||||||
Bytes.SIZEOF_INT, 7 };
|
Bytes.SIZEOF_INT, 7};
|
||||||
splits = separator.split(arr, sizes2);
|
splits = separator.split(arr, sizes2);
|
||||||
fail("Exception should have been thrown.");
|
fail("Exception should have been thrown.");
|
||||||
} catch (IllegalArgumentException e) {}
|
} catch (IllegalArgumentException e) {}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
int[] sizes2 = { Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, 2,
|
int[] sizes2 = {Separator.VARIABLE_SIZE, Separator.VARIABLE_SIZE, 2,
|
||||||
Bytes.SIZEOF_LONG };
|
Bytes.SIZEOF_LONG};
|
||||||
splits = separator.split(arr, sizes2);
|
splits = separator.split(arr, sizes2);
|
||||||
fail("Exception should have been thrown.");
|
fail("Exception should have been thrown.");
|
||||||
} catch (IllegalArgumentException e) {}
|
} catch (IllegalArgumentException e) {}
|
||||||
@ -196,12 +196,12 @@ public void testJoinStripped() {
|
|||||||
split = Separator.VALUES.splitEncoded(joined);
|
split = Separator.VALUES.splitEncoded(joined);
|
||||||
assertTrue(Iterables.elementsEqual(stringList, split));
|
assertTrue(Iterables.elementsEqual(stringList, split));
|
||||||
|
|
||||||
String[] stringArray1 = { "else" };
|
String[] stringArray1 = {"else"};
|
||||||
joined = Separator.VALUES.joinEncoded(stringArray1);
|
joined = Separator.VALUES.joinEncoded(stringArray1);
|
||||||
split = Separator.VALUES.splitEncoded(joined);
|
split = Separator.VALUES.splitEncoded(joined);
|
||||||
assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray1), split));
|
assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray1), split));
|
||||||
|
|
||||||
String[] stringArray2 = { "d", "e?", "f" };
|
String[] stringArray2 = {"d", "e?", "f"};
|
||||||
joined = Separator.VALUES.joinEncoded(stringArray2);
|
joined = Separator.VALUES.joinEncoded(stringArray2);
|
||||||
split = Separator.VALUES.splitEncoded(joined);
|
split = Separator.VALUES.splitEncoded(joined);
|
||||||
assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray2), split));
|
assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray2), split));
|
||||||
|
@ -1197,4 +1197,3 @@ container ID. Similarly, application attempt can be queried by specifying entity
|
|||||||
1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned.
|
1. If any problem occurs in parsing request, HTTP 400 (Bad Request) is returned.
|
||||||
1. If flow context information cannot be retrieved or entity for the given entity id cannot be found, HTTP 404 (Not Found) is returned.
|
1. If flow context information cannot be retrieved or entity for the given entity id cannot be found, HTTP 404 (Not Found) is returned.
|
||||||
1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned.
|
1. For non-recoverable errors while retrieving data, HTTP 500 (Internal Server Error) is returned.
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user