big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-7507. Allow ganglia metrics to include the metrics system tags in the gmetric names. Contributed by Alejandro Abdelnur.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1166460 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Todd Lipcon 2011-09-07 22:59:22 +00:00
parent 1c15670d53
commit 2ca9c8d926
5 changed files with 131 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -368,6 +368,9 @@ Release 0.23.0 - Unreleased
HADOOP-7580. Add a version of getLocalPathForWrite to LocalDirAllocator
which doesn't create dirs. (Chris Douglas & Siddharth Seth via acmurthy)
HADOOP-7507. Allow ganglia metrics to include the metrics system tags
in the gmetric names. (Alejandro Abdelnur via todd)
OPTIMIZATIONS
HADOOP-7333. Performance improvement in PureJavaCrc32. (Eric Caspole

10
hadoop-common-project/hadoop-common/src/main/conf/hadoop-metrics2.properties
View File

@ -43,6 +43,16 @@
#*.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
#*.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
# Tag values to use for the ganglia prefix. If not defined no tags are used.
# If '*' all tags are used. If specifiying multiple tags separate them with
# commas. Note that the last segment of the property name is the context name.
#
#*.sink.ganglia.tagsForPrefix.jvm=ProcesName
#*.sink.ganglia.tagsForPrefix.dfs=
#*.sink.ganglia.tagsForPrefix.rpc=
#*.sink.ganglia.tagsForPrefix.mapred=
#*.sink.ganglia.tagsForPrefix.fairscheduler=
#namenode.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
#datanode.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/ganglia/GangliaContext.java
View File

@ -132,6 +132,12 @@ public void emitRecord(String contextName, String recordName,
StringBuilder sb = new StringBuilder();
sb.append(contextName);
sb.append('.');
if (contextName.equals("jvm") && outRec.getTag("processName") != null) {
sb.append(outRec.getTag("processName"));
sb.append('.');
}
sb.append(recordName);
sb.append('.');
int sbBaseLen = sb.length();

69
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/GangliaSink30.java
View File

@ -20,13 +20,21 @@
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import org.apache.commons.configuration.SubsetConfiguration;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricsException;
import org.apache.hadoop.metrics2.MetricsRecord;
import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.impl.MsInfo;
import org.apache.hadoop.metrics2.util.MetricsCache;
import org.apache.hadoop.metrics2.util.MetricsCache.Record;
@ -38,8 +46,67 @@ public class GangliaSink30 extends AbstractGangliaSink {
public final Log LOG = LogFactory.getLog(this.getClass());
private static final String TAGS_FOR_PREFIX_PROPERTY_PREFIX = "tagsForPrefix.";
private MetricsCache metricsCache = new MetricsCache();
// a key with a NULL value means ALL
private Map<String,Set<String>> useTagsMap = new HashMap<String,Set<String>>();
@Override
@SuppressWarnings("unchecked")
public void init(SubsetConfiguration conf) {
super.init(conf);
conf.setListDelimiter(',');
Iterator<String> it = (Iterator<String>) conf.getKeys();
while (it.hasNext()) {
String propertyName = it.next();
if (propertyName.startsWith(TAGS_FOR_PREFIX_PROPERTY_PREFIX)) {
String contextName = propertyName.substring(TAGS_FOR_PREFIX_PROPERTY_PREFIX.length());
String[] tags = conf.getStringArray(propertyName);
boolean useAllTags = false;
Set<String> set = null;
if (tags.length > 0) {
set = new HashSet<String>();
for (String tag : tags) {
tag = tag.trim();
useAllTags |= tag.equals("*");
if (tag.length() > 0) {
set.add(tag);
}
}
if (useAllTags) {
set = null;
}
}
useTagsMap.put(contextName, set);
}
}
}
@InterfaceAudience.Private
public void appendPrefix(MetricsRecord record, StringBuilder sb) {
String contextName = record.context();
Collection<MetricsTag> tags = record.tags();
if (useTagsMap.containsKey(contextName)) {
Set<String> useTags = useTagsMap.get(contextName);
for (MetricsTag t : tags) {
if (useTags == null || useTags.contains(t.name())) {
// the context is always skipped here because it is always added
// the hostname is always skipped to avoid case-mismatches
// from different DNSes.
if (t.info() != MsInfo.Context && t.info() != MsInfo.Hostname && t.value() != null) {
sb.append('.').append(t.name()).append('=').append(t.value());
}
}
}
}
}
@Override
public void putMetrics(MetricsRecord record) {
// The method handles both cases whether Ganglia support dense publish
@ -53,6 +120,8 @@ public void putMetrics(MetricsRecord record) {
sb.append('.');
sb.append(recordName);
appendPrefix(record, sb);
String groupName = sb.toString();
sb.append('.');
int sbBaseLen = sb.length();

43
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
View File

@ -26,12 +26,17 @@
import java.net.DatagramSocket;
import java.net.SocketException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricsRecord;
import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
@ -54,6 +59,44 @@ public class TestGangliaMetrics {
"test.s1rec.S1NumOps",
"test.s1rec.S1AvgTime" };
@Test
public void testTagsForPrefix() throws Exception {
ConfigBuilder cb = new ConfigBuilder()
.add("test.sink.ganglia.tagsForPrefix.all", "*")
.add("test.sink.ganglia.tagsForPrefix.some", "NumActiveSinks, NumActiveSources")
.add("test.sink.ganglia.tagsForPrefix.none", "");
GangliaSink30 sink = new GangliaSink30();
sink.init(cb.subset("test.sink.ganglia"));
List<MetricsTag> tags = new ArrayList<MetricsTag>();
tags.add(new MetricsTag(MsInfo.Context, "all"));
tags.add(new MetricsTag(MsInfo.NumActiveSources, "foo"));
tags.add(new MetricsTag(MsInfo.NumActiveSinks, "bar"));
tags.add(new MetricsTag(MsInfo.NumAllSinks, "haa"));
tags.add(new MetricsTag(MsInfo.Hostname, "host"));
Set<AbstractMetric> metrics = new HashSet<AbstractMetric>();
MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 1, tags, metrics);
StringBuilder sb = new StringBuilder();
sink.appendPrefix(record, sb);
assertEquals(".NumActiveSources=foo.NumActiveSinks=bar.NumAllSinks=haa", sb.toString());
tags.set(0, new MetricsTag(MsInfo.Context, "some"));
sb = new StringBuilder();
sink.appendPrefix(record, sb);
assertEquals(".NumActiveSources=foo.NumActiveSinks=bar", sb.toString());
tags.set(0, new MetricsTag(MsInfo.Context, "none"));
sb = new StringBuilder();
sink.appendPrefix(record, sb);
assertEquals("", sb.toString());
tags.set(0, new MetricsTag(MsInfo.Context, "nada"));
sb = new StringBuilder();
sink.appendPrefix(record, sb);
assertEquals("", sb.toString());
}
@Test public void testGangliaMetrics2() throws Exception {
ConfigBuilder cb = new ConfigBuilder().add("default.period", 10)
.add("test.sink.gsink30.context", "test") // filter out only "test"