HDDS-1568 : Add RocksDB metrics to OM. Contributed by Aravindan Vijayan

This commit is contained in:
avijayanhwx 2019-05-30 08:48:14 -07:00 committed by Anu Engineer
parent 0b907bc0e8
commit 33419a980a
7 changed files with 262 additions and 18 deletions

View File

@ -75,7 +75,8 @@ public RocksDBStore(File dbFile, Options options)
jmxProperties.put("dbName", dbFile.getName()); jmxProperties.put("dbName", dbFile.getName());
statMBeanName = HddsUtils.registerWithJmxProperties( statMBeanName = HddsUtils.registerWithJmxProperties(
"Ozone", "RocksDbStore", jmxProperties, "Ozone", "RocksDbStore", jmxProperties,
new RocksDBStoreMBean(dbOptions.statistics())); RocksDBStoreMBean.create(dbOptions.statistics(),
dbFile.getName()));
if (statMBeanName == null) { if (statMBeanName == null) {
LOG.warn("jmx registration failed during RocksDB init, db path :{}", LOG.warn("jmx registration failed during RocksDB init, db path :{}",
dbFile.getAbsolutePath()); dbFile.getAbsolutePath());

View File

@ -18,10 +18,18 @@
package org.apache.hadoop.utils; package org.apache.hadoop.utils;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.MetricsSource;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.lib.Interns;
import org.rocksdb.HistogramData; import org.rocksdb.HistogramData;
import org.rocksdb.HistogramType; import org.rocksdb.HistogramType;
import org.rocksdb.Statistics; import org.rocksdb.Statistics;
import org.rocksdb.TickerType; import org.rocksdb.TickerType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.management.Attribute; import javax.management.Attribute;
import javax.management.AttributeList; import javax.management.AttributeList;
@ -41,13 +49,21 @@
/** /**
* Adapter JMX bean to publish all the Rocksdb metrics. * Adapter JMX bean to publish all the Rocksdb metrics.
*/ */
public class RocksDBStoreMBean implements DynamicMBean { public class RocksDBStoreMBean implements DynamicMBean, MetricsSource {
private Statistics statistics; private Statistics statistics;
private Set<String> histogramAttributes = new HashSet<>(); private Set<String> histogramAttributes = new HashSet<>();
public RocksDBStoreMBean(Statistics statistics) { private String contextName;
private static final Logger LOG =
LoggerFactory.getLogger(RocksDBStoreMBean.class);
public final static String ROCKSDB_CONTEXT_PREFIX = "Rocksdb_";
public RocksDBStoreMBean(Statistics statistics, String dbName) {
this.contextName = ROCKSDB_CONTEXT_PREFIX + dbName;
this.statistics = statistics; this.statistics = statistics;
histogramAttributes.add("Average"); histogramAttributes.add("Average");
histogramAttributes.add("Median"); histogramAttributes.add("Median");
@ -56,6 +72,22 @@ public RocksDBStoreMBean(Statistics statistics) {
histogramAttributes.add("StandardDeviation"); histogramAttributes.add("StandardDeviation");
} }
public static RocksDBStoreMBean create(Statistics statistics,
String contextName) {
RocksDBStoreMBean rocksDBStoreMBean = new RocksDBStoreMBean(
statistics, contextName);
MetricsSystem ms = DefaultMetricsSystem.instance();
MetricsSource metricsSource = ms.getSource(rocksDBStoreMBean.contextName);
if (metricsSource != null) {
return (RocksDBStoreMBean)metricsSource;
} else {
return ms.register(rocksDBStoreMBean.contextName,
"RocksDB Metrics",
rocksDBStoreMBean);
}
}
@Override @Override
public Object getAttribute(String attribute) public Object getAttribute(String attribute)
throws AttributeNotFoundException, MBeanException, ReflectionException { throws AttributeNotFoundException, MBeanException, ReflectionException {
@ -141,4 +173,47 @@ public MBeanInfo getMBeanInfo() {
attributes.toArray(new MBeanAttributeInfo[0]), null, null, null); attributes.toArray(new MBeanAttributeInfo[0]), null, null, null);
} }
@Override
public void getMetrics(MetricsCollector metricsCollector, boolean b) {
MetricsRecordBuilder rb = metricsCollector.addRecord(contextName);
getHistogramData(rb);
getTickerTypeData(rb);
}
/**
* Collect all histogram metrics from RocksDB statistics.
* @param rb Metrics Record Builder.
*/
private void getHistogramData(MetricsRecordBuilder rb) {
for (HistogramType histogramType : HistogramType.values()) {
HistogramData histogram =
statistics.getHistogramData(
HistogramType.valueOf(histogramType.name()));
for (String histogramAttribute : histogramAttributes) {
try {
Method method =
HistogramData.class.getMethod("get" + histogramAttribute);
double metricValue = (double) method.invoke(histogram);
rb.addGauge(Interns.info(histogramType.name() + "_" +
histogramAttribute.toUpperCase(), "RocksDBStat"),
metricValue);
} catch (Exception e) {
LOG.error("Error reading histogram data {} ", e);
}
}
}
}
/**
* Collect all Counter metrics from RocksDB statistics.
* @param rb Metrics Record Builder.
*/
private void getTickerTypeData(MetricsRecordBuilder rb) {
for (TickerType tickerType : TickerType.values()) {
rb.addCounter(Interns.info(tickerType.name(), "RocksDBStat"),
statistics.getTickerCount(tickerType));
}
}
} }

View File

@ -28,6 +28,8 @@
import org.rocksdb.ColumnFamilyOptions; import org.rocksdb.ColumnFamilyOptions;
import org.rocksdb.DBOptions; import org.rocksdb.DBOptions;
import org.rocksdb.RocksDB; import org.rocksdb.RocksDB;
import org.rocksdb.Statistics;
import org.rocksdb.StatsLevel;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -42,6 +44,9 @@
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_DB_PROFILE; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_DB_PROFILE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF;
/** /**
* DBStore Builder. * DBStore Builder.
@ -57,12 +62,16 @@ public final class DBStoreBuilder {
private List<String> tableNames; private List<String> tableNames;
private Configuration configuration; private Configuration configuration;
private CodecRegistry registry; private CodecRegistry registry;
private String rocksDbStat;
private DBStoreBuilder(Configuration configuration) { private DBStoreBuilder(Configuration configuration) {
tables = new HashSet<>(); tables = new HashSet<>();
tableNames = new LinkedList<>(); tableNames = new LinkedList<>();
this.configuration = configuration; this.configuration = configuration;
this.registry = new CodecRegistry(); this.registry = new CodecRegistry();
this.rocksDbStat = configuration.getTrimmed(
OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT);
} }
public static DBStoreBuilder newBuilder(Configuration configuration) { public static DBStoreBuilder newBuilder(Configuration configuration) {
@ -187,7 +196,13 @@ private DBOptions getDbProfile() {
if (option == null) { if (option == null) {
LOG.info("Using default options. {}", dbProfile.toString()); LOG.info("Using default options. {}", dbProfile.toString());
return dbProfile.getDBOptions(); option = dbProfile.getDBOptions();
}
if (!rocksDbStat.equals(OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF)) {
Statistics statistics = new Statistics();
statistics.setStatsLevel(StatsLevel.valueOf(rocksDbStat));
option = option.setStatistics(statistics);
} }
return option; return option;
} }

View File

@ -108,7 +108,8 @@ public RDBStore(File dbFile, DBOptions options, Set<TableConfig> families,
jmxProperties.put("dbName", dbFile.getName()); jmxProperties.put("dbName", dbFile.getName());
statMBeanName = HddsUtils.registerWithJmxProperties( statMBeanName = HddsUtils.registerWithJmxProperties(
"Ozone", "RocksDbStore", jmxProperties, "Ozone", "RocksDbStore", jmxProperties,
new RocksDBStoreMBean(dbOptions.statistics())); RocksDBStoreMBean.create(dbOptions.statistics(),
dbFile.getName()));
if (statMBeanName == null) { if (statMBeanName == null) {
LOG.warn("jmx registration failed during RocksDB init, db path :{}", LOG.warn("jmx registration failed during RocksDB init, db path :{}",
dbFile.getAbsolutePath()); dbFile.getAbsolutePath());

View File

@ -19,6 +19,14 @@
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsInfo;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.MetricsSource;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert; import org.junit.Assert;
@ -27,9 +35,14 @@
import javax.management.MBeanServer; import javax.management.MBeanServer;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory; import java.lang.management.ManagementFactory;
import java.util.HashMap;
import java.util.Map;
import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/** /**
* Test the JMX interface for the rocksdb metastore implementation. * Test the JMX interface for the rocksdb metastore implementation.
@ -49,18 +62,8 @@ public void init() throws Exception {
@Test @Test
public void testJmxBeans() throws Exception { public void testJmxBeans() throws Exception {
File testDir =
GenericTestUtils.getTestDir(getClass().getSimpleName() + "-withstat");
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS, "ALL"); RocksDBStore metadataStore = getTestRocksDBStoreWithData();
RocksDBStore metadataStore =
(RocksDBStore) MetadataStoreBuilder.newBuilder().setConf(conf)
.setCreateIfMissing(true).setDbFile(testDir).build();
for (int i = 0; i < 10; i++) {
metadataStore.put("key".getBytes(UTF_8), "value".getBytes(UTF_8));
}
MBeanServer platformMBeanServer = MBeanServer platformMBeanServer =
ManagementFactory.getPlatformMBeanServer(); ManagementFactory.getPlatformMBeanServer();
@ -69,11 +72,11 @@ public void testJmxBeans() throws Exception {
Object keysWritten = platformMBeanServer Object keysWritten = platformMBeanServer
.getAttribute(metadataStore.getStatMBeanName(), "NUMBER_KEYS_WRITTEN"); .getAttribute(metadataStore.getStatMBeanName(), "NUMBER_KEYS_WRITTEN");
Assert.assertEquals(10L, keysWritten); assertEquals(10L, keysWritten);
Object dbWriteAverage = platformMBeanServer Object dbWriteAverage = platformMBeanServer
.getAttribute(metadataStore.getStatMBeanName(), "DB_WRITE_AVERAGE"); .getAttribute(metadataStore.getStatMBeanName(), "DB_WRITE_AVERAGE");
Assert.assertTrue((double) dbWriteAverage > 0); assertTrue((double) dbWriteAverage > 0);
metadataStore.close(); metadataStore.close();
@ -93,4 +96,139 @@ public void testDisabledStat() throws Exception {
Assert.assertNull(metadataStore.getStatMBeanName()); Assert.assertNull(metadataStore.getStatMBeanName());
} }
@Test
public void testMetricsSystemIntegration() throws Exception {
RocksDBStore metadataStore = getTestRocksDBStoreWithData();
Thread.sleep(2000);
MetricsSystem ms = DefaultMetricsSystem.instance();
MetricsSource rdbSource =
ms.getSource("Rocksdb_TestRocksDBStoreMBean-withstat");
BufferedMetricsCollector metricsCollector = new BufferedMetricsCollector();
rdbSource.getMetrics(metricsCollector, true);
Map<String, Double> metrics = metricsCollector.getMetricsRecordBuilder()
.getMetrics();
assertTrue(10.0 == metrics.get("NUMBER_KEYS_WRITTEN"));
assertTrue(metrics.get("DB_WRITE_AVERAGE") > 0);
metadataStore.close();
}
private RocksDBStore getTestRocksDBStoreWithData() throws IOException {
File testDir =
GenericTestUtils.getTestDir(getClass().getSimpleName() + "-withstat");
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS, "ALL");
RocksDBStore metadataStore =
(RocksDBStore) MetadataStoreBuilder.newBuilder().setConf(conf)
.setCreateIfMissing(true).setDbFile(testDir).build();
for (int i = 0; i < 10; i++) {
metadataStore.put("key".getBytes(UTF_8), "value".getBytes(UTF_8));
}
return metadataStore;
}
}
/**
* Test class to buffer a single MetricsRecordBuilder instance.
*/
class BufferedMetricsCollector implements MetricsCollector {
private BufferedMetricsRecordBuilderImpl metricsRecordBuilder;
BufferedMetricsCollector() {
metricsRecordBuilder = new BufferedMetricsRecordBuilderImpl();
}
public BufferedMetricsRecordBuilderImpl getMetricsRecordBuilder() {
return metricsRecordBuilder;
}
@Override
public MetricsRecordBuilder addRecord(String s) {
metricsRecordBuilder.setContext(s);
return metricsRecordBuilder;
}
@Override
public MetricsRecordBuilder addRecord(MetricsInfo metricsInfo) {
return metricsRecordBuilder;
}
/**
* Test class to buffer a single snapshot of metrics.
*/
class BufferedMetricsRecordBuilderImpl extends MetricsRecordBuilder {
private Map<String, Double> metrics = new HashMap<>();
private String contextName;
public Map<String, Double> getMetrics() {
return metrics;
}
@Override
public MetricsRecordBuilder tag(MetricsInfo metricsInfo, String s) {
return null;
}
@Override
public MetricsRecordBuilder add(MetricsTag metricsTag) {
return null;
}
@Override
public MetricsRecordBuilder add(AbstractMetric abstractMetric) {
return null;
}
@Override
public MetricsRecordBuilder setContext(String s) {
this.contextName = s;
return this;
}
@Override
public MetricsRecordBuilder addCounter(MetricsInfo metricsInfo, int i) {
return null;
}
@Override
public MetricsRecordBuilder addCounter(MetricsInfo metricsInfo, long l) {
metrics.put(metricsInfo.name(), (double)l);
return this;
}
@Override
public MetricsRecordBuilder addGauge(MetricsInfo metricsInfo, int i) {
return null;
}
@Override
public MetricsRecordBuilder addGauge(MetricsInfo metricsInfo, long l) {
return null;
}
@Override
public MetricsRecordBuilder addGauge(MetricsInfo metricsInfo, float v) {
return null;
}
@Override
public MetricsRecordBuilder addGauge(MetricsInfo metricsInfo, double v) {
metrics.put(metricsInfo.name(), v);
return this;
}
@Override
public MetricsCollector parent() {
return null;
}
}
} }

View File

@ -17,6 +17,8 @@
*/ */
package org.apache.hadoop.hdds.server; package org.apache.hadoop.hdds.server;
import static org.apache.hadoop.utils.RocksDBStoreMBean.ROCKSDB_CONTEXT_PREFIX;
import java.io.IOException; import java.io.IOException;
import java.io.Writer; import java.io.Writer;
import java.util.HashMap; import java.util.HashMap;
@ -24,6 +26,7 @@
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.metrics2.AbstractMetric; import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricType; import org.apache.hadoop.metrics2.MetricType;
import org.apache.hadoop.metrics2.MetricsRecord; import org.apache.hadoop.metrics2.MetricsRecord;
@ -90,6 +93,12 @@ public void putMetrics(MetricsRecord metricsRecord) {
*/ */
public String prometheusName(String recordName, public String prometheusName(String recordName,
String metricName) { String metricName) {
//RocksDB metric names already have underscores as delimiters.
if (StringUtils.isNotEmpty(recordName) &&
recordName.startsWith(ROCKSDB_CONTEXT_PREFIX)) {
return recordName.toLowerCase() + "_" + metricName.toLowerCase();
}
String baseName = upperFirst(recordName) + upperFirst(metricName); String baseName = upperFirst(recordName) + upperFirst(metricName);
Matcher m = UPPER_CASE_SEQ.matcher(baseName); Matcher m = UPPER_CASE_SEQ.matcher(baseName);
StringBuffer sb = new StringBuffer(); StringBuffer sb = new StringBuffer();

View File

@ -82,6 +82,11 @@ public void testNaming() throws IOException {
Assert.assertEquals("rpc_time_small", Assert.assertEquals("rpc_time_small",
sink.prometheusName("RpcTime", "small")); sink.prometheusName("RpcTime", "small"));
//RocksDB metrics are handled differently.
Assert.assertEquals("rocksdb_om.db_num_open_connections",
sink.prometheusName("Rocksdb_om.db", "num_open_connections"));
} }
/** /**