HDFS-12807. Ozone: Expose RockDB stats via JMX for Ozone metadata stores. Contributed by Elek, Marton.

This commit is contained in:
Anu Engineer 2017-11-27 21:40:15 -08:00 committed by Owen O'Malley
parent 86802a461c
commit 8023ba50b5
7 changed files with 309 additions and 4 deletions

View File

@ -91,6 +91,14 @@ public final class OzoneConfigKeys {
public static final String OZONE_METADATA_STORE_IMPL_DEFAULT = public static final String OZONE_METADATA_STORE_IMPL_DEFAULT =
OZONE_METADATA_STORE_IMPL_ROCKSDB; OZONE_METADATA_STORE_IMPL_ROCKSDB;
public static final String OZONE_METADATA_STORE_ROCKSDB_STATISTICS =
"ozone.metastore.rocksdb.statistics";
public static final String OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT =
"ALL";
public static final String OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF =
"OFF";
public static final String OZONE_CONTAINER_CACHE_SIZE = public static final String OZONE_CONTAINER_CACHE_SIZE =
"ozone.container.cache.size"; "ozone.container.cache.size";
public static final int OZONE_CONTAINER_CACHE_DEFAULT = 1024; public static final int OZONE_CONTAINER_CACHE_DEFAULT = 1024;

View File

@ -190,7 +190,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<dependency> <dependency>
<groupId>org.rocksdb</groupId> <groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId> <artifactId>rocksdbjni</artifactId>
<version>5.5.5</version> <version>5.8.0</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>io.swagger</groupId> <groupId>io.swagger</groupId>

View File

@ -20,6 +20,12 @@
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConfigKeys;
import static org.apache.hadoop.ozone.OzoneConfigKeys
.OZONE_METADATA_STORE_ROCKSDB_STATISTICS;
import static org.apache.hadoop.ozone.OzoneConfigKeys
.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys
.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF;
import org.iq80.leveldb.Options; import org.iq80.leveldb.Options;
import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.BlockBasedTableConfig;
@ -29,6 +35,9 @@
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB;
import org.rocksdb.Statistics;
import org.rocksdb.StatsLevel;
/** /**
* Builder for metadata store. * Builder for metadata store.
*/ */
@ -91,6 +100,18 @@ public MetadataStore build() throws IOException {
tableConfig.setBlockCacheSize(cacheSize); tableConfig.setBlockCacheSize(cacheSize);
opts.setTableFormatConfig(tableConfig); opts.setTableFormatConfig(tableConfig);
} }
String rocksDbStat = conf == null ?
OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT :
conf.getTrimmed(OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT);
if (!rocksDbStat.equals(OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF)) {
Statistics statistics = new Statistics();
statistics.setStatsLevel(StatsLevel.valueOf(rocksDbStat));
opts = opts.setStatistics(statistics);
}
store = new RocksDBStore(dbFile, opts); store = new RocksDBStore(dbFile, opts);
} else { } else {
throw new IllegalArgumentException("Invalid argument for " throw new IllegalArgumentException("Invalid argument for "

View File

@ -21,8 +21,13 @@
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting;
import org.rocksdb.DBOptions;
import org.rocksdb.RocksIterator; import org.rocksdb.RocksIterator;
import org.rocksdb.Options; import org.rocksdb.Options;
import org.rocksdb.Statistics;
import org.rocksdb.StatsLevel;
import org.rocksdb.WriteOptions; import org.rocksdb.WriteOptions;
import org.rocksdb.RocksDB; import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException; import org.rocksdb.RocksDBException;
@ -31,8 +36,10 @@
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import javax.management.ObjectName;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.List; import java.util.List;
import java.util.ArrayList; import java.util.ArrayList;
@ -51,8 +58,10 @@ public class RocksDBStore implements MetadataStore {
private File dbLocation; private File dbLocation;
private WriteOptions writeOptions; private WriteOptions writeOptions;
private Options dbOptions; private Options dbOptions;
private ObjectName statMBeanName;
public RocksDBStore(File dbFile, Options options) throws IOException { public RocksDBStore(File dbFile, Options options)
throws IOException {
Preconditions.checkNotNull(dbFile, "DB file location cannot be null"); Preconditions.checkNotNull(dbFile, "DB file location cannot be null");
RocksDB.loadLibrary(); RocksDB.loadLibrary();
dbOptions = options; dbOptions = options;
@ -61,10 +70,18 @@ public RocksDBStore(File dbFile, Options options) throws IOException {
writeOptions.setSync(true); writeOptions.setSync(true);
writeOptions.setNoSlowdown(true); writeOptions.setNoSlowdown(true);
try { try {
db = RocksDB.open(dbOptions, dbLocation.getAbsolutePath()); db = RocksDB.open(dbOptions, dbLocation.getAbsolutePath());
if (dbOptions.statistics() != null) {
Map<String, String> jmxProperties = new HashMap<String, String>();
jmxProperties.put("dbName", dbFile.getName());
statMBeanName = MBeans.register("Ozone", "RocksDbStore", jmxProperties,
new RocksDBStoreMBean(dbOptions.statistics()));
}
} catch (RocksDBException e) { } catch (RocksDBException e) {
throw new IOException("Failed init RocksDB, db path : " throw new IOException(
+ dbFile.getAbsolutePath(), e); "Failed init RocksDB, db path : " + dbFile.getAbsolutePath(), e);
} }
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
@ -349,8 +366,18 @@ public void iterate(byte[] from, EntryConsumer consumer)
@Override @Override
public void close() throws IOException { public void close() throws IOException {
if (statMBeanName != null) {
MBeans.unregister(statMBeanName);
}
if (db != null) { if (db != null) {
db.close(); db.close();
} }
} }
@VisibleForTesting
protected ObjectName getStatMBeanName() {
return statMBeanName;
}
} }

View File

@ -0,0 +1,148 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.utils;
import org.rocksdb.HistogramData;
import org.rocksdb.HistogramType;
import org.rocksdb.Statistics;
import org.rocksdb.TickerType;
import javax.management.Attribute;
import javax.management.AttributeList;
import javax.management.AttributeNotFoundException;
import javax.management.DynamicMBean;
import javax.management.InvalidAttributeValueException;
import javax.management.MBeanAttributeInfo;
import javax.management.MBeanException;
import javax.management.MBeanInfo;
import javax.management.ReflectionException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Stream;
/**
* Adapter JMX bean to publish all the Rocksdb metrics.
*/
public class RocksDBStoreMBean implements DynamicMBean {
private Statistics statistics;
private Set<String> histogramAttributes = new HashSet<>();
public RocksDBStoreMBean(Statistics statistics) {
this.statistics = statistics;
histogramAttributes.add("Average");
histogramAttributes.add("Median");
histogramAttributes.add("Percentile95");
histogramAttributes.add("Percentile99");
histogramAttributes.add("StandardDeviation");
}
@Override
public Object getAttribute(String attribute)
throws AttributeNotFoundException, MBeanException, ReflectionException {
for (String histogramAttribute : histogramAttributes) {
if (attribute.endsWith("_" + histogramAttribute.toUpperCase())) {
String keyName = attribute
.substring(0, attribute.length() - histogramAttribute.length() -1);
try {
HistogramData histogram =
statistics.getHistogramData(HistogramType.valueOf(keyName));
try {
Method method =
HistogramData.class.getMethod("get" + histogramAttribute);
return method.invoke(histogram);
} catch (Exception e) {
throw new ReflectionException(e,
"Can't read attribute " + attribute);
}
} catch (IllegalArgumentException exception) {
throw new AttributeNotFoundException(
"No such attribute in RocksDB stats: " + attribute);
}
}
}
try {
return statistics.getTickerCount(TickerType.valueOf(attribute));
} catch (IllegalArgumentException ex) {
throw new AttributeNotFoundException(
"No such attribute in RocksDB stats: " + attribute);
}
}
@Override
public void setAttribute(Attribute attribute)
throws AttributeNotFoundException, InvalidAttributeValueException,
MBeanException, ReflectionException {
}
@Override
public AttributeList getAttributes(String[] attributes) {
AttributeList result = new AttributeList();
for (String attributeName : attributes) {
try {
Object value = getAttribute(attributeName);
result.add(value);
} catch (Exception e) {
//TODO
}
}
return result;
}
@Override
public AttributeList setAttributes(AttributeList attributes) {
return null;
}
@Override
public Object invoke(String actionName, Object[] params, String[] signature)
throws MBeanException, ReflectionException {
return null;
}
@Override
public MBeanInfo getMBeanInfo() {
List<MBeanAttributeInfo> attributes = new ArrayList<>();
for (TickerType tickerType : TickerType.values()) {
attributes.add(new MBeanAttributeInfo(tickerType.name(), "long",
"RocksDBStat: " + tickerType.name(), true, false, false));
}
for (HistogramType histogramType : HistogramType.values()) {
for (String histogramAttribute : histogramAttributes) {
attributes.add(new MBeanAttributeInfo(
histogramType.name() + "_" + histogramAttribute.toUpperCase(),
"long", "RocksDBStat: " + histogramType.name(), true, false,
false));
}
}
return new MBeanInfo("", "RocksDBStat",
attributes.toArray(new MBeanAttributeInfo[0]), null, null, null);
}
}

View File

@ -629,6 +629,20 @@
</description> </description>
</property> </property>
<property>
<name>ozone.metastore.rocksdb.statistics</name>
<value>ALL</value>
<tag>OZONE, KSM, SCM, STORAGE, PERFORMANCE</tag>
<description>
The statistics level of the rocksdb store. If you use any value from
org.rocksdb.StatsLevel (eg. ALL or EXCEPT_DETAILED_TIMERS), the rocksdb
statistics will be exposed over JMX bean with the choosed setting. Set
it to OFF to not initialize rocksdb statistics at all. Please note that
collection of statistics could have 5-10% performance penalty.
Check the rocksdb documentation for more details.
</description>
</property>
<property> <property>
<name>ozone.scm.block.client.address</name> <name>ozone.scm.block.client.address</name>
<value/> <value/>

View File

@ -0,0 +1,87 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.utils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import static org.apache.hadoop.ozone.OzoneConfigKeys
.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Test;
import javax.management.AttributeNotFoundException;
import javax.management.MBeanServer;
import java.io.File;
import java.lang.management.ManagementFactory;
public class TestRocksDBStoreMBean {
@Test
public void testJmxBeans() throws Exception {
File testDir =
GenericTestUtils.getTestDir(getClass().getSimpleName() + "-withstat");
Configuration conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB);
RocksDBStore metadataStore =
(RocksDBStore) MetadataStoreBuilder.newBuilder().setConf(conf)
.setCreateIfMissing(true).setDbFile(testDir).build();
for (int i = 0; i < 10; i++) {
metadataStore.put("key".getBytes(), "value".getBytes());
}
MBeanServer platformMBeanServer =
ManagementFactory.getPlatformMBeanServer();
Thread.sleep(2000);
Object keysWritten = platformMBeanServer
.getAttribute(metadataStore.getStatMBeanName(), "NUMBER_KEYS_WRITTEN");
Assert.assertEquals(10l, keysWritten);
Object dbWriteAverage = platformMBeanServer
.getAttribute(metadataStore.getStatMBeanName(), "DB_WRITE_AVERAGE");
Assert.assertTrue((double) dbWriteAverage > 0);
metadataStore.close();
}
@Test()
public void testDisabledStat() throws Exception {
File testDir = GenericTestUtils
.getTestDir(getClass().getSimpleName() + "-withoutstat");
Configuration conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB);
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
OzoneConfigKeys.OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF);
RocksDBStore metadataStore =
(RocksDBStore) MetadataStoreBuilder.newBuilder().setConf(conf)
.setCreateIfMissing(true).setDbFile(testDir).build();
Assert.assertNull(metadataStore.getStatMBeanName());
}
}