big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-16278. With S3A Filesystem, Long Running services End up Doing lot of GC and eventually die.

Browse Source 
Contributed by Rajat Khandelwal
This commit is contained in:
Rajat Khandelwal 2019-05-09 21:12:14 +01:00 committed by Steve Loughran
parent 2d31ccc1b3
commit 591ca69823
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
View File

@ -120,7 +120,8 @@ public class S3AInstrumentation implements Closeable, MetricsSource {
private final MutableCounterLong streamBytesReadInClose; private final MutableCounterLong streamBytesReadInClose;
private final MutableCounterLong streamBytesDiscardedInAbort; private final MutableCounterLong streamBytesDiscardedInAbort;
private final MutableCounterLong ignoredErrors; private final MutableCounterLong ignoredErrors;
private final MutableQuantiles putLatencyQuantile;
private final MutableQuantiles throttleRateQuantile;
private final MutableCounterLong numberOfFilesCreated; private final MutableCounterLong numberOfFilesCreated;
private final MutableCounterLong numberOfFilesCopied; private final MutableCounterLong numberOfFilesCopied;
private final MutableCounterLong bytesOfFilesCopied; private final MutableCounterLong bytesOfFilesCopied;
@ -239,9 +240,9 @@ public S3AInstrumentation(URI name) {
} }
//todo need a config for the quantiles interval? //todo need a config for the quantiles interval?
int interval = 1; int interval = 1;
quantiles(S3GUARD_METADATASTORE_PUT_PATH_LATENCY, putLatencyQuantile = quantiles(S3GUARD_METADATASTORE_PUT_PATH_LATENCY,
"ops", "latency", interval); "ops", "latency", interval);
quantiles(S3GUARD_METADATASTORE_THROTTLE_RATE, throttleRateQuantile = quantiles(S3GUARD_METADATASTORE_THROTTLE_RATE,
"events", "frequency (Hz)", interval); "events", "frequency (Hz)", interval);
registerAsMetricsSource(name); registerAsMetricsSource(name);
@ -606,6 +607,8 @@ public void getMetrics(MetricsCollector collector, boolean all) {
public void close() { public void close() {
synchronized (metricsSystemLock) { synchronized (metricsSystemLock) {
putLatencyQuantile.stop();
throttleRateQuantile.stop();
metricsSystem.unregisterSource(metricsSourceName); metricsSystem.unregisterSource(metricsSourceName);
int activeSources = --metricsSourceActiveCounter; int activeSources = --metricsSourceActiveCounter;
if (activeSources == 0) { if (activeSources == 0) {