From 1a118d6a32da63e7c94fb7bf43547c341a6c8b59 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Thu, 26 Apr 2012 18:39:22 +0000 Subject: [PATCH] MAPREDUCE-4138. Reduce memory usage of counters due to non-static nested classes. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1331023 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/mapred/Counters.java | 20 ++----------------- .../org/apache/hadoop/mapreduce/Counters.java | 4 ++-- .../counters/FileSystemCounterGroup.java | 5 ++--- .../counters/FrameworkCounterGroup.java | 14 ++++++------- 5 files changed, 15 insertions(+), 31 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 0f00d00a41..0256620021 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -154,6 +154,9 @@ Release 2.0.0 - UNRELEASED MAPREDUCE-4093. Improve RM WebApp start up when proxy address is not set (Devaraj K vai bobby) + MAPREDUCE-4138. Reduce memory usage of counters due to non-static nested + classes. (tomwhite) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java index 3c95f976af..9a7b9963b1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java @@ -387,21 +387,13 @@ public CounterGroupBase getUnderlyingGroup() { private static class FrameworkGroupImpl> extends FrameworkCounterGroup { - // Mix the framework counter implementation into the Counter interface - class FrameworkCounterImpl extends FrameworkCounter { - FrameworkCounterImpl(T key) { - super(key); - } - - } - FrameworkGroupImpl(Class cls) { super(cls); } @Override protected Counter newCounter(T key) { - return new Counter(new FrameworkCounterImpl(key)); + return new Counter(new FrameworkCounter(key, getName())); } @Override @@ -413,17 +405,9 @@ public CounterGroupBase getUnderlyingGroup() { // Mix the file system counter group implementation into the Group interface private static class FSGroupImpl extends FileSystemCounterGroup { - private class FSCounterImpl extends FSCounter { - - FSCounterImpl(String scheme, FileSystemCounter key) { - super(scheme, key); - } - - } - @Override protected Counter newCounter(String scheme, FileSystemCounter key) { - return new Counter(new FSCounterImpl(scheme, key)); + return new Counter(new FSCounter(scheme, key)); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Counters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Counters.java index 4422fad8b5..be08f8b539 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Counters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Counters.java @@ -49,8 +49,8 @@ private static class FrameworkGroupImpl> } @Override - protected FrameworkCounter newCounter(T key) { - return new FrameworkCounter(key); + protected FrameworkCounter newCounter(T key) { + return new FrameworkCounter(key, getName()); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java index 91f5f499c3..7c23561b65 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java @@ -61,7 +61,7 @@ public abstract class FileSystemCounterGroup private static final Joiner DISP_JOINER = Joiner.on(": "); @InterfaceAudience.Private - public class FSCounter extends AbstractCounter { + public static class FSCounter extends AbstractCounter { final String scheme; final FileSystemCounter key; private long value; @@ -139,8 +139,7 @@ public void setDisplayName(String displayName) { @Override public void addCounter(C counter) { C ours; - if (counter instanceof FileSystemCounterGroup.FSCounter) { - @SuppressWarnings("unchecked") + if (counter instanceof FileSystemCounterGroup.FSCounter) { FSCounter c = (FSCounter) counter; ours = findCounter(c.scheme, c.key); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java index 5dcf31bc09..64e4cc8163 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FrameworkCounterGroup.java @@ -57,12 +57,14 @@ public abstract class FrameworkCounterGroup, * Use old (which extends new) interface to make compatibility easier. */ @InterfaceAudience.Private - public class FrameworkCounter extends AbstractCounter { + public static class FrameworkCounter> extends AbstractCounter { final T key; + final String groupName; private long value; - public FrameworkCounter(T ref) { + public FrameworkCounter(T ref, String groupName) { key = ref; + this.groupName = groupName; } @Override @@ -72,7 +74,7 @@ public String getName() { @Override public String getDisplayName() { - return localizeCounterName(getName()); + return ResourceBundles.getCounterName(groupName, getName(), getName()); } @Override @@ -131,10 +133,6 @@ public void setDisplayName(String displayName) { this.displayName = displayName; } - private String localizeCounterName(String counterName) { - return ResourceBundles.getCounterName(getName(), counterName, counterName); - } - private T valueOf(String name) { return Enum.valueOf(enumClass, name); } @@ -204,7 +202,7 @@ public void incrAllCounters(CounterGroupBase other) { if (checkNotNull(other, "other counter group") instanceof FrameworkCounterGroup) { for (Counter counter : other) { - findCounter(((FrameworkCounter) counter).key) + findCounter(((FrameworkCounter) counter).key.name()) .increment(counter.getValue()); } }