big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-4379. Node Manager throws java.lang.OutOfMemoryError: Java heap space due to org.apache.hadoop.fs.LocalDirAllocator.contexts (Devaraj K via bobby)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1358305 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans 2012-07-06 17:19:13 +00:00
parent 072292219e
commit 0e7204c9e7
4 changed files with 37 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java
View File

@ -208,6 +208,19 @@ public static boolean isContextValid(String contextCfgItemName) {
} }
} }
/**
* Removes the context from the context config items
*
* @param contextCfgItemName
*/
@Deprecated
@InterfaceAudience.LimitedPrivate({"MapReduce"})
public static void removeContext(String contextCfgItemName) {
synchronized (contexts) {
contexts.remove(contextCfgItemName);
}
}
/** We search through all the configured dirs for the file's existence /** We search through all the configured dirs for the file's existence
* and return true when we find * and return true when we find
* @param pathStr the requested file (this will be searched) * @param pathStr the requested file (this will be searched)

13
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
View File

@ -340,4 +340,17 @@ public void testGetLocalPathToRead() throws IOException {
} }
@Test
public void testRemoveContext() throws IOException {
String dir = buildBufferDir(ROOT, 0);
String contextCfgItemName = "application_1340842292563_0004.app.cache.dirs";
conf.set(contextCfgItemName, dir);
LocalDirAllocator localDirAllocator = new LocalDirAllocator(
contextCfgItemName);
localDirAllocator.getLocalPathForWrite("p1/x", SMALL_FILE_SIZE, conf);
assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName));
LocalDirAllocator.removeContext(contextCfgItemName);
assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName));
}
} }

4
hadoop-mapreduce-project/CHANGES.txt
View File

@ -643,6 +643,10 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4387. RM gets fatal error and exits during TestRM MAPREDUCE-4387. RM gets fatal error and exits during TestRM
(Kihwal Lee via tgraves) (Kihwal Lee via tgraves)
MAPREDUCE-4379. Node Manager throws java.lang.OutOfMemoryError: Java heap
space due to org.apache.hadoop.fs.LocalDirAllocator.contexts (Devaraj K
via bobby)
Release 0.23.2 - UNRELEASED Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

12
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
View File

@ -91,6 +91,7 @@ public class ContainerLocalizer {
private final LocalDirAllocator userDirs; private final LocalDirAllocator userDirs;
private final RecordFactory recordFactory; private final RecordFactory recordFactory;
private final Map<LocalResource,Future<Path>> pendingResources; private final Map<LocalResource,Future<Path>> pendingResources;
private final String appCacheDirContextName;
public ContainerLocalizer(FileContext lfs, String user, String appId, public ContainerLocalizer(FileContext lfs, String user, String appId,
String localizerId, List<Path> localDirs, String localizerId, List<Path> localDirs,
@ -108,10 +109,9 @@ public ContainerLocalizer(FileContext lfs, String user, String appId,
this.localizerId = localizerId; this.localizerId = localizerId;
this.recordFactory = recordFactory; this.recordFactory = recordFactory;
this.conf = new Configuration(); this.conf = new Configuration();
this.appDirs = this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId);
new LocalDirAllocator(String.format(APPCACHE_CTXT_FMT, appId)); this.appDirs = new LocalDirAllocator(appCacheDirContextName);
this.userDirs = this.userDirs = new LocalDirAllocator(String.format(USERCACHE_CTXT_FMT, user));
new LocalDirAllocator(String.format(USERCACHE_CTXT_FMT, appId));
this.pendingResources = new HashMap<LocalResource,Future<Path>>(); this.pendingResources = new HashMap<LocalResource,Future<Path>>();
} }
@ -121,6 +121,7 @@ LocalizationProtocol getProxy(final InetSocketAddress nmAddr) {
rpc.getProxy(LocalizationProtocol.class, nmAddr, conf); rpc.getProxy(LocalizationProtocol.class, nmAddr, conf);
} }
@SuppressWarnings("deprecation")
public int runLocalization(final InetSocketAddress nmAddr) public int runLocalization(final InetSocketAddress nmAddr)
throws IOException, InterruptedException { throws IOException, InterruptedException {
// load credentials // load credentials
@ -177,6 +178,7 @@ public LocalizationProtocol run() {
if (exec != null) { if (exec != null) {
exec.shutdownNow(); exec.shutdownNow();
} }
LocalDirAllocator.removeContext(appCacheDirContextName);
} }
} }
@ -373,7 +375,7 @@ private static void initDirs(Configuration conf, String user, String appId,
lfs.mkdir(appFileCacheDir, null, false); lfs.mkdir(appFileCacheDir, null, false);
} }
conf.setStrings(String.format(APPCACHE_CTXT_FMT, appId), appsFileCacheDirs); conf.setStrings(String.format(APPCACHE_CTXT_FMT, appId), appsFileCacheDirs);
conf.setStrings(String.format(USERCACHE_CTXT_FMT, appId), usersFileCacheDirs); conf.setStrings(String.format(USERCACHE_CTXT_FMT, user), usersFileCacheDirs);
} }
} }