big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2730. DefaultContainerExecutor runs only one localizer at a time. Contributed by Siqi Li

Browse Source
This commit is contained in:
Jason Lowe 2014-11-03 20:37:47 +00:00
parent 67f13b58e4
commit 6157ace547
2 changed files with 15 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -839,6 +839,9 @@ Release 2.6.0 - UNRELEASED
YARN-2785. Fixed intermittent TestContainerResourceUsage failure. (Varun Vasudev YARN-2785. Fixed intermittent TestContainerResourceUsage failure. (Varun Vasudev
via zjshen) via zjshen)
YARN-2730. DefaultContainerExecutor runs only one localizer at a time
(Siqi Li via jlowe)
Release 2.5.1 - 2014-09-05 Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
View File

@ -94,7 +94,7 @@ public void init() throws IOException {
} }
@Override @Override
public synchronized void startLocalizer(Path nmPrivateContainerTokensPath, public void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId, InetSocketAddress nmAddr, String user, String appId, String locId,
LocalDirsHandlerService dirsHandler) LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException { throws IOException, InterruptedException {
@ -102,10 +102,6 @@ public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
List<String> localDirs = dirsHandler.getLocalDirs(); List<String> localDirs = dirsHandler.getLocalDirs();
List<String> logDirs = dirsHandler.getLogDirs(); List<String> logDirs = dirsHandler.getLogDirs();
ContainerLocalizer localizer =
new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
RecordFactoryProvider.getRecordFactory(getConf()));
createUserLocalDirs(localDirs, user); createUserLocalDirs(localDirs, user);
createUserCacheDirs(localDirs, user); createUserCacheDirs(localDirs, user);
createAppDirs(localDirs, user, appId); createAppDirs(localDirs, user, appId);
@ -118,8 +114,17 @@ public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
Path tokenDst = new Path(appStorageDir, tokenFn); Path tokenDst = new Path(appStorageDir, tokenFn);
copyFile(nmPrivateContainerTokensPath, tokenDst, user); copyFile(nmPrivateContainerTokensPath, tokenDst, user);
LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst); LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
lfs.setWorkingDirectory(appStorageDir);
LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory());
FileContext localizerFc = FileContext.getFileContext(
lfs.getDefaultFileSystem(), getConf());
localizerFc.setUMask(lfs.getUMask());
localizerFc.setWorkingDirectory(appStorageDir);
LOG.info("Localizer CWD set to " + appStorageDir + " = "
+ localizerFc.getWorkingDirectory());
ContainerLocalizer localizer =
new ContainerLocalizer(localizerFc, user, appId, locId,
getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf()));
// TODO: DO it over RPC for maintaining similarity? // TODO: DO it over RPC for maintaining similarity?
localizer.runLocalization(nmAddr); localizer.runLocalization(nmAddr);
} }