big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-1386. NodeManager mistakenly loses resources and relocalizes them (Jason Lowe via jeagles)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1541375 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Turner Eagles 2013-11-13 03:17:23 +00:00
parent cccee537cb
commit 656e859f5c
3 changed files with 18 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -1636,6 +1636,9 @@ Release 0.23.10 - UNRELEASED
YARN-1176. RM web services ClusterMetricsInfo total nodes doesn't include YARN-1176. RM web services ClusterMetricsInfo total nodes doesn't include
unhealthy nodes (Jonathan Eagles via tgraves) unhealthy nodes (Jonathan Eagles via tgraves)
YARN-1386. NodeManager mistakenly loses resources and relocalizes them
(Jason Lowe via jeagles)
Release 0.23.9 - 2013-07-08 Release 0.23.9 - 2013-07-08
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
View File

@ -46,6 +46,7 @@
import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.Token;
@ -82,6 +83,8 @@ public class ContainerLocalizer {
public static final String WORKDIR = "work"; public static final String WORKDIR = "work";
private static final String APPCACHE_CTXT_FMT = "%s.app.cache.dirs"; private static final String APPCACHE_CTXT_FMT = "%s.app.cache.dirs";
private static final String USERCACHE_CTXT_FMT = "%s.user.cache.dirs"; private static final String USERCACHE_CTXT_FMT = "%s.user.cache.dirs";
private static final FsPermission FILECACHE_PERMS =
new FsPermission((short)0710);
private final String user; private final String user;
private final String appId; private final String appId;
@ -363,16 +366,23 @@ private static void initDirs(Configuration conf, String user, String appId,
// $x/usercache/$user/filecache // $x/usercache/$user/filecache
Path userFileCacheDir = new Path(base, FILECACHE); Path userFileCacheDir = new Path(base, FILECACHE);
usersFileCacheDirs[i] = userFileCacheDir.toString(); usersFileCacheDirs[i] = userFileCacheDir.toString();
lfs.mkdir(userFileCacheDir, null, false); createDir(lfs, userFileCacheDir, FILECACHE_PERMS, false);
// $x/usercache/$user/appcache/$appId // $x/usercache/$user/appcache/$appId
Path appBase = new Path(base, new Path(APPCACHE, appId)); Path appBase = new Path(base, new Path(APPCACHE, appId));
// $x/usercache/$user/appcache/$appId/filecache // $x/usercache/$user/appcache/$appId/filecache
Path appFileCacheDir = new Path(appBase, FILECACHE); Path appFileCacheDir = new Path(appBase, FILECACHE);
appsFileCacheDirs[i] = appFileCacheDir.toString(); appsFileCacheDirs[i] = appFileCacheDir.toString();
lfs.mkdir(appFileCacheDir, null, false); createDir(lfs, appFileCacheDir, FILECACHE_PERMS, false);
} }
conf.setStrings(String.format(APPCACHE_CTXT_FMT, appId), appsFileCacheDirs); conf.setStrings(String.format(APPCACHE_CTXT_FMT, appId), appsFileCacheDirs);
conf.setStrings(String.format(USERCACHE_CTXT_FMT, user), usersFileCacheDirs); conf.setStrings(String.format(USERCACHE_CTXT_FMT, user), usersFileCacheDirs);
} }
private static void createDir(FileContext lfs, Path dirPath,
FsPermission perms, boolean createParent) throws IOException {
lfs.mkdir(dirPath, perms, createParent);
if (!perms.equals(perms.applyUMask(lfs.getUMask()))) {
lfs.setPermission(dirPath, perms);
}
}
} }

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
View File

@ -83,6 +83,7 @@ public class TestContainerLocalizer {
static final Log LOG = LogFactory.getLog(TestContainerLocalizer.class); static final Log LOG = LogFactory.getLog(TestContainerLocalizer.class);
static final Path basedir = static final Path basedir =
new Path("target", TestContainerLocalizer.class.getName()); new Path("target", TestContainerLocalizer.class.getName());
static final FsPermission CACHE_DIR_PERM = new FsPermission((short)0710);
static final String appUser = "yak"; static final String appUser = "yak";
static final String appId = "app_RM_0"; static final String appId = "app_RM_0";
@ -171,12 +172,12 @@ public void testContainerLocalizerMain() throws Exception {
Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser); Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser);
Path privcache = new Path(base, ContainerLocalizer.FILECACHE); Path privcache = new Path(base, ContainerLocalizer.FILECACHE);
// $x/usercache/$user/filecache // $x/usercache/$user/filecache
verify(spylfs).mkdir(eq(privcache), isA(FsPermission.class), eq(false)); verify(spylfs).mkdir(eq(privcache), eq(CACHE_DIR_PERM), eq(false));
Path appDir = Path appDir =
new Path(base, new Path(ContainerLocalizer.APPCACHE, appId)); new Path(base, new Path(ContainerLocalizer.APPCACHE, appId));
// $x/usercache/$user/appcache/$appId/filecache // $x/usercache/$user/appcache/$appId/filecache
Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE); Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE);
verify(spylfs).mkdir(eq(appcache), isA(FsPermission.class), eq(false)); verify(spylfs).mkdir(eq(appcache), eq(CACHE_DIR_PERM), eq(false));
} }
// verify tokens read at expected location // verify tokens read at expected location
verify(spylfs).open(tokenPath); verify(spylfs).open(tokenPath);