big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-3205. FileSystemRMStateStore should disable FileSystem Cache to avoid get a Filesystem with an old configuration. Contributed by Zhihai Xu.

Browse Source
This commit is contained in:
Tsuyoshi Ozawa 2015-03-18 11:53:14 +09:00
parent fc90bf7b27
commit 3bc72cc16d
3 changed files with 25 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -72,6 +72,9 @@ Release 2.8.0 - UNRELEASED
YARN-3305. Normalize AM resource request on app submission. (Rohith Sharmaks
via jianhe)
YARN-3205 FileSystemRMStateStore should disable FileSystem Cache to avoid
get a Filesystem with an old configuration. (Zhihai Xu via ozawa)
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
View File

@ -84,7 +84,10 @@ public class FileSystemRMStateStore extends RMStateStore {
protected static final String AMRMTOKEN_SECRET_MANAGER_NODE =
"AMRMTokenSecretManagerNode";
@VisibleForTesting
protected FileSystem fs;
@VisibleForTesting
protected Configuration fsConf;
private Path rootDirPath;
@Private
@ -121,14 +124,23 @@ protected synchronized void startInternal() throws Exception {
// create filesystem only now, as part of service-start. By this time, RM is
// authenticated with kerberos so we are good to create a file-system
// handle.
Configuration conf = new Configuration(getConfig());
conf.setBoolean("dfs.client.retry.policy.enabled", true);
fsConf = new Configuration(getConfig());
fsConf.setBoolean("dfs.client.retry.policy.enabled", true);
String retryPolicy =
conf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
fsConf.get(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC,
YarnConfiguration.DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC);
conf.set("dfs.client.retry.policy.spec", retryPolicy);
fsConf.set("dfs.client.retry.policy.spec", retryPolicy);
fs = fsWorkingPath.getFileSystem(conf);
String scheme = fsWorkingPath.toUri().getScheme();
if (scheme == null) {
scheme = FileSystem.getDefaultUri(fsConf).getScheme();
}
if (scheme != null) {
String disableCacheName = String.format("fs.%s.impl.disable.cache", scheme);
fsConf.setBoolean(disableCacheName, true);
}
fs = fsWorkingPath.getFileSystem(fsConf);
mkdirsWithRetries(rmDTSecretManagerRoot);
mkdirsWithRetries(rmAppRoot);
mkdirsWithRetries(amrmTokenSecretManagerRoot);

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
View File

@ -106,6 +106,11 @@ public RMStateStore getRMStateStore() throws Exception {
this.store = new TestFileSystemRMStore(conf);
Assert.assertEquals(store.getNumRetries(), 8);
Assert.assertEquals(store.getRetryInterval(), 900L);
Assert.assertTrue(store.fs.getConf() == store.fsConf);
FileSystem previousFs = store.fs;
store.startInternal();
Assert.assertTrue(store.fs != previousFs);
Assert.assertTrue(store.fs.getConf() == store.fsConf);
return store;
}