From 717c853873dd3b9112f5c15059a24655b8654607 Mon Sep 17 00:00:00 2001 From: Eric Yang Date: Tue, 27 Aug 2019 13:14:59 -0400 Subject: [PATCH] YARN-9755. Fixed RM failing to start when FileSystemBasedConfigurationProvider is configured. Contributed by Prabhu Joseph --- .../FileSystemBasedConfigurationProvider.java | 14 ++++++- .../resourcemanager/ResourceManager.java | 32 ++++++++-------- .../resourcemanager/TestRMAdminService.java | 37 +++++++++++++++++++ 3 files changed, 65 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java index b6ba660e52..3532d13f4a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java @@ -71,10 +71,20 @@ public synchronized InputStream getConfigurationInputStream( @Override public synchronized void initInternal(Configuration bootstrapConf) throws Exception { + Configuration conf = new Configuration(bootstrapConf); configDir = - new Path(bootstrapConf.get(YarnConfiguration.FS_BASED_RM_CONF_STORE, + new Path(conf.get(YarnConfiguration.FS_BASED_RM_CONF_STORE, YarnConfiguration.DEFAULT_FS_BASED_RM_CONF_STORE)); - fs = configDir.getFileSystem(bootstrapConf); + String scheme = configDir.toUri().getScheme(); + if (scheme == null) { + scheme = FileSystem.getDefaultUri(conf).getScheme(); + } + if (scheme != null) { + String disableCacheName = String.format("fs.%s.impl.disable.cache", + scheme); + conf.setBoolean(disableCacheName, true); + } + fs = configDir.getFileSystem(conf); fs.mkdirs(configDir); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index c0a9133eba..57be468bc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -267,6 +267,22 @@ protected void serviceInit(Configuration conf) throws Exception { this.rmContext = new RMContextImpl(); rmContext.setResourceManager(this); + // Set HA configuration should be done before login + this.rmContext.setHAEnabled(HAUtil.isHAEnabled(this.conf)); + if (this.rmContext.isHAEnabled()) { + HAUtil.verifyAndSetConfiguration(this.conf); + } + + // Set UGI and do login + // If security is enabled, use login user + // If security is not enabled, use current user + this.rmLoginUGI = UserGroupInformation.getCurrentUser(); + try { + doSecureLogin(); + } catch(IOException ie) { + throw new YarnRuntimeException("Failed to login", ie); + } + this.configurationProvider = ConfigurationProviderFactory.getConfigurationProvider(conf); this.configurationProvider.init(this.conf); @@ -285,22 +301,6 @@ protected void serviceInit(Configuration conf) throws Exception { loadConfigurationXml(YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); validateConfigs(this.conf); - - // Set HA configuration should be done before login - this.rmContext.setHAEnabled(HAUtil.isHAEnabled(this.conf)); - if (this.rmContext.isHAEnabled()) { - HAUtil.verifyAndSetConfiguration(this.conf); - } - - // Set UGI and do login - // If security is enabled, use login user - // If security is not enabled, use current user - this.rmLoginUGI = UserGroupInformation.getCurrentUser(); - try { - doSecureLogin(); - } catch(IOException ie) { - throw new YarnRuntimeException("Failed to login", ie); - } // register the handlers for all AlwaysOn services using setupDispatcher(). rmDispatcher = setupDispatcher(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 6659f107ef..e235632f75 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -39,6 +39,8 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo; @@ -50,6 +52,7 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; +import org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; @@ -59,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.ConfigurationProvider; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -172,6 +176,39 @@ public void testAdminRefreshQueuesWithLocalConfigurationProvider() } } + + @Test + public void testFileSystemCloseWithFileSystemBasedConfigurationProvider() + throws Exception { + MiniDFSCluster hdfsCluster = null; + try { + HdfsConfiguration hdfsConfig = new HdfsConfiguration(); + hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig) + .numDataNodes(1).build(); + FileSystem fs1 = hdfsCluster.getFileSystem(); + ConfigurationProvider configurationProvider = new + FileSystemBasedConfigurationProvider(); + configurationProvider.init(hdfsConfig); + fs1.close(); + try { + configurationProvider.getConfigurationInputStream(hdfsConfig, + "yarn-site.xml"); + } catch (IOException e) { + if (e.getMessage().contains("Filesystem closed")) { + fail("FileSystemBasedConfigurationProvider failed to handle " + + "FileSystem close"); + } else { + fail("Should not get any exceptions"); + } + } + } finally { + if (hdfsCluster != null) { + hdfsCluster.shutdown(); + } + } + } + + @Test public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider() throws IOException, YarnException {