From a60434368cdf0fd119d024c2ed10942abdb2e7f1 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 3 Mar 2014 19:41:52 +0000 Subject: [PATCH] YARN-1758. Fixed ResourceManager to not mandate the presence of site specific configuration files and thus fix failures in downstream tests. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1573695 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 ++ .../FileSystemBasedConfigurationProvider.java | 7 +- .../server/resourcemanager/AdminService.java | 8 ++- .../ApplicationMasterService.java | 11 ++- .../resourcemanager/ClientRMService.java | 11 ++- .../resourcemanager/ResourceManager.java | 20 ++++-- .../ResourceTrackerService.java | 11 ++- .../scheduler/capacity/CapacityScheduler.java | 14 ++-- .../resourcemanager/TestRMAdminService.java | 69 ++++++------------- 9 files changed, 83 insertions(+), 72 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 646ed1a3a0..a16af830fc 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -368,6 +368,10 @@ Release 2.4.0 - UNRELEASED YARN-1760. TestRMAdminService assumes CapacityScheduler. (kasha) + YARN-1758. Fixed ResourceManager to not mandate the presence of site specific + configuration files and thus fix failures in downstream tests. (Xuan Gong via + vinodkv) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java index d87759e354..bf50cadd3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java @@ -54,13 +54,14 @@ public synchronized InputStream getConfigurationInputStream( if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) { filePath = new Path(this.configDir, name); if (!fs.exists(filePath)) { - throw new YarnException("Can not find Configuration: " + name + " in " - + configDir); + LOG.info(filePath + " not found"); + return null; } } else { filePath = new Path(name); if (!fs.exists(filePath)) { - throw new YarnException("Can not find file: " + name); + LOG.info(filePath + " not found"); + return null; } } return fs.open(filePath); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index c53d40f54a..c5b2651319 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; +import java.io.InputStream; import java.net.InetSocketAddress; import java.util.Map; import java.util.Set; @@ -538,8 +539,11 @@ public UpdateNodeResourceResponse updateNodeResource( private synchronized Configuration getConfiguration(Configuration conf, String confFileName) throws YarnException, IOException { - conf.addResource(this.rmContext.getConfigurationProvider() - .getConfigurationInputStream(conf, confFileName)); + InputStream confFileInputStream = this.rmContext.getConfigurationProvider() + .getConfigurationInputStream(conf, confFileName); + if (confFileInputStream != null) { + conf.addResource(confFileInputStream); + } return conf; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index 70fd69fdad..17ec32aaa3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; +import java.io.InputStream; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.ArrayList; @@ -142,9 +143,13 @@ protected void serviceStart() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - conf.addResource(this.rmContext.getConfigurationProvider() - .getConfigurationInputStream(conf, - YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE)); + InputStream inputStream = + this.rmContext.getConfigurationProvider() + .getConfigurationInputStream(conf, + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); + if (inputStream != null) { + conf.addResource(inputStream); + } refreshServiceAcls(conf, RMPolicyProvider.getInstance()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index fdde381884..351f4bd1ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; +import java.io.InputStream; import java.net.InetSocketAddress; import java.security.AccessControlException; import java.util.ArrayList; @@ -170,9 +171,13 @@ protected void serviceStart() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - conf.addResource(this.rmContext.getConfigurationProvider() - .getConfigurationInputStream(conf, - YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE)); + InputStream inputStream = + this.rmContext.getConfigurationProvider() + .getConfigurationInputStream(conf, + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); + if (inputStream != null) { + conf.addResource(inputStream); + } refreshServiceAcls(conf, RMPolicyProvider.getInstance()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index b0fa8f3d11..d408298c6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; +import java.io.InputStream; import java.net.InetSocketAddress; import java.util.List; import java.util.concurrent.BlockingQueue; @@ -190,13 +191,20 @@ protected void serviceInit(Configuration conf) throws Exception { rmContext.setConfigurationProvider(configurationProvider); // load yarn-site.xml - this.conf.addResource(this.configurationProvider - .getConfigurationInputStream(this.conf, - YarnConfiguration.YARN_SITE_CONFIGURATION_FILE)); + InputStream yarnSiteXMLInputStream = + this.configurationProvider.getConfigurationInputStream(this.conf, + YarnConfiguration.YARN_SITE_CONFIGURATION_FILE); + if (yarnSiteXMLInputStream != null) { + this.conf.addResource(yarnSiteXMLInputStream); + } // load core-site.xml - this.conf.addResource(this.configurationProvider - .getConfigurationInputStream(this.conf, - YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)); + InputStream coreSiteXMLInputStream = + this.configurationProvider.getConfigurationInputStream(this.conf, + YarnConfiguration.CORE_SITE_CONFIGURATION_FILE); + if (coreSiteXMLInputStream != null) { + this.conf.addResource(coreSiteXMLInputStream); + } + // Do refreshUserToGroupsMappings with loaded core-site.xml Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf) .refresh(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index fe889df13a..aad4d92c73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; +import java.io.InputStream; import java.net.InetSocketAddress; import org.apache.commons.logging.Log; @@ -163,9 +164,13 @@ protected void serviceStart() throws Exception { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - conf.addResource(this.rmContext.getConfigurationProvider() - .getConfigurationInputStream(conf, - YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE)); + InputStream inputStream = + this.rmContext.getConfigurationProvider() + .getConfigurationInputStream(conf, + YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); + if (inputStream != null) { + conf.addResource(inputStream); + } refreshServiceAcls(conf, RMPolicyProvider.getInstance()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index b8f2376358..6c392b5a72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; @@ -1024,10 +1025,15 @@ public List getAppsInQueue(String queueName) { private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration( Configuration configuration) throws IOException { try { - configuration.addResource(this.rmContext.getConfigurationProvider() - .getConfigurationInputStream(configuration, - YarnConfiguration.CS_CONFIGURATION_FILE)); - return new CapacitySchedulerConfiguration(configuration, false); + InputStream CSInputStream = + this.rmContext.getConfigurationProvider() + .getConfigurationInputStream(configuration, + YarnConfiguration.CS_CONFIGURATION_FILE); + if (CSInputStream != null) { + configuration.addResource(CSInputStream); + return new CapacitySchedulerConfiguration(configuration, false); + } + return new CapacitySchedulerConfiguration(configuration, true); } catch (Exception e) { throw new IOException(e); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index dfca9eb543..cb9f90d178 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -126,14 +126,6 @@ public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - try { - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); - fail("Should throw an exception"); - } catch(Exception ex) { - // Expect exception here - } //upload default configurations uploadDefaultConfiguration(); @@ -180,14 +172,6 @@ public void testAdminAclsWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - try { - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); - fail("Should throw an exception"); - } catch(Exception ex) { - // Expect exception here - } //upload default configurations uploadDefaultConfiguration(); @@ -246,14 +230,6 @@ public void testServiceAclsRefreshWithFileSystemBasedConfigurationProvider() "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); ResourceManager resourceManager = null; try { - try { - resourceManager = new ResourceManager(); - resourceManager.init(configuration); - resourceManager.start(); - fail("Should throw an exception"); - } catch (Exception ex) { - // expect to get an exception here - } //upload default configurations uploadDefaultConfiguration(); @@ -350,14 +326,6 @@ private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager, throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - try { - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); - fail("Should throw an exception"); - } catch(Exception ex) { - // Expect exception here - } //upload default configurations uploadDefaultConfiguration(); @@ -408,14 +376,6 @@ public void testRefreshUserToGroupsMappingsWithLocalConfigurationProvider() { throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - try { - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); - fail("Should throw an exception"); - } catch (Exception ex) { - // Expect exception here - } String user = UserGroupInformation.getCurrentUser().getUserName(); List groupWithInit = @@ -484,14 +444,6 @@ public void testRefreshNodesWithFileSystemBasedConfigurationProvider() throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); - try { - rm = new MockRM(configuration); - rm.init(configuration); - rm.start(); - fail("Should throw an exception"); - } catch (Exception ex) { - // Expect exception here - } // upload default configurations uploadDefaultConfiguration(); @@ -617,6 +569,27 @@ public void testRMHAWithFileSystemBasedConfiguration() throws IOException, } } + @Test + public void testRMStartsWithoutConfigurationFilesProvided() { + // enable FileSystemBasedConfigurationProvider without uploading + // any configuration files into Remote File System. + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + + // The configurationProvider will return NULL instead of + // throwing out Exceptions, if there are no configuration files provided. + // RM will not load the remote Configuration files, + // and should start successfully. + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch (Exception ex) { + fail("Should not get any exceptions"); + } + + } + private String writeConfigurationXML(Configuration conf, String confXMLName) throws IOException { DataOutputStream output = null;