YARN-1758. Fixed ResourceManager to not mandate the presence of site specific configuration files and thus fix failures in downstream tests. Contributed by Xuan Gong.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1573695 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -368,6 +368,10 @@ Release 2.4.0 - UNRELEASED
 
     YARN-1760. TestRMAdminService assumes CapacityScheduler. (kasha)
 
+    YARN-1758. Fixed ResourceManager to not mandate the presence of site specific
+    configuration files and thus fix failures in downstream tests. (Xuan Gong via
+    vinodkv)
+
 Release 2.3.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java

@@ -54,13 +54,14 @@ public synchronized InputStream getConfigurationInputStream(
     if (YarnConfiguration.RM_CONFIGURATION_FILES.contains(name)) {
       filePath = new Path(this.configDir, name);
       if (!fs.exists(filePath)) {
-        throw new YarnException("Can not find Configuration: " + name + " in "
+        LOG.info(filePath + " not found");
-            + configDir);
+        return null;
       }
     } else {
       filePath = new Path(name);
       if (!fs.exists(filePath)) {
-        throw new YarnException("Can not find file: " + name);
+        LOG.info(filePath + " not found");
+        return null;
       }
     }
     return fs.open(filePath);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.InetSocketAddress;
 import java.util.Map;
 import java.util.Set;
@@ -538,8 +539,11 @@ public UpdateNodeResourceResponse updateNodeResource(
 
   private synchronized Configuration getConfiguration(Configuration conf,
       String confFileName) throws YarnException, IOException {
-    conf.addResource(this.rmContext.getConfigurationProvider()
+    InputStream confFileInputStream = this.rmContext.getConfigurationProvider()
-        .getConfigurationInputStream(conf, confFileName));
+        .getConfigurationInputStream(conf, confFileName);
+    if (confFileInputStream != null) {
+      conf.addResource(confFileInputStream);
+    }
     return conf;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
@@ -142,9 +143,13 @@ protected void serviceStart() throws Exception {
     if (conf.getBoolean(
         CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
         false)) {
-      conf.addResource(this.rmContext.getConfigurationProvider()
+      InputStream inputStream =
-          .getConfigurationInputStream(conf,
+          this.rmContext.getConfigurationProvider()
-              YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE));
+              .getConfigurationInputStream(conf,
+                  YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
+      if (inputStream != null) {
+        conf.addResource(inputStream);
+      }
       refreshServiceAcls(conf, RMPolicyProvider.getInstance());
     }
     

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.InetSocketAddress;
 import java.security.AccessControlException;
 import java.util.ArrayList;
@@ -170,9 +171,13 @@ protected void serviceStart() throws Exception {
     if (conf.getBoolean(
         CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
         false)) {
-      conf.addResource(this.rmContext.getConfigurationProvider()
+      InputStream inputStream =
-          .getConfigurationInputStream(conf,
+          this.rmContext.getConfigurationProvider()
-              YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE));
+              .getConfigurationInputStream(conf,
+                  YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
+      if (inputStream != null) {
+        conf.addResource(inputStream);
+      }
       refreshServiceAcls(conf, RMPolicyProvider.getInstance());
     }
     

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.InetSocketAddress;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
@@ -190,13 +191,20 @@ protected void serviceInit(Configuration conf) throws Exception {
     rmContext.setConfigurationProvider(configurationProvider);
 
     // load yarn-site.xml
-    this.conf.addResource(this.configurationProvider
+    InputStream yarnSiteXMLInputStream =
-        .getConfigurationInputStream(this.conf,
+        this.configurationProvider.getConfigurationInputStream(this.conf,
-            YarnConfiguration.YARN_SITE_CONFIGURATION_FILE));
+            YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
+    if (yarnSiteXMLInputStream != null) {
+      this.conf.addResource(yarnSiteXMLInputStream);
+    }
     // load core-site.xml
-    this.conf.addResource(this.configurationProvider
+    InputStream coreSiteXMLInputStream =
-        .getConfigurationInputStream(this.conf,
+        this.configurationProvider.getConfigurationInputStream(this.conf,
-            YarnConfiguration.CORE_SITE_CONFIGURATION_FILE));
+            YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
+    if (coreSiteXMLInputStream != null) {
+      this.conf.addResource(coreSiteXMLInputStream);
+    }
+
     // Do refreshUserToGroupsMappings with loaded core-site.xml
     Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
         .refresh();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.InetSocketAddress;
 
 import org.apache.commons.logging.Log;
@@ -163,9 +164,13 @@ protected void serviceStart() throws Exception {
     if (conf.getBoolean(
         CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
         false)) {
-      conf.addResource(this.rmContext.getConfigurationProvider()
+      InputStream inputStream =
-          .getConfigurationInputStream(conf,
+          this.rmContext.getConfigurationProvider()
-              YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE));
+              .getConfigurationInputStream(conf,
+                  YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
+      if (inputStream != null) {
+        conf.addResource(inputStream);
+      }
       refreshServiceAcls(conf, RMPolicyProvider.getInstance());
     }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -1024,10 +1025,15 @@ public List<ApplicationAttemptId> getAppsInQueue(String queueName) {
   private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration(
       Configuration configuration) throws IOException {
     try {
-      configuration.addResource(this.rmContext.getConfigurationProvider()
+      InputStream CSInputStream =
-          .getConfigurationInputStream(configuration,
+          this.rmContext.getConfigurationProvider()
-              YarnConfiguration.CS_CONFIGURATION_FILE));
+              .getConfigurationInputStream(configuration,
-      return new CapacitySchedulerConfiguration(configuration, false);
+                  YarnConfiguration.CS_CONFIGURATION_FILE);
+      if (CSInputStream != null) {
+        configuration.addResource(CSInputStream);
+        return new CapacitySchedulerConfiguration(configuration, false);
+      }
+      return new CapacitySchedulerConfiguration(configuration, true);
     } catch (Exception e) {
       throw new IOException(e);
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java

@@ -126,14 +126,6 @@ public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider()
       throws IOException, YarnException {
     configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
         "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
-    try {
-      rm = new MockRM(configuration);
-      rm.init(configuration);
-      rm.start();
-      fail("Should throw an exception");
-    } catch(Exception ex) {
-      // Expect exception here
-    }
 
     //upload default configurations
     uploadDefaultConfiguration();
@@ -180,14 +172,6 @@ public void testAdminAclsWithFileSystemBasedConfigurationProvider()
       throws IOException, YarnException {
     configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
         "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
-    try {
-      rm = new MockRM(configuration);
-      rm.init(configuration);
-      rm.start();
-      fail("Should throw an exception");
-    } catch(Exception ex) {
-      // Expect exception here
-    }
 
     //upload default configurations
     uploadDefaultConfiguration();
@@ -246,14 +230,6 @@ public void testServiceAclsRefreshWithFileSystemBasedConfigurationProvider()
         "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
     ResourceManager resourceManager = null;
     try {
-      try {
-        resourceManager = new ResourceManager();
-        resourceManager.init(configuration);
-        resourceManager.start();
-        fail("Should throw an exception");
-      } catch (Exception ex) {
-        // expect to get an exception here
-      }
 
       //upload default configurations
       uploadDefaultConfiguration();
@@ -350,14 +326,6 @@ private void verifyServiceACLsRefresh(ServiceAuthorizationManager manager,
       throws IOException, YarnException {
     configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
         "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
-    try {
-      rm = new MockRM(configuration);
-      rm.init(configuration);
-      rm.start();
-      fail("Should throw an exception");
-    } catch(Exception ex) {
-      // Expect exception here
-    }
 
     //upload default configurations
     uploadDefaultConfiguration();
@@ -408,14 +376,6 @@ public void testRefreshUserToGroupsMappingsWithLocalConfigurationProvider() {
           throws IOException, YarnException {
     configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
         "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
-    try {
-      rm = new MockRM(configuration);
-      rm.init(configuration);
-      rm.start();
-      fail("Should throw an exception");
-    } catch (Exception ex) {
-      // Expect exception here
-    }
 
     String user = UserGroupInformation.getCurrentUser().getUserName();
     List<String> groupWithInit =
@@ -484,14 +444,6 @@ public void testRefreshNodesWithFileSystemBasedConfigurationProvider()
       throws IOException, YarnException {
     configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
         "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
-    try {
-      rm = new MockRM(configuration);
-      rm.init(configuration);
-      rm.start();
-      fail("Should throw an exception");
-    } catch (Exception ex) {
-      // Expect exception here
-    }
 
     // upload default configurations
     uploadDefaultConfiguration();
@@ -617,6 +569,27 @@ public void testRMHAWithFileSystemBasedConfiguration() throws IOException,
     }
   }
 
+  @Test
+  public void testRMStartsWithoutConfigurationFilesProvided() {
+    // enable FileSystemBasedConfigurationProvider without uploading
+    // any configuration files into Remote File System.
+    configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+        "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
+
+    // The configurationProvider will return NULL instead of
+    // throwing out Exceptions, if there are no configuration files provided.
+    // RM will not load the remote Configuration files,
+    // and should start successfully.
+    try {
+      rm = new MockRM(configuration);
+      rm.init(configuration);
+      rm.start();
+    } catch (Exception ex) {
+      fail("Should not get any exceptions");
+    }
+
+  }
+
   private String writeConfigurationXML(Configuration conf, String confXMLName)
       throws IOException {
     DataOutputStream output = null;