diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f2a42d056b..30b56f6723 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -85,6 +85,8 @@ Release 2.1.2 - UNRELEASED YARN-899. Added back queue level administrator-acls so that there is no regression w.r.t 1.x. (Xuan Gong via vinodkv) + YARN-1228. Clean up Fair Scheduler configuration loading. (Sandy Ryza) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index a3de8942ea..edfc8fa83e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -18,9 +18,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; import java.io.File; +import java.net.URL; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; @@ -33,6 +36,9 @@ @Evolving public class FairSchedulerConfiguration extends Configuration { + public static final Log LOG = LogFactory.getLog( + FairSchedulerConfiguration.class.getName()); + /** Increment request grant-able by the RM scheduler. * These properties are looked up in the yarn-site.xml */ public static final String RM_SCHEDULER_INCREMENT_ALLOCATION_MB = @@ -42,11 +48,10 @@ public class FairSchedulerConfiguration extends Configuration { YarnConfiguration.YARN_PREFIX + "scheduler.increment-allocation-vcores"; public static final int DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES = 1; - public static final String FS_CONFIGURATION_FILE = "fair-scheduler.xml"; - private static final String CONF_PREFIX = "yarn.scheduler.fair."; protected static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file"; + protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml"; protected static final String EVENT_LOG_DIR = "eventlog.dir"; /** Whether to use the user name as the queue name (instead of "default") if @@ -105,7 +110,6 @@ public class FairSchedulerConfiguration extends Configuration { public FairSchedulerConfiguration(Configuration conf) { super(conf); - addResource(FS_CONFIGURATION_FILE); } public Resource getMinimumAllocation() { @@ -182,8 +186,28 @@ public boolean getSizeBasedWeight() { return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT); } - public String getAllocationFile() { - return get(ALLOCATION_FILE); + /** + * Path to XML file containing allocations. If the + * path is relative, it is searched for in the + * classpath, but loaded like a regular File. + */ + public File getAllocationFile() { + String allocFilePath = get(ALLOCATION_FILE, DEFAULT_ALLOCATION_FILE); + File allocFile = new File(allocFilePath); + if (!allocFile.isAbsolute()) { + URL url = Thread.currentThread().getContextClassLoader() + .getResource(allocFilePath); + if (url == null) { + LOG.warn(allocFilePath + " not found on the classpath."); + allocFile = null; + } else if (!url.getProtocol().equalsIgnoreCase("file")) { + throw new RuntimeException("Allocation file " + url + + " found on the classpath is not on the local filesystem."); + } else { + allocFile = new File(url.getPath()); + } + } + return allocFile; } public String getEventlogDir() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index 839b190bb1..7560309f5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -75,11 +75,8 @@ public class QueueManager { private final FairScheduler scheduler; - private Object allocFile; // Path to XML file containing allocations. This - // is either a URL to specify a classpath resource - // (if the fair-scheduler.xml on the classpath is - // used) or a String to specify an absolute path (if - // mapred.fairscheduler.allocation.file is used). + // Path to XML file containing allocations. + private File allocFile; private final Collection leafQueues = new CopyOnWriteArrayList(); @@ -107,16 +104,7 @@ public void initialize() throws IOException, SAXException, queues.put(rootQueue.getName(), rootQueue); this.allocFile = conf.getAllocationFile(); - if (allocFile == null) { - // No allocation file specified in jobconf. Use the default allocation - // file, fair-scheduler.xml, looking for it on the classpath. - allocFile = new Configuration().getResource("fair-scheduler.xml"); - if (allocFile == null) { - LOG.error("The fair scheduler allocation file fair-scheduler.xml was " - + "not found on the classpath, and no other config file is given " - + "through mapred.fairscheduler.allocation.file."); - } - } + reloadAllocs(); lastSuccessfulReload = scheduler.getClock().getTime(); lastReloadAttempt = scheduler.getClock().getTime(); @@ -255,14 +243,7 @@ public void reloadAllocsIfNecessary() { try { // Get last modified time of alloc file depending whether it's a String // (for a path name) or an URL (for a classloader resource) - long lastModified; - if (allocFile instanceof String) { - File file = new File((String) allocFile); - lastModified = file.lastModified(); - } else { // allocFile is an URL - URLConnection conn = ((URL) allocFile).openConnection(); - lastModified = conn.getLastModified(); - } + long lastModified = allocFile.lastModified(); if (lastModified > lastSuccessfulReload && time > lastModified + ALLOC_RELOAD_WAIT) { reloadAllocs(); @@ -321,12 +302,7 @@ public void reloadAllocs() throws IOException, ParserConfigurationException, DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); - Document doc; - if (allocFile instanceof String) { - doc = builder.parse(new File((String) allocFile)); - } else { - doc = builder.parse(allocFile.toString()); - } + Document doc = builder.parse(allocFile); Element root = doc.getDocumentElement(); if (!"allocations".equals(root.getTagName())) throw new AllocationConfigurationException("Bad fair scheduler config " + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java index 12ed779267..da8a183c10 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java @@ -20,6 +20,11 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue; import static org.junit.Assert.assertEquals; +import java.io.File; + +import junit.framework.Assert; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.Test; @@ -55,4 +60,15 @@ public void testOnlyCPU() throws Exception { public void testGibberish() throws Exception { parseResourceConfigValue("1o24vc0res"); } + + @Test + public void testGetAllocationFileFromClasspath() { + FairSchedulerConfiguration conf = new FairSchedulerConfiguration( + new Configuration()); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, + "test-fair-scheduler.xml"); + File allocationFile = conf.getAllocationFile(); + Assert.assertEquals("test-fair-scheduler.xml", allocationFile.getName()); + Assert.assertTrue(allocationFile.exists()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/test-fair-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/test-fair-scheduler.xml new file mode 100644 index 0000000000..db160c9063 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/test-fair-scheduler.xml @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm index 84d6ca2911..7008c20768 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm @@ -81,7 +81,7 @@ Hadoop MapReduce Next Generation - Fair Scheduler distribute the resources assigned to them to their children in the same fashion. Applications may only be scheduled on leaf queues. Queues can be specified as children of other queues by placing them as sub-elements of - their parents in the fair scheduler configuration file. + their parents in the fair scheduler allocation file. A queue's name starts with the names of its parents, with periods as separators. So a queue named "queue1" under the root queue, would be referred @@ -118,16 +118,20 @@ Hadoop MapReduce Next Generation - Fair Scheduler Customizing the Fair Scheduler typically involves altering two files. First, scheduler-wide options can be set by adding configuration properties in the yarn-site.xml file in your existing configuration directory. Second, in - most cases users will want to create a manifest file listing which queues - exist and their respective weights and capacities. The location of this file - is flexible - but it must be declared in yarn-site.xml. + most cases users will want to create an allocation file listing which queues + exist and their respective weights and capacities. The allocation file + is reloaded every 10 seconds, allowing changes to be made on the fly. + +Properties that can be placed in yarn-site.xml * <<>> * Path to allocation file. An allocation file is an XML manifest describing queues and their properties, in addition to certain policy defaults. This file - must be in XML format as described in the next section. - Defaults to fair-scheduler.xml in configuration directory. + must be in the XML format described in the next section. If a relative path is + given, the file is searched for on the classpath (which typically includes + the Hadoop conf directory). + Defaults to fair-scheduler.xml. * <<>>