big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-1228. Clean up Fair Scheduler configuration loading. (Sandy Ryza)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1528201 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Sanford Ryza 2013-10-01 20:14:18 +00:00
parent cc8a0cab59
commit aa2745abe5
6 changed files with 84 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -85,6 +85,8 @@ Release 2.1.2 - UNRELEASED
YARN-899. Added back queue level administrator-acls so that there is no
regression w.r.t 1.x. (Xuan Gong via vinodkv)
YARN-1228. Clean up Fair Scheduler configuration loading. (Sandy Ryza)
OPTIMIZATIONS
BUG FIXES

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
View File

@ -18,9 +18,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.File;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
@ -33,6 +36,9 @@
@Evolving
public class FairSchedulerConfiguration extends Configuration {
public static final Log LOG = LogFactory.getLog(
FairSchedulerConfiguration.class.getName());
/** Increment request grant-able by the RM scheduler.
* These properties are looked up in the yarn-site.xml */
public static final String RM_SCHEDULER_INCREMENT_ALLOCATION_MB =
@ -42,11 +48,10 @@ public class FairSchedulerConfiguration extends Configuration {
YarnConfiguration.YARN_PREFIX + "scheduler.increment-allocation-vcores";
public static final int DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES = 1;
public static final String FS_CONFIGURATION_FILE = "fair-scheduler.xml";
private static final String CONF_PREFIX = "yarn.scheduler.fair.";
protected static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file";
protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml";
protected static final String EVENT_LOG_DIR = "eventlog.dir";
/** Whether to use the user name as the queue name (instead of "default") if
@ -105,7 +110,6 @@ public class FairSchedulerConfiguration extends Configuration {
public FairSchedulerConfiguration(Configuration conf) {
super(conf);
addResource(FS_CONFIGURATION_FILE);
}
public Resource getMinimumAllocation() {
@ -182,8 +186,28 @@ public boolean getSizeBasedWeight() {
return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT);
}
public String getAllocationFile() {
return get(ALLOCATION_FILE);
/**
* Path to XML file containing allocations. If the
* path is relative, it is searched for in the
* classpath, but loaded like a regular File.
*/
public File getAllocationFile() {
String allocFilePath = get(ALLOCATION_FILE, DEFAULT_ALLOCATION_FILE);
File allocFile = new File(allocFilePath);
if (!allocFile.isAbsolute()) {
URL url = Thread.currentThread().getContextClassLoader()
.getResource(allocFilePath);
if (url == null) {
LOG.warn(allocFilePath + " not found on the classpath.");
allocFile = null;
} else if (!url.getProtocol().equalsIgnoreCase("file")) {
throw new RuntimeException("Allocation file " + url
+ " found on the classpath is not on the local filesystem.");
} else {
allocFile = new File(url.getPath());
}
}
return allocFile;
}
public String getEventlogDir() {

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
View File

@ -75,11 +75,8 @@ public class QueueManager {
private final FairScheduler scheduler;
private Object allocFile; // Path to XML file containing allocations. This
// is either a URL to specify a classpath resource
// (if the fair-scheduler.xml on the classpath is
// used) or a String to specify an absolute path (if
// mapred.fairscheduler.allocation.file is used).
// Path to XML file containing allocations.
private File allocFile;
private final Collection<FSLeafQueue> leafQueues =
new CopyOnWriteArrayList<FSLeafQueue>();
@ -107,16 +104,7 @@ public void initialize() throws IOException, SAXException,
queues.put(rootQueue.getName(), rootQueue);
this.allocFile = conf.getAllocationFile();
if (allocFile == null) {
// No allocation file specified in jobconf. Use the default allocation
// file, fair-scheduler.xml, looking for it on the classpath.
allocFile = new Configuration().getResource("fair-scheduler.xml");
if (allocFile == null) {
LOG.error("The fair scheduler allocation file fair-scheduler.xml was "
+ "not found on the classpath, and no other config file is given "
+ "through mapred.fairscheduler.allocation.file.");
}
}
reloadAllocs();
lastSuccessfulReload = scheduler.getClock().getTime();
lastReloadAttempt = scheduler.getClock().getTime();
@ -255,14 +243,7 @@ public void reloadAllocsIfNecessary() {
try {
// Get last modified time of alloc file depending whether it's a String
// (for a path name) or an URL (for a classloader resource)
long lastModified;
if (allocFile instanceof String) {
File file = new File((String) allocFile);
lastModified = file.lastModified();
} else { // allocFile is an URL
URLConnection conn = ((URL) allocFile).openConnection();
lastModified = conn.getLastModified();
}
long lastModified = allocFile.lastModified();
if (lastModified > lastSuccessfulReload &&
time > lastModified + ALLOC_RELOAD_WAIT) {
reloadAllocs();
@ -321,12 +302,7 @@ public void reloadAllocs() throws IOException, ParserConfigurationException,
DocumentBuilderFactory.newInstance();
docBuilderFactory.setIgnoringComments(true);
DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
Document doc;
if (allocFile instanceof String) {
doc = builder.parse(new File((String) allocFile));
} else {
doc = builder.parse(allocFile.toString());
}
Document doc = builder.parse(allocFile);
Element root = doc.getDocumentElement();
if (!"allocations".equals(root.getTagName()))
throw new AllocationConfigurationException("Bad fair scheduler config " +

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
View File

@ -20,6 +20,11 @@
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration.parseResourceConfigValue;
import static org.junit.Assert.assertEquals;
import java.io.File;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Test;
@ -55,4 +60,15 @@ public void testOnlyCPU() throws Exception {
public void testGibberish() throws Exception {
parseResourceConfigValue("1o24vc0res");
}
@Test
public void testGetAllocationFileFromClasspath() {
FairSchedulerConfiguration conf = new FairSchedulerConfiguration(
new Configuration());
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE,
"test-fair-scheduler.xml");
File allocationFile = conf.getAllocationFile();
Assert.assertEquals("test-fair-scheduler.xml", allocationFile.getName());
Assert.assertTrue(allocationFile.exists());
}
}

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/test-fair-scheduler.xml Normal file
View File

@ -0,0 +1,22 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
<?xml version="1.0"?>
<allocations>
<queue name="queue1"></queue>
</allocations>

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
View File

@ -81,7 +81,7 @@ Hadoop MapReduce Next Generation - Fair Scheduler
distribute the resources assigned to them to their children in the same
fashion. Applications may only be scheduled on leaf queues. Queues can be
specified as children of other queues by placing them as sub-elements of
their parents in the fair scheduler configuration file.
their parents in the fair scheduler allocation file.
A queue's name starts with the names of its parents, with periods as
separators. So a queue named "queue1" under the root queue, would be referred
@ -118,16 +118,20 @@ Hadoop MapReduce Next Generation - Fair Scheduler
Customizing the Fair Scheduler typically involves altering two files. First,
scheduler-wide options can be set by adding configuration properties in the
yarn-site.xml file in your existing configuration directory. Second, in
most cases users will want to create a manifest file listing which queues
exist and their respective weights and capacities. The location of this file
is flexible - but it must be declared in yarn-site.xml.
most cases users will want to create an allocation file listing which queues
exist and their respective weights and capacities. The allocation file
is reloaded every 10 seconds, allowing changes to be made on the fly.
Properties that can be placed in yarn-site.xml
* <<<yarn.scheduler.fair.allocation.file>>>
* Path to allocation file. An allocation file is an XML manifest describing
queues and their properties, in addition to certain policy defaults. This file
must be in XML format as described in the next section.
Defaults to fair-scheduler.xml in configuration directory.
must be in the XML format described in the next section. If a relative path is
given, the file is searched for on the classpath (which typically includes
the Hadoop conf directory).
Defaults to fair-scheduler.xml.
* <<<yarn.scheduler.fair.user-as-default-queue>>>