big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-546. Allow disabling the Fair Scheduler event log. (Sandy Ryza)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1548360 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Sanford Ryza 2013-12-06 01:32:27 +00:00
parent 6b459506c3
commit eccd7b2093
4 changed files with 34 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -143,6 +143,8 @@ Release 2.4.0 - UNRELEASED
YARN-1181. Augment MiniYARNCluster to support HA mode (Karthik Kambatla)
YARN-546. Allow disabling the Fair Scheduler event log (Sandy Ryza)
OPTIMIZATIONS
BUG FIXES

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
View File

@ -53,6 +53,11 @@ public class FairSchedulerConfiguration extends Configuration {
public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file";
protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml";
/** Whether to enable the Fair Scheduler event log */
public static final String EVENT_LOG_ENABLED = CONF_PREFIX + "event-log-enabled";
public static final boolean DEFAULT_EVENT_LOG_ENABLED = false;
protected static final String EVENT_LOG_DIR = "eventlog.dir";
/** Whether pools can be created that were not specified in the FS configuration file
@ -192,6 +197,10 @@ public boolean getSizeBasedWeight() {
return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT);
}
public boolean isEventLogEnabled() {
return getBoolean(EVENT_LOG_ENABLED, DEFAULT_EVENT_LOG_ENABLED);
}
public String getEventlogDir() {
return get(EVENT_LOG_DIR, new File(System.getProperty("hadoop.log.dir",
"/tmp/")).getAbsolutePath() + File.separator + "fairscheduler");

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
View File

@ -75,26 +75,30 @@ class FairSchedulerEventLog {
private DailyRollingFileAppender appender;
boolean init(FairSchedulerConfiguration conf) {
try {
logDir = conf.getEventlogDir();
File logDirFile = new File(logDir);
if (!logDirFile.exists()) {
if (!logDirFile.mkdirs()) {
throw new IOException(
"Mkdirs failed to create " + logDirFile.toString());
if (conf.isEventLogEnabled()) {
try {
logDir = conf.getEventlogDir();
File logDirFile = new File(logDir);
if (!logDirFile.exists()) {
if (!logDirFile.mkdirs()) {
throw new IOException(
"Mkdirs failed to create " + logDirFile.toString());
}
}
String username = System.getProperty("user.name");
logFile = String.format("%s%shadoop-%s-fairscheduler.log",
logDir, File.separator, username);
logDisabled = false;
PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n");
appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd");
appender.activateOptions();
LOG.info("Initialized fair scheduler event log, logging to " + logFile);
} catch (IOException e) {
LOG.error(
"Failed to initialize fair scheduler event log. Disabling it.", e);
logDisabled = true;
}
String username = System.getProperty("user.name");
logFile = String.format("%s%shadoop-%s-fairscheduler.log",
logDir, File.separator, username);
logDisabled = false;
PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n");
appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd");
appender.activateOptions();
LOG.info("Initialized fair scheduler event log, logging to " + logFile);
} catch (IOException e) {
LOG.error(
"Failed to initialize fair scheduler event log. Disabling it.", e);
} else {
logDisabled = true;
}
return !(logDisabled);

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
View File

@ -44,7 +44,7 @@ public void setUp() throws IOException {
Configuration conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
ResourceScheduler.class);
conf.set("mapred.fairscheduler.eventlog.enabled", "true");
conf.set("yarn.scheduler.fair.event-log-enabled", "true");
// All tests assume only one assignment per node update
conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");