big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7524. Remove unused FairSchedulerEventLog. (Contributed by Wilfred Spiegelenburg)

Browse Source
This commit is contained in:
Yufei Gu 2017-11-22 14:18:36 -08:00
parent 738d1a206a
commit 4cc9479dae
4 changed files with 0 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
View File

@ -177,7 +177,6 @@ public class FairScheduler extends
protected double rackLocalityThreshold; // Cluster threshold for rack locality
protected long nodeLocalityDelayMs; // Delay for node locality
protected long rackLocalityDelayMs; // Delay for rack locality
private FairSchedulerEventLog eventLog; // Machine-readable event log
protected boolean assignMultiple; // Allocate multiple containers per
// heartbeat
@VisibleForTesting
@ -404,10 +403,6 @@ public int getContinuousSchedulingSleepMs() {
return continuousSchedulingSleepMs;
}
public FairSchedulerEventLog getEventLog() {
return eventLog;
}
/**
* Add a new application to the scheduler, with a given id, queue name, and
* user. This will accept a new app even if the user or queue is above
@ -875,7 +870,6 @@ protected void nodeUpdate(RMNode nm) {
try {
writeLock.lock();
long start = getClock().getTime();
eventLog.log("HEARTBEAT", nm.getHostName());
super.nodeUpdate(nm);
FSSchedulerNode fsNode = getFSSchedulerNode(nm.getNodeID());
@ -1284,8 +1278,6 @@ private void initScheduler(Configuration conf) throws IOException {
// This stores per-application scheduling information
this.applications = new ConcurrentHashMap<>();
this.eventLog = new FairSchedulerEventLog();
eventLog.init(this.conf);
allocConf = new AllocationConfiguration(conf);
try {

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
View File

@ -17,7 +17,6 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.File;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -64,12 +63,6 @@ public class FairSchedulerConfiguration extends Configuration {
public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file";
protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml";
/** Whether to enable the Fair Scheduler event log */
public static final String EVENT_LOG_ENABLED = CONF_PREFIX + "event-log-enabled";
public static final boolean DEFAULT_EVENT_LOG_ENABLED = false;
protected static final String EVENT_LOG_DIR = "eventlog.dir";
/** Whether pools can be created that were not specified in the FS configuration file
*/
protected static final String ALLOW_UNDECLARED_POOLS = CONF_PREFIX + "allow-undeclared-pools";
@ -255,15 +248,6 @@ public boolean getSizeBasedWeight() {
return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT);
}
public boolean isEventLogEnabled() {
return getBoolean(EVENT_LOG_ENABLED, DEFAULT_EVENT_LOG_ENABLED);
}
public String getEventlogDir() {
return get(EVENT_LOG_DIR, new File(System.getProperty("hadoop.log.dir",
"/tmp/")).getAbsolutePath() + File.separator + "fairscheduler");
}
public long getWaitTimeBeforeNextStarvationCheck() {
return getLong(WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS,
DEFAULT_WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS);

152
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
View File

@ -1,152 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.File;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.DailyRollingFileAppender;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.PatternLayout;
import org.apache.log4j.spi.LoggingEvent;
/**
* Event log used by the fair scheduler for machine-readable debug info.
* This class uses a log4j rolling file appender to write the log, but uses
* a custom tab-separated event format of the form:
* <pre>
* DATE EVENT_TYPE PARAM_1 PARAM_2 ...
* </pre>
* Various event types are used by the fair scheduler. The purpose of logging
* in this format is to enable tools to parse the history log easily and read
* internal scheduler variables, rather than trying to make the log human
* readable. The fair scheduler also logs human readable messages in the
* JobTracker's main log.
*
* Constructing this class creates a disabled log. It must be initialized
* using {@link FairSchedulerEventLog#init(FairSchedulerConfiguration)} to
* begin writing to the file.
*/
@Private
@Unstable
class FairSchedulerEventLog {
private static final Log LOG = LogFactory.getLog(FairSchedulerEventLog.class.getName());
/** Set to true if logging is disabled due to an error. */
private boolean logDisabled = true;
/**
* Log directory, set by mapred.fairscheduler.eventlog.location in conf file;
* defaults to {hadoop.log.dir}/fairscheduler.
*/
private String logDir;
/**
* Active log file, which is {LOG_DIR}/hadoop-{user}-fairscheduler.log.
* Older files are also stored as {LOG_FILE}.date (date format YYYY-MM-DD).
*/
private String logFile;
/** Log4j appender used to write to the log file */
private DailyRollingFileAppender appender;
boolean init(FairSchedulerConfiguration conf) {
if (conf.isEventLogEnabled()) {
try {
logDir = conf.getEventlogDir();
File logDirFile = new File(logDir);
if (!logDirFile.exists()) {
if (!logDirFile.mkdirs()) {
throw new IOException(
"Mkdirs failed to create " + logDirFile.toString());
}
}
String username = System.getProperty("user.name");
logFile = String.format("%s%shadoop-%s-fairscheduler.log",
logDir, File.separator, username);
logDisabled = false;
PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n");
appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd");
appender.activateOptions();
LOG.info("Initialized fair scheduler event log, logging to " + logFile);
} catch (IOException e) {
LOG.error(
"Failed to initialize fair scheduler event log. Disabling it.", e);
logDisabled = true;
}
} else {
logDisabled = true;
}
return !(logDisabled);
}
/**
* Log an event, writing a line in the log file of the form
* <pre>
* DATE EVENT_TYPE PARAM_1 PARAM_2 ...
* </pre>
*/
synchronized void log(String eventType, Object... params) {
try {
if (logDisabled)
return;
StringBuffer buffer = new StringBuffer();
buffer.append(eventType);
for (Object param: params) {
buffer.append("\t");
buffer.append(param);
}
String message = buffer.toString();
Logger logger = Logger.getLogger(getClass());
appender.append(new LoggingEvent("", logger, Level.INFO, message, null));
} catch (Exception e) {
LOG.error("Failed to append to fair scheduler event log", e);
logDisabled = true;
}
}
/**
* Flush and close the log.
*/
synchronized void shutdown() {
try {
if (appender != null)
appender.close();
} catch (Exception e) {
LOG.error("Failed to close fair scheduler event log", e);
logDisabled = true;
}
}
synchronized boolean isEnabled() {
return !logDisabled;
}
public String getLogFile() {
return logFile;
}
}

83
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
View File

@ -1,83 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.File;
import java.io.IOException;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class TestFairSchedulerEventLog {
private File logFile;
private FairScheduler scheduler;
private ResourceManager resourceManager;
@Before
public void setUp() throws IOException {
scheduler = new FairScheduler();
Configuration conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
ResourceScheduler.class);
conf.set(FairSchedulerConfiguration.EVENT_LOG_ENABLED, "true");
// All tests assume only one assignment per node update
conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
resourceManager = new MockRM(conf);
((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
scheduler.init(conf);
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
}
/**
* Make sure the scheduler creates the event log.
*/
@Test
public void testCreateEventLog() throws IOException {
FairSchedulerEventLog eventLog = scheduler.getEventLog();
logFile = new File(eventLog.getLogFile());
Assert.assertTrue(logFile.exists());
}
@After
public void tearDown() {
logFile.delete();
logFile.getParentFile().delete(); // fairscheduler/
if (scheduler != null) {
scheduler.stop();
scheduler = null;
}
if (resourceManager != null) {
resourceManager.stop();
resourceManager = null;
}
}
}