From 4cc9479dae2bfb7d14d29b55d103eea9fa35a586 Mon Sep 17 00:00:00 2001 From: Yufei Gu Date: Wed, 22 Nov 2017 14:18:36 -0800 Subject: [PATCH] YARN-7524. Remove unused FairSchedulerEventLog. (Contributed by Wilfred Spiegelenburg) --- .../scheduler/fair/FairScheduler.java | 8 - .../fair/FairSchedulerConfiguration.java | 16 -- .../scheduler/fair/FairSchedulerEventLog.java | 152 ------------------ .../fair/TestFairSchedulerEventLog.java | 83 ---------- 4 files changed, 259 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index b2978d4ce3..661d0a091a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -177,7 +177,6 @@ public class FairScheduler extends protected double rackLocalityThreshold; // Cluster threshold for rack locality protected long nodeLocalityDelayMs; // Delay for node locality protected long rackLocalityDelayMs; // Delay for rack locality - private FairSchedulerEventLog eventLog; // Machine-readable event log protected boolean assignMultiple; // Allocate multiple containers per // heartbeat @VisibleForTesting @@ -404,10 +403,6 @@ public int getContinuousSchedulingSleepMs() { return continuousSchedulingSleepMs; } - public FairSchedulerEventLog getEventLog() { - return eventLog; - } - /** * Add a new application to the scheduler, with a given id, queue name, and * user. This will accept a new app even if the user or queue is above @@ -875,7 +870,6 @@ protected void nodeUpdate(RMNode nm) { try { writeLock.lock(); long start = getClock().getTime(); - eventLog.log("HEARTBEAT", nm.getHostName()); super.nodeUpdate(nm); FSSchedulerNode fsNode = getFSSchedulerNode(nm.getNodeID()); @@ -1284,8 +1278,6 @@ private void initScheduler(Configuration conf) throws IOException { // This stores per-application scheduling information this.applications = new ConcurrentHashMap<>(); - this.eventLog = new FairSchedulerEventLog(); - eventLog.init(this.conf); allocConf = new AllocationConfiguration(conf); try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 9c9eee627a..38e71a75f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; -import java.io.File; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -64,12 +63,6 @@ public class FairSchedulerConfiguration extends Configuration { public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file"; protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml"; - /** Whether to enable the Fair Scheduler event log */ - public static final String EVENT_LOG_ENABLED = CONF_PREFIX + "event-log-enabled"; - public static final boolean DEFAULT_EVENT_LOG_ENABLED = false; - - protected static final String EVENT_LOG_DIR = "eventlog.dir"; - /** Whether pools can be created that were not specified in the FS configuration file */ protected static final String ALLOW_UNDECLARED_POOLS = CONF_PREFIX + "allow-undeclared-pools"; @@ -255,15 +248,6 @@ public boolean getSizeBasedWeight() { return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT); } - public boolean isEventLogEnabled() { - return getBoolean(EVENT_LOG_ENABLED, DEFAULT_EVENT_LOG_ENABLED); - } - - public String getEventlogDir() { - return get(EVENT_LOG_DIR, new File(System.getProperty("hadoop.log.dir", - "/tmp/")).getAbsolutePath() + File.separator + "fairscheduler"); - } - public long getWaitTimeBeforeNextStarvationCheck() { return getLong(WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS, DEFAULT_WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java deleted file mode 100644 index 803ce780af..0000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import java.io.File; -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.log4j.DailyRollingFileAppender; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.log4j.PatternLayout; -import org.apache.log4j.spi.LoggingEvent; - -/** - * Event log used by the fair scheduler for machine-readable debug info. - * This class uses a log4j rolling file appender to write the log, but uses - * a custom tab-separated event format of the form: - * 
- * DATE    EVENT_TYPE   PARAM_1   PARAM_2   ...
- *
- * Various event types are used by the fair scheduler. The purpose of logging - * in this format is to enable tools to parse the history log easily and read - * internal scheduler variables, rather than trying to make the log human - * readable. The fair scheduler also logs human readable messages in the - * JobTracker's main log. - * - * Constructing this class creates a disabled log. It must be initialized - * using {@link FairSchedulerEventLog#init(FairSchedulerConfiguration)} to - * begin writing to the file. - */ -@Private -@Unstable -class FairSchedulerEventLog { - private static final Log LOG = LogFactory.getLog(FairSchedulerEventLog.class.getName()); - - /** Set to true if logging is disabled due to an error. */ - private boolean logDisabled = true; - - /** - * Log directory, set by mapred.fairscheduler.eventlog.location in conf file; - * defaults to {hadoop.log.dir}/fairscheduler. - */ - private String logDir; - - /** - * Active log file, which is {LOG_DIR}/hadoop-{user}-fairscheduler.log. - * Older files are also stored as {LOG_FILE}.date (date format YYYY-MM-DD). - */ - private String logFile; - - /** Log4j appender used to write to the log file */ - private DailyRollingFileAppender appender; - - boolean init(FairSchedulerConfiguration conf) { - if (conf.isEventLogEnabled()) { - try { - logDir = conf.getEventlogDir(); - File logDirFile = new File(logDir); - if (!logDirFile.exists()) { - if (!logDirFile.mkdirs()) { - throw new IOException( - "Mkdirs failed to create " + logDirFile.toString()); - } - } - String username = System.getProperty("user.name"); - logFile = String.format("%s%shadoop-%s-fairscheduler.log", - logDir, File.separator, username); - logDisabled = false; - PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n"); - appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd"); - appender.activateOptions(); - LOG.info("Initialized fair scheduler event log, logging to " + logFile); - } catch (IOException e) { - LOG.error( - "Failed to initialize fair scheduler event log. Disabling it.", e); - logDisabled = true; - } - } else { - logDisabled = true; - } - return !(logDisabled); - } - - /** - * Log an event, writing a line in the log file of the form - * 
-   * DATE    EVENT_TYPE   PARAM_1   PARAM_2   ...
-   *
- */ - synchronized void log(String eventType, Object... params) { - try { - if (logDisabled) - return; - StringBuffer buffer = new StringBuffer(); - buffer.append(eventType); - for (Object param: params) { - buffer.append("\t"); - buffer.append(param); - } - String message = buffer.toString(); - Logger logger = Logger.getLogger(getClass()); - appender.append(new LoggingEvent("", logger, Level.INFO, message, null)); - } catch (Exception e) { - LOG.error("Failed to append to fair scheduler event log", e); - logDisabled = true; - } - } - - /** - * Flush and close the log. - */ - synchronized void shutdown() { - try { - if (appender != null) - appender.close(); - } catch (Exception e) { - LOG.error("Failed to close fair scheduler event log", e); - logDisabled = true; - } - } - - synchronized boolean isEnabled() { - return !logDisabled; - } - - public String getLogFile() { - return logFile; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java deleted file mode 100644 index 09c2370395..0000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; - -import java.io.File; -import java.io.IOException; - -import org.apache.hadoop.yarn.server.resourcemanager.MockRM; -import org.junit.Assert; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.AsyncDispatcher; -import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class TestFairSchedulerEventLog { - private File logFile; - private FairScheduler scheduler; - private ResourceManager resourceManager; - - @Before - public void setUp() throws IOException { - scheduler = new FairScheduler(); - - Configuration conf = new YarnConfiguration(); - conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class, - ResourceScheduler.class); - conf.set(FairSchedulerConfiguration.EVENT_LOG_ENABLED, "true"); - - // All tests assume only one assignment per node update - conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false"); - resourceManager = new MockRM(conf); - ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); - scheduler.init(conf); - scheduler.start(); - scheduler.reinitialize(conf, resourceManager.getRMContext()); - } - - /** - * Make sure the scheduler creates the event log. - */ - @Test - public void testCreateEventLog() throws IOException { - FairSchedulerEventLog eventLog = scheduler.getEventLog(); - - logFile = new File(eventLog.getLogFile()); - Assert.assertTrue(logFile.exists()); - } - - @After - public void tearDown() { - logFile.delete(); - logFile.getParentFile().delete(); // fairscheduler/ - if (scheduler != null) { - scheduler.stop(); - scheduler = null; - } - if (resourceManager != null) { - resourceManager.stop(); - resourceManager = null; - } - } -}