big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2565. Fixed RM to not use FileSystemApplicationHistoryStore unless explicitly set. Contributed by Zhijie Shen

Browse Source
This commit is contained in:
Jian He 2014-09-19 11:26:29 -07:00
parent 25fd69a8f9
commit 444acf8ea7
4 changed files with 50 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -400,6 +400,9 @@ Release 2.6.0 - UNRELEASED
YARN-2568. Fixed the potential test failures due to race conditions when RM
work-preserving recovery is enabled. (Jian He via zjshen)
YARN-2565. Fixed RM to not use FileSystemApplicationHistoryStore unless
explicitly set. (Zhijie Shen via jianhe)
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
View File

@ -182,7 +182,9 @@ private ApplicationHistoryManager createApplicationHistoryManager(
// APPLICATION_HISTORY_STORE is neither null nor empty, it means that the
// user has enabled it explicitly.
if (conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE) == null ||
conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0) {
conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0 ||
conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).equals(
NullApplicationHistoryStore.class.getName())) {
return new ApplicationHistoryManagerOnTimelineStore(
timelineDataManager, aclsManager);
} else {

49
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
View File

@ -52,6 +52,8 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import com.google.common.annotations.VisibleForTesting;
/**
* <p>
* {@link ResourceManager} uses this class to write the information of
@ -71,8 +73,10 @@ public class RMApplicationHistoryWriter extends CompositeService {
.getLog(RMApplicationHistoryWriter.class);
private Dispatcher dispatcher;
private ApplicationHistoryWriter writer;
private boolean historyServiceEnabled;
@VisibleForTesting
ApplicationHistoryWriter writer;
@VisibleForTesting
boolean historyServiceEnabled;
public RMApplicationHistoryWriter() {
super(RMApplicationHistoryWriter.class.getName());
@ -80,13 +84,18 @@ public RMApplicationHistoryWriter() {
@Override
protected synchronized void serviceInit(Configuration conf) throws Exception {
historyServiceEnabled =
conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED);
if (conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE) == null ||
conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0 ||
conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).equals(
NullApplicationHistoryStore.class.getName())) {
historyServiceEnabled = false;
}
// Only create the services when the history service is enabled, preventing
// wasting the system resources.
// Only create the services when the history service is enabled and not
// using the null store, preventing wasting the system resources.
if (historyServiceEnabled) {
writer = createApplicationHistoryStore(conf);
addIfService(writer);
@ -112,25 +121,19 @@ protected Dispatcher createDispatcher(Configuration conf) {
protected ApplicationHistoryStore createApplicationHistoryStore(
Configuration conf) {
// If the history writer is not enabled, a dummy store will be used to
// write nothing
if (historyServiceEnabled) {
try {
Class<? extends ApplicationHistoryStore> storeClass =
conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
FileSystemApplicationHistoryStore.class,
try {
Class<? extends ApplicationHistoryStore> storeClass =
conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
NullApplicationHistoryStore.class,
ApplicationHistoryStore.class);
return storeClass.newInstance();
} catch (Exception e) {
String msg =
"Could not instantiate ApplicationHistoryWriter: "
+ conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE,
FileSystemApplicationHistoryStore.class.getName());
LOG.error(msg, e);
throw new YarnRuntimeException(msg, e);
}
} else {
return new NullApplicationHistoryStore();
return storeClass.newInstance();
} catch (Exception e) {
String msg =
"Could not instantiate ApplicationHistoryWriter: "
+ conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE,
NullApplicationHistoryStore.class.getName());
LOG.error(msg, e);
throw new YarnRuntimeException(msg, e);
}
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
View File

@ -78,6 +78,8 @@ public void setup() {
store = new MemoryApplicationHistoryStore();
Configuration conf = new Configuration();
conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
conf.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
writer = new RMApplicationHistoryWriter() {
@Override
@ -174,6 +176,22 @@ private static RMContainer createRMContainer(ContainerId containerId) {
return container;
}
@Test
public void testDefaultStoreSetup() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
RMApplicationHistoryWriter writer = new RMApplicationHistoryWriter();
writer.init(conf);
writer.start();
try {
Assert.assertFalse(writer.historyServiceEnabled);
Assert.assertNull(writer.writer);
} finally {
writer.stop();
writer.close();
}
}
@Test
public void testWriteApplication() throws Exception {
RMApp app = createRMApp(ApplicationId.newInstance(0, 1));