From 989db9ee556e2239d1c1669883a624e01b8d3abc Mon Sep 17 00:00:00 2001 From: Sanford Ryza Date: Mon, 14 Oct 2013 21:42:31 +0000 Subject: [PATCH] YARN-1182. MiniYARNCluster creates and inits the RM/NM only on start() (Karthik Kambatla via Sandy Ryza) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1532109 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/server/MiniYARNCluster.java | 127 +++++++++--------- 2 files changed, 68 insertions(+), 62 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a54cdc5475..459a41e0f0 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -63,6 +63,9 @@ Release 2.3.0 - UNRELEASED YARN-1258. Allow configuring the Fair Scheduler root queue (Sandy Ryza) + YARN-1182. MiniYARNCluster creates and inits the RM/NM only on start() + (Karthik Kambatla via Sandy Ryza) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 0c8338301f..6cf9644246 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -189,31 +189,33 @@ public ResourceManagerWrapper() { } @Override - public synchronized void serviceStart() throws Exception { - try { - getConfig().setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true); - if (!getConfig().getBoolean( - YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, - YarnConfiguration.DEFAULT_YARN_MINICLUSTER_FIXED_PORTS)) { - // pick free random ports. - String hostname = MiniYARNCluster.getHostname(); - getConfig().set(YarnConfiguration.RM_ADDRESS, - hostname + ":0"); - getConfig().set(YarnConfiguration.RM_ADMIN_ADDRESS, - hostname + ":0"); - getConfig().set(YarnConfiguration.RM_SCHEDULER_ADDRESS, - hostname + ":0"); - getConfig().set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, - hostname + ":0"); - WebAppUtils.setRMWebAppHostnameAndPort(getConfig(), hostname, 0); - } - resourceManager = new ResourceManager() { - @Override - protected void doSecureLogin() throws IOException { - // Don't try to login using keytab in the testcase. - }; + protected synchronized void serviceInit(Configuration conf) + throws Exception { + conf.setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true); + if (!conf.getBoolean( + YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, + YarnConfiguration.DEFAULT_YARN_MINICLUSTER_FIXED_PORTS)) { + // pick free random ports. + String hostname = MiniYARNCluster.getHostname(); + conf.set(YarnConfiguration.RM_ADDRESS, hostname + ":0"); + conf.set(YarnConfiguration.RM_ADMIN_ADDRESS, hostname + ":0"); + conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, hostname + ":0"); + conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, hostname + ":0"); + WebAppUtils.setRMWebAppHostnameAndPort(conf, hostname, 0); + } + resourceManager = new ResourceManager() { + @Override + protected void doSecureLogin() throws IOException { + // Don't try to login using keytab in the testcase. }; - resourceManager.init(getConfig()); + }; + resourceManager.init(conf); + super.serviceInit(conf); + } + + @Override + protected synchronized void serviceStart() throws Exception { + try { new Thread() { public void run() { resourceManager.start(); @@ -242,7 +244,7 @@ public void run() { } @Override - public synchronized void serviceStop() throws Exception { + protected synchronized void serviceStop() throws Exception { if (resourceManager != null) { resourceManager.stop(); } @@ -271,8 +273,43 @@ public NodeManagerWrapper(int i) { index = i; } - public synchronized void serviceInit(Configuration conf) throws Exception { + protected synchronized void serviceInit(Configuration conf) + throws Exception { Configuration config = new YarnConfiguration(conf); + // create nm-local-dirs and configure them for the nodemanager + String localDirsString = prepareDirs("local", numLocalDirs); + config.set(YarnConfiguration.NM_LOCAL_DIRS, localDirsString); + // create nm-log-dirs and configure them for the nodemanager + String logDirsString = prepareDirs("log", numLogDirs); + config.set(YarnConfiguration.NM_LOG_DIRS, logDirsString); + + File remoteLogDir = + new File(testWorkDir, MiniYARNCluster.this.getName() + + "-remoteLogDir-nm-" + index); + remoteLogDir.mkdir(); + config.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + remoteLogDir.getAbsolutePath()); + // By default AM + 2 containers + config.setInt(YarnConfiguration.NM_PMEM_MB, 4*1024); + config.set(YarnConfiguration.NM_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + config.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, + MiniYARNCluster.getHostname() + ":0"); + WebAppUtils + .setNMWebAppHostNameAndPort(config, + MiniYARNCluster.getHostname(), 0); + + // Disable resource checks by default + if (!config.getBoolean( + YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, + YarnConfiguration. + DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) { + config.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false); + config.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false); + } + + LOG.info("Starting NM: " + index); + nodeManagers[index].init(config); super.serviceInit(config); } @@ -296,42 +333,8 @@ private String prepareDirs(String dirType, int numDirs) { return dirsString; } - public synchronized void serviceStart() throws Exception { + protected synchronized void serviceStart() throws Exception { try { - // create nm-local-dirs and configure them for the nodemanager - String localDirsString = prepareDirs("local", numLocalDirs); - getConfig().set(YarnConfiguration.NM_LOCAL_DIRS, localDirsString); - // create nm-log-dirs and configure them for the nodemanager - String logDirsString = prepareDirs("log", numLogDirs); - getConfig().set(YarnConfiguration.NM_LOG_DIRS, logDirsString); - - File remoteLogDir = - new File(testWorkDir, MiniYARNCluster.this.getName() - + "-remoteLogDir-nm-" + index); - remoteLogDir.mkdir(); - getConfig().set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - remoteLogDir.getAbsolutePath()); - // By default AM + 2 containers - getConfig().setInt(YarnConfiguration.NM_PMEM_MB, 4*1024); - getConfig().set(YarnConfiguration.NM_ADDRESS, - MiniYARNCluster.getHostname() + ":0"); - getConfig().set(YarnConfiguration.NM_LOCALIZER_ADDRESS, - MiniYARNCluster.getHostname() + ":0"); - WebAppUtils - .setNMWebAppHostNameAndPort(getConfig(), - MiniYARNCluster.getHostname(), 0); - - // Disable resource checks by default - if (!getConfig().getBoolean( - YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, - YarnConfiguration. - DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) { - getConfig().setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false); - getConfig().setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false); - } - - LOG.info("Starting NM: " + index); - nodeManagers[index].init(getConfig()); new Thread() { public void run() { nodeManagers[index].start(); @@ -354,7 +357,7 @@ public void run() { } @Override - public synchronized void serviceStop() throws Exception { + protected synchronized void serviceStop() throws Exception { if (nodeManagers[index] != null) { nodeManagers[index].stop(); }