big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-4095. Avoid sharing AllocatorPerContext object in LocalDirAllocator between ShuffleHandler and LocalDirsHandlerService. Contributed by Zhihai Xu

Browse Source
This commit is contained in:
Jason Lowe 2015-09-23 15:42:01 +00:00
parent a2c76e5f26
commit c890c51a91
3 changed files with 50 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -460,6 +460,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12428. Fix inconsistency between log-level guards and statements.
(Jagadesh Kiran N and Jackie Chang via ozawa)
YARN-4095. Avoid sharing AllocatorPerContext object in LocalDirAllocator
between ShuffleHandler and LocalDirsHandlerService. (Zhihai Xu via jlowe)
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
View File

@ -30,6 +30,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
@ -52,6 +53,22 @@ public class LocalDirsHandlerService extends AbstractService {
private static Log LOG = LogFactory.getLog(LocalDirsHandlerService.class);
/**
* Good local directories, use internally,
* initial value is the same as NM_LOCAL_DIRS.
*/
@Private
static final String NM_GOOD_LOCAL_DIRS =
YarnConfiguration.NM_PREFIX + "good-local-dirs";
/**
* Good log directories, use internally,
* initial value is the same as NM_LOG_DIRS.
*/
@Private
static final String NM_GOOD_LOG_DIRS =
YarnConfiguration.NM_PREFIX + "good-log-dirs";
/** Timer used to schedule disk health monitoring code execution */
private Timer dirsHandlerScheduler;
private long diskHealthCheckInterval;
@ -113,9 +130,17 @@ public MonitoringTimerTask(Configuration conf) throws YarnRuntimeException {
new DirectoryCollection(
validatePaths(conf.getTrimmedStrings(YarnConfiguration.NM_LOG_DIRS)),
maxUsableSpacePercentagePerDisk, minFreeSpacePerDiskMB);
String local = conf.get(YarnConfiguration.NM_LOCAL_DIRS);
conf.set(NM_GOOD_LOCAL_DIRS,
(local != null) ? local : "");
localDirsAllocator = new LocalDirAllocator(
YarnConfiguration.NM_LOCAL_DIRS);
logDirsAllocator = new LocalDirAllocator(YarnConfiguration.NM_LOG_DIRS);
NM_GOOD_LOCAL_DIRS);
String log = conf.get(YarnConfiguration.NM_LOG_DIRS);
conf.set(NM_GOOD_LOG_DIRS,
(log != null) ? log : "");
logDirsAllocator = new LocalDirAllocator(
NM_GOOD_LOG_DIRS);
}
@Override
@ -373,10 +398,10 @@ private void updateDirsAfterTest() {
Configuration conf = getConfig();
List<String> localDirs = getLocalDirs();
conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS,
conf.setStrings(NM_GOOD_LOCAL_DIRS,
localDirs.toArray(new String[localDirs.size()]));
List<String> logDirs = getLogDirs();
conf.setStrings(YarnConfiguration.NM_LOG_DIRS,
conf.setStrings(NM_GOOD_LOG_DIRS,
logDirs.toArray(new String[logDirs.size()]));
if (!areDisksHealthy()) {
// Just log.

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
View File

@ -120,6 +120,15 @@ public void testGetFullDirs() throws Exception {
Assert.assertEquals(0, nm.getGoodLocalDirsDiskUtilizationPerc());
Assert.assertEquals(0, nm.getGoodLogDirsDiskUtilizationPerc());
Assert.assertEquals("",
dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOCAL_DIRS));
Assert.assertEquals("",
dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOG_DIRS));
Assert.assertEquals(localDir1 + "," + localDir2,
dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS));
Assert.assertEquals(logDir1 + "," + logDir2,
dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS));
conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE,
100.0f);
nm = NodeManagerMetrics.create();
@ -141,6 +150,15 @@ public void testGetFullDirs() throws Exception {
Assert
.assertEquals(utilizationPerc, nm.getGoodLogDirsDiskUtilizationPerc());
Assert.assertEquals(localDir2,
dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOCAL_DIRS));
Assert.assertEquals(logDir2,
dirSvc.getConfig().get(LocalDirsHandlerService.NM_GOOD_LOG_DIRS));
Assert.assertEquals(localDir1 + "," + localDir2,
dirSvc.getConfig().get(YarnConfiguration.NM_LOCAL_DIRS));
Assert.assertEquals(logDir1 + "," + logDir2,
dirSvc.getConfig().get(YarnConfiguration.NM_LOG_DIRS));
FileUtils.deleteDirectory(new File(localDir1));
FileUtils.deleteDirectory(new File(localDir2));
FileUtils.deleteDirectory(new File(logDir1));