From 6b5838ed3220f992092c7348f92f1d9d0d4a3061 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Wed, 19 Sep 2018 16:44:51 -0500 Subject: [PATCH] YARN-8784. DockerLinuxContainerRuntime prevents access to distributed cache entries on a full disk. Contributed by Eric Badger --- .../launcher/ContainerLaunch.java | 5 +- .../launcher/TestContainerLaunch.java | 81 +++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 2aca5f8f2c..6347d4e13e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -250,12 +250,13 @@ public class ContainerLaunch implements Callable { // accessible by users pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath); List localDirs = dirsHandler.getLocalDirs(); + List localDirsForRead = dirsHandler.getLocalDirsForRead(); List logDirs = dirsHandler.getLogDirs(); - List filecacheDirs = getNMFilecacheDirs(localDirs); + List filecacheDirs = getNMFilecacheDirs(localDirsForRead); List userLocalDirs = getUserLocalDirs(localDirs); List containerLocalDirs = getContainerLocalDirs(localDirs); List containerLogDirs = getContainerLogDirs(logDirs); - List userFilecacheDirs = getUserFilecacheDirs(localDirs); + List userFilecacheDirs = getUserFilecacheDirs(localDirsForRead); List applicationLocalDirs = getApplicationLocalDirs(localDirs, appIdStr); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index ddf46a63ac..5714a1c185 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -96,6 +96,7 @@ import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext; import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; @@ -107,6 +108,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.Conta import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; +import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; @@ -121,6 +123,7 @@ import org.junit.Assert; import org.junit.Assume; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; public class TestContainerLaunch extends BaseContainerManagerTest { @@ -2362,4 +2365,82 @@ public class TestContainerLaunch extends BaseContainerManagerTest { } } } + + @Test + public void testDistributedCacheDirs() throws Exception { + Container container = mock(Container.class); + ApplicationId appId = + ApplicationId.newInstance(System.currentTimeMillis(), 1); + ContainerId containerId = ContainerId + .newContainerId(ApplicationAttemptId.newInstance(appId, 1), 1); + when(container.getContainerId()).thenReturn(containerId); + when(container.getUser()).thenReturn("test"); + + when(container.getLocalizedResources()) + .thenReturn(Collections.> emptyMap()); + Dispatcher dispatcher = mock(Dispatcher.class); + + ContainerLaunchContext clc = mock(ContainerLaunchContext.class); + when(clc.getCommands()).thenReturn(Collections.emptyList()); + when(container.getLaunchContext()).thenReturn(clc); + + @SuppressWarnings("rawtypes") + ContainerExitHandler eventHandler = + mock(ContainerExitHandler.class); + when(dispatcher.getEventHandler()).thenReturn(eventHandler); + + Application app = mock(Application.class); + when(app.getAppId()).thenReturn(appId); + when(app.getUser()).thenReturn("test"); + + Credentials creds = mock(Credentials.class); + when(container.getCredentials()).thenReturn(creds); + + ((NMContext) context).setNodeId(NodeId.newInstance("127.0.0.1", HTTP_PORT)); + ContainerExecutor mockExecutor = mock(ContainerExecutor.class); + + LocalDirsHandlerService mockDirsHandler = + mock(LocalDirsHandlerService.class); + + List localDirsForRead = new ArrayList(); + String localDir1 = + new File("target", this.getClass().getSimpleName() + "-localDir1") + .getAbsoluteFile().toString(); + String localDir2 = + new File("target", this.getClass().getSimpleName() + "-localDir2") + .getAbsoluteFile().toString(); + localDirsForRead.add(localDir1); + localDirsForRead.add(localDir2); + + List localDirs = new ArrayList(); + localDirs.add(localDir1); + Path logPathForWrite = new Path(localDirs.get(0)); + + when(mockDirsHandler.areDisksHealthy()).thenReturn(true); + when(mockDirsHandler.getLocalDirsForRead()).thenReturn(localDirsForRead); + when(mockDirsHandler.getLocalDirs()).thenReturn(localDirs); + when(mockDirsHandler.getLogDirs()).thenReturn(localDirs); + when(mockDirsHandler.getLogPathForWrite(anyString(), + anyBoolean())).thenReturn(logPathForWrite); + when(mockDirsHandler.getLocalPathForWrite(anyString())) + .thenReturn(logPathForWrite); + when(mockDirsHandler.getLocalPathForWrite(anyString(), anyLong(), + anyBoolean())).thenReturn(logPathForWrite); + + ContainerLaunch launch = new ContainerLaunch(context, conf, dispatcher, + mockExecutor, app, container, mockDirsHandler, containerManager); + launch.call(); + + ArgumentCaptor ctxCaptor = + ArgumentCaptor.forClass(ContainerStartContext.class); + verify(mockExecutor, times(1)).launchContainer(ctxCaptor.capture()); + ContainerStartContext ctx = ctxCaptor.getValue(); + + Assert.assertEquals(StringUtils.join(",", + launch.getNMFilecacheDirs(localDirsForRead)), + StringUtils.join(",", ctx.getFilecacheDirs())); + Assert.assertEquals(StringUtils.join(",", + launch.getUserFilecacheDirs(localDirsForRead)), + StringUtils.join(",", ctx.getUserFilecacheDirs())); + } }