big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2741. Made NM web UI serve logs on the drive other than C: on Windows. Contributed by Craig Welch.

Browse Source
This commit is contained in:
Zhijie Shen 2014-10-28 14:10:16 -07:00
parent c6f04f391b
commit 8984e9b177
3 changed files with 87 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -784,6 +784,9 @@ Release 2.6.0 - UNRELEASED
YARN-2758. Update TestApplicationHistoryClientService to use the new generic
history store. (Zhijie Shen via xgong)
YARN-2741. Made NM web UI serve logs on the drive other than C: on Windows. (Craig
Welch via zjshen)
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
View File

@ -77,11 +77,7 @@ static List<File> getContainerLogDirs(ContainerId containerId,
List<String> logDirs = dirsHandler.getLogDirs();
List<File> containerLogDirs = new ArrayList<File>(logDirs.size());
for (String logDir : logDirs) {
try {
logDir = new URI(logDir).getPath();
} catch (URISyntaxException e) {
throw new YarnException("Internal error", e);
}
logDir = new File(logDir).toURI().getPath();
String appIdStr = ConverterUtils.toString(containerId
.getApplicationAttemptId().getApplicationId());
File appLogDir = new File(logDir, appIdStr);
@ -109,11 +105,9 @@ public static File getContainerLogFile(ContainerId containerId,
application.getAppId().toString(), containerId.toString());
Path logPath = dirsHandler.getLogPathToRead(
relativeContainerLogDir + Path.SEPARATOR + fileName);
URI logPathURI = new URI(logPath.toString());
URI logPathURI = new File(logPath.toString()).toURI();
File logFile = new File(logPathURI.getPath());
return logFile;
} catch (URISyntaxException e) {
throw new YarnException("Internal error", e);
} catch (IOException e) {
LOG.warn("Failed to find log file", e);
throw new NotFoundException("Cannot find this log on the local disk.");

82
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
View File

@ -28,6 +28,7 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -36,11 +37,16 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
@ -210,4 +216,80 @@ public void testContainerLogPageAccess() throws IOException {
}
}
}
@Test
public void testLogDirWithDriveLetter() throws Exception {
//To verify that logs paths which include drive letters (Windows)
//do not lose their drive letter specification
LocalDirsHandlerService localDirs = mock(LocalDirsHandlerService.class);
List<String> logDirs = new ArrayList<String>();
logDirs.add("F:/nmlogs");
when(localDirs.getLogDirs()).thenReturn(logDirs);
ApplicationIdPBImpl appId = mock(ApplicationIdPBImpl.class);
when(appId.toString()).thenReturn("app_id_1");
ApplicationAttemptIdPBImpl appAttemptId =
mock(ApplicationAttemptIdPBImpl.class);
when(appAttemptId.getApplicationId()).thenReturn(appId);
ContainerId containerId = mock(ContainerIdPBImpl.class);
when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId);
List<File> logDirFiles = ContainerLogsUtils.getContainerLogDirs(
containerId, localDirs);
Assert.assertTrue("logDir lost drive letter " +
logDirFiles.get(0),
logDirFiles.get(0).toString().indexOf("F:" + File.separator +
"nmlogs") > -1);
}
@Test
public void testLogFileWithDriveLetter() throws Exception {
ContainerImpl container = mock(ContainerImpl.class);
ApplicationIdPBImpl appId = mock(ApplicationIdPBImpl.class);
when(appId.toString()).thenReturn("appId");
Application app = mock(Application.class);
when(app.getAppId()).thenReturn(appId);
ApplicationAttemptIdPBImpl appAttemptId =
mock(ApplicationAttemptIdPBImpl.class);
when(appAttemptId.getApplicationId()).thenReturn(appId);
ConcurrentMap<ApplicationId, Application> applications =
new ConcurrentHashMap<ApplicationId, Application>();
applications.put(appId, app);
ContainerId containerId = mock(ContainerIdPBImpl.class);
when(containerId.toString()).thenReturn("containerId");
when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId);
ConcurrentMap<ContainerId, Container> containers =
new ConcurrentHashMap<ContainerId, Container>();
containers.put(containerId, container);
LocalDirsHandlerService localDirs = mock(LocalDirsHandlerService.class);
when(localDirs.getLogPathToRead("appId" + Path.SEPARATOR + "containerId" +
Path.SEPARATOR + "fileName"))
.thenReturn(new Path("F:/nmlogs/appId/containerId/fileName"));
NMContext context = mock(NMContext.class);
when(context.getLocalDirsHandler()).thenReturn(localDirs);
when(context.getApplications()).thenReturn(applications);
when(context.getContainers()).thenReturn(containers);
File logFile = ContainerLogsUtils.getContainerLogFile(containerId,
"fileName", null, context);
Assert.assertTrue("logFile lost drive letter " +
logFile,
logFile.toString().indexOf("F:" + File.separator + "nmlogs") > -1);
}
}