diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 73080757b1..aef0d319e9 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -428,6 +428,8 @@ Release 2.8.0 - UNRELEASED YARN-4121. Fix typos in capacity scheduler documentation. (Kai Sasaki via vvasudev) + YARN-4086. Allow Aggregated Log readers to handle HAR files (rkanter) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml index 77a6583dfb..90f2bc7e9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml @@ -134,6 +134,18 @@ + + org.apache.rat + apache-rat-plugin + + + src/test/resources/application_1440536969523_0001.har/_index + src/test/resources/application_1440536969523_0001.har/part-0 + src/test/resources/application_1440536969523_0001.har/_masterindex + src/test/resources/application_1440536969523_0001.har/_SUCCESS + + + org.apache.hadoop hadoop-maven-plugins diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 7d20cf2e5a..a3538116ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.client.cli; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doReturn; @@ -32,6 +33,7 @@ import java.io.PrintStream; import java.io.PrintWriter; import java.io.Writer; +import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -318,6 +320,54 @@ public void testFetchApplictionLogs() throws Exception { fs.delete(new Path(rootLogDir), true); } + @Test (timeout = 15000) + public void testFetchApplictionLogsHar() throws Exception { + String remoteLogRootDir = "target/logs/"; + Configuration configuration = new Configuration(); + configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); + configuration + .set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); + configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); + FileSystem fs = FileSystem.get(configuration); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + URL harUrl = ClassLoader.getSystemClassLoader() + .getResource("application_1440536969523_0001.har"); + assertNotNull(harUrl); + Path path = + new Path(remoteLogRootDir + ugi.getShortUserName() + + "/logs/application_1440536969523_0001"); + if (fs.exists(path)) { + fs.delete(path, true); + } + assertTrue(fs.mkdirs(path)); + Path harPath = new Path(path, "application_1440536969523_0001.har"); + fs.copyFromLocalFile(false, new Path(harUrl.toURI()), harPath); + assertTrue(fs.exists(harPath)); + + YarnClient mockYarnClient = + createMockYarnClient(YarnApplicationState.FINISHED); + LogsCLI cli = new LogsCLIForTest(mockYarnClient); + cli.setConf(configuration); + int exitCode = cli.run(new String[]{"-applicationId", + "application_1440536969523_0001"}); + assertTrue(exitCode == 0); + String out = sysOutStream.toString(); + assertTrue( + out.contains("container_1440536969523_0001_01_000001 on host1_1111")); + assertTrue(out.contains("Hello stderr")); + assertTrue(out.contains("Hello stdout")); + assertTrue(out.contains("Hello syslog")); + assertTrue( + out.contains("container_1440536969523_0001_01_000002 on host2_2222")); + assertTrue(out.contains("Goodbye stderr")); + assertTrue(out.contains("Goodbye stdout")); + assertTrue(out.contains("Goodbye syslog")); + sysOutStream.reset(); + + fs.delete(new Path(remoteLogRootDir), true); + } + private static void createContainerLogInLocalDir(Path appLogsDir, ContainerId containerId, FileSystem fs, List logTypes) throws Exception { Path containerLogsDir = new Path(appLogsDir, containerId.toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_SUCCESS b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_SUCCESS new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_index b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_index new file mode 100644 index 0000000000..92ee728478 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_index @@ -0,0 +1,3 @@ +%2F dir 1440540845855+504+rkanter+supergroup 0 0 host1_1111 host2_2222 +%2Fhost1_1111 file part-0 0 394 1440540845834+420+rkanter+supergroup +%2Fhost2_2222 file part-0 394 400 1440540845854+420+rkanter+supergroup diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_masterindex b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_masterindex new file mode 100644 index 0000000000..086d2b852e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/_masterindex @@ -0,0 +1,2 @@ +3 +0 1520266628 0 214 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/part-0 b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/part-0 new file mode 100644 index 0000000000..fef262a815 Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/resources/application_1440536969523_0001.har/part-0 differ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml index 3b47cdda4b..d4913b27b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -215,6 +215,10 @@ src/main/resources/webapps/static/dt-1.9.4/css/demo_table.css src/main/resources/webapps/static/dt-1.9.4/images/Sorting icons.psd src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css + src/test/resources/application_1440536969523_0001.har/_index + src/test/resources/application_1440536969523_0001.har/part-0 + src/test/resources/application_1440536969523_0001.har/_masterindex + src/test/resources/application_1440536969523_0001.har/_SUCCESS diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 39fd95eda5..fb4d3cd943 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.HarFs; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -61,8 +62,9 @@ public int dumpAContainersLogsForALogType(String appId, String containerId, YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); + ApplicationId applicationId = ConverterUtils.toApplicationId(appId); Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir( - remoteRootLogDir, ConverterUtils.toApplicationId(appId), jobOwner, + remoteRootLogDir, applicationId, jobOwner, suffix); RemoteIterator nodeFiles; try { @@ -80,6 +82,12 @@ public int dumpAContainersLogsForALogType(String appId, String containerId, while (nodeFiles.hasNext()) { FileStatus thisNodeFile = nodeFiles.next(); String fileName = thisNodeFile.getPath().getName(); + if (fileName.equals(applicationId + ".har")) { + Path p = new Path("har:///" + + thisNodeFile.getPath().toUri().getRawPath()); + nodeFiles = HarFs.get(p.toUri(), conf).listStatusIterator(p); + continue; + } if (fileName.contains(LogAggregationUtils.getNodeString(nodeId)) && !fileName.endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { AggregatedLogFormat.LogReader reader = null; @@ -207,6 +215,12 @@ public int dumpAllContainersLogs(ApplicationId appId, String appOwner, boolean foundAnyLogs = false; while (nodeFiles.hasNext()) { FileStatus thisNodeFile = nodeFiles.next(); + if (thisNodeFile.getPath().getName().equals(appId + ".har")) { + Path p = new Path("har:///" + + thisNodeFile.getPath().toUri().getRawPath()); + nodeFiles = HarFs.get(p.toUri(), conf).listStatusIterator(p); + continue; + } if (!thisNodeFile.getPath().getName() .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { AggregatedLogFormat.LogReader reader = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java index 620d097bdb..69fc347b1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.HarFs; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.security.UserGroupInformation; @@ -120,6 +121,12 @@ protected void render(Block html) { AggregatedLogFormat.LogReader reader = null; try { FileStatus thisNodeFile = nodeFiles.next(); + if (thisNodeFile.getPath().getName().equals(applicationId + ".har")) { + Path p = new Path("har:///" + + thisNodeFile.getPath().toUri().getRawPath()); + nodeFiles = HarFs.get(p.toUri(), conf).listStatusIterator(p); + continue; + } if (!thisNodeFile.getPath().getName() .contains(LogAggregationUtils.getNodeString(nodeId)) || thisNodeFile.getPath().getName() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java index 2a5762c302..798406de99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java @@ -23,6 +23,7 @@ import java.io.FileWriter; import java.io.PrintWriter; import java.io.Writer; +import java.net.URL; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -30,6 +31,7 @@ import javax.servlet.http.HttpServletRequest; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; @@ -117,7 +119,8 @@ public void testBadLogs() throws Exception { } /** - * All ok and the AggregatedLogsBlockFor should aggregate logs and show it. + * Reading from logs should succeed and they should be shown in the + * AggregatedLogsBlock html. * * @throws Exception */ @@ -144,8 +147,56 @@ public void testAggregatedLogsBlock() throws Exception { assertTrue(out.contains("test log1")); assertTrue(out.contains("test log2")); assertTrue(out.contains("test log3")); - } + + /** + * Reading from logs should succeed (from a HAR archive) and they should be + * shown in the AggregatedLogsBlock html. + * + * @throws Exception + */ + @Test + public void testAggregatedLogsBlockHar() throws Exception { + FileUtil.fullyDelete(new File("target/logs")); + Configuration configuration = getConfiguration(); + + URL harUrl = ClassLoader.getSystemClassLoader() + .getResource("application_1440536969523_0001.har"); + assertNotNull(harUrl); + String path = "target/logs/admin/logs/application_1440536969523_0001" + + "/application_1440536969523_0001.har"; + FileUtils.copyDirectory(new File(harUrl.getPath()), new File(path)); + + AggregatedLogsBlockForTest aggregatedBlock = getAggregatedLogsBlockForTest( + configuration, "admin", + "container_1440536969523_0001_01_000001", "host1:1111"); + ByteArrayOutputStream data = new ByteArrayOutputStream(); + PrintWriter printWriter = new PrintWriter(data); + HtmlBlock html = new HtmlBlockForTest(); + HtmlBlock.Block block = new BlockForTest(html, printWriter, 10, false); + aggregatedBlock.render(block); + + block.getWriter().flush(); + String out = data.toString(); + assertTrue(out.contains("Hello stderr")); + assertTrue(out.contains("Hello stdout")); + assertTrue(out.contains("Hello syslog")); + + aggregatedBlock = getAggregatedLogsBlockForTest( + configuration, "admin", + "container_1440536969523_0001_01_000002", "host2:2222"); + data = new ByteArrayOutputStream(); + printWriter = new PrintWriter(data); + html = new HtmlBlockForTest(); + block = new BlockForTest(html, printWriter, 10, false); + aggregatedBlock.render(block); + block.getWriter().flush(); + out = data.toString(); + assertTrue(out.contains("Goodbye stderr")); + assertTrue(out.contains("Goodbye stdout")); + assertTrue(out.contains("Goodbye syslog")); + } + /** * Log files was deleted. * @throws Exception @@ -188,14 +239,20 @@ private Configuration getConfiguration() { private AggregatedLogsBlockForTest getAggregatedLogsBlockForTest( Configuration configuration, String user, String containerId) { + return getAggregatedLogsBlockForTest(configuration, user, containerId, + "localhost:1234"); + } + + private AggregatedLogsBlockForTest getAggregatedLogsBlockForTest( + Configuration configuration, String user, String containerId, + String nodeName) { HttpServletRequest request = mock(HttpServletRequest.class); when(request.getRemoteUser()).thenReturn(user); AggregatedLogsBlockForTest aggregatedBlock = new AggregatedLogsBlockForTest( configuration); aggregatedBlock.setRequest(request); aggregatedBlock.moreParams().put(YarnWebParams.CONTAINER_ID, containerId); - aggregatedBlock.moreParams().put(YarnWebParams.NM_NODENAME, - "localhost:1234"); + aggregatedBlock.moreParams().put(YarnWebParams.NM_NODENAME, nodeName); aggregatedBlock.moreParams().put(YarnWebParams.APP_OWNER, user); aggregatedBlock.moreParams().put("start", ""); aggregatedBlock.moreParams().put("end", ""); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_SUCCESS b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_SUCCESS new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_index b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_index new file mode 100644 index 0000000000..92ee728478 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_index @@ -0,0 +1,3 @@ +%2F dir 1440540845855+504+rkanter+supergroup 0 0 host1_1111 host2_2222 +%2Fhost1_1111 file part-0 0 394 1440540845834+420+rkanter+supergroup +%2Fhost2_2222 file part-0 394 400 1440540845854+420+rkanter+supergroup diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_masterindex b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_masterindex new file mode 100644 index 0000000000..086d2b852e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/_masterindex @@ -0,0 +1,2 @@ +3 +0 1520266628 0 214 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/part-0 b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/part-0 new file mode 100644 index 0000000000..fef262a815 Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/application_1440536969523_0001.har/part-0 differ