diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index df1c6b7dd0..6683fc75c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -400,7 +400,7 @@ public ContainerReport getContainerReport(String containerIdStr) String logMessage = "Hello container_0_0001_01_000003 in stdout!"; int fileContentSize = logMessage.getBytes().length; - int tailContentSize = "End of LogType:syslog\n\n".getBytes().length; + int tailContentSize = "\nEnd of LogType:syslog\n\n".getBytes().length; // specify how many bytes we should get from logs // specify a position number, it would get the first n bytes from diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 2fb18ed03d..7508dd5bb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -787,20 +787,19 @@ private static void readContainerLogs(DataInputStream valueStream, long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { toSkip = fileLength - absBytes; totalBytesToRead = absBytes; } - long skippedBytes = valueStream.skip(toSkip); - if (skippedBytes != toSkip) { - throw new IOException("The bytes were skipped are " - + "different from the caller requested"); - } + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, toSkip); } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -818,7 +817,9 @@ private static void readContainerLogs(DataInputStream valueStream, pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } - out.println("End of LogType:" + fileType); + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, skipAfterRead); + out.println("\nEnd of LogType:" + fileType); out.println(""); } @@ -913,20 +914,19 @@ public static int readContainerLogsForALogType( long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { toSkip = fileLength - absBytes; totalBytesToRead = absBytes; } - long skippedBytes = valueStream.skip(toSkip); - if (skippedBytes != toSkip) { - throw new IOException("The bytes were skipped are " - + "different from the caller requested"); - } + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, toSkip); } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -942,7 +942,9 @@ public static int readContainerLogsForALogType( toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } - out.println("End of LogType:" + fileType); + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, skipAfterRead); + out.println("\nEnd of LogType:" + fileType); out.println(""); return 0; } else { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java index 21e1655cce..45dd8abca1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java @@ -258,7 +258,7 @@ private void testReadAcontainerLog(boolean logUploadedTime) throws Exception { .currentTimeMillis())).length() : 0) + ("\nLogLength:" + numChars).length() + "\nLog Contents:\n".length() + numChars + "\n".length() - + "End of LogType:stdout\n".length(); + + "\nEnd of LogType:stdout\n".length(); Assert.assertTrue("LogType not matched", s.contains("LogType:stdout")); Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr")); Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java index 692b1720c8..b56fae971e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java @@ -404,20 +404,19 @@ public void write(OutputStream os) throws IOException, long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { toSkip = fileLength - absBytes; totalBytesToRead = absBytes; } - long skippedBytes = valueStream.skip(toSkip); - if (skippedBytes != toSkip) { - throw new IOException("The bytes were skipped are " - + "different from the caller requested"); - } + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, toSkip); } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -435,6 +434,8 @@ public void write(OutputStream os) throws IOException, : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } + org.apache.hadoop.io.IOUtils.skipFully( + valueStream, skipAfterRead); sb = new StringBuilder(); sb.append("\nEnd of LogType:" + fileType + "\n"); b = sb.toString().getBytes(Charset.forName("UTF-8")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java index 3a30392ee5..4e38e906b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java @@ -264,20 +264,18 @@ public void write(OutputStream os) throws IOException, byte[] buf = new byte[bufferSize]; long toSkip = 0; long totalBytesToRead = fileLength; + long skipAfterRead = 0; if (bytes < 0) { long absBytes = Math.abs(bytes); if (absBytes < fileLength) { toSkip = fileLength - absBytes; totalBytesToRead = absBytes; } - long skippedBytes = fis.skip(toSkip); - if (skippedBytes != toSkip) { - throw new IOException("The bytes were skipped are different " - + "from the caller requested"); - } + org.apache.hadoop.io.IOUtils.skipFully(fis, toSkip); } else { if (bytes < fileLength) { totalBytesToRead = bytes; + skipAfterRead = fileLength - bytes; } } @@ -295,6 +293,7 @@ public void write(OutputStream os) throws IOException, : (int) pendingRead; len = fis.read(buf, 0, toRead); } + org.apache.hadoop.io.IOUtils.skipFully(fis, skipAfterRead); os.flush(); } finally { IOUtils.closeQuietly(fis); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index c98e3664bd..1edb841f11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -948,7 +948,7 @@ private LogFileStatusInLastCycle verifyContainerLogs( Assert.assertEquals(numOfLogsPerContainer, thisContainerMap.size()); for (String fileType : logFiles) { String expectedValue = - containerStr + " Hello " + fileType + "!End of LogType:" + containerStr + " Hello " + fileType + "!\nEnd of LogType:" + fileType; LOG.info("Expected log-content : " + new String(expectedValue)); String foundValue = thisContainerMap.remove(fileType);