YARN-1145. Fixed a potential file-handle leak in the web interface for displaying aggregated logs. Contributed by Rohith Sharma.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1551326 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -249,6 +249,9 @@ Release 2.4.0 - UNRELEASED
     YARN-1505. Fixed Webapplication proxy server to not hardcode its bind
     address. (Xuan Gong via vinodkv)
 
+    YARN-1145. Fixed a potential file-handle leak in the web interface for
+    displaying aggregated logs. (Rohith Sharma via vinodkv)
+
 Release 2.3.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java

@@ -53,6 +53,7 @@
 import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.SecureIOUtils;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.file.tfile.TFile;
@@ -294,7 +295,7 @@ public void append(LogKey logKey, LogValue logValue) throws IOException {
       out.close();
     }
 
-    public void closeWriter() {
+    public void close() {
       try {
         this.writer.close();
       } catch (IOException e) {
@@ -569,9 +570,8 @@ public static void readAContainerLogsForALogType(
       out.println("");
     }
 
-    public void close() throws IOException {
+    public void close() {
-      this.scanner.close();
+      IOUtils.cleanup(LOG, scanner, reader, fsDataIStream);
-      this.fsDataIStream.close();
     }
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java

@@ -59,6 +59,8 @@ public class AggregatedLogsBlock extends HtmlBlock {
 
   @Override
   protected void render(Block html) {
+    AggregatedLogFormat.LogReader reader = null;
+    try {
       ContainerId containerId = verifyAndGetContainerId(html);
       NodeId nodeId = verifyAndGetNodeId(html);
       String appOwner = verifyAndGetAppOwner(html);
@@ -68,8 +70,8 @@ protected void render(Block html) {
         return;
       }
 
-    ApplicationId applicationId =
+      ApplicationId applicationId = containerId.getApplicationAttemptId()
-        containerId.getApplicationAttemptId().getApplicationId();
+          .getApplicationId();
       String logEntity = $(ENTITY_STRING);
       if (logEntity == null || logEntity.isEmpty()) {
         logEntity = containerId.toString();
@@ -83,24 +85,21 @@ protected void render(Block html) {
         return;
       }
 
-    Path remoteRootLogDir =
+      Path remoteRootLogDir = new Path(conf.get(
-        new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
+          YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
           YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
-    AggregatedLogFormat.LogReader reader = null;
+
       try {
-      reader =
+        reader = new AggregatedLogFormat.LogReader(conf,
-          new AggregatedLogFormat.LogReader(conf,
+            LogAggregationUtils.getRemoteNodeLogFileForApp(remoteRootLogDir,
-              LogAggregationUtils.getRemoteNodeLogFileForApp(
+                applicationId, appOwner, nodeId,
-                  remoteRootLogDir, applicationId, appOwner, nodeId,
                 LogAggregationUtils.getRemoteNodeLogDirSuffix(conf)));
       } catch (FileNotFoundException e) {
         // ACLs not available till the log file is opened.
         html.h1()
-          ._("Logs not available for "
+            ._("Logs not available for " + logEntity
-              + logEntity
                 + ". Aggregation may not be complete, "
-              + "Check back later or try the nodemanager at "
+                + "Check back later or try the nodemanager at " + nodeId)._();
-              + nodeId)._();
         return;
       } catch (IOException e) {
         html.h1()._("Error getting logs for " + logEntity)._();
@@ -127,8 +126,8 @@ protected void render(Block html) {
         callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
       }
       if (callerUGI != null
-        && !aclsManager.checkAccess(callerUGI, ApplicationAccessType.VIEW_APP,
+          && !aclsManager.checkAccess(callerUGI,
-            owner, applicationId)) {
+              ApplicationAccessType.VIEW_APP, owner, applicationId)) {
         html.h1()
             ._("User [" + remoteUser
                 + "] is not authorized to view the logs for " + logEntity)._();
@@ -137,11 +136,11 @@ protected void render(Block html) {
 
       String desiredLogType = $(CONTAINER_LOG_TYPE);
       try {
-      AggregatedLogFormat.ContainerLogsReader logReader =
+        AggregatedLogFormat.ContainerLogsReader logReader = reader
-          reader.getContainerLogsReader(containerId);
+            .getContainerLogsReader(containerId);
         if (logReader == null) {
-        html.h1()._(
+          html.h1()
-            "Logs not available for " + logEntity
+              ._("Logs not available for " + logEntity
                   + ". Could be caused by the rentention policy")._();
           return;
         }
@@ -163,6 +162,11 @@ protected void render(Block html) {
         LOG.error("Error getting logs for " + logEntity, e);
         return;
       }
+    } finally {
+      if (reader != null) {
+        reader.close();
+      }
+    }
   }
 
   private boolean readContainerLogs(Block html,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java

@@ -114,7 +114,7 @@ public void testReadAcontainerLogs1() throws Exception {
             testContainerId, ugi.getShortUserName());
 
     logWriter.append(logKey, logValue);
-    logWriter.closeWriter();
+    logWriter.close();
 
     // make sure permission are correct on the file
     FileStatus fsStatus =  fs.getFileStatus(remoteAppLogFile);
@@ -194,7 +194,7 @@ public void testContainerLogsFileAccess() throws IOException {
         ugi.getShortUserName());
     logWriter.append(logKey, logValue);
 
-    logWriter.closeWriter();
+    logWriter.close();
     
     BufferedReader in =
         new BufferedReader(new FileReader(new File(remoteAppLogFile

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java

@@ -229,7 +229,7 @@ private void writeLog(Configuration configuration, String user)
 
     writer.append(new AggregatedLogFormat.LogKey("container_0_0001_01_000001"),
         new AggregatedLogFormat.LogValue(rootLogDirs, containerId,UserGroupInformation.getCurrentUser().getShortUserName()));
-    writer.closeWriter();
+    writer.close();
   }
 
   private void writeLogs(String dirName) throws Exception {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java

@@ -178,7 +178,7 @@ private void doAppLogAggregation() {
         localAppLogDirs);
 
     if (this.writer != null) {
-      this.writer.closeWriter();
+      this.writer.close();
       LOG.info("Finished aggregate log-file for app " + this.applicationId);
     }