MAPREDUCE-7201.Make Job History File Permissions configurable (#4507)

* MAPREDUCE-7201.Make Job History File Permissions configurable

Co-authored-by: Ashutosh Gupta <ashugpt@amazon.com>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java

@@ -1481,7 +1481,7 @@ protected void processDoneFiles(JobId jobId) throws IOException {
       summaryFileOut.writeUTF(mi.getJobSummary().getJobSummaryString());
       summaryFileOut.close();
       doneDirFS.setPermission(qualifiedSummaryDoneFile, new FsPermission(
-          JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
+          JobHistoryUtils.getConfiguredHistoryIntermediateUserDoneDirPermissions(getConfig())));
     } catch (IOException e) {
       LOG.info("Unable to write out JobSummaryInfo to ["
           + qualifiedSummaryDoneFile + "]", e);
@@ -1738,8 +1738,9 @@ protected boolean moveToDoneNow(Path fromPath, Path toPath)
       boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath,
           false, getConfig());
 
-      doneDirFS.setPermission(toPath, new FsPermission(
+      doneDirFS.setPermission(toPath, new FsPermission(JobHistoryUtils.
-          JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
+          getConfiguredHistoryIntermediateUserDoneDirPermissions(
+          getConfig())));
       if (copied) {
         LOG.info("Copied from: " + fromPath.toString()
             + " to done location: " + toPath.toString());

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.mapreduce.jobhistory;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
@@ -43,6 +44,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.mapreduce.CounterGroup;
@@ -1036,6 +1038,48 @@ public void testDontSetTrackingURLIfHistoryWriteThrows() throws Exception {
       jheh.stop();
     }
   }
+
+  @Test(timeout = 50000)
+  public void testJobHistoryFilePermissions() throws Exception {
+    TestParams t = new TestParams(true);
+    Configuration conf = new Configuration();
+    String setFilePermission = "777";
+    conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS, setFilePermission);
+
+    conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, dfsCluster.getURI().toString());
+
+    JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext,
+        0, false);
+    JHEvenHandlerForTest jheh = spy(realJheh);
+    jheh.init(conf);
+
+    try {
+      jheh.start();
+      handleEvent(jheh, new JobHistoryEvent(t.jobId,
+          new AMStartedEvent(t.appAttemptId, 200, t.containerId, "nmhost",
+              3000, 4000, -1)));
+
+      // Job finishes and successfully writes history
+      handleEvent(jheh, new JobHistoryEvent(t.jobId,
+          new JobFinishedEvent(TypeConverter.fromYarn(t.jobId), 0, 0,
+              0, 0, 0, 0, 0,
+              new Counters(),
+              new Counters(), new Counters())));
+
+      verify(jheh, times(1)).processDoneFiles(any(JobId.class));
+
+      String intermediateSummaryFileName = JobHistoryUtils.getIntermediateSummaryFileName(t.jobId);
+      String doneDir = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
+      FileSystem fs = FileSystem.get(dfsCluster.getConfiguration(0));
+      Path intermediateSummaryFileNamePath = new Path(doneDir, intermediateSummaryFileName);
+      FsPermission getIntermediateSummaryFilePermission =
+          fs.getFileStatus(intermediateSummaryFileNamePath).getPermission();
+      assertEquals(setFilePermission,
+          String.valueOf(getIntermediateSummaryFilePermission.toOctal()));
+    } finally {
+      jheh.stop();
+    }
+  }
 }
 
 class JHEvenHandlerForTest extends JobHistoryEventHandler {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java

@@ -91,9 +91,6 @@ public class JobHistoryUtils {
   public static final FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS = 
     FsPermission.createImmutable((short) 01777);
 
-  public static final FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS = 
-    FsPermission.createImmutable((short) 0770); // rwx------
-  
   /**
    * Suffix for configuration files.
    */
@@ -206,7 +203,7 @@ public static String getConfiguredHistoryIntermediateDoneDirPrefix(
 
   /**
    * Gets the configured directory permissions for the user directories in the
-   * directory of the intermediate done history files. The user and the group
+   * Gets the configured permissions for the user directories and files in the
    * both need full permissions, this is enforced by this method.
    * @param conf The configuration object
    * @return FsPermission of the user directories