big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-7201.Make Job History File Permissions configurable (#4507)

Browse Source 
* MAPREDUCE-7201.Make Job History File Permissions configurable

Co-authored-by: Ashutosh Gupta <ashugpt@amazon.com>
This commit is contained in:
Ashutosh Gupta 2022-07-11 07:04:52 +01:00 committed by GitHub
parent e11ba5930e
commit 4e8c0b902e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 49 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
View File

@ -1481,7 +1481,7 @@ protected void processDoneFiles(JobId jobId) throws IOException {
summaryFileOut.writeUTF(mi.getJobSummary().getJobSummaryString());
summaryFileOut.close();
doneDirFS.setPermission(qualifiedSummaryDoneFile, new FsPermission(
JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
JobHistoryUtils.getConfiguredHistoryIntermediateUserDoneDirPermissions(getConfig())));
} catch (IOException e) {
LOG.info("Unable to write out JobSummaryInfo to ["
+ qualifiedSummaryDoneFile + "]", e);
@ -1738,8 +1738,9 @@ protected boolean moveToDoneNow(Path fromPath, Path toPath)
boolean copied = FileUtil.copy(stagingDirFS, fromPath, doneDirFS, toPath,
false, getConfig());
doneDirFS.setPermission(toPath, new FsPermission(
JobHistoryUtils.HISTORY_INTERMEDIATE_FILE_PERMISSIONS));
doneDirFS.setPermission(toPath, new FsPermission(JobHistoryUtils.
getConfiguredHistoryIntermediateUserDoneDirPermissions(
getConfig())));
if (copied) {
LOG.info("Copied from: " + fromPath.toString()
+ " to done location: " + toPath.toString());

44
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.mapreduce.jobhistory;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@ -43,6 +44,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.mapreduce.CounterGroup;
@ -1036,6 +1038,48 @@ public void testDontSetTrackingURLIfHistoryWriteThrows() throws Exception {
jheh.stop();
}
}
@Test(timeout = 50000)
public void testJobHistoryFilePermissions() throws Exception {
TestParams t = new TestParams(true);
Configuration conf = new Configuration();
String setFilePermission = "777";
conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS, setFilePermission);
conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, dfsCluster.getURI().toString());
JHEvenHandlerForTest realJheh = new JHEvenHandlerForTest(t.mockAppContext,
0, false);
JHEvenHandlerForTest jheh = spy(realJheh);
jheh.init(conf);
try {
jheh.start();
handleEvent(jheh, new JobHistoryEvent(t.jobId,
new AMStartedEvent(t.appAttemptId, 200, t.containerId, "nmhost",
3000, 4000, -1)));
// Job finishes and successfully writes history
handleEvent(jheh, new JobHistoryEvent(t.jobId,
new JobFinishedEvent(TypeConverter.fromYarn(t.jobId), 0, 0,
0, 0, 0, 0, 0,
new Counters(),
new Counters(), new Counters())));
verify(jheh, times(1)).processDoneFiles(any(JobId.class));
String intermediateSummaryFileName = JobHistoryUtils.getIntermediateSummaryFileName(t.jobId);
String doneDir = JobHistoryUtils.getHistoryIntermediateDoneDirForUser(conf);
FileSystem fs = FileSystem.get(dfsCluster.getConfiguration(0));
Path intermediateSummaryFileNamePath = new Path(doneDir, intermediateSummaryFileName);
FsPermission getIntermediateSummaryFilePermission =
fs.getFileStatus(intermediateSummaryFileNamePath).getPermission();
assertEquals(setFilePermission,
String.valueOf(getIntermediateSummaryFilePermission.toOctal()));
} finally {
jheh.stop();
}
}
}
class JHEvenHandlerForTest extends JobHistoryEventHandler {

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
View File

@ -91,9 +91,6 @@ public class JobHistoryUtils {
public static final FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS =
FsPermission.createImmutable((short) 01777);
public static final FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS =
FsPermission.createImmutable((short) 0770); // rwx------
/**
* Suffix for configuration files.
*/
@ -206,7 +203,7 @@ public static String getConfiguredHistoryIntermediateDoneDirPrefix(
/**
* Gets the configured directory permissions for the user directories in the
* directory of the intermediate done history files. The user and the group
* Gets the configured permissions for the user directories and files in the
* both need full permissions, this is enforced by this method.
* @param conf The configuration object
* @return FsPermission of the user directories