From 7dd385098c7a3046e6b049e70669d5b726de79c9 Mon Sep 17 00:00:00 2001 From: Billie Rinaldi Date: Mon, 26 Feb 2018 14:32:46 -0800 Subject: [PATCH] =?UTF-8?q?MAPREDUCE-7010.=20Make=20Job=20History=20File?= =?UTF-8?q?=20Permissions=20configurable.=20Contributed=20by=20Gergely=20N?= =?UTF-8?q?ov=C3=A1k?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jobhistory/JobHistoryEventHandler.java | 4 +- .../v2/jobhistory/JHAdminConfig.java | 4 ++ .../v2/jobhistory/JobHistoryUtils.java | 38 +++++++++++++++---- .../v2/jobhistory/TestJobHistoryUtils.java | 24 ++++++++++++ .../src/main/resources/mapred-default.xml | 9 +++++ 5 files changed, 70 insertions(+), 9 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index ae461295d1..fd93d078ce 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -231,8 +231,8 @@ protected void serviceInit(Configuration conf) throws Exception { try { doneDirPrefixPath = FileContext.getFileContext(conf).makeQualified(new Path(userDoneDirStr)); - mkdir(doneDirFS, doneDirPrefixPath, new FsPermission( - JobHistoryUtils.HISTORY_INTERMEDIATE_USER_DIR_PERMISSIONS)); + mkdir(doneDirFS, doneDirPrefixPath, JobHistoryUtils. + getConfiguredHistoryIntermediateUserDoneDirPermissions(conf)); } catch (IOException e) { LOG.error("Error creating user intermediate history done directory: [ " + doneDirPrefixPath + "]", e); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java index 50979467f9..1cadf84bf6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java @@ -93,6 +93,10 @@ public class JHAdminConfig { **/ public static final String MR_HISTORY_INTERMEDIATE_DONE_DIR = MR_HISTORY_PREFIX + "intermediate-done-dir"; + public static final String MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS = + MR_HISTORY_PREFIX + "intermediate-user-done-dir.permissions"; + public static final short + DEFAULT_MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS = 0770; /** Size of the job list cache.*/ public static final String MR_HISTORY_JOBLIST_CACHE_SIZE = diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java index 58ac1db69b..5160ce82f8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -89,13 +90,7 @@ public class JobHistoryUtils { */ public static final FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS = FsPermission.createImmutable((short) 01777); - - /** - * Permissions for the user directory under the intermediate done directory. - */ - public static final FsPermission HISTORY_INTERMEDIATE_USER_DIR_PERMISSIONS = - FsPermission.createImmutable((short) 0770); - + public static final FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS = FsPermission.createImmutable((short) 0770); // rwx------ @@ -208,6 +203,35 @@ public static String getConfiguredHistoryIntermediateDoneDirPrefix( } return ensurePathInDefaultFileSystem(doneDirPrefix, conf); } + + /** + * Gets the configured directory permissions for the user directories in the + * directory of the intermediate done history files. The user and the group + * both need full permissions, this is enforced by this method. + * @param conf The configuration object + * @return FsPermission of the user directories + */ + public static FsPermission + getConfiguredHistoryIntermediateUserDoneDirPermissions( + Configuration conf) { + String userDoneDirPermissions = conf.get( + JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS); + if (userDoneDirPermissions == null) { + return new FsPermission( + JHAdminConfig.DEFAULT_MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS); + } + FsPermission permission = new FsPermission(userDoneDirPermissions); + if (permission.getUserAction() != FsAction.ALL || + permission.getGroupAction() != FsAction.ALL) { + permission = new FsPermission(FsAction.ALL, FsAction.ALL, + permission.getOtherAction(), permission.getStickyBit()); + LOG.warn("Unsupported permission configured in " + + JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS + + ", the user and the group permission must be 7 (rwx). " + + "The permission was set to " + permission.toString()); + } + return permission; + } /** * Gets the configured directory prefix for Done history files. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestJobHistoryUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestJobHistoryUtils.java index 6878cce7d4..cee401e179 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestJobHistoryUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestJobHistoryUtils.java @@ -24,6 +24,10 @@ import java.util.Calendar; import java.util.Collections; import java.util.List; +import java.util.Map; + +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -31,6 +35,9 @@ import org.junit.Assert; import org.junit.Test; +import static org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils.getConfiguredHistoryIntermediateUserDoneDirPermissions; + + public class TestJobHistoryUtils { final static String TEST_DIR = new File(System.getProperty("test.build.data")) @@ -140,4 +147,21 @@ private Path createPath(FileContext fc, Path root, String year, String month, fc.mkdir(path, FsPermission.getDirDefault(), true); return path; } + + @Test + public void testGetConfiguredHistoryIntermediateUserDoneDirPermissions() { + Configuration conf = new Configuration(); + Map parameters = ImmutableMap.of( + "775", new FsPermission(0775), + "123", new FsPermission(0773), + "-rwx", new FsPermission(0770) , + "+rwx", new FsPermission(0777) + ); + for (Map.Entry entry : parameters.entrySet()) { + conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_USER_DONE_DIR_PERMISSIONS, + entry.getKey()); + Assert.assertEquals(entry.getValue(), + getConfiguredHistoryIntermediateUserDoneDirPermissions(conf)); + } + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 72f509c1d8..d0e5a2dfd7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1730,6 +1730,15 @@ + + mapreduce.jobhistory.intermediate-user-done-dir.permissions + 770 + The permissions of the user directories in + ${mapreduce.jobhistory.intermediate-done-dir}. The user and the group + permission must be 7, this is enforced. + + + mapreduce.jobhistory.done-dir ${yarn.app.mapreduce.am.staging-dir}/history/done