From cc0a01c50308fe599b4bdd5a80068d6244062e4f Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 30 Mar 2015 14:48:21 -0700 Subject: [PATCH] Revert "MAPREDUCE-6288. Changed permissions on JobHistory server's done directory so that user's client can load the conf files directly. Contributed by Robert Kanter." This reverts commit 5358b83167777a7108b32c9900fb0d01ca0fe961. --- hadoop-mapreduce-project/CHANGES.txt | 4 - .../v2/jobhistory/JobHistoryUtils.java | 4 +- .../mapreduce/v2/hs/HistoryFileManager.java | 31 +------- .../v2/hs/TestHistoryFileManager.java | 73 ------------------- 4 files changed, 4 insertions(+), 108 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 69ff96bd6d..b0367a7dd1 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -510,10 +510,6 @@ Release 2.7.0 - UNRELEASED MAPREDUCE-6285. ClientServiceDelegate should not retry upon AuthenticationException. (Jonathan Eagles via ozawa) - MAPREDUCE-6288. Changed permissions on JobHistory server's done directory - so that user's client can load the conf files directly. (Robert Kanter via - vinodkv) - Release 2.6.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java index 8966e4e078..e279c03ac1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java @@ -72,7 +72,7 @@ public class JobHistoryUtils { * Permissions for the history done dir and derivatives. */ public static final FsPermission HISTORY_DONE_DIR_PERMISSION = - FsPermission.createImmutable((short) 0771); + FsPermission.createImmutable((short) 0770); public static final FsPermission HISTORY_DONE_FILE_PERMISSION = FsPermission.createImmutable((short) 0770); // rwx------ @@ -81,7 +81,7 @@ public class JobHistoryUtils { * Umask for the done dir and derivatives. */ public static final FsPermission HISTORY_DONE_DIR_UMASK = FsPermission - .createImmutable((short) (0771 ^ 0777)); + .createImmutable((short) (0770 ^ 0777)); /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 53770759d0..65f8a4febf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -571,10 +571,8 @@ boolean tryCreatingHistoryDirs(boolean logWait) throws IOException { new Path(doneDirPrefix)); doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf); doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK); - FsPermission doneDirPerm = new FsPermission( - JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION); - mkdir(doneDirFc, doneDirPrefixPath, doneDirPerm); - checkHistoryDirsPermissions(doneDirFc, doneDirPrefixPath, doneDirPerm); + mkdir(doneDirFc, doneDirPrefixPath, new FsPermission( + JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION)); } catch (ConnectException ex) { if (logWait) { LOG.info("Waiting for FileSystem at " + @@ -661,31 +659,6 @@ private void mkdir(FileContext fc, Path path, FsPermission fsp) } } - private void checkHistoryDirsPermissions(FileContext fc, Path donePath, - FsPermission fsp) throws IOException { - FileStatus fsStatus = fc.getFileStatus(donePath); - if (fsStatus.getPermission().toShort() != fsp.toShort()) { - fc.setPermission(donePath, fsp); - } - List dirs = findTimestampedDirectories(); - for (FileStatus dir : dirs) { - setPermission(fc, donePath, fsp, dir); - } - } - - private void setPermission(FileContext fc, Path donePath, FsPermission fsp, - FileStatus dir) throws IOException { - FsPermission fsPerm = dir.getPermission(); - if (fsPerm.toShort() != fsp.toShort()) { - fc.setPermission(dir.getPath(), fsp); - Path parentPath = dir.getPath().getParent(); - if (!parentPath.equals(donePath)) { - FileStatus parentDir = fc.getFileStatus(parentPath); - setPermission(fc, donePath, fsp, parentDir); - } - } - } - /** * Populates index data structures. Should only be called at initialization * times. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java index 032bb72dd9..e2e943a3d6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java @@ -23,9 +23,6 @@ import java.io.FileOutputStream; import java.util.UUID; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -111,76 +108,6 @@ public void testCreateDirsWithFileSystem() throws Exception { testTryCreateHistoryDirs(dfsCluster.getConfiguration(0), true); } - @Test - public void testUpdateDirPermissions() throws Exception { - DistributedFileSystem fs = dfsCluster.getFileSystem(); - fs.setSafeMode( HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); - Assert.assertFalse(dfsCluster.getFileSystem().isInSafeMode()); - Configuration conf = dfsCluster.getConfiguration(0); - conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR, getDoneDirNameForTest()); - conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, getIntermediateDoneDirNameForTest()); - Path p1a = new Path(getDoneDirNameForTest(), "2013"); - Path p1b = new Path(p1a, "02"); - Path p1c = new Path(p1b, "15"); - Path p1d = new Path(p1c, "000000"); - Path p2a = new Path(getDoneDirNameForTest(), "2013"); - Path p2b = new Path(p2a, "03"); - Path p2c = new Path(p2b, "14"); - Path p2d = new Path(p2c, "000001"); - FsPermission oldPerms = new FsPermission((short) 0770); - fs.mkdirs(p1d); - fs.mkdirs(p2d); - fs.setPermission(p1a, oldPerms); - fs.setPermission(p1b, oldPerms); - fs.setPermission(p1c, oldPerms); - fs.setPermission(p1d, oldPerms); - fs.setPermission(p2a, oldPerms); - fs.setPermission(p2b, oldPerms); - fs.setPermission(p2c, oldPerms); - fs.setPermission(p2d, oldPerms); - Path p1File = new Path(p1d, "foo.jhist"); - Assert.assertTrue(fs.createNewFile(p1File)); - fs.setPermission(p1File, JobHistoryUtils.HISTORY_DONE_FILE_PERMISSION); - Path p2File = new Path(p2d, "bar.jhist"); - Assert.assertTrue(fs.createNewFile(p2File)); - fs.setPermission(p2File, JobHistoryUtils.HISTORY_DONE_FILE_PERMISSION); - Assert.assertEquals(oldPerms, fs.getFileStatus(p1a).getPermission()); - Assert.assertEquals(oldPerms, fs.getFileStatus(p1b).getPermission()); - Assert.assertEquals(oldPerms, fs.getFileStatus(p1c).getPermission()); - Assert.assertEquals(oldPerms, fs.getFileStatus(p1d).getPermission()); - Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_FILE_PERMISSION, - fs.getFileStatus(p1File).getPermission()); - Assert.assertEquals(oldPerms, fs.getFileStatus(p2a).getPermission()); - Assert.assertEquals(oldPerms, fs.getFileStatus(p2b).getPermission()); - Assert.assertEquals(oldPerms, fs.getFileStatus(p2c).getPermission()); - Assert.assertEquals(oldPerms, fs.getFileStatus(p2d).getPermission()); - Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_FILE_PERMISSION, - fs.getFileStatus(p2File).getPermission()); - HistoryFileManager hfm = new HistoryFileManager(); - hfm.conf = conf; - Assert.assertEquals(true, hfm.tryCreatingHistoryDirs(false)); - Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION, - fs.getFileStatus(p1a).getPermission()); - Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION, - fs.getFileStatus(p1b).getPermission()); - Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION, - fs.getFileStatus(p1c).getPermission()); - Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION, - fs.getFileStatus(p1d).getPermission()); - Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_FILE_PERMISSION, - fs.getFileStatus(p2File).getPermission()); - Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION, - fs.getFileStatus(p2a).getPermission()); - Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION, - fs.getFileStatus(p2b).getPermission()); - Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION, - fs.getFileStatus(p2c).getPermission()); - Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION, - fs.getFileStatus(p2d).getPermission()); - Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_FILE_PERMISSION, - fs.getFileStatus(p2File).getPermission()); - } - @Test public void testCreateDirsWithAdditionalFileSystem() throws Exception { dfsCluster.getFileSystem().setSafeMode(