diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 26e35325d4..d13a247dd9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -316,6 +316,9 @@ Branch-2 ( Unreleased changes ) HDFS-3517. TestStartup should bind ephemeral ports. (eli) + HDFS-3522. If a namenode is in safemode, it should throw SafeModeException + when getBlockLocations has zero locations. (Brandon Li via szetszwo) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 22013e0feb..f965ed04d2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1089,7 +1089,8 @@ void setOwner(String src, String username, String group) LocatedBlocks getBlockLocations(String clientMachine, String src, long offset, long length) throws AccessControlException, FileNotFoundException, UnresolvedLinkException, IOException { - LocatedBlocks blocks = getBlockLocations(src, offset, length, true, true); + LocatedBlocks blocks = getBlockLocations(src, offset, length, true, true, + true); if (blocks != null) { blockManager.getDatanodeManager().sortLocatedBlocks( clientMachine, blocks.getLocatedBlocks()); @@ -1103,8 +1104,8 @@ LocatedBlocks getBlockLocations(String clientMachine, String src, * @throws FileNotFoundException, UnresolvedLinkException, IOException */ LocatedBlocks getBlockLocations(String src, long offset, long length, - boolean doAccessTime, boolean needBlockToken) throws FileNotFoundException, - UnresolvedLinkException, IOException { + boolean doAccessTime, boolean needBlockToken, boolean checkSafeMode) + throws FileNotFoundException, UnresolvedLinkException, IOException { if (isPermissionEnabled) { checkPathAccess(src, FsAction.READ); } @@ -1124,6 +1125,15 @@ LocatedBlocks getBlockLocations(String src, long offset, long length, Server.getRemoteIp(), "open", src, null, null); } + if (checkSafeMode && isInSafeMode()) { + for (LocatedBlock b : ret.getLocatedBlocks()) { + // if safemode & no block locations yet then throw safemodeException + if ((b.getLocations() == null) || (b.getLocations().length == 0)) { + throw new SafeModeException("Zero blocklocations for " + src, + safeMode); + } + } + } return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java index 2100b2c0cf..dd377cab4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java @@ -277,7 +277,7 @@ void check(String parent, HdfsFileStatus file, Result res) throws IOException { // Get block locations without updating the file access time // and without block access tokens LocatedBlocks blocks = namenode.getNamesystem().getBlockLocations(path, 0, - fileLen, false, false); + fileLen, false, false, false); if (blocks == null) { // the file is deleted return; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java index 3da68643d7..d88001240e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.permission.FsPermission; @@ -31,9 +32,11 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; +import org.apache.hadoop.hdfs.server.namenode.SafeModeException; import org.apache.hadoop.test.GenericTestUtils; import static org.junit.Assert.*; @@ -372,4 +375,76 @@ public void testSafeModeUtils() throws IOException { dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); assertFalse("State was expected to be out of safemode.", dfs.isInSafeMode()); } + + @Test + public void testSafeModeWhenZeroBlockLocations() throws IOException { + + try { + Path file1 = new Path("/tmp/testManualSafeMode/file1"); + Path file2 = new Path("/tmp/testManualSafeMode/file2"); + + System.out.println("Created file1 and file2."); + + // create two files with one block each. + DFSTestUtil.createFile(fs, file1, 1000, (short)1, 0); + DFSTestUtil.createFile(fs, file2, 2000, (short)1, 0); + checkGetBlockLocationsWorks(fs, file1); + + NameNode namenode = cluster.getNameNode(); + + // manually set safemode. + dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + assertTrue("should still be in SafeMode", namenode.isInSafeMode()); + // getBlock locations should still work since block locations exists + checkGetBlockLocationsWorks(fs, file1); + dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + assertFalse("should not be in SafeMode", namenode.isInSafeMode()); + + + // Now 2nd part of the tests where there aren't block locations + cluster.shutdownDataNodes(); + cluster.shutdownNameNode(0); + + // now bring up just the NameNode. + cluster.restartNameNode(); + cluster.waitActive(); + + System.out.println("Restarted cluster with just the NameNode"); + + namenode = cluster.getNameNode(); + + assertTrue("No datanode is started. Should be in SafeMode", + namenode.isInSafeMode()); + FileStatus stat = fs.getFileStatus(file1); + try { + fs.getFileBlockLocations(stat, 0, 1000); + assertTrue("Should have got safemode exception", false); + } catch (SafeModeException e) { + // as expected + } catch (RemoteException re) { + if (!re.getClassName().equals(SafeModeException.class.getName())) + assertTrue("Should have got safemode exception", false); + } + + + dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + assertFalse("Should not be in safemode", namenode.isInSafeMode()); + checkGetBlockLocationsWorks(fs, file1); + + } finally { + if(fs != null) fs.close(); + if(cluster!= null) cluster.shutdown(); + } + } + + void checkGetBlockLocationsWorks(FileSystem fs, Path fileName) throws IOException { + FileStatus stat = fs.getFileStatus(fileName); + try { + fs.getFileBlockLocations(stat, 0, 1000); + } catch (SafeModeException e) { + assertTrue("Should have not got safemode exception", false); + } catch (RemoteException re) { + assertTrue("Should have not got safemode exception", false); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java index ab2640d872..6794591e71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java @@ -56,7 +56,7 @@ public static FSNamesystem getNamesystem(NameNode namenode) { public static LocatedBlocks getBlockLocations(NameNode namenode, String src, long offset, long length) throws IOException { return namenode.getNamesystem().getBlockLocations( - src, offset, length, false, true); + src, offset, length, false, true, true); } public static HdfsFileStatus getFileInfo(NameNode namenode, String src,