big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-5427. Not able to read deleted files from snapshot directly under snapshottable dir after checkpoint and NN restart. Contributed by Vinay.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1538875 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jing Zhao 2013-11-05 06:38:50 +00:00
parent c47b3d7502
commit a6250a4943
4 changed files with 80 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -495,6 +495,9 @@ Release 2.3.0 - UNRELEASED
HDFS-5257. addBlock() retry should return LocatedBlock with locations else client
will get AIOBE. (Vinay via jing9)
HDFS-5427. Not able to read deleted files from snapshot directly under
snapshottable dir after checkpoint and NN restart. (Vinay via jing9)
Release 2.2.1 - UNRELEASED
INCOMPATIBLE CHANGES

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
View File

@ -588,8 +588,12 @@ private void addToParent(INodeDirectory parent, INode child) {
namesystem.dir.cacheName(child);
if (child.isFile()) {
updateBlocksMap(child.asFile());
}
}
public void updateBlocksMap(INodeFile file) {
// Add file->block mapping
final INodeFile file = child.asFile();
final BlockInfo[] blocks = file.getBlocks();
if (blocks != null) {
final BlockManager bm = namesystem.getBlockManager();
@ -598,7 +602,6 @@ private void addToParent(INodeDirectory parent, INode child) {
}
}
}
}
/** @return The FSDirectory of the namesystem where the fsimage is loaded */
public FSDirectory getFSDirectoryInLoading() {

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotFSImageFormat.java
View File

@ -203,6 +203,9 @@ private static List<INode> loadDeletedList(INodeDirectoryWithSnapshot parent,
// useful, but set the parent here to be consistent with the original
// fsdir tree.
deleted.setParent(parent);
if (deleted.isFile()) {
loader.updateBlocksMap(deleted.asFile());
}
}
return deletedList;
}

69
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
View File

@ -21,6 +21,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
@ -36,6 +37,8 @@
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -205,4 +208,70 @@ public void testDeletionWithSnapshots() throws Exception {
assertExceptionContains("File does not exist: " + s1f0, e);
}
}
/*
* Try to read the files inside snapshot but deleted in original place after
* restarting post checkpoint. refer HDFS-5427
*/
@Test(timeout = 30000)
public void testReadSnapshotFileWithCheckpoint() throws Exception {
Path foo = new Path("/foo");
hdfs.mkdirs(foo);
hdfs.allowSnapshot(foo);
Path bar = new Path("/foo/bar");
DFSTestUtil.createFile(hdfs, bar, 100, (short) 2, 100024L);
hdfs.createSnapshot(foo, "s1");
assertTrue(hdfs.delete(bar, true));
// checkpoint
NameNode nameNode = cluster.getNameNode();
NameNodeAdapter.enterSafeMode(nameNode, false);
NameNodeAdapter.saveNamespace(nameNode);
NameNodeAdapter.leaveSafeMode(nameNode);
// restart namenode to load snapshot files from fsimage
cluster.restartNameNode(true);
String snapshotPath = Snapshot.getSnapshotPath(foo.toString(), "s1/bar");
DFSTestUtil.readFile(hdfs, new Path(snapshotPath));
}
/*
* Try to read the files inside snapshot but renamed to different file and
* deleted after restarting post checkpoint. refer HDFS-5427
*/
@Test(timeout = 30000)
public void testReadRenamedSnapshotFileWithCheckpoint() throws Exception {
final Path foo = new Path("/foo");
final Path foo2 = new Path("/foo2");
hdfs.mkdirs(foo);
hdfs.mkdirs(foo2);
hdfs.allowSnapshot(foo);
hdfs.allowSnapshot(foo2);
final Path bar = new Path(foo, "bar");
final Path bar2 = new Path(foo2, "bar");
DFSTestUtil.createFile(hdfs, bar, 100, (short) 2, 100024L);
hdfs.createSnapshot(foo, "s1");
// rename to another snapshottable directory and take snapshot
assertTrue(hdfs.rename(bar, bar2));
hdfs.createSnapshot(foo2, "s2");
// delete the original renamed file to make sure blocks are not updated by
// the original file
assertTrue(hdfs.delete(bar2, true));
// checkpoint
NameNode nameNode = cluster.getNameNode();
NameNodeAdapter.enterSafeMode(nameNode, false);
NameNodeAdapter.saveNamespace(nameNode);
NameNodeAdapter.leaveSafeMode(nameNode);
// restart namenode to load snapshot files from fsimage
cluster.restartNameNode(true);
// file in first snapshot
String barSnapshotPath = Snapshot.getSnapshotPath(foo.toString(), "s1/bar");
DFSTestUtil.readFile(hdfs, new Path(barSnapshotPath));
// file in second snapshot after rename+delete
String bar2SnapshotPath = Snapshot.getSnapshotPath(foo2.toString(),
"s2/bar");
DFSTestUtil.readFile(hdfs, new Path(bar2SnapshotPath));
}
}