big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-10283. o.a.h.hdfs.server.namenode.TestFSImageWithSnapshot#testSaveLoadImageWithAppending fails intermittently. Contributed by Mingliang Liu.

Browse Source
This commit is contained in:
Jing Zhao 2016-04-15 10:49:21 -07:00
parent 4feed9b2db
commit 89a838769f
1 changed files with 22 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java
View File

@ -47,6 +47,7 @@
import org.apache.hadoop.hdfs.util.Canceler; import org.apache.hadoop.hdfs.util.Canceler;
import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level; import org.apache.log4j.Level;
import org.junit.After; import org.junit.After;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
@ -62,7 +63,7 @@ public class TestFSImageWithSnapshot {
} }
static final long seed = 0; static final long seed = 0;
static final short REPLICATION = 3; static final short NUM_DATANODES = 3;
static final int BLOCKSIZE = 1024; static final int BLOCKSIZE = 1024;
static final long txid = 1; static final long txid = 1;
@ -78,7 +79,7 @@ public class TestFSImageWithSnapshot {
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
conf = new Configuration(); conf = new Configuration();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION) cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES)
.build(); .build();
cluster.waitActive(); cluster.waitActive();
fsn = cluster.getNamesystem(); fsn = cluster.getNamesystem();
@ -177,7 +178,7 @@ public void testSnapshotOnRoot() throws Exception {
cluster.shutdown(); cluster.shutdown();
cluster = new MiniDFSCluster.Builder(conf).format(false) cluster = new MiniDFSCluster.Builder(conf).format(false)
.numDataNodes(REPLICATION).build(); .numDataNodes(NUM_DATANODES).build();
cluster.waitActive(); cluster.waitActive();
fsn = cluster.getNamesystem(); fsn = cluster.getNamesystem();
hdfs = cluster.getFileSystem(); hdfs = cluster.getFileSystem();
@ -188,7 +189,7 @@ public void testSnapshotOnRoot() throws Exception {
hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
cluster.shutdown(); cluster.shutdown();
cluster = new MiniDFSCluster.Builder(conf).format(false) cluster = new MiniDFSCluster.Builder(conf).format(false)
.numDataNodes(REPLICATION).build(); .numDataNodes(NUM_DATANODES).build();
cluster.waitActive(); cluster.waitActive();
fsn = cluster.getNamesystem(); fsn = cluster.getNamesystem();
hdfs = cluster.getFileSystem(); hdfs = cluster.getFileSystem();
@ -215,7 +216,7 @@ public void testSnapshotOnRoot() throws Exception {
hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
cluster.shutdown(); cluster.shutdown();
cluster = new MiniDFSCluster.Builder(conf).format(false) cluster = new MiniDFSCluster.Builder(conf).format(false)
.numDataNodes(REPLICATION).build(); .numDataNodes(NUM_DATANODES).build();
cluster.waitActive(); cluster.waitActive();
fsn = cluster.getNamesystem(); fsn = cluster.getNamesystem();
hdfs = cluster.getFileSystem(); hdfs = cluster.getFileSystem();
@ -248,20 +249,20 @@ public void testSaveLoadImage() throws Exception {
hdfs.createSnapshot(dir, "s" + ++s); hdfs.createSnapshot(dir, "s" + ++s);
Path sub1file1 = new Path(sub1, "sub1file1"); Path sub1file1 = new Path(sub1, "sub1file1");
Path sub1file2 = new Path(sub1, "sub1file2"); Path sub1file2 = new Path(sub1, "sub1file2");
DFSTestUtil.createFile(hdfs, sub1file1, BLOCKSIZE, REPLICATION, seed); DFSTestUtil.createFile(hdfs, sub1file1, BLOCKSIZE, (short) 1, seed);
DFSTestUtil.createFile(hdfs, sub1file2, BLOCKSIZE, REPLICATION, seed); DFSTestUtil.createFile(hdfs, sub1file2, BLOCKSIZE, (short) 1, seed);
checkImage(s); checkImage(s);
hdfs.createSnapshot(dir, "s" + ++s); hdfs.createSnapshot(dir, "s" + ++s);
Path sub2 = new Path(dir, "sub2"); Path sub2 = new Path(dir, "sub2");
Path sub2file1 = new Path(sub2, "sub2file1"); Path sub2file1 = new Path(sub2, "sub2file1");
Path sub2file2 = new Path(sub2, "sub2file2"); Path sub2file2 = new Path(sub2, "sub2file2");
DFSTestUtil.createFile(hdfs, sub2file1, BLOCKSIZE, REPLICATION, seed); DFSTestUtil.createFile(hdfs, sub2file1, BLOCKSIZE, (short) 1, seed);
DFSTestUtil.createFile(hdfs, sub2file2, BLOCKSIZE, REPLICATION, seed); DFSTestUtil.createFile(hdfs, sub2file2, BLOCKSIZE, (short) 1, seed);
checkImage(s); checkImage(s);
hdfs.createSnapshot(dir, "s" + ++s); hdfs.createSnapshot(dir, "s" + ++s);
hdfs.setReplication(sub1file1, (short) (REPLICATION - 1)); hdfs.setReplication(sub1file1, (short) 1);
hdfs.delete(sub1file2, true); hdfs.delete(sub1file2, true);
hdfs.setOwner(sub2, "dr.who", "unknown"); hdfs.setOwner(sub2, "dr.who", "unknown");
hdfs.delete(sub2file1, true); hdfs.delete(sub2file1, true);
@ -300,7 +301,7 @@ void checkImage(int s) throws IOException {
// restart the cluster, and format the cluster // restart the cluster, and format the cluster
cluster = new MiniDFSCluster.Builder(conf).format(true) cluster = new MiniDFSCluster.Builder(conf).format(true)
.numDataNodes(REPLICATION).build(); .numDataNodes(NUM_DATANODES).build();
cluster.waitActive(); cluster.waitActive();
fsn = cluster.getNamesystem(); fsn = cluster.getNamesystem();
hdfs = cluster.getFileSystem(); hdfs = cluster.getFileSystem();
@ -338,8 +339,8 @@ public void testSaveLoadImageWithAppending() throws Exception {
Path sub1 = new Path(dir, "sub1"); Path sub1 = new Path(dir, "sub1");
Path sub1file1 = new Path(sub1, "sub1file1"); Path sub1file1 = new Path(sub1, "sub1file1");
Path sub1file2 = new Path(sub1, "sub1file2"); Path sub1file2 = new Path(sub1, "sub1file2");
DFSTestUtil.createFile(hdfs, sub1file1, BLOCKSIZE, REPLICATION, seed); DFSTestUtil.createFile(hdfs, sub1file1, BLOCKSIZE, (short) 1, seed);
DFSTestUtil.createFile(hdfs, sub1file2, BLOCKSIZE, REPLICATION, seed); DFSTestUtil.createFile(hdfs, sub1file2, BLOCKSIZE, (short) 1, seed);
// 1. create snapshot s0 // 1. create snapshot s0
hdfs.allowSnapshot(dir); hdfs.allowSnapshot(dir);
@ -372,7 +373,7 @@ public void testSaveLoadImageWithAppending() throws Exception {
out.close(); out.close();
cluster.shutdown(); cluster.shutdown();
cluster = new MiniDFSCluster.Builder(conf).format(true) cluster = new MiniDFSCluster.Builder(conf).format(true)
.numDataNodes(REPLICATION).build(); .numDataNodes(NUM_DATANODES).build();
cluster.waitActive(); cluster.waitActive();
fsn = cluster.getNamesystem(); fsn = cluster.getNamesystem();
hdfs = cluster.getFileSystem(); hdfs = cluster.getFileSystem();
@ -394,8 +395,8 @@ public void testLoadImageWithAppending() throws Exception {
Path sub1 = new Path(dir, "sub1"); Path sub1 = new Path(dir, "sub1");
Path sub1file1 = new Path(sub1, "sub1file1"); Path sub1file1 = new Path(sub1, "sub1file1");
Path sub1file2 = new Path(sub1, "sub1file2"); Path sub1file2 = new Path(sub1, "sub1file2");
DFSTestUtil.createFile(hdfs, sub1file1, BLOCKSIZE, REPLICATION, seed); DFSTestUtil.createFile(hdfs, sub1file1, BLOCKSIZE, (short) 1, seed);
DFSTestUtil.createFile(hdfs, sub1file2, BLOCKSIZE, REPLICATION, seed); DFSTestUtil.createFile(hdfs, sub1file2, BLOCKSIZE, (short) 1, seed);
hdfs.allowSnapshot(dir); hdfs.allowSnapshot(dir);
hdfs.createSnapshot(dir, "s0"); hdfs.createSnapshot(dir, "s0");
@ -410,7 +411,7 @@ public void testLoadImageWithAppending() throws Exception {
cluster.shutdown(); cluster.shutdown();
cluster = new MiniDFSCluster.Builder(conf).format(false) cluster = new MiniDFSCluster.Builder(conf).format(false)
.numDataNodes(REPLICATION).build(); .numDataNodes(NUM_DATANODES).build();
cluster.waitActive(); cluster.waitActive();
fsn = cluster.getNamesystem(); fsn = cluster.getNamesystem();
hdfs = cluster.getFileSystem(); hdfs = cluster.getFileSystem();
@ -440,7 +441,7 @@ public void testLoadImageWithEmptyFile() throws Exception {
// restart cluster // restart cluster
cluster.shutdown(); cluster.shutdown();
cluster = new MiniDFSCluster.Builder(conf).format(false) cluster = new MiniDFSCluster.Builder(conf).format(false)
.numDataNodes(REPLICATION).build(); .numDataNodes(NUM_DATANODES).build();
cluster.waitActive(); cluster.waitActive();
hdfs = cluster.getFileSystem(); hdfs = cluster.getFileSystem();
@ -478,7 +479,7 @@ public void testSaveLoadImageAfterSnapshotDeletion()
Path newDir = new Path(subsubDir, "newdir"); Path newDir = new Path(subsubDir, "newdir");
Path newFile = new Path(newDir, "newfile"); Path newFile = new Path(newDir, "newfile");
hdfs.mkdirs(newDir); hdfs.mkdirs(newDir);
DFSTestUtil.createFile(hdfs, newFile, BLOCKSIZE, REPLICATION, seed); DFSTestUtil.createFile(hdfs, newFile, BLOCKSIZE, (short) 1, seed);
// create another snapshot // create another snapshot
SnapshotTestHelper.createSnapshot(hdfs, dir, "s2"); SnapshotTestHelper.createSnapshot(hdfs, dir, "s2");
@ -491,7 +492,7 @@ public void testSaveLoadImageAfterSnapshotDeletion()
// restart cluster // restart cluster
cluster.shutdown(); cluster.shutdown();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION) cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES)
.format(false).build(); .format(false).build();
cluster.waitActive(); cluster.waitActive();
fsn = cluster.getNamesystem(); fsn = cluster.getNamesystem();
@ -504,7 +505,7 @@ public void testSaveLoadImageAfterSnapshotDeletion()
cluster.shutdown(); cluster.shutdown();
cluster = new MiniDFSCluster.Builder(conf).format(false) cluster = new MiniDFSCluster.Builder(conf).format(false)
.numDataNodes(REPLICATION).build(); .numDataNodes(NUM_DATANODES).build();
cluster.waitActive(); cluster.waitActive();
fsn = cluster.getNamesystem(); fsn = cluster.getNamesystem();
hdfs = cluster.getFileSystem(); hdfs = cluster.getFileSystem();