big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-4187. Add tests for replication handling in snapshots. Contributed by Jing Zhao

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2802@1410097 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2012-11-15 23:08:25 +00:00
parent 8ca8687fb2
commit b5a2dd19c4
2 changed files with 232 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt
View File

@ -73,3 +73,6 @@ Branch-2802 Snapshot (Unreleased)
HDFS-4188. Add Snapshot.ID_COMPARATOR for comparing IDs and fix a bug in
ReadOnlyList.Util.binarySearch(..). (szetszwo)
HDFS-4187. Add tests for replication handling in snapshots. (Jing Zhao via
szetszwo)

229
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotReplication.java Normal file
View File

@ -0,0 +1,229 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode.snapshot;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
/**
* This class tests the replication handling/calculation of snapshots. In
* particular, {@link INodeFile#getFileReplication()} and
* {@link INodeFileWithLink#getBlockReplication()} are tested to make sure
* the number of replication is calculated correctly with/without snapshots.
*/
public class TestSnapshotReplication {
private static final long seed = 0;
private static final short REPLICATION = 3;
private static final int NUMDATANODE = 5;
private static final long BLOCKSIZE = 1024;
private final Path dir = new Path("/TestSnapshot");
private final Path sub1 = new Path(dir, "sub1");
private final Path file1 = new Path(sub1, "file1");
Configuration conf;
MiniDFSCluster cluster;
FSNamesystem fsn;
DistributedFileSystem hdfs;
FSDirectory fsdir;
@Before
public void setUp() throws Exception {
conf = new Configuration();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUMDATANODE)
.build();
cluster.waitActive();
fsn = cluster.getNamesystem();
hdfs = cluster.getFileSystem();
fsdir = fsn.getFSDirectory();
}
@After
public void tearDown() throws Exception {
if (cluster != null) {
cluster.shutdown();
}
}
/**
* Check the replication of a given file. We test both
* {@link INodeFile#getFileReplication()} and
* {@link INodeFile#getBlockReplication()}.
*
* @param file The given file
* @param replication The expected replication number
* @param blockReplication The expected replication number for the block
* @throws Exception
*/
private void checkFileReplication(Path file, short replication,
short blockReplication) throws Exception {
// Get FileStatus of file1, and identify the replication number of file1.
// Note that the replication number in FileStatus was derived from
// INodeFile#getFileReplication().
short fileReplication = hdfs.getFileStatus(file1).getReplication();
assertEquals(replication, fileReplication);
// Check the correctness of getBlockReplication()
INode inode = fsdir.getINode(file1.toString());
assertTrue(inode instanceof INodeFile);
assertEquals(blockReplication, ((INodeFile) inode).getBlockReplication());
}
/**
* Test replication number calculation for a normal file without snapshots.
*/
@Test
public void testReplicationWithoutSnapshot() throws Exception {
// Create file1, set its replication to REPLICATION
DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, seed);
// Check the replication of file1
checkFileReplication(file1, REPLICATION, REPLICATION);
// Change the replication factor of file1 from 3 to 2
hdfs.setReplication(file1, (short) (REPLICATION - 1));
// Check the replication again
checkFileReplication(file1, (short) (REPLICATION - 1),
(short) (REPLICATION - 1));
}
/**
* Check the replication for both the current file and all its prior snapshots
*
* @param currentFile
* the Path of the current file
* @param snapshotRepMap
* A map maintaining all the snapshots of the current file, as well
* as their expected replication number stored in their corresponding
* INodes
* @param expectedBlockRep
* The expected replication number that should be returned by
* {@link INodeFileWithLink#getBlockReplication()} of all the INodes
* @throws Exception
*/
private void checkSnapshotFileReplication(Path currentFile,
Map<Path, Short> snapshotRepMap, short expectedBlockRep) throws Exception {
// First check the getBlockReplication for the INode of the currentFile
INodeFileWithLink inodeOfCurrentFile = (INodeFileWithLink) fsdir
.getINode(currentFile.toString());
assertEquals(expectedBlockRep, inodeOfCurrentFile.getBlockReplication());
// Then check replication for every snapshot
for (Path ss : snapshotRepMap.keySet()) {
INodeFileWithLink ssInode = (INodeFileWithLink) fsdir.getINode(ss
.toString());
// The replication number derived from the
// INodeFileWithLink#getBlockReplication should always == expectedBlockRep
assertEquals(expectedBlockRep, ssInode.getBlockReplication());
// Also check the number derived from INodeFile#getFileReplication
assertEquals(snapshotRepMap.get(ss).shortValue(),
ssInode.getFileReplication());
}
}
/**
* Test replication number calculation for a file with snapshots.
*/
@Test
public void testReplicationWithSnapshot() throws Exception {
short fileRep = 1;
// Create file1, set its replication to 1
DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, fileRep, seed);
Map<Path, Short> snapshotRepMap = new HashMap<Path, Short>();
// Change replication factor from 1 to 5. In the meanwhile, keep taking
// snapshots for sub1
for (; fileRep < NUMDATANODE; ) {
// Create snapshot for sub1
Path snapshotRoot = SnapshotTestHelper.createSnapshot(hdfs, sub1, "s"
+ fileRep);
Path snapshot = new Path(snapshotRoot, file1.getName());
// Check the replication stored in the INode of the snapshot of file1
INode inode = fsdir.getINode(snapshot.toString());
assertTrue(inode instanceof INodeFileWithLink);
assertEquals(fileRep, ((INodeFileWithLink) inode).getFileReplication());
snapshotRepMap.put(snapshot, fileRep);
// Increase the replication factor by 1
hdfs.setReplication(file1, ++fileRep);
// Check the replication for file1
checkFileReplication(file1, fileRep, fileRep);
// Also check the replication for all the prior snapshots of file1
checkSnapshotFileReplication(file1, snapshotRepMap, fileRep);
}
// Change replication factor back to 3.
hdfs.setReplication(file1, REPLICATION);
// Check the replication for file1
// Currently the max replication among snapshots should be 4
checkFileReplication(file1, REPLICATION, (short) (NUMDATANODE - 1));
// Also check the replication for all the prior snapshots of file1.
// Currently the max replication among snapshots should be 4
checkSnapshotFileReplication(file1, snapshotRepMap,
(short) (NUMDATANODE - 1));
}
/**
* Test replication for a file with snapshots, also including the scenario
* where the original file is deleted
*/
@Test
public void testReplicationAfterDeletion() throws Exception {
// Create file1, set its replication to 3
DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, seed);
Map<Path, Short> snapshotRepMap = new HashMap<Path, Short>();
// Take 3 snapshots of sub1
for (int i = 1; i <= 3; i++) {
Path root = SnapshotTestHelper.createSnapshot(hdfs, sub1, "s" + i);
Path ssFile = new Path(root, file1.getName());
snapshotRepMap.put(ssFile, REPLICATION);
}
// Check replication
checkFileReplication(file1, REPLICATION, REPLICATION);
checkSnapshotFileReplication(file1, snapshotRepMap, REPLICATION);
// TODO: check replication after deleting snapshot(s)
// Delete file1
hdfs.delete(file1, true);
// Check replication of snapshots
for (Path ss : snapshotRepMap.keySet()) {
INodeFileWithLink ssInode = (INodeFileWithLink) fsdir.getINode(ss
.toString());
// The replication number derived from the
// INodeFileWithLink#getBlockReplication should always == expectedBlockRep
assertEquals(REPLICATION, ssInode.getBlockReplication());
// Also check the number derived from INodeFile#getFileReplication
assertEquals(snapshotRepMap.get(ss).shortValue(),
ssInode.getFileReplication());
}
}
}