big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Name node cannot start if the path of a file under construction contains .snapshot. (wang)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613329 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrew Wang 2014-07-24 23:42:06 +00:00
parent 8c6e172a0a
commit e171254d56
5 changed files with 151 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -908,6 +908,9 @@ Release 2.5.0 - UNRELEASED
HDFS-6422. getfattr in CLI doesn't throw exception or return non-0 return code
when xattr doesn't exist. (Charles Lamb via umamahesh)
HDFS-6696. Name node cannot start if the path of a file under
construction contains ".snapshot". (wang)
BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS
HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh)

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
View File

@ -614,6 +614,16 @@ private void loadFullNameINodes(long numFiles, DataInput in, Counter counter)
INodeDirectory parentINode = fsDir.rootDir;
for (long i = 0; i < numFiles; i++) {
pathComponents = FSImageSerialization.readPathComponents(in);
for (int j=0; j < pathComponents.length; j++) {
byte[] newComponent = renameReservedComponentOnUpgrade
(pathComponents[j], getLayoutVersion());
if (!Arrays.equals(newComponent, pathComponents[j])) {
String oldPath = DFSUtil.byteArray2PathString(pathComponents);
pathComponents[j] = newComponent;
String newPath = DFSUtil.byteArray2PathString(pathComponents);
LOG.info("Renaming reserved path " + oldPath + " to " + newPath);
}
}
final INode newNode = loadINode(
pathComponents[pathComponents.length-1], false, in, counter);
@ -926,6 +936,7 @@ LayoutVersion.Feature.ADD_INODE_ID, getLayoutVersion())) {
oldnode = namesystem.dir.getInode(cons.getId()).asFile();
inSnapshot = true;
} else {
path = renameReservedPathsOnUpgrade(path, getLayoutVersion());
final INodesInPath iip = fsDir.getLastINodeInPath(path);
oldnode = INodeFile.valueOf(iip.getINode(0), path);
}

137
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
View File

@ -70,6 +70,9 @@ public class TestDFSUpgradeFromImage {
private static final String HADOOP_DFS_DIR_TXT = "hadoop-dfs-dir.txt";
private static final String HADOOP22_IMAGE = "hadoop-22-dfs-dir.tgz";
private static final String HADOOP1_BBW_IMAGE = "hadoop1-bbw.tgz";
private static final String HADOOP1_RESERVED_IMAGE = "hadoop-1-reserved.tgz";
private static final String HADOOP023_RESERVED_IMAGE =
"hadoop-0.23-reserved.tgz";
private static final String HADOOP2_RESERVED_IMAGE = "hadoop-2-reserved.tgz";
private static class ReferenceFileInfo {
@ -325,6 +328,140 @@ public void testUpgradeFromCorruptRel22Image() throws IOException {
}
}
/**
* Test upgrade from a branch-1.2 image with reserved paths
*/
@Test
public void testUpgradeFromRel1ReservedImage() throws Exception {
unpackStorage(HADOOP1_RESERVED_IMAGE);
MiniDFSCluster cluster = null;
// Try it once without setting the upgrade flag to ensure it fails
final Configuration conf = new Configuration();
// Try it again with a custom rename string
try {
FSImageFormat.setRenameReservedPairs(
".snapshot=.user-snapshot," +
".reserved=.my-reserved");
cluster =
new MiniDFSCluster.Builder(conf)
.format(false)
.startupOption(StartupOption.UPGRADE)
.numDataNodes(0).build();
DistributedFileSystem dfs = cluster.getFileSystem();
// Make sure the paths were renamed as expected
// Also check that paths are present after a restart, checks that the
// upgraded fsimage has the same state.
final String[] expected = new String[] {
"/.my-reserved",
"/.user-snapshot",
"/.user-snapshot/.user-snapshot",
"/.user-snapshot/open",
"/dir1",
"/dir1/.user-snapshot",
"/dir2",
"/dir2/.user-snapshot",
"/user",
"/user/andrew",
"/user/andrew/.user-snapshot",
};
for (int i=0; i<2; i++) {
// Restart the second time through this loop
if (i==1) {
cluster.finalizeCluster(conf);
cluster.restartNameNode(true);
}
ArrayList<Path> toList = new ArrayList<Path>();
toList.add(new Path("/"));
ArrayList<String> found = new ArrayList<String>();
while (!toList.isEmpty()) {
Path p = toList.remove(0);
FileStatus[] statuses = dfs.listStatus(p);
for (FileStatus status: statuses) {
final String path = status.getPath().toUri().getPath();
System.out.println("Found path " + path);
found.add(path);
if (status.isDirectory()) {
toList.add(status.getPath());
}
}
}
for (String s: expected) {
assertTrue("Did not find expected path " + s, found.contains(s));
}
assertEquals("Found an unexpected path while listing filesystem",
found.size(), expected.length);
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
/**
* Test upgrade from a 0.23.11 image with reserved paths
*/
@Test
public void testUpgradeFromRel023ReservedImage() throws Exception {
unpackStorage(HADOOP023_RESERVED_IMAGE);
MiniDFSCluster cluster = null;
// Try it once without setting the upgrade flag to ensure it fails
final Configuration conf = new Configuration();
// Try it again with a custom rename string
try {
FSImageFormat.setRenameReservedPairs(
".snapshot=.user-snapshot," +
".reserved=.my-reserved");
cluster =
new MiniDFSCluster.Builder(conf)
.format(false)
.startupOption(StartupOption.UPGRADE)
.numDataNodes(0).build();
DistributedFileSystem dfs = cluster.getFileSystem();
// Make sure the paths were renamed as expected
// Also check that paths are present after a restart, checks that the
// upgraded fsimage has the same state.
final String[] expected = new String[] {
"/.user-snapshot",
"/dir1",
"/dir1/.user-snapshot",
"/dir2",
"/dir2/.user-snapshot"
};
for (int i=0; i<2; i++) {
// Restart the second time through this loop
if (i==1) {
cluster.finalizeCluster(conf);
cluster.restartNameNode(true);
}
ArrayList<Path> toList = new ArrayList<Path>();
toList.add(new Path("/"));
ArrayList<String> found = new ArrayList<String>();
while (!toList.isEmpty()) {
Path p = toList.remove(0);
FileStatus[] statuses = dfs.listStatus(p);
for (FileStatus status: statuses) {
final String path = status.getPath().toUri().getPath();
System.out.println("Found path " + path);
found.add(path);
if (status.isDirectory()) {
toList.add(status.getPath());
}
}
}
for (String s: expected) {
assertTrue("Did not find expected path " + s, found.contains(s));
}
assertEquals("Found an unexpected path while listing filesystem",
found.size(), expected.length);
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
/**
* Test upgrade from 2.0 image with a variety of .snapshot and .reserved
* paths to test renaming on upgrade

BIN
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-0.23-reserved.tgz Normal file
View File

Binary file not shown.

BIN
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-1-reserved.tgz Normal file
View File

Binary file not shown.