From d6bd920bba0d7cb77ca76c3a79d1ba1e039da9e5 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 5 Feb 2014 06:48:00 +0000 Subject: [PATCH] HDFS-5709. Improve NameNode upgrade with existing reserved paths and path components. Contributed by Andrew Wang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1564645 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../java/org/apache/hadoop/hdfs/DFSUtil.java | 61 +++++- .../hadoop/hdfs/protocol/HdfsConstants.java | 13 +- .../server/common/HdfsServerConstants.java | 3 +- .../hdfs/server/namenode/FSEditLogLoader.java | 144 ++++++++------ .../hdfs/server/namenode/FSImageFormat.java | 179 +++++++++++++++++- .../hadoop/hdfs/server/namenode/NameNode.java | 39 +++- .../src/site/apt/HdfsUserGuide.apt.vm | 29 ++- .../src/site/xdoc/HdfsSnapshots.xml | 21 +- .../hadoop/hdfs/TestDFSUpgradeFromImage.java | 85 +++++++++ .../namenode/TestNameNodeOptionParsing.java | 88 +++++++++ .../src/test/resources/hadoop-2-reserved.tgz | Bin 0 -> 2838 bytes 12 files changed, 584 insertions(+), 81 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-2-reserved.tgz diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0b68cd35a0..5b39abfeea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -334,6 +334,9 @@ Release 2.4.0 - UNRELEASED HDFS-5791. TestHttpsFileSystem should use a random port to avoid binding error during testing (Haohui Mai via brandonli) + HDFS-5709. Improve NameNode upgrade with existing reserved paths and path + components. (Andrew Wang via atm) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index b58f2732d0..9274f505a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -261,6 +261,47 @@ public class DFSUtil { return true; } + /** + * Checks if a string is a valid path component. For instance, components + * cannot contain a ":" or "/", and cannot be equal to a reserved component + * like ".snapshot". + *

+ * The primary use of this method is for validating paths when loading the + * FSImage. During normal NN operation, paths are sometimes allowed to + * contain reserved components. + * + * @return If component is valid + */ + public static boolean isValidNameForComponent(String component) { + if (component.equals(".") || + component.equals("..") || + component.indexOf(":") >= 0 || + component.indexOf("/") >= 0) { + return false; + } + return !isReservedPathComponent(component); + } + + + /** + * Returns if the component is reserved. + * + *

+ * Note that some components are only reserved under certain directories, e.g. + * "/.reserved" is reserved, while "/hadoop/.reserved" is not. + * + * @param component + * @return if the component is reserved + */ + public static boolean isReservedPathComponent(String component) { + for (String reserved : HdfsConstants.RESERVED_PATH_COMPONENTS) { + if (component.equals(reserved)) { + return true; + } + } + return false; + } + /** * Converts a byte array to a string using UTF8 encoding. */ @@ -312,7 +353,25 @@ public class DFSUtil { } return result.toString(); } - + + /** + * Converts a list of path components into a path using Path.SEPARATOR. + * + * @param components Path components + * @return Combined path as a UTF-8 string + */ + public static String strings2PathString(String[] components) { + if (components.length == 0) { + return ""; + } + if (components.length == 1) { + if (components[0] == null || components[0].isEmpty()) { + return Path.SEPARATOR; + } + } + return Joiner.on(Path.SEPARATOR).join(components); + } + /** * Given a list of path components returns a byte array */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index da934c3699..d1c7e143c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.server.namenode.FSDirectory; /************************************ * Some handy constants @@ -108,7 +109,17 @@ public class HdfsConstants { */ public static final int LAYOUT_VERSION = LayoutVersion .getCurrentLayoutVersion(); - + + /** + * Path components that are reserved in HDFS. + *

+ * .reserved is only reserved under root ("/"). + */ + public static final String[] RESERVED_PATH_COMPONENTS = new String[] { + HdfsConstants.DOT_SNAPSHOT_DIR, + FSDirectory.DOT_RESERVED_STRING + }; + /** * A special path component contained in the path for a snapshot file/dir */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index 50f6e73040..77fe7c6dd0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -59,7 +59,8 @@ public final class HdfsServerConstants { INITIALIZESHAREDEDITS("-initializeSharedEdits"), RECOVER ("-recover"), FORCE("-force"), - NONINTERACTIVE("-nonInteractive"); + NONINTERACTIVE("-nonInteractive"), + RENAMERESERVED("-renameReserved"); private final String name; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index ce2cf2f7be..7433b6b109 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import static org.apache.hadoop.hdfs.server.namenode.FSImageFormat.renameReservedPathsOnUpgrade; import static org.apache.hadoop.util.Time.now; import java.io.FilterInputStream; @@ -292,8 +293,10 @@ public class FSEditLogLoader { switch (op.opCode) { case OP_ADD: { AddCloseOp addCloseOp = (AddCloseOp)op; + final String path = + renameReservedPathsOnUpgrade(addCloseOp.path, logVersion); if (FSNamesystem.LOG.isDebugEnabled()) { - FSNamesystem.LOG.debug(op.opCode + ": " + addCloseOp.path + + FSNamesystem.LOG.debug(op.opCode + ": " + path + " numblocks : " + addCloseOp.blocks.length + " clientHolder " + addCloseOp.clientName + " clientMachine " + addCloseOp.clientMachine); @@ -304,9 +307,9 @@ public class FSEditLogLoader { // 3. OP_ADD to open file for append // See if the file already exists (persistBlocks call) - final INodesInPath iip = fsDir.getLastINodeInPath(addCloseOp.path); + final INodesInPath iip = fsDir.getLastINodeInPath(path); final INodeFile oldFile = INodeFile.valueOf( - iip.getINode(0), addCloseOp.path, true); + iip.getINode(0), path, true); INodeFile newFile = oldFile; if (oldFile == null) { // this is OP_ADD on a new file (case 1) // versions > 0 support per file replication @@ -319,10 +322,10 @@ public class FSEditLogLoader { inodeId = getAndUpdateLastInodeId(addCloseOp.inodeId, logVersion, lastInodeId); newFile = fsDir.unprotectedAddFile(inodeId, - addCloseOp.path, addCloseOp.permissions, replication, + path, addCloseOp.permissions, replication, addCloseOp.mtime, addCloseOp.atime, addCloseOp.blockSize, true, addCloseOp.clientName, addCloseOp.clientMachine); - fsNamesys.leaseManager.addLease(addCloseOp.clientName, addCloseOp.path); + fsNamesys.leaseManager.addLease(addCloseOp.clientName, path); // add the op into retry cache if necessary if (toAddRetryCache) { @@ -338,11 +341,11 @@ public class FSEditLogLoader { FSNamesystem.LOG.debug("Reopening an already-closed file " + "for append"); } - LocatedBlock lb = fsNamesys.prepareFileForWrite(addCloseOp.path, + LocatedBlock lb = fsNamesys.prepareFileForWrite(path, oldFile, addCloseOp.clientName, addCloseOp.clientMachine, null, false, iip.getLatestSnapshotId(), false); - newFile = INodeFile.valueOf(fsDir.getINode(addCloseOp.path), - addCloseOp.path, true); + newFile = INodeFile.valueOf(fsDir.getINode(path), + path, true); // add the op into retry cache is necessary if (toAddRetryCache) { @@ -363,16 +366,17 @@ public class FSEditLogLoader { } case OP_CLOSE: { AddCloseOp addCloseOp = (AddCloseOp)op; - + final String path = + renameReservedPathsOnUpgrade(addCloseOp.path, logVersion); if (FSNamesystem.LOG.isDebugEnabled()) { - FSNamesystem.LOG.debug(op.opCode + ": " + addCloseOp.path + + FSNamesystem.LOG.debug(op.opCode + ": " + path + " numblocks : " + addCloseOp.blocks.length + " clientHolder " + addCloseOp.clientName + " clientMachine " + addCloseOp.clientMachine); } - final INodesInPath iip = fsDir.getLastINodeInPath(addCloseOp.path); - final INodeFile file = INodeFile.valueOf(iip.getINode(0), addCloseOp.path); + final INodesInPath iip = fsDir.getLastINodeInPath(path); + final INodeFile file = INodeFile.valueOf(iip.getINode(0), path); // Update the salient file attributes. file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID); @@ -386,24 +390,26 @@ public class FSEditLogLoader { // could show up twice in a row. But after that version, this // should be fixed, so we should treat it as an error. throw new IOException( - "File is not under construction: " + addCloseOp.path); + "File is not under construction: " + path); } // One might expect that you could use removeLease(holder, path) here, // but OP_CLOSE doesn't serialize the holder. So, remove by path. if (file.isUnderConstruction()) { - fsNamesys.leaseManager.removeLeaseWithPrefixPath(addCloseOp.path); + fsNamesys.leaseManager.removeLeaseWithPrefixPath(path); file.toCompleteFile(file.getModificationTime()); } break; } case OP_UPDATE_BLOCKS: { UpdateBlocksOp updateOp = (UpdateBlocksOp)op; + final String path = + renameReservedPathsOnUpgrade(updateOp.path, logVersion); if (FSNamesystem.LOG.isDebugEnabled()) { - FSNamesystem.LOG.debug(op.opCode + ": " + updateOp.path + + FSNamesystem.LOG.debug(op.opCode + ": " + path + " numblocks : " + updateOp.blocks.length); } - INodeFile oldFile = INodeFile.valueOf(fsDir.getINode(updateOp.path), - updateOp.path); + INodeFile oldFile = INodeFile.valueOf(fsDir.getINode(path), + path); // Update in-memory data structures updateBlocks(fsDir, updateOp, oldFile); @@ -414,7 +420,7 @@ public class FSEditLogLoader { } case OP_ADD_BLOCK: { AddBlockOp addBlockOp = (AddBlockOp) op; - String path = addBlockOp.getPath(); + String path = renameReservedPathsOnUpgrade(addBlockOp.getPath(), logVersion); if (FSNamesystem.LOG.isDebugEnabled()) { FSNamesystem.LOG.debug(op.opCode + ": " + path + " new block id : " + addBlockOp.getLastBlock().getBlockId()); @@ -428,14 +434,20 @@ public class FSEditLogLoader { SetReplicationOp setReplicationOp = (SetReplicationOp)op; short replication = fsNamesys.getBlockManager().adjustReplication( setReplicationOp.replication); - fsDir.unprotectedSetReplication(setReplicationOp.path, + fsDir.unprotectedSetReplication( + renameReservedPathsOnUpgrade(setReplicationOp.path, logVersion), replication, null); break; } case OP_CONCAT_DELETE: { ConcatDeleteOp concatDeleteOp = (ConcatDeleteOp)op; - fsDir.unprotectedConcat(concatDeleteOp.trg, concatDeleteOp.srcs, - concatDeleteOp.timestamp); + String trg = renameReservedPathsOnUpgrade(concatDeleteOp.trg, logVersion); + String[] srcs = new String[concatDeleteOp.srcs.length]; + for (int i=0; i removedINodes = new ChunkedArrayList(); + final String snapshotRoot = + renameReservedPathsOnUpgrade(deleteSnapshotOp.snapshotRoot, + logVersion); fsNamesys.getSnapshotManager().deleteSnapshot( - deleteSnapshotOp.snapshotRoot, deleteSnapshotOp.snapshotName, + snapshotRoot, deleteSnapshotOp.snapshotName, collectedBlocks, removedINodes); fsNamesys.removeBlocksAndUpdateSafemodeTotal(collectedBlocks); collectedBlocks.clear(); @@ -617,8 +647,11 @@ public class FSEditLogLoader { } case OP_RENAME_SNAPSHOT: { RenameSnapshotOp renameSnapshotOp = (RenameSnapshotOp) op; + final String snapshotRoot = + renameReservedPathsOnUpgrade(renameSnapshotOp.snapshotRoot, + logVersion); fsNamesys.getSnapshotManager().renameSnapshot( - renameSnapshotOp.snapshotRoot, renameSnapshotOp.snapshotOldName, + snapshotRoot, renameSnapshotOp.snapshotOldName, renameSnapshotOp.snapshotNewName); if (toAddRetryCache) { @@ -629,14 +662,19 @@ public class FSEditLogLoader { } case OP_ALLOW_SNAPSHOT: { AllowSnapshotOp allowSnapshotOp = (AllowSnapshotOp) op; + final String snapshotRoot = + renameReservedPathsOnUpgrade(allowSnapshotOp.snapshotRoot, logVersion); fsNamesys.getSnapshotManager().setSnapshottable( - allowSnapshotOp.snapshotRoot, false); + snapshotRoot, false); break; } case OP_DISALLOW_SNAPSHOT: { DisallowSnapshotOp disallowSnapshotOp = (DisallowSnapshotOp) op; + final String snapshotRoot = + renameReservedPathsOnUpgrade(disallowSnapshotOp.snapshotRoot, + logVersion); fsNamesys.getSnapshotManager().resetSnapshottable( - disallowSnapshotOp.snapshotRoot); + snapshotRoot); break; } case OP_SET_GENSTAMP_V2: { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java index 385917e8e0..3ad258a451 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java @@ -32,12 +32,13 @@ import java.security.DigestOutputStream; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import org.apache.commons.logging.Log; -import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -45,13 +46,15 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsNotDirectoryException; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.PermissionStatus; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.LayoutFlags; import org.apache.hadoop.hdfs.protocol.LayoutVersion; import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; -import org.apache.hadoop.hdfs.protocol.LayoutFlags; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature; import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList; @@ -67,6 +70,10 @@ import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType; import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.io.Text; +import org.apache.hadoop.util.StringUtils; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; /** * Contains inner classes for reading or writing the on-disk format for @@ -405,7 +412,8 @@ public class FSImageFormat { } /** - * load fsimage files assuming only local names are stored + * load fsimage files assuming only local names are stored. Used when + * snapshots are not supported by the layout version. * * @param numFiles number of files expected to be read * @param in image input stream @@ -521,6 +529,8 @@ public class FSImageFormat { */ private int loadDirectory(DataInput in, Counter counter) throws IOException { String parentPath = FSImageSerialization.readString(in); + // Rename .snapshot paths if we're doing an upgrade + parentPath = renameReservedPathsOnUpgrade(parentPath, getLayoutVersion()); final INodeDirectory parent = INodeDirectory.valueOf( namesystem.dir.rootDir.getNode(parentPath, true), parentPath); return loadChildren(parent, in, counter); @@ -580,11 +590,9 @@ public class FSImageFormat { */ private void addToParent(INodeDirectory parent, INode child) { FSDirectory fsDir = namesystem.dir; - if (parent == fsDir.rootDir && FSDirectory.isReservedName(child)) { - throw new HadoopIllegalArgumentException("File name \"" - + child.getLocalName() + "\" is reserved. Please " - + " change the name of the existing file or directory to another " - + "name before upgrading to this release."); + if (parent == fsDir.rootDir) { + child.setLocalName(renameReservedRootComponentOnUpgrade( + child.getLocalNameBytes(), getLayoutVersion())); } // NOTE: This does not update space counts for parents if (!parent.addChild(child)) { @@ -621,7 +629,9 @@ public class FSImageFormat { public INode loadINodeWithLocalName(boolean isSnapshotINode, DataInput in, boolean updateINodeMap, Counter counter) throws IOException { - final byte[] localName = FSImageSerialization.readLocalName(in); + byte[] localName = FSImageSerialization.readLocalName(in); + localName = + renameReservedComponentOnUpgrade(localName, getLayoutVersion()); INode inode = loadINode(localName, isSnapshotINode, in, counter); if (updateINodeMap && LayoutVersion.supports(Feature.ADD_INODE_ID, getLayoutVersion())) { @@ -926,7 +936,156 @@ public class FSImageFormat { return snapshotMap.get(in.readInt()); } } - + + @VisibleForTesting + public static TreeMap renameReservedMap = + new TreeMap(); + + /** + * Use the default key-value pairs that will be used to determine how to + * rename reserved paths on upgrade. + */ + @VisibleForTesting + public static void useDefaultRenameReservedPairs() { + renameReservedMap.clear(); + for (String key: HdfsConstants.RESERVED_PATH_COMPONENTS) { + renameReservedMap.put( + key, + key + "." + LayoutVersion.getCurrentLayoutVersion() + "." + + "UPGRADE_RENAMED"); + } + } + + /** + * Set the key-value pairs that will be used to determine how to rename + * reserved paths on upgrade. + */ + @VisibleForTesting + public static void setRenameReservedPairs(String renameReserved) { + // Clear and set the default values + useDefaultRenameReservedPairs(); + // Overwrite with provided values + setRenameReservedMapInternal(renameReserved); + } + + private static void setRenameReservedMapInternal(String renameReserved) { + Collection pairs = + StringUtils.getTrimmedStringCollection(renameReserved); + for (String p : pairs) { + String[] pair = StringUtils.split(p, '/', '='); + Preconditions.checkArgument(pair.length == 2, + "Could not parse key-value pair " + p); + String key = pair[0]; + String value = pair[1]; + Preconditions.checkArgument(DFSUtil.isReservedPathComponent(key), + "Unknown reserved path " + key); + Preconditions.checkArgument(DFSUtil.isValidNameForComponent(value), + "Invalid rename path for " + key + ": " + value); + LOG.info("Will rename reserved path " + key + " to " + value); + renameReservedMap.put(key, value); + } + } + + /** + * When upgrading from an old version, the filesystem could contain paths + * that are now reserved in the new version (e.g. .snapshot). This renames + * these new reserved paths to a user-specified value to avoid collisions + * with the reserved name. + * + * @param path Old path potentially containing a reserved path + * @return New path with reserved path components renamed to user value + */ + static String renameReservedPathsOnUpgrade(String path, + final int layoutVersion) { + final String oldPath = path; + // If any known LVs aren't supported, we're doing an upgrade + if (!LayoutVersion.supports(Feature.ADD_INODE_ID, layoutVersion)) { + String[] components = INode.getPathNames(path); + // Only need to worry about the root directory + if (components.length > 1) { + components[1] = DFSUtil.bytes2String( + renameReservedRootComponentOnUpgrade( + DFSUtil.string2Bytes(components[1]), + layoutVersion)); + path = DFSUtil.strings2PathString(components); + } + } + if (!LayoutVersion.supports(Feature.SNAPSHOT, layoutVersion)) { + String[] components = INode.getPathNames(path); + // Special case the root path + if (components.length == 0) { + return path; + } + for (int i=0; i] ] | [" + StartupOption.ROLLBACK.getName() + "] | [" + StartupOption.FINALIZE.getName() + "] | [" + StartupOption.IMPORT.getName() + "] | [" @@ -1056,7 +1058,8 @@ public class NameNode implements NameNodeStatusMXBean { out.println(USAGE + "\n"); } - private static StartupOption parseArguments(String args[]) { + @VisibleForTesting + static StartupOption parseArguments(String args[]) { int argsLen = (args == null) ? 0 : args.length; StartupOption startOpt = StartupOption.REGULAR; for(int i=0; i < argsLen; i++) { @@ -1103,11 +1106,33 @@ public class NameNode implements NameNodeStatusMXBean { startOpt = StartupOption.CHECKPOINT; } else if (StartupOption.UPGRADE.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.UPGRADE; - // might be followed by two args - if (i + 2 < argsLen - && args[i + 1].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) { - i += 2; - startOpt.setClusterId(args[i]); + /* Can be followed by CLUSTERID with a required parameter or + * RENAMERESERVED with an optional parameter + */ + while (i + 1 < argsLen) { + String flag = args[i + 1]; + if (flag.equalsIgnoreCase(StartupOption.CLUSTERID.getName())) { + if (i + 2 < argsLen) { + i += 2; + startOpt.setClusterId(args[i]); + } else { + LOG.fatal("Must specify a valid cluster ID after the " + + StartupOption.CLUSTERID.getName() + " flag"); + return null; + } + } else if (flag.equalsIgnoreCase(StartupOption.RENAMERESERVED + .getName())) { + if (i + 2 < argsLen) { + FSImageFormat.setRenameReservedPairs(args[i + 2]); + i += 2; + } else { + FSImageFormat.useDefaultRenameReservedPairs(); + i += 1; + } + } else { + LOG.fatal("Unknown upgrade flag " + flag); + return null; + } } } else if (StartupOption.ROLLBACK.getName().equalsIgnoreCase(cmd)) { startOpt = StartupOption.ROLLBACK; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm index b84da5991b..9d6aeb97fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm @@ -435,7 +435,7 @@ HDFS Users Guide state it was in before the upgrade. HDFS upgrade is described in more detail in {{{http://wiki.apache.org/hadoop/Hadoop_Upgrade}Hadoop Upgrade}} Wiki page. HDFS can have one such backup at a time. Before upgrading, - administrators need to remove existing backupusing bin/hadoop dfsadmin + administrators need to remove existing backup using bin/hadoop dfsadmin <<<-finalizeUpgrade>>> command. The following briefly describes the typical upgrade procedure: @@ -459,6 +459,33 @@ HDFS Users Guide * start the cluster with rollback option. (<<>>). + When upgrading to a new version of HDFS, it is necessary to rename or + delete any paths that are reserved in the new version of HDFS. If the + NameNode encounters a reserved path during upgrade, it will print an + error like the following: + + <<< /.reserved is a reserved path and .snapshot is a + reserved path component in this version of HDFS. Please rollback and delete + or rename this path, or upgrade with the -renameReserved [key-value pairs] + option to automatically rename these paths during upgrade.>>> + + Specifying <<<-upgrade -renameReserved [optional key-value pairs]>>> causes + the NameNode to automatically rename any reserved paths found during + startup. For example, to rename all paths named <<<.snapshot>>> to + <<<.my-snapshot>>> and <<<.reserved>>> to <<<.my-reserved>>>, a user would + specify <<<-upgrade -renameReserved + .snapshot=.my-snapshot,.reserved=.my-reserved>>>. + + If no key-value pairs are specified with <<<-renameReserved>>>, the + NameNode will then suffix reserved paths with + <<<..UPGRADE_RENAMED>>>, e.g. + <<<.snapshot.-51.UPGRADE_RENAMED>>>. + + There are some caveats to this renaming process. It's recommended, + if possible, to first <<>> before upgrading. + This is because data inconsistency can result if an edit log operation + refers to the destination of an automatically renamed file. + * File Permissions and Security The file permissions are designed to be similar to file permissions on diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml index 9aecf9cd55..bd499c79c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml @@ -20,7 +20,7 @@ xsi:schemaLocation="http://maven.apache.org/XDOC/2.0 http://maven.apache.org/xsd/xdoc-2.0.xsd"> - HFDS Snapshots + HDFS Snapshots @@ -99,15 +99,22 @@

  • Copying a file from snapshot s0: hdfs dfs -cp /foo/.snapshot/s0/bar /tmp
    • -

    - Note that the name ".snapshot" is now a reserved file name in HDFS - so that users cannot create a file/directory with ".snapshot" as the name. - If ".snapshot" is used in a previous version of HDFS, it must be renamed before upgrade; - otherwise, upgrade will fail. -

    +
    + +

    + The HDFS snapshot feature introduces a new reserved path name used to + interact with snapshots: .snapshot. When upgrading from an + older version of HDFS, existing paths named .snapshot need + to first be renamed or deleted to avoid conflicting with the reserved path. + See the upgrade section in + the HDFS user guide + for more information.

    + +
    +

    diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java index c686e4ebe7..fda4e83530 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; +import java.util.ArrayList; import java.util.Iterator; import java.util.LinkedList; import java.util.TreeMap; @@ -43,7 +44,9 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.hdfs.server.namenode.FSImageFormat; import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Logger; import org.junit.Test; @@ -67,6 +70,7 @@ public class TestDFSUpgradeFromImage { private static final String HADOOP_DFS_DIR_TXT = "hadoop-dfs-dir.txt"; private static final String HADOOP22_IMAGE = "hadoop-22-dfs-dir.tgz"; private static final String HADOOP1_BBW_IMAGE = "hadoop1-bbw.tgz"; + private static final String HADOOP2_RESERVED_IMAGE = "hadoop-2-reserved.tgz"; private static class ReferenceFileInfo { String path; @@ -320,6 +324,87 @@ public class TestDFSUpgradeFromImage { assertEquals("Upgrade did not fail with bad MD5", 1, md5failures); } } + + /** + * Test upgrade from 2.0 image with a variety of .snapshot and .reserved + * paths to test renaming on upgrade + */ + @Test + public void testUpgradeFromRel2ReservedImage() throws IOException { + unpackStorage(HADOOP2_RESERVED_IMAGE); + MiniDFSCluster cluster = null; + // Try it once without setting the upgrade flag to ensure it fails + try { + cluster = + new MiniDFSCluster.Builder(new Configuration()) + .format(false) + .startupOption(StartupOption.UPGRADE) + .numDataNodes(0).build(); + } catch (IllegalArgumentException e) { + GenericTestUtils.assertExceptionContains( + "reserved path component in this version", + e); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + // Try it again with a custom rename string + try { + FSImageFormat.setRenameReservedPairs( + ".snapshot=.user-snapshot," + + ".reserved=.my-reserved"); + cluster = + new MiniDFSCluster.Builder(new Configuration()) + .format(false) + .startupOption(StartupOption.UPGRADE) + .numDataNodes(0).build(); + // Make sure the paths were renamed as expected + DistributedFileSystem dfs = cluster.getFileSystem(); + ArrayList toList = new ArrayList(); + ArrayList found = new ArrayList(); + toList.add(new Path("/")); + while (!toList.isEmpty()) { + Path p = toList.remove(0); + FileStatus[] statuses = dfs.listStatus(p); + for (FileStatus status: statuses) { + final String path = status.getPath().toUri().getPath(); + System.out.println("Found path " + path); + found.add(path); + if (status.isDirectory()) { + toList.add(status.getPath()); + } + } + } + String[] expected = new String[] { + "/edits", + "/edits/.reserved", + "/edits/.user-snapshot", + "/edits/.user-snapshot/editsdir", + "/edits/.user-snapshot/editsdir/editscontents", + "/edits/.user-snapshot/editsdir/editsdir2", + "/image", + "/image/.reserved", + "/image/.user-snapshot", + "/image/.user-snapshot/imagedir", + "/image/.user-snapshot/imagedir/imagecontents", + "/image/.user-snapshot/imagedir/imagedir2", + "/.my-reserved", + "/.my-reserved/edits-touch", + "/.my-reserved/image-touch" + }; + + for (String s: expected) { + assertTrue("Did not find expected path " + s, found.contains(s)); + } + assertEquals("Found an unexpected path while listing filesystem", + found.size(), expected.length); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } static void recoverAllLeases(DFSClient dfs, Path path) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java new file mode 100644 index 0000000000..d43eda0cef --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeOptionParsing.java @@ -0,0 +1,88 @@ +package org.apache.hadoop.hdfs.server.namenode; + +import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hdfs.protocol.LayoutVersion; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.junit.Test; + +public class TestNameNodeOptionParsing { + + @Test(timeout = 10000) + public void testUpgrade() { + StartupOption opt = null; + // UPGRADE is set, but nothing else + opt = NameNode.parseArguments(new String[] {"-upgrade"}); + assertEquals(opt, StartupOption.UPGRADE); + assertNull(opt.getClusterId()); + assertTrue(FSImageFormat.renameReservedMap.isEmpty()); + // cluster ID is set + opt = NameNode.parseArguments(new String[] { "-upgrade", "-clusterid", + "mycid" }); + assertEquals(StartupOption.UPGRADE, opt); + assertEquals("mycid", opt.getClusterId()); + assertTrue(FSImageFormat.renameReservedMap.isEmpty()); + // Everything is set + opt = NameNode.parseArguments(new String[] { "-upgrade", "-clusterid", + "mycid", "-renameReserved", + ".snapshot=.my-snapshot,.reserved=.my-reserved" }); + assertEquals(StartupOption.UPGRADE, opt); + assertEquals("mycid", opt.getClusterId()); + assertEquals(".my-snapshot", + FSImageFormat.renameReservedMap.get(".snapshot")); + assertEquals(".my-reserved", + FSImageFormat.renameReservedMap.get(".reserved")); + // Reset the map + FSImageFormat.renameReservedMap.clear(); + // Everything is set, but in a different order + opt = NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved", + ".reserved=.my-reserved,.snapshot=.my-snapshot", "-clusterid", + "mycid"}); + assertEquals(StartupOption.UPGRADE, opt); + assertEquals("mycid", opt.getClusterId()); + assertEquals(".my-snapshot", + FSImageFormat.renameReservedMap.get(".snapshot")); + assertEquals(".my-reserved", + FSImageFormat.renameReservedMap.get(".reserved")); + // Try the default renameReserved + opt = NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved"}); + assertEquals(StartupOption.UPGRADE, opt); + assertEquals( + ".snapshot." + LayoutVersion.getCurrentLayoutVersion() + + ".UPGRADE_RENAMED", + FSImageFormat.renameReservedMap.get(".snapshot")); + assertEquals( + ".reserved." + LayoutVersion.getCurrentLayoutVersion() + + ".UPGRADE_RENAMED", + FSImageFormat.renameReservedMap.get(".reserved")); + + // Try some error conditions + try { + opt = + NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved", + ".reserved=.my-reserved,.not-reserved=.my-not-reserved" }); + } catch (IllegalArgumentException e) { + assertExceptionContains("Unknown reserved path", e); + } + try { + opt = + NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved", + ".reserved=.my-reserved,.snapshot=.snapshot" }); + } catch (IllegalArgumentException e) { + assertExceptionContains("Invalid rename path", e); + } + try { + opt = + NameNode.parseArguments(new String[] { "-upgrade", "-renameReserved", + ".snapshot=.reserved" }); + } catch (IllegalArgumentException e) { + assertExceptionContains("Invalid rename path", e); + } + opt = NameNode.parseArguments(new String[] { "-upgrade", "-cid"}); + assertNull(opt); + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-2-reserved.tgz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-2-reserved.tgz new file mode 100644 index 0000000000000000000000000000000000000000..3cb2ee63f0e2624968dcd97836a608b88e1a36c2 GIT binary patch literal 2838 zcmeH}`&-h79>+bM<<2pUo|(tI!Etlt(J8e|ElM`+pw5)E(o_`FO{XAeGR@FfD{RG- z@{)-;miHT8P*hYTC3R+sN`i_4MqU9|+|i2L1|o8sJzD6$S+xhuK;10RkLUaSD(1-s5?Gay#(gf;`He z#5OH<6JZN1y}Z7Tubim%Xx45EweO5B#fwOlgicHJKwXr4i&~aZst>oVBW>=jCOuxv z&3GJMO`=@@7)_UfbyQhkNqj_>BI>g(~37=EaW!aI(3KbS9 z;=b=_d#}s#{6Ds|<{V}8$Y#h-c?>N|^X-^)WMi&E6UkX6QInvCV(*q`4~;vlmU|QZ zHaI@N-F&ckF1{_DDS2*K)DV6)2Q% zUsw<$|JC}%iZJU=#iycMXGCWB1uY&P&7$QAj<;lM@2UlumO(^QQ?rM2n?7$PuRl%! zclzkctYmqq9$*{=blfhTHjB<(4esF*S0~ko*!iI4(-`Ehzt2R%R~y0C*T~9eR$^i{ z*%Ug9c|i|Lcf}CN*8tvIyU?E-S#GAoo`f88SJRDSJyJT06TPb)gCdwjfe7Xg4D#_r zg&zA7b_*7w`fVdxgZOR{O;L4Iuou-zcoO#AOM*^0#Pi+{>D*rSGSxxA^hdu;xYPY6 zjur+WdcRr%h0psC-u%HP5!&Z^8%&liK~VTeSfg&Zo%v6RjWV>1KI(z-xT+9`iZQkA zXMT7$j;+PN3D{yQ1QGby3z-|Q7Me1k8H=3$)a;X-n1YiwIqQMW^xLYMh%al95TY71 zT`sx$z*vID8xlzbUiP#*cid)4s^1I(da#e%4PoZ;G;O%{$ZZIzX$p6}mF#JWxbhf% zYKF>(YvNsJ#KISH!Pep$KV+DzVwD*he7Fw;J`ngnBd}Z%gvlAr4+tq6|DVsZg)~8x z-;vt9`Ky7au6E{vTygr>F20N6>iZxl7G@NGpg-9*AjLNqPl7s>$2%V~avK+AjhqOt z7#KT$o>t>hi>%jkSRP!@g1j)xrEdy6FZ;`*8HXlwo11HDTJsl;77`j<2%`Ts-09`R zi286x5sCHXTtY*A>LHWujy=N3K}!WZXMJsA3vx%(8nufjtC3G(U0sd38=V_zQq@~9 z$_xvMxV)p<9_07zjbbHr$&>S%2Hk}`=nI5h);*& ze+aPA6NPG{wV9XVv^d*_{kSl2X$qpM#U7(b-GKwcZmUC&>Pg-rM zWuup1%iZ>(xG>e^7#0-wr0mskO5%}zvb!UUbgUx=xEUj_suaf|!5&w4#yH*(+7z(5 z^Xzp}KJ;vIZe`3L-ySqR`~IoG0+dKVf@w4+nCd15Oo=$oU6v$cJ#N| zbZN|c6*Q1E+LegVmp7x52<3r^!B*4|$CA92E+-7J1ZBA+m(-u1YXf+R?SPIS`lP`m z)fu6*bDgc_7a(JK@YR9nE<}X7KBzF&Keyu%C1ov_*-c0xV1ge)8;b_I6#nvs=Iw@& ztc-}X-yy2g8+}NoEt=bL!=v~)4hqU(PT};Q8s~YUUYHyCA*{~jkH(hmK7V6zx+9Te zVaH`J%r@Yj+OWmju6ufZ4A$%2>L!bZgrF+N!g}fnI>4J{E&&-HzPoxnxmsom{%cDz R(m2VWuLLw?@YMl>KLII!>~8=7 literal 0 HcmV?d00001