diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1a976592ba..40891bfc51 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -772,6 +772,11 @@ Release 2.6.0 - UNRELEASED HDFS-7157. Using Time.now() for recording start/end time of reconfiguration tasks (Lei Xu via cmccabe) + HDFS-6664. HDFS permissions guide documentation states incorrect default + group mapping class. (Ray Chiang via aw) + + HDFS-4227. Document dfs.namenode.resource.* (Daisuke Kobayashi via aw) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating & deleting an @@ -1003,10 +1008,7 @@ Release 2.6.0 - UNRELEASED HDFS-7140. Add a tool to list all the existing block storage policies. (jing9) - HDFS-6664. HDFS permissions guide documentation states incorrect default - group mapping class. (Ray Chiang via aw) - - HDFS-4227. Document dfs.namenode.resource.* (Daisuke Kobayashi via aw) + HDFS-7167. NPE while running Mover if the given path is for a file. (jing9) Release 2.5.1 - 2014-09-05 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java index 4db0df69d3..04133bd23c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java @@ -252,14 +252,9 @@ private boolean isSnapshotPathInCurrent(String path) throws IOException { */ private boolean processNamespace() { getSnapshottableDirs(); - boolean hasRemaining = true; - try { - for (Path target : targetPaths) { - hasRemaining = processDirRecursively("", dfs.getFileInfo(target - .toUri().getPath())); - } - } catch (IOException e) { - LOG.warn("Failed to get root directory status. Ignore and continue.", e); + boolean hasRemaining = false; + for (Path target : targetPaths) { + hasRemaining |= processPath(target.toUri().getPath()); } // wait for pending move to finish and retry the failed migration hasRemaining |= Dispatcher.waitForMoveCompletion(storages.targets.values()); @@ -270,7 +265,7 @@ private boolean processNamespace() { * @return whether there is still remaing migration work for the next * round */ - private boolean processChildrenList(String fullPath) { + private boolean processPath(String fullPath) { boolean hasRemaining = false; for (byte[] lastReturnedName = HdfsFileStatus.EMPTY_NAME;;) { final DirectoryListing children; @@ -285,7 +280,7 @@ private boolean processChildrenList(String fullPath) { return hasRemaining; } for (HdfsFileStatus child : children.getPartialListing()) { - hasRemaining |= processDirRecursively(fullPath, child); + hasRemaining |= processRecursively(fullPath, child); } if (children.hasMore()) { lastReturnedName = children.getLastName(); @@ -296,8 +291,7 @@ private boolean processChildrenList(String fullPath) { } /** @return whether the migration requires next round */ - private boolean processDirRecursively(String parent, - HdfsFileStatus status) { + private boolean processRecursively(String parent, HdfsFileStatus status) { String fullPath = status.getFullName(parent); boolean hasRemaining = false; if (status.isDir()) { @@ -305,11 +299,11 @@ private boolean processDirRecursively(String parent, fullPath = fullPath + Path.SEPARATOR; } - hasRemaining = processChildrenList(fullPath); + hasRemaining = processPath(fullPath); // process snapshots if this is a snapshottable directory if (snapshottableDirs.contains(fullPath)) { final String dirSnapshot = fullPath + HdfsConstants.DOT_SNAPSHOT_DIR; - hasRemaining |= processChildrenList(dirSnapshot); + hasRemaining |= processPath(dirSnapshot); } } else if (!status.isSymlink()) { // file try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java index 0001e3b5a7..a6edd8090e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.hdfs.server.mover; -import java.io.File; import java.io.IOException; import java.net.URI; import java.util.ArrayList; @@ -27,12 +26,10 @@ import java.util.List; import java.util.Map; -import com.google.common.base.Joiner; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.ReconfigurationException; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; @@ -46,7 +43,6 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.DirectoryListing; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; @@ -514,6 +510,42 @@ static void banner(String string) { "==================================================\n\n"); } + /** + * Run Mover with arguments specifying files and directories + */ + @Test + public void testMoveSpecificPaths() throws Exception { + LOG.info("testMoveSpecificPaths"); + final Path foo = new Path("/foo"); + final Path barFile = new Path(foo, "bar"); + final Path foo2 = new Path("/foo2"); + final Path bar2File = new Path(foo2, "bar2"); + Map policyMap = Maps.newHashMap(); + policyMap.put(foo, COLD); + policyMap.put(foo2, WARM); + NamespaceScheme nsScheme = new NamespaceScheme(Arrays.asList(foo, foo2), + Arrays.asList(barFile, bar2File), BLOCK_SIZE, null, policyMap); + ClusterScheme clusterScheme = new ClusterScheme(DEFAULT_CONF, + NUM_DATANODES, REPL, genStorageTypes(NUM_DATANODES), null); + MigrationTest test = new MigrationTest(clusterScheme, nsScheme); + test.setupCluster(); + + try { + test.prepareNamespace(); + test.setStoragePolicy(); + + Map> map = Mover.Cli.getNameNodePathsToMove(test.conf, + "-p", "/foo/bar", "/foo2"); + int result = Mover.run(map, test.conf); + Assert.assertEquals(ExitStatus.SUCCESS.getExitCode(), result); + + Thread.sleep(5000); + test.verify(true); + } finally { + test.shutdownCluster(); + } + } + /** * Move an open file into archival storage */