diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java index 3a5f919b78..1f65fed913 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/SetReplication.java @@ -39,11 +39,14 @@ class SetReplication extends FsCommand { } public static final String NAME = "setrep"; - public static final String USAGE = "[-R] [-w] ..."; + public static final String USAGE = "[-R] [-w] ..."; public static final String DESCRIPTION = - "Set the replication level of a file.\n" + - "The -R flag requests a recursive change of replication level\n" + - "for an entire tree."; + "Set the replication level of a file. If is a directory\n" + + "then the command recursively changes the replication factor of\n" + + "all files under the directory tree rooted at .\n" + + "The -w flag requests that the command wait for the replication\n" + + "to complete. This can potentially take a very long time.\n" + + "The -R flag is accepted for backwards compatibility. It has no effect."; protected short newRep = 0; protected List waitList = new LinkedList(); @@ -54,7 +57,7 @@ class SetReplication extends FsCommand { CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "R", "w"); cf.parse(args); waitOpt = cf.getOpt("w"); - setRecursive(cf.getOpt("R")); + setRecursive(true); try { newRep = Short.parseShort(args.removeFirst()); @@ -126,4 +129,4 @@ class SetReplication extends FsCommand { out.println(" done"); } } -} \ No newline at end of file +} diff --git a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm index 5c0869c0ae..78cd880a67 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm @@ -381,17 +381,22 @@ rmr setrep - Usage: << >>> + Usage: << >>> - Changes the replication factor of a file. + Changes the replication factor of a file. If is a directory then + the command recursively changes the replication factor of all files under + the directory tree rooted at . Options: - * The -R option will recursively increase the replication factor of files within a directory. + * The -w flag requests that the command wait for the replication + to complete. This can potentially take a very long time. + + * The -R flag is accepted for backwards compatibility. It has no effect. Example: - * <<>> + * <<>> Exit Code: diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index 62d94474f5..544494d01b 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -601,16 +601,28 @@ RegexpComparator - ^-setrep \[-R\] \[-w\] <rep> <path/file> \.\.\.:( |\t)*Set the replication level of a file.( )* + ^-setrep \[-R\] \[-w\] <rep> <path> \.\.\.:( |\t)*Set the replication level of a file. If <path> is a directory( )* RegexpComparator - ^( |\t)*The -R flag requests a recursive change of replication level( )* + ^( |\t)*then the command recursively changes the replication factor of( )* RegexpComparator - ^( |\t)*for an entire tree.( )* + ^( |\t)*all files under the directory tree rooted at <path>\.( )* + + RegexpComparator + ^( |\t)*The -w flag requests that the command wait for the replication( )* + + + RegexpComparator + ^( |\t)*to complete. This can potentially take a very long time\.( )* + + + RegexpComparator + ^( |\t)*The -R flag is accepted for backwards compatibility\. It has no effect\.( )* + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index 42873785f3..63bf317a5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -61,6 +61,8 @@ import static org.junit.Assert.*; public class TestDFSShell { private static final Log LOG = LogFactory.getLog(TestDFSShell.class); private static AtomicInteger counter = new AtomicInteger(); + private final int SUCCESS = 0; + private final int ERROR = 1; static final String TEST_ROOT_DIR = PathUtils.getTestDirName(TestDFSShell.class); @@ -1619,9 +1621,6 @@ public class TestDFSShell { // force Copy Option is -f @Test (timeout = 30000) public void testCopyCommandsWithForceOption() throws Exception { - final int SUCCESS = 0; - final int ERROR = 1; - Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) .format(true).build(); @@ -1682,7 +1681,55 @@ public class TestDFSShell { } cluster.shutdown(); } + } + // setrep for file and directory. + @Test (timeout = 30000) + public void testSetrep() throws Exception { + + Configuration conf = new Configuration(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) + .format(true).build(); + FsShell shell = null; + FileSystem fs = null; + + final String testdir1 = "/tmp/TestDFSShell-testSetrep-" + counter.getAndIncrement(); + final String testdir2 = testdir1 + "/nestedDir"; + final Path hdfsFile1 = new Path(testdir1, "testFileForSetrep"); + final Path hdfsFile2 = new Path(testdir2, "testFileForSetrep"); + final Short oldRepFactor = new Short((short) 1); + final Short newRepFactor = new Short((short) 3); + try { + String[] argv; + cluster.waitActive(); + fs = cluster.getFileSystem(); + assertThat(fs.mkdirs(new Path(testdir2)), is(true)); + shell = new FsShell(conf); + + fs.create(hdfsFile1, true).close(); + fs.create(hdfsFile2, true).close(); + + // Tests for setrep on a file. + argv = new String[] { "-setrep", newRepFactor.toString(), hdfsFile1.toString() }; + assertThat(shell.run(argv), is(SUCCESS)); + assertThat(fs.getFileStatus(hdfsFile1).getReplication(), is(newRepFactor)); + assertThat(fs.getFileStatus(hdfsFile2).getReplication(), is(oldRepFactor)); + + // Tests for setrep + + // Tests for setrep on a directory and make sure it is applied recursively. + argv = new String[] { "-setrep", newRepFactor.toString(), testdir1 }; + assertThat(shell.run(argv), is(SUCCESS)); + assertThat(fs.getFileStatus(hdfsFile1).getReplication(), is(newRepFactor)); + assertThat(fs.getFileStatus(hdfsFile2).getReplication(), is(newRepFactor)); + + } finally { + if (shell != null) { + shell.close(); + } + + cluster.shutdown(); + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 563d51a841..16a44ff4cf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -6049,7 +6049,7 @@ -fs NAMENODE -mkdir /dir0 -fs NAMENODE -touchz /dir0/file0 -fs NAMENODE -touchz /dir0/file1 - -fs NAMENODE -setrep -R 2 /dir0 + -fs NAMENODE -setrep 2 /dir0 -fs NAMENODE -rm -r /user @@ -6072,7 +6072,7 @@ -fs NAMENODE -mkdir -p dir0 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/file1 - -fs NAMENODE -setrep -R 2 dir0 + -fs NAMENODE -setrep 2 dir0 -fs NAMENODE -rm -r /user @@ -6089,6 +6089,24 @@ + + setrep: -R ignored for existing file + + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -touchz dir0/file0 + -fs NAMENODE -setrep -R 2 dir0/file0 + + + -fs NAMENODE -rm -r /user + + + + RegexpComparator + ^Replication 2 set: dir0/file0 + + + + setrep: non existent file (absolute path) @@ -6145,7 +6163,7 @@ -fs NAMENODE -mkdir hdfs:///dir0/ -fs NAMENODE -touchz hdfs:///dir0/file0 -fs NAMENODE -touchz hdfs:///dir0/file1 - -fs NAMENODE -setrep -R 2 hdfs:///dir0 + -fs NAMENODE -setrep 2 hdfs:///dir0 -fs NAMENODE -rm -r hdfs:///* @@ -6203,7 +6221,7 @@ -fs NAMENODE -mkdir -p NAMENODE/dir0 -fs NAMENODE -touchz NAMENODE/dir0/file0 -fs NAMENODE -touchz NAMENODE/dir0/file1 - -fs NAMENODE -setrep -R 2 NAMENODE/dir0 + -fs NAMENODE -setrep 2 NAMENODE/dir0 -fs NAMENODE -rm -r NAMENODE/*