From 48a2e2b4e279800b4371e6715e942392aa991ffe Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Mon, 30 Jul 2012 18:11:32 +0000 Subject: [PATCH] HADOOP-8550. hadoop fs -touchz automatically created parent directories (John George via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1367202 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../org/apache/hadoop/fs/shell/Touchz.java | 4 + .../org/apache/hadoop/hdfs/TestDFSShell.java | 13 ++++ .../src/test/resources/testHDFSConf.xml | 77 ++++++++++++++++++- 4 files changed, 95 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index a2f3ff80e5..d1faef370a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -862,6 +862,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8634. Ensure FileSystem#close doesn't squawk for deleteOnExit paths (daryn via bobby) + HADOOP-8550. hadoop fs -touchz automatically created parent directories + (John George via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java index 18c9aa74c2..9e0844e0f4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Touchz.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.shell.PathExceptions.PathIOException; import org.apache.hadoop.fs.shell.PathExceptions.PathIsDirectoryException; +import org.apache.hadoop.fs.shell.PathExceptions.PathNotFoundException; /** * Unix touch like commands @@ -70,6 +71,9 @@ protected void processPath(PathData item) throws IOException { @Override protected void processNonexistentPath(PathData item) throws IOException { + if (!item.parentExists()) { + throw new PathNotFoundException(item.toString()); + } touchz(item); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index ce402b1878..4387fb75a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -1108,6 +1108,19 @@ public void testDFSShell() throws IOException { } assertEquals(0, val); + args = new String[2]; + args[0] = "-touchz"; + args[1] = "/test/mkdirs/thisDirNotExists/noFileHere"; + val = -1; + try { + val = shell.run(args); + } catch (Exception e) { + System.err.println("Exception raised from DFSShell.run " + + e.getLocalizedMessage()); + } + assertEquals(1, val); + + args = new String[3]; args[0] = "-test"; args[1] = "-e"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index a611c2e3c3..40e3d4f919 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -55,6 +55,7 @@ ls: file using relative path + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -ls file1 @@ -76,6 +77,7 @@ ls: files using globbing + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -touchz file2 -fs NAMENODE -touchz file3 @@ -937,6 +939,7 @@ -fs NAMENODE -mkdir /dir0 -fs NAMENODE -mkdir /dir0/\* -fs NAMENODE -touchz /dir0/\*/file + -fs NAMENODE -mkdir /dir0/dir1/ -fs NAMENODE -touchz /dir0/dir1/file1 -fs NAMENODE -rm -r /dir0/\* -fs NAMENODE -ls -R /dir0 @@ -977,7 +980,7 @@ du: file using relative path - -fs NAMENODE -touchz test + -fs NAMENODE -mkdir -p dir -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytesZZ -fs NAMENODE -du data15bytesZZ @@ -995,6 +998,7 @@ du: files using globbing + -fs NAMENODE -mkdir -p CLITEST_DATA -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -put CLITEST_DATA/data30bytes data30bytes -fs NAMENODE -put CLITEST_DATA/data60bytes data60bytes @@ -1724,6 +1728,7 @@ mv: file (relative) to file (relative) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -mv file1 file2 -fs NAMENODE -ls file* @@ -2127,6 +2132,7 @@ cp: file (absolute path) to file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz /file1 -fs NAMENODE -cp /file1 file2 -fs NAMENODE -ls /file1 file2 @@ -2149,6 +2155,7 @@ cp: file (relative path) to file (absolute path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -cp file1 /file2 -fs NAMENODE -ls file1 /file2 @@ -2171,6 +2178,7 @@ cp: file (relative path) to file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -cp file1 file2 -fs NAMENODE -ls file1 file2 @@ -2720,6 +2728,7 @@ cp: putting file into an already existing destination with -f option(absolute path) + -fs NAMENODE -mkdir /user -fs NAMENODE -touchz /user/file0 -fs NAMENODE -cp -f CLITEST_DATA/data120bytes /user/file0 -fs NAMENODE -cat /user/file0 @@ -3160,6 +3169,7 @@ rm: removing a file (absolute path) + -fs NAMENODE -mkdir /dir0 -fs NAMENODE -touchz /dir0/file0 -fs NAMENODE -rm /dir0/file0 @@ -3177,6 +3187,7 @@ rm: removing a file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -rm file0 @@ -3194,6 +3205,7 @@ rm: removing files by globbing (absolute path) + -fs NAMENODE -mkdir /dir0 -fs NAMENODE -touchz /dir0/file0 -fs NAMENODE -touchz /dir0/file1 -fs NAMENODE -touchz /dir0/file2 @@ -3226,6 +3238,7 @@ rm: removing files by globbing (relative path) + -fs NAMENODE -mkdir dir -fs NAMENODE -touchz file0 -fs NAMENODE -touchz file1 -fs NAMENODE -touchz file2 @@ -3555,6 +3568,7 @@ rm: removing a file (absolute path) + -fs NAMENODE -mkdir /dir0 -fs NAMENODE -touchz /dir0/file0 -fs NAMENODE -rm -r /dir0/file0 @@ -3572,6 +3586,7 @@ rm: removing a file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -rm -r file0 @@ -3968,6 +3983,7 @@ put: putting file into a file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -du data15bytes @@ -4066,6 +4082,7 @@ put: putting non existent file(absolute path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put /user/wrongdata file @@ -4083,6 +4100,7 @@ put: putting non existent file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put wrongdata file @@ -4100,6 +4118,7 @@ put: putting file into an already existing destination(absolute path) + -fs NAMENODE -mkdir /user -fs NAMENODE -touchz /user/file0 -fs NAMENODE -put CLITEST_DATA/data15bytes /user/file0 @@ -4117,6 +4136,7 @@ put: putting file into an already existing destination with -f option(absolute path) + -fs NAMENODE -mkdir /user -fs NAMENODE -touchz /user/file0 -fs NAMENODE -put -f CLITEST_DATA/data120bytes /user/file0 -fs NAMENODE -cat /user/file0 @@ -4135,6 +4155,7 @@ put: putting file into an already existing destination(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -put CLITEST_DATA/data15bytes file0 @@ -4154,6 +4175,7 @@ -fs NAMENODE -put CLITEST_DATA/data15bytes /data15bytes -fs NAMENODE -put CLITEST_DATA/data30bytes /data30bytes + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -put /data15bytes /data30bytes file0 @@ -4262,6 +4284,7 @@ put: putting local file into an already existing destination in hdfs:// path + -fs NAMENODE -mkdir hdfs:///user/ -fs NAMENODE -touchz hdfs:///user/file0 -fs NAMENODE -put CLITEST_DATA/data15bytes hdfs:///user/file0 @@ -4385,6 +4408,7 @@ put: putting local file into an already existing destination in Namenode's path + -fs NAMENODE -mkdir NAMENODE/user/ -fs NAMENODE -touchz NAMENODE/user/file0 -fs NAMENODE -put CLITEST_DATA/data15bytes NAMENODE/user/file0 @@ -4453,6 +4477,7 @@ copyFromLocal: copying file into a file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -du data15bytes @@ -4551,6 +4576,7 @@ copyFromLocal: copying non existent file(absolute path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -copyFromLocal /user/wrongdata file @@ -4568,6 +4594,7 @@ copyFromLocal: copying non existent file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -copyFromLocal wrongdata file @@ -4585,6 +4612,7 @@ copyFromLocal: copying file into an already existing destination(absolute path) + -fs NAMENODE -mkdir /user -fs NAMENODE -touchz /user/file0 -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes /user/file0 @@ -4602,6 +4630,7 @@ copyFromLocal: copying file into an already existing destination with -f option(absolute path) + -fs NAMENODE -mkdir /user -fs NAMENODE -touchz /user/file0 -fs NAMENODE -copyFromLocal -f CLITEST_DATA/data120bytes /user/file0 -fs NAMENODE -cat /user/file0 @@ -4621,6 +4650,7 @@ copyFromLocal: copying file into an already existing destination(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes file0 @@ -4640,6 +4670,7 @@ -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes /data15bytes -fs NAMENODE -copyFromLocal CLITEST_DATA/data30bytes /data30bytes + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -copyFromLocal /data15bytes /data30bytes file0 @@ -4750,6 +4781,7 @@ copyFromLocal: Test for hdfs:// path - copying local file into an already existing destination + -fs NAMENODE -mkdir hdfs:///user/ -fs NAMENODE -touchz hdfs:///user/file0 -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes hdfs:///user/file0 @@ -4876,6 +4908,7 @@ copyFromLocal: Test for Namenode's path - copying local file into an already existing destination + -fs NAMENODE -mkdir NAMENODE/user/ -fs NAMENODE -touchz NAMENODE/user/file0 -fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes NAMENODE/user/file0 @@ -5012,6 +5045,7 @@ cat: contents of file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -cat data15bytes @@ -5796,6 +5830,7 @@ setrep: existent file (absolute path) + -fs NAMENODE -mkdir /dir0 -fs NAMENODE -touchz /dir0/file0 -fs NAMENODE -setrep 2 /dir0/file0 @@ -5813,6 +5848,7 @@ setrep: existent file (relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -setrep 2 file0 @@ -5830,6 +5866,7 @@ setrep: existent directory (absolute path) + -fs NAMENODE -mkdir /dir0 -fs NAMENODE -touchz /dir0/file0 -fs NAMENODE -touchz /dir0/file1 -fs NAMENODE -setrep -R 2 /dir0 @@ -5852,6 +5889,7 @@ setrep: existent directory (relative path) + -fs NAMENODE -mkdir -p dir0 -fs NAMENODE -touchz dir0/file0 -fs NAMENODE -touchz dir0/file1 -fs NAMENODE -setrep -R 2 dir0 @@ -5906,6 +5944,7 @@ setrep: Test for hdfs:// path - existent file + -fs NAMENODE -mkdir hdfs:///dir0/ -fs NAMENODE -touchz hdfs:///dir0/file0 -fs NAMENODE -setrep 2 hdfs:///dir0/file0 @@ -5923,6 +5962,7 @@ setrep: Test for hdfs:// path - existent directory + -fs NAMENODE -mkdir hdfs:///dir0/ -fs NAMENODE -touchz hdfs:///dir0/file0 -fs NAMENODE -touchz hdfs:///dir0/file1 -fs NAMENODE -setrep -R 2 hdfs:///dir0 @@ -5945,6 +5985,7 @@ setrep: Test for hdfs:// path - non existent file + -fs NAMENODE -mkdir hdfs:///dir0/ -fs NAMENODE -setrep 2 hdfs:///dir0/file @@ -5961,6 +6002,7 @@ setrep: Test for Namenode's path - existent file + -fs NAMENODE -mkdir NAMENODE/dir0/ -fs NAMENODE -touchz NAMENODE/dir0/file0 -fs NAMENODE -setrep 2 NAMENODE/dir0/file0 @@ -5978,6 +6020,7 @@ setrep: Test for Namenode's path - existent directory + -fs NAMENODE -mkdir -p NAMENODE/dir0 -fs NAMENODE -touchz NAMENODE/dir0/file0 -fs NAMENODE -touchz NAMENODE/dir0/file1 -fs NAMENODE -setrep -R 2 NAMENODE/dir0 @@ -6017,6 +6060,7 @@ touchz: touching file (absolute path) + -fs NAMENODE -mkdir /user -fs NAMENODE -touchz /user/file0 -fs NAMENODE -du /user/file0 @@ -6030,10 +6074,26 @@ - + + + touchz: touching file in non-existent directory + + -fs NAMENODE -touchz file0 + + + + + + RegexpComparator + touchz: `file0': No such file or directory + + + + touchz: touching file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -du file0 @@ -6051,6 +6111,7 @@ touchz: touching many files + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 file1 file2 -fs NAMENODE -du file* @@ -6070,6 +6131,7 @@ touchz: touching already existing file + -fs NAMENODE -mkdir -p dir -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -touchz data15bytes @@ -6087,6 +6149,7 @@ touchz: Test for hdfs:// path - touching file + -fs NAMENODE -mkdir -p hdfs:///user/ -fs NAMENODE -touchz hdfs:///user/file0 -fs NAMENODE -du hdfs:///user/file0 @@ -6340,6 +6403,7 @@ stat: statistics about file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put CLITEST_DATA/data60bytes data60bytes -fs NAMENODE -stat "%n-%b" data60bytes @@ -6430,6 +6494,7 @@ stat: statistics about files (relative path) using globbing + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -put CLITEST_DATA/data30bytes data30bytes @@ -6690,6 +6755,7 @@ tail: contents of file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -put CLITEST_DATA/data15bytes data15bytes -fs NAMENODE -tail data15bytes @@ -6968,6 +7034,7 @@ count: file using relative path + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -count file1 @@ -7145,6 +7212,7 @@ count: relative path to multiple files without globbing + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -touchz file2 -fs NAMENODE -touchz file3 @@ -7322,6 +7390,7 @@ count: file using relative path with -q option + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -count -q file1 @@ -7503,6 +7572,7 @@ count: relative path to multiple files without globbing with -q option + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -touchz file2 -fs NAMENODE -touchz file3 @@ -8462,6 +8532,7 @@ chmod: change permission(octal mode) of file in relative path + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file1 -fs NAMENODE -chmod 666 file1 -fs NAMENODE -ls file1 @@ -15988,6 +16059,7 @@ moveFromLocal: moving non existent file(relative path) + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz test -fs NAMENODE -moveFromLocal wrongdata file @@ -16007,6 +16079,7 @@ -fs NAMENODE -moveFromLocal CLITEST_DATA/data15bytes /data15bytes -fs NAMENODE -moveFromLocal CLITEST_DATA/data30bytes /data30bytes + -fs NAMENODE -mkdir -p dir -fs NAMENODE -touchz file0 -fs NAMENODE -moveFromLocal /data15bytes /data30bytes file0