From 2deaca4415863fb20cee539878fd9acc3fc2fa82 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Thu, 8 Dec 2011 19:57:14 +0000 Subject: [PATCH] HADOOP-6886. LocalFileSystem Needs createNonRecursive API. Contributed by Nicolas Spiegelberg and Eli Collins git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1212073 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../apache/hadoop/fs/ChecksumFileSystem.java | 29 +++++++++++++++---- .../java/org/apache/hadoop/fs/FileSystem.java | 6 ++-- .../apache/hadoop/fs/RawLocalFileSystem.java | 22 ++++++++++++-- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 3a74ce3dca..02baf9bfad 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -164,6 +164,9 @@ Release 0.23.1 - Unreleased HADOOP-6840. Support non-recursive create() in FileSystem and SequenceFile.Writer. (jitendra and eli via eli) + HADOOP-6886. LocalFileSystem Needs createNonRecursive API. + (Nicolas Spiegelberg and eli via eli) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java index 3c2a6d7c1f..f24c3924ca 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java @@ -20,8 +20,6 @@ import java.io.*; import java.util.Arrays; -import java.util.Iterator; -import java.util.zip.CRC32; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -31,7 +29,6 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.PureJavaCrc32; -import org.apache.hadoop.util.StringUtils; /**************************************************************** * Abstract Checksumed FileSystem. @@ -389,9 +386,22 @@ protected void writeChunk(byte[] b, int offset, int len, byte[] checksum) public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { + return create(f, permission, overwrite, true, bufferSize, + replication, blockSize, progress); + } + + private FSDataOutputStream create(Path f, FsPermission permission, + boolean overwrite, boolean createParent, int bufferSize, + short replication, long blockSize, + Progressable progress) throws IOException { Path parent = f.getParent(); - if (parent != null && !mkdirs(parent)) { - throw new IOException("Mkdirs failed to create " + parent); + if (parent != null) { + if (!createParent && !exists(parent)) { + throw new FileNotFoundException("Parent directory doesn't exist: " + + parent); + } else if (!mkdirs(parent)) { + throw new IOException("Mkdirs failed to create " + parent); + } } final FSDataOutputStream out = new FSDataOutputStream( new ChecksumFSOutputSummer(this, f, overwrite, bufferSize, replication, @@ -402,6 +412,15 @@ public FSDataOutputStream create(Path f, FsPermission permission, return out; } + /** {@inheritDoc} */ + @Override + public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, + boolean overwrite, int bufferSize, short replication, long blockSize, + Progressable progress) throws IOException { + return create(f, permission, overwrite, false, bufferSize, replication, + blockSize, progress); + } + /** * Set replication for an existing file. * Implement the abstract setReplication of FileSystem diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 49814b596f..71d05f2c3b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -871,10 +871,10 @@ public FSDataOutputStream createNonRecursive(Path f, */ @Deprecated public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, - boolean overwrite, - int bufferSize, short replication, long blockSize, + boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { - throw new IOException("createNonRecursive unsupported for this filesystem"); + throw new IOException("createNonRecursive unsupported for this filesystem " + + this.getClass()); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index fe40d8bc4b..502847960d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -29,7 +29,6 @@ import java.net.URI; import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.EnumSet; import java.util.StringTokenizer; import org.apache.hadoop.classification.InterfaceAudience; @@ -238,9 +237,16 @@ public FSDataOutputStream append(Path f, int bufferSize, } /** {@inheritDoc} */ + @Override public FSDataOutputStream create(Path f, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { + return create(f, overwrite, true, bufferSize, replication, blockSize, progress); + } + + private FSDataOutputStream create(Path f, boolean overwrite, + boolean createParent, int bufferSize, short replication, long blockSize, + Progressable progress) throws IOException { if (exists(f) && !overwrite) { throw new IOException("File already exists: "+f); } @@ -263,7 +269,19 @@ public FSDataOutputStream create(Path f, FsPermission permission, setPermission(f, permission); return out; } - + + /** {@inheritDoc} */ + @Override + public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, + boolean overwrite, + int bufferSize, short replication, long blockSize, + Progressable progress) throws IOException { + FSDataOutputStream out = create(f, + overwrite, false, bufferSize, replication, blockSize, progress); + setPermission(f, permission); + return out; + } + public boolean rename(Path src, Path dst) throws IOException { if (pathToFile(src).renameTo(pathToFile(dst))) { return true;