From 9ad19eec6f20530c9f2b9b80ee858ac9ca02827b Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Thu, 1 Aug 2013 01:04:29 +0000 Subject: [PATCH] HADOOP-9758. Provide configuration option for FS/FC symlink resolution. (Andrew Wang via Colin Patrick McCabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1509064 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../fs/CommonConfigurationKeysPublic.java | 5 ++ .../org/apache/hadoop/fs/FSLinkResolver.java | 5 ++ .../org/apache/hadoop/fs/FileContext.java | 6 +- .../java/org/apache/hadoop/fs/FileSystem.java | 4 ++ .../hadoop/fs/FileSystemLinkResolver.java | 7 ++ .../src/main/resources/core-default.xml | 11 ++++ .../hadoop/fs/TestSymlinkHdfsDisable.java | 66 +++++++++++++++++++ 8 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f899bac4d4..ac4687a744 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -297,6 +297,9 @@ Release 2.3.0 - UNRELEASED HADOOP-9435. Support building the JNI code against the IBM JVM. (Tian Hong Wang via Colin Patrick McCabe) + HADOOP-9758. Provide configuration option for FileSystem/FileContext + symlink resolution. (Andrew Wang via Colin Patrick McCabe) + OPTIMIZATIONS HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 42bd92858c..ab30003ed3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -58,6 +58,11 @@ public class CommonConfigurationKeysPublic { public static final String FS_DU_INTERVAL_KEY = "fs.du.interval"; /** Default value for FS_DU_INTERVAL_KEY */ public static final long FS_DU_INTERVAL_DEFAULT = 600000; + /** See core-default.xml */ + public static final String FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY = + "fs.client.resolve.remote.symlinks"; + /** Default value for FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY */ + public static final boolean FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT = true; //Defaults are not specified for following keys diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java index e5718bedf8..831d4cadcf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java @@ -90,6 +90,11 @@ public T resolve(final FileContext fc, final Path path) throws IOException { in = next(fs, p); isLink = false; } catch (UnresolvedLinkException e) { + if (!fc.resolveSymlinks) { + throw new IOException("Path " + path + " contains a symlink" + + " and symlink resolution is disabled (" + + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY + ").", e); + } if (count++ > FsConstants.MAX_PATH_LINKS) { throw new IOException("Possible cyclic loop while " + "following symbolic link " + path); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index 3d7b2e50cb..7564e58183 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -215,6 +215,7 @@ public boolean accept(final Path file) { private FsPermission umask; private final Configuration conf; private final UserGroupInformation ugi; + final boolean resolveSymlinks; private FileContext(final AbstractFileSystem defFs, final FsPermission theUmask, final Configuration aConf) { @@ -240,9 +241,12 @@ private FileContext(final AbstractFileSystem defFs, if (workingDir == null) { workingDir = defaultFS.getHomeDirectory(); } + resolveSymlinks = conf.getBoolean( + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT); util = new Util(); // for the inner class } - + /* * Remove relative part - return "absolute": * If input is relative path ("foo/bar") add wd: ie "//foo/bar" diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 7eaa2c2cea..8f8bc8752f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -120,6 +120,7 @@ public abstract class FileSystem extends Configured implements Closeable { */ private Set deleteOnExit = new TreeSet(); + boolean resolveSymlinks; /** * This method adds a file system for testing so that we can find it later. It * is only for testing. @@ -196,6 +197,9 @@ public static void setDefaultUri(Configuration conf, String uri) { */ public void initialize(URI name, Configuration conf) throws IOException { statistics = getStatistics(name.getScheme(), getClass()); + resolveSymlinks = conf.getBoolean( + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java index 4d67b348f6..fce2891750 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs; import java.io.IOException; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -78,6 +79,12 @@ public T resolve(final FileSystem filesys, final Path path) in = doCall(p); isLink = false; } catch (UnresolvedLinkException e) { + if (!filesys.resolveSymlinks) { + throw new IOException("Path " + path + " contains a symlink" + + " and symlink resolution is disabled (" + + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY + + ").", e); + } if (count++ > FsConstants.MAX_PATH_LINKS) { throw new IOException("Possible cyclic loop while " + "following symbolic link " + path); diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index b23503779a..8b2b0e1f17 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1215,4 +1215,15 @@ + + fs.client.resolve.remote.symlinks + true + + Whether to resolve symlinks when accessing a remote Hadoop filesystem. + Setting this to false causes an exception to be thrown upon encountering + a symlink. This setting does not apply to local filesystems, which + automatically resolve local symlinks. + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java new file mode 100644 index 0000000000..2ba8911607 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Test; + +public class TestSymlinkHdfsDisable { + + @Test(timeout=60000) + public void testSymlinkHdfsDisable() throws Exception { + Configuration conf = new HdfsConfiguration(); + // disable symlink resolution + conf.setBoolean( + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, false); + // spin up minicluster, get dfs and filecontext + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + DistributedFileSystem dfs = cluster.getFileSystem(); + FileContext fc = FileContext.getFileContext(cluster.getURI(0), conf); + // Create test files/links + FileContextTestHelper helper = new FileContextTestHelper(); + Path root = helper.getTestRootPath(fc); + Path target = new Path(root, "target"); + Path link = new Path(root, "link"); + DFSTestUtil.createFile(dfs, target, 4096, (short)1, 0xDEADDEAD); + fc.createSymlink(target, link, false); + + // Try to resolve links with FileSystem and FileContext + try { + fc.open(link); + fail("Expected error when attempting to resolve link"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("resolution is disabled", e); + } + try { + dfs.open(link); + fail("Expected error when attempting to resolve link"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("resolution is disabled", e); + } + } +}