HADOOP-9758. Provide configuration option for FS/FC symlink resolution. (Andrew Wang via Colin Patrick McCabe)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1509064 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -297,6 +297,9 @@ Release 2.3.0 - UNRELEASED
     HADOOP-9435.  Support building the JNI code against the IBM JVM.
     (Tian Hong Wang via Colin Patrick McCabe)
 
+    HADOOP-9758.  Provide configuration option for FileSystem/FileContext
+    symlink resolution.  (Andrew Wang via Colin Patrick McCabe)
+
   OPTIMIZATIONS
 
     HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java

@@ -58,6 +58,11 @@ public class CommonConfigurationKeysPublic {
   public static final String  FS_DU_INTERVAL_KEY = "fs.du.interval";
   /** Default value for FS_DU_INTERVAL_KEY */
   public static final long    FS_DU_INTERVAL_DEFAULT = 600000;
+  /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+  public static final String  FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY =
+    "fs.client.resolve.remote.symlinks";
+  /** Default value for FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY */
+  public static final boolean FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT = true;
 
 
   //Defaults are not specified for following keys

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSLinkResolver.java

@@ -90,6 +90,11 @@ public T resolve(final FileContext fc, final Path path) throws IOException {
         in = next(fs, p);
         isLink = false;
       } catch (UnresolvedLinkException e) {
+        if (!fc.resolveSymlinks) {
+          throw new IOException("Path " + path + " contains a symlink"
+              + " and symlink resolution is disabled ("
+              + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY + ").", e);
+        }
         if (count++ > FsConstants.MAX_PATH_LINKS) {
           throw new IOException("Possible cyclic loop while " +
                                 "following symbolic link " + path);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java

@@ -215,6 +215,7 @@ public boolean accept(final Path file) {
   private FsPermission umask;
   private final Configuration conf;
   private final UserGroupInformation ugi;
+  final boolean resolveSymlinks;
 
   private FileContext(final AbstractFileSystem defFs,
     final FsPermission theUmask, final Configuration aConf) {
@@ -240,6 +241,9 @@ private FileContext(final AbstractFileSystem defFs,
     if (workingDir == null) {
       workingDir = defaultFS.getHomeDirectory();
     }
+    resolveSymlinks = conf.getBoolean(
+        CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
+        CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
     util = new Util(); // for the inner class
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -120,6 +120,7 @@ public abstract class FileSystem extends Configured implements Closeable {
    */
   private Set<Path> deleteOnExit = new TreeSet<Path>();
   
+  boolean resolveSymlinks;
   /**
    * This method adds a file system for testing so that we can find it later. It
    * is only for testing.
@@ -196,6 +197,9 @@ public static void setDefaultUri(Configuration conf, String uri) {
    */
   public void initialize(URI name, Configuration conf) throws IOException {
     statistics = getStatistics(name.getScheme(), getClass());    
+    resolveSymlinks = conf.getBoolean(
+        CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY,
+        CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT);
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemLinkResolver.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.fs;
 
 import java.io.IOException;
+
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
@@ -78,6 +79,12 @@ public T resolve(final FileSystem filesys, final Path path)
         in = doCall(p);
         isLink = false;
       } catch (UnresolvedLinkException e) {
+        if (!filesys.resolveSymlinks) {
+          throw new IOException("Path " + path + " contains a symlink"
+              + " and symlink resolution is disabled ("
+              + CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY
+              + ").", e);
+        }
         if (count++ > FsConstants.MAX_PATH_LINKS) {
           throw new IOException("Possible cyclic loop while " +
                                 "following symbolic link " + path);

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -1215,4 +1215,15 @@
   </description>
 </property>
 
+<property>
+  <name>fs.client.resolve.remote.symlinks</name>
+  <value>true</value>
+  <description>
+      Whether to resolve symlinks when accessing a remote Hadoop filesystem.
+      Setting this to false causes an exception to be thrown upon encountering
+      a symlink. This setting does not apply to local filesystems, which
+      automatically resolve local symlinks.
+  </description>
+</property>
+
 </configuration>

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestSymlinkHdfsDisable.java

@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.Test;
+
+public class TestSymlinkHdfsDisable {
+
+  @Test(timeout=60000)
+  public void testSymlinkHdfsDisable() throws Exception {
+    Configuration conf = new HdfsConfiguration();
+    // disable symlink resolution
+    conf.setBoolean(
+        CommonConfigurationKeys.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, false);
+    // spin up minicluster, get dfs and filecontext
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+    DistributedFileSystem dfs = cluster.getFileSystem();
+    FileContext fc = FileContext.getFileContext(cluster.getURI(0), conf);
+    // Create test files/links
+    FileContextTestHelper helper = new FileContextTestHelper();
+    Path root = helper.getTestRootPath(fc);
+    Path target = new Path(root, "target");
+    Path link = new Path(root, "link");
+    DFSTestUtil.createFile(dfs, target, 4096, (short)1, 0xDEADDEAD);
+    fc.createSymlink(target, link, false);
+
+    // Try to resolve links with FileSystem and FileContext
+    try {
+      fc.open(link);
+      fail("Expected error when attempting to resolve link");
+    } catch (IOException e) {
+      GenericTestUtils.assertExceptionContains("resolution is disabled", e);
+    }
+    try {
+      dfs.open(link);
+      fail("Expected error when attempting to resolve link");
+    } catch (IOException e) {
+      GenericTestUtils.assertExceptionContains("resolution is disabled", e);
+    }
+  }
+}