big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14455. ViewFileSystem#rename should support be supported within same nameservice with different mountpoints. Contributed by Brahma Reddy Battula.

Browse Source
This commit is contained in:
Brahma Reddy Battula 2017-07-25 23:20:35 +08:00
parent 1a79dcfc45
commit 6d983cca52
9 changed files with 308 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
View File

@ -66,4 +66,6 @@ public interface Constants {
static public final FsPermission PERMISSION_555 =
new FsPermission((short) 0555);
String CONFIG_VIEWFS_RENAME_STRATEGY = "fs.viewfs.rename.strategy";
}

79
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
View File

@ -126,7 +126,8 @@ public URI[] getTargetFileSystemURIs() {
Configuration config;
InodeTree<FileSystem> fsState; // the fs state; ie the mount table
Path homeDir = null;
// Default to rename within same mountpoint
private RenameStrategy renameStrategy = RenameStrategy.SAME_MOUNTPOINT;
/**
* Make the path Absolute and get the path-part of a pathname.
* Checks that URI matches this file system
@ -207,6 +208,9 @@ FileSystem getTargetFileSystem(URI[] mergeFsURIList)
}
};
workingDir = this.getHomeDirectory();
renameStrategy = RenameStrategy.valueOf(
conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
RenameStrategy.SAME_MOUNTPOINT.toString()));
} catch (URISyntaxException e) {
throw new IOException("URISyntax exception: " + theUri);
}
@ -490,27 +494,55 @@ public boolean rename(final Path src, final Path dst) throws IOException {
if (resDst.isInternalDir()) {
throw readOnlyMountTable("rename", dst);
}
/**
// Alternate 1: renames within same file system - valid but we disallow
// Alternate 2: (as described in next para - valid but we have disallowed it
//
// Note we compare the URIs. the URIs include the link targets.
// hence we allow renames across mount links as long as the mount links
// point to the same target.
if (!resSrc.targetFileSystem.getUri().equals(
resDst.targetFileSystem.getUri())) {
throw new IOException("Renames across Mount points not supported");
URI srcUri = resSrc.targetFileSystem.getUri();
URI dstUri = resDst.targetFileSystem.getUri();
verifyRenameStrategy(srcUri, dstUri,
resSrc.targetFileSystem == resDst.targetFileSystem, renameStrategy);
ChRootedFileSystem srcFS = (ChRootedFileSystem) resSrc.targetFileSystem;
ChRootedFileSystem dstFS = (ChRootedFileSystem) resDst.targetFileSystem;
return srcFS.getMyFs().rename(srcFS.fullPath(resSrc.remainingPath),
dstFS.fullPath(resDst.remainingPath));
}
static void verifyRenameStrategy(URI srcUri, URI dstUri,
boolean isSrcDestSame, ViewFileSystem.RenameStrategy renameStrategy)
throws IOException {
switch (renameStrategy) {
case SAME_FILESYSTEM_ACROSS_MOUNTPOINT:
if (srcUri.getAuthority() != null) {
if (!(srcUri.getScheme().equals(dstUri.getScheme()) && srcUri
.getAuthority().equals(dstUri.getAuthority()))) {
throw new IOException("Renames across Mount points not supported");
}
}
break;
case SAME_TARGET_URI_ACROSS_MOUNTPOINT:
// Alternate 2: Rename across mountpoints with same target.
// i.e. Rename across alias mountpoints.
//
// Note we compare the URIs. the URIs include the link targets.
// hence we allow renames across mount links as long as the mount links
// point to the same target.
if (!srcUri.equals(dstUri)) {
throw new IOException("Renames across Mount points not supported");
}
break;
case SAME_MOUNTPOINT:
//
// Alternate 3 : renames ONLY within the the same mount links.
//
if (!isSrcDestSame) {
throw new IOException("Renames across Mount points not supported");
}
break;
default:
throw new IllegalArgumentException ("Unexpected rename strategy");
}
*/
//
// Alternate 3 : renames ONLY within the the same mount links.
//
if (resSrc.targetFileSystem !=resDst.targetFileSystem) {
throw new IOException("Renames across Mount points not supported");
}
return resSrc.targetFileSystem.rename(resSrc.remainingPath,
resDst.remainingPath);
}
@Override
@ -1241,4 +1273,9 @@ public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
return allPolicies;
}
}
enum RenameStrategy {
SAME_MOUNTPOINT, SAME_TARGET_URI_ACROSS_MOUNTPOINT,
SAME_FILESYSTEM_ACROSS_MOUNTPOINT
}
}

41
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
View File

@ -157,7 +157,9 @@ public class ViewFs extends AbstractFileSystem {
final Configuration config;
InodeTree<AbstractFileSystem> fsState; // the fs state; ie the mount table
Path homeDir = null;
private ViewFileSystem.RenameStrategy renameStrategy =
ViewFileSystem.RenameStrategy.SAME_MOUNTPOINT;
static AccessControlException readOnlyMountTable(final String operation,
final String p) {
return new AccessControlException(
@ -237,6 +239,9 @@ AbstractFileSystem getTargetFileSystem(URI[] mergeFsURIList)
// return MergeFs.createMergeFs(mergeFsURIList, config);
}
};
renameStrategy = ViewFileSystem.RenameStrategy.valueOf(
conf.get(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
ViewFileSystem.RenameStrategy.SAME_MOUNTPOINT.toString()));
}
@Override
@ -495,37 +500,23 @@ public void renameInternal(final Path src, final Path dst,
+ " is readOnly");
}
InodeTree.ResolveResult<AbstractFileSystem> resDst =
InodeTree.ResolveResult<AbstractFileSystem> resDst =
fsState.resolve(getUriPath(dst), false);
if (resDst.isInternalDir()) {
throw new AccessControlException(
"Cannot Rename within internal dirs of mount table: dest=" + dst
+ " is readOnly");
}
/**
// Alternate 1: renames within same file system - valid but we disallow
// Alternate 2: (as described in next para - valid but we have disallowed it
//
// Note we compare the URIs. the URIs include the link targets.
// hence we allow renames across mount links as long as the mount links
// point to the same target.
if (!resSrc.targetFileSystem.getUri().equals(
resDst.targetFileSystem.getUri())) {
throw new IOException("Renames across Mount points not supported");
}
*/
//
// Alternate 3 : renames ONLY within the the same mount links.
//
//Alternate 1: renames within same file system
URI srcUri = resSrc.targetFileSystem.getUri();
URI dstUri = resDst.targetFileSystem.getUri();
ViewFileSystem.verifyRenameStrategy(srcUri, dstUri,
resSrc.targetFileSystem == resDst.targetFileSystem, renameStrategy);
if (resSrc.targetFileSystem !=resDst.targetFileSystem) {
throw new IOException("Renames across Mount points not supported");
}
resSrc.targetFileSystem.renameInternal(resSrc.remainingPath,
resDst.remainingPath, overwrite);
ChRootedFs srcFS = (ChRootedFs) resSrc.targetFileSystem;
ChRootedFs dstFS = (ChRootedFs) resDst.targetFileSystem;
srcFS.getMyFs().renameInternal(srcFS.fullPath(resSrc.remainingPath),
dstFS.fullPath(resDst.remainingPath), overwrite);
}
@Override

9
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -800,6 +800,15 @@
(ie client side mount table:).</description>
</property>
<property>
<name>fs.viewfs.rename.strategy</name>
<value>SAME_MOUNTPOINT</value>
<description>Allowed rename strategy to rename between multiple mountpoints.
Allowed values are SAME_MOUNTPOINT,SAME_TARGET_URI_ACROSS_MOUNTPOINT and
SAME_FILESYSTEM_ACROSS_MOUNTPOINT.
</description>
</property>
<property>
<name>fs.AbstractFileSystem.ftp.impl</name>
<value>org.apache.hadoop.fs.ftp.FtpFs</value>

1
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
View File

@ -95,6 +95,7 @@ public void initializeMemberVariables() {
xmlPropsToSkipCompare.add("nfs3.mountd.port");
xmlPropsToSkipCompare.add("nfs3.server.port");
xmlPropsToSkipCompare.add("test.fs.s3n.name");
xmlPropsToSkipCompare.add("fs.viewfs.rename.strategy");
// S3N/S3A properties are in a different subtree.
// - org.apache.hadoop.fs.s3native.S3NativeFileSystemConfigKeys

54
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
View File

@ -20,6 +20,7 @@
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
@ -715,6 +716,21 @@ public static void assertIsFile(FileSystem fileSystem, Path filename)
assertIsFile(filename, status);
}
/**
* Assert that a file exists and whose {@link FileStatus} entry
* declares that this is a file and not a symlink or directory.
*
* @param fileContext filesystem to resolve path against
* @param filename name of the file
* @throws IOException IO problems during file operations
*/
public static void assertIsFile(FileContext fileContext, Path filename)
throws IOException {
assertPathExists(fileContext, "Expected file", filename);
FileStatus status = fileContext.getFileStatus(filename);
assertIsFile(filename, status);
}
/**
* Assert that a file exists and whose {@link FileStatus} entry
* declares that this is a file and not a symlink or directory.
@ -765,6 +781,25 @@ public static void assertPathExists(FileSystem fileSystem, String message,
}
}
/**
* Assert that a path exists -but make no assertions as to the
* type of that entry.
*
* @param fileContext fileContext to examine
* @param message message to include in the assertion failure message
* @param path path in the filesystem
* @throws FileNotFoundException raised if the path is missing
* @throws IOException IO problems
*/
public static void assertPathExists(FileContext fileContext, String message,
Path path) throws IOException {
if (!fileContext.util().exists(path)) {
//failure, report it
throw new FileNotFoundException(
message + ": not found " + path + " in " + path.getParent());
}
}
/**
* Assert that a path does not exist.
*
@ -785,6 +820,25 @@ public static void assertPathDoesNotExist(FileSystem fileSystem,
}
}
/**
* Assert that a path does not exist.
*
* @param fileContext fileContext to examine
* @param message message to include in the assertion failure message
* @param path path in the filesystem
* @throws IOException IO problems
*/
public static void assertPathDoesNotExist(FileContext fileContext,
String message, Path path) throws IOException {
try {
FileStatus status = fileContext.getFileStatus(path);
fail(message + ": unexpectedly found " + path + " as " + status);
} catch (FileNotFoundException expected) {
//this is expected
}
}
/**
* Assert that a FileSystem.listStatus on a dir finds the subdir/child entry.
* @param fs filesystem

79
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java
View File

@ -41,6 +41,7 @@
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.AclUtil;
@ -51,6 +52,7 @@
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assume;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -366,28 +368,83 @@ private void testOperationsThroughMountLinksInternal(boolean located)
}
// rename across mount points that point to same target also fail
@Test(expected=IOException.class)
@Test
public void testRenameAcrossMounts1() throws IOException {
fileSystemTestHelper.createFile(fsView, "/user/foo");
fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
/* - code if we had wanted this to succeed
Assert.assertFalse(fSys.exists(new Path("/user/foo")));
Assert.assertFalse(fSysLocal.exists(new Path(targetTestRoot,"user/foo")));
Assert.assertTrue(fSys.isFile(FileSystemTestHelper.getTestRootPath(fSys,"/user2/fooBarBar")));
Assert.assertTrue(fSysLocal.isFile(new Path(targetTestRoot,"user/fooBarBar")));
*/
try {
fsView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
ContractTestUtils.fail("IOException is not thrown on rename operation");
} catch (IOException e) {
GenericTestUtils
.assertExceptionContains("Renames across Mount points not supported",
e);
}
}
// rename across mount points fail if the mount link targets are different
// even if the targets are part of the same target FS
@Test(expected=IOException.class)
@Test
public void testRenameAcrossMounts2() throws IOException {
fileSystemTestHelper.createFile(fsView, "/user/foo");
fsView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
try {
fsView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
ContractTestUtils.fail("IOException is not thrown on rename operation");
} catch (IOException e) {
GenericTestUtils
.assertExceptionContains("Renames across Mount points not supported",
e);
}
}
// RenameStrategy SAME_TARGET_URI_ACROSS_MOUNTPOINT enabled
// to rename across mount points that point to same target URI
@Test
public void testRenameAcrossMounts3() throws IOException {
Configuration conf2 = new Configuration(conf);
conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
ViewFileSystem.RenameStrategy.SAME_TARGET_URI_ACROSS_MOUNTPOINT
.toString());
FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2);
fileSystemTestHelper.createFile(fsView2, "/user/foo");
fsView2.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
ContractTestUtils
.assertPathDoesNotExist(fsView2, "src should not exist after rename",
new Path("/user/foo"));
ContractTestUtils
.assertPathDoesNotExist(fsTarget, "src should not exist after rename",
new Path(targetTestRoot, "user/foo"));
ContractTestUtils.assertIsFile(fsView2,
fileSystemTestHelper.getTestRootPath(fsView2, "/user2/fooBarBar"));
ContractTestUtils
.assertIsFile(fsTarget, new Path(targetTestRoot, "user/fooBarBar"));
}
// RenameStrategy SAME_FILESYSTEM_ACROSS_MOUNTPOINT enabled
// to rename across mount points where the mount link targets are different
// but are part of the same target FS
@Test
public void testRenameAcrossMounts4() throws IOException {
Configuration conf2 = new Configuration(conf);
conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
ViewFileSystem.RenameStrategy.SAME_FILESYSTEM_ACROSS_MOUNTPOINT
.toString());
FileSystem fsView2 = FileSystem.newInstance(FsConstants.VIEWFS_URI, conf2);
fileSystemTestHelper.createFile(fsView2, "/user/foo");
fsView2.rename(new Path("/user/foo"), new Path("/data/fooBar"));
ContractTestUtils
.assertPathDoesNotExist(fsView2, "src should not exist after rename",
new Path("/user/foo"));
ContractTestUtils
.assertPathDoesNotExist(fsTarget, "src should not exist after rename",
new Path(targetTestRoot, "user/foo"));
ContractTestUtils.assertIsFile(fsView2,
fileSystemTestHelper.getTestRootPath(fsView2, "/data/fooBar"));
ContractTestUtils
.assertIsFile(fsTarget, new Path(targetTestRoot, "data/fooBar"));
}
static protected boolean SupportsBlocks = false; // local fs use 1 block
// override for HDFS
@Test

94
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java
View File

@ -58,6 +58,7 @@
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.local.LocalConfigKeys;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
@ -66,6 +67,7 @@
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -345,33 +347,93 @@ public void testOperationsThroughMountLinks() throws IOException {
}
// rename across mount points that point to same target also fail
@Test(expected=IOException.class)
@Test
public void testRenameAcrossMounts1() throws IOException {
fileContextTestHelper.createFile(fcView, "/user/foo");
fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
/* - code if we had wanted this to succeed
Assert.assertFalse(exists(fc, new Path("/user/foo")));
Assert.assertFalse(exists(fclocal, new Path(targetTestRoot,"user/foo")));
Assert.assertTrue(isFile(fc,
FileContextTestHelper.getTestRootPath(fc,"/user2/fooBarBar")));
Assert.assertTrue(isFile(fclocal,
new Path(targetTestRoot,"user/fooBarBar")));
*/
try {
fcView.rename(new Path("/user/foo"), new Path("/user2/fooBarBar"));
ContractTestUtils.fail("IOException is not thrown on rename operation");
} catch (IOException e) {
GenericTestUtils
.assertExceptionContains("Renames across Mount points not supported",
e);
}
}
// rename across mount points fail if the mount link targets are different
// even if the targets are part of the same target FS
@Test(expected=IOException.class)
@Test
public void testRenameAcrossMounts2() throws IOException {
fileContextTestHelper.createFile(fcView, "/user/foo");
fcView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
try {
fcView.rename(new Path("/user/foo"), new Path("/data/fooBar"));
ContractTestUtils.fail("IOException is not thrown on rename operation");
} catch (IOException e) {
GenericTestUtils
.assertExceptionContains("Renames across Mount points not supported",
e);
}
}
// RenameStrategy SAME_TARGET_URI_ACROSS_MOUNTPOINT enabled
// to rename across mount points that point to same target URI
@Test
public void testRenameAcrossMounts3() throws IOException {
Configuration conf2 = new Configuration(conf);
conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
ViewFileSystem.RenameStrategy.SAME_TARGET_URI_ACROSS_MOUNTPOINT
.toString());
FileContext fcView2 =
FileContext.getFileContext(FsConstants.VIEWFS_URI, conf2);
String user1Path = "/user/foo";
fileContextTestHelper.createFile(fcView2, user1Path);
String user2Path = "/user2/fooBarBar";
Path user2Dst = new Path(user2Path);
fcView2.rename(new Path(user1Path), user2Dst);
ContractTestUtils
.assertPathDoesNotExist(fcView2, "src should not exist after rename",
new Path(user1Path));
ContractTestUtils
.assertPathDoesNotExist(fcTarget, "src should not exist after rename",
new Path(targetTestRoot, "user/foo"));
ContractTestUtils.assertIsFile(fcView2,
fileContextTestHelper.getTestRootPath(fcView2, user2Path));
ContractTestUtils
.assertIsFile(fcTarget, new Path(targetTestRoot, "user/fooBarBar"));
}
// RenameStrategy SAME_FILESYSTEM_ACROSS_MOUNTPOINT enabled
// to rename across mount points if the mount link targets are different
// but are part of the same target FS
@Test
public void testRenameAcrossMounts4() throws IOException {
Configuration conf2 = new Configuration(conf);
conf2.set(Constants.CONFIG_VIEWFS_RENAME_STRATEGY,
ViewFileSystem.RenameStrategy.SAME_FILESYSTEM_ACROSS_MOUNTPOINT
.toString());
FileContext fcView2 =
FileContext.getFileContext(FsConstants.VIEWFS_URI, conf2);
String userPath = "/user/foo";
fileContextTestHelper.createFile(fcView2, userPath);
String anotherMountPath = "/data/fooBar";
Path anotherDst = new Path(anotherMountPath);
fcView2.rename(new Path(userPath), anotherDst);
ContractTestUtils
.assertPathDoesNotExist(fcView2, "src should not exist after rename",
new Path(userPath));
ContractTestUtils
.assertPathDoesNotExist(fcTarget, "src should not exist after rename",
new Path(targetTestRoot, "user/foo"));
ContractTestUtils.assertIsFile(fcView2,
fileContextTestHelper.getTestRootPath(fcView2, anotherMountPath));
ContractTestUtils
.assertIsFile(fcView2, new Path(targetTestRoot, "data/fooBar"));
}
static protected boolean SupportsBlocks = false; // local fs use 1 block
// override for HDFS
@Test

22
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
View File

@ -38,6 +38,7 @@
import org.apache.hadoop.fs.FsConstants;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@ -45,6 +46,7 @@
import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
@ -247,4 +249,24 @@ public void testFileChecksum() throws IOException {
Assert.assertTrue("File checksum not matching!",
fileChecksumViaViewFs.equals(fileChecksumViaTargetFs));
}
//Rename should fail on across different fileSystems
@Test
public void testRenameAccorssFilesystem() throws IOException {
//data is mountpoint in nn1
Path mountDataRootPath = new Path("/data");
//mountOnNn2 is nn2 mountpoint
Path fsTargetFilePath = new Path("/mountOnNn2");
Path filePath = new Path(mountDataRootPath + "/ttest");
Path hdfFilepath = new Path(fsTargetFilePath + "/ttest2");
fsView.create(filePath);
try {
fsView.rename(filePath, hdfFilepath);
ContractTestUtils.fail("Should thrown IOE on Renames across filesytems");
} catch (IOException e) {
GenericTestUtils
.assertExceptionContains("Renames across Mount points not supported",
e);
}
}
}