big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-13162. Consider reducing number of getFileStatus calls in S3AFileSystem.mkdirs. (Rajesh Balamohan via stevel)

Browse Source
This commit is contained in:
Steve Loughran 2016-06-01 14:17:18 +01:00
parent d749cf65e1
commit 5870611030
2 changed files with 73 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextCreateMkdirBaseTest.java
View File

@ -20,14 +20,15 @@
import java.io.IOException;
import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import static org.apache.hadoop.fs.FileContextTestHelper.*;
import org.apache.commons.logging.impl.Log4JLogger;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile;
import org.apache.hadoop.test.GenericTestUtils;
/**
@ -116,7 +117,73 @@ public void testMkdirRecursiveWithNonExistingDir() throws IOException {
fc.mkdir(f, FileContext.DEFAULT_PERM, true);
Assert.assertTrue(isDir(fc, f));
}
@Test
public void testMkdirsRecursiveWithExistingDir() throws IOException {
Path f = getTestRootPath(fc, "aDir/bDir/cDir");
fc.mkdir(f, FileContext.DEFAULT_PERM, true);
assertIsDirectory(fc.getFileStatus(f));
assertIsDirectory(fc.getFileStatus(f.getParent()));
assertIsDirectory(fc.getFileStatus(f.getParent().getParent()));
}
@Test
public void testMkdirRecursiveWithExistingFile() throws IOException {
Path f = getTestRootPath(fc, "NonExistant3/aDir");
fc.mkdir(f, FileContext.DEFAULT_PERM, true);
assertIsDirectory(fc.getFileStatus(f));
assertIsDirectory(fc.getFileStatus(f.getParent()));
// create a sample file
Path filePath = new Path(f.getParent(), "test.txt");
createFile(fc, filePath);
assertIsFile(filePath, fc.getFileStatus(filePath));
// try creating another folder which conflicts with filePath
Path dirPath = new Path(filePath, "bDir/cDir");
try {
fc.mkdir(dirPath, FileContext.DEFAULT_PERM, true);
Assert.fail("Mkdir for " + dirPath
+ " should have failed as a file was present");
} catch(IOException e) {
// failed as expected
}
}
@Test
public void testWithRename() throws IOException, InterruptedException {
Path root = getTestRootPath(fc);
Path f = new Path(root, "d1/d2/d3");
fc.mkdir(f, FileContext.DEFAULT_PERM, true);
assertIsDirectory(fc.getFileStatus(new Path(root, "d1")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2/d3")));
// create a sample file f.txt
Path fPath = new Path(root, "d1/d2/f.txt");
createFile(fc, fPath);
assertIsFile(fPath, fc.getFileStatus(fPath));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2/d3")));
// create a sample file f2.txt
Path f2Path = new Path(getTestRootPath(fc), "d1/d2/d3/f2.txt");
createFile(fc, f2Path);
assertIsFile(fPath, fc.getFileStatus(f2Path));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d2/d3")));
//rename d1/d2/d3 d1/d4
fc.rename(new Path(root, "d1/d2/d3"), new Path(root, "d1/d4"));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1")));
assertIsDirectory(fc.getFileStatus(new Path(root, "d1/d4")));
Path f2NewPath = new Path(root, "d1/d4/f2.txt");
assertIsFile(f2NewPath, fc.getFileStatus(f2NewPath));
}
///////////////////////
// Test Create
////////////////////////

3
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -1088,6 +1088,9 @@ private boolean innerMkdirs(Path f, FsPermission permission)
do {
try {
FileStatus fileStatus = getFileStatus(fPart);
if (fileStatus.isDirectory()) {
break;
}
if (fileStatus.isFile()) {
throw new FileAlreadyExistsException(String.format(
"Can't make directory for path '%s' since it is a file.",