big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15932. Oozie unable to create sharelib in s3a filesystem.

Browse Source 
Contributed by Steve Loughran.
This commit is contained in:
Steve Loughran 2018-11-27 20:39:54 +00:00
parent 300f772c36
commit 4c106fca0c
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
2 changed files with 18 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View File

@ -2357,7 +2357,10 @@ private boolean s3Exists(final Path f) throws IOException {
@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
Path dst) throws IOException {
innerCopyFromLocalFile(delSrc, overwrite, src, dst);
entryPoint(INVOCATION_COPY_FROM_LOCAL_FILE);
LOG.debug("Copying local file from {} to {}", src, dst);
// innerCopyFromLocalFile(delSrc, overwrite, src, dst);
super.copyFromLocalFile(delSrc, overwrite, src, dst);
}
/**
@ -2367,6 +2370,9 @@ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
* This version doesn't need to create a temporary file to calculate the md5.
* Sadly this doesn't seem to be used by the shell cp :(
*
* <i>HADOOP-15932:</i> this method has been unwired from
* {@link #copyFromLocalFile(boolean, boolean, Path, Path)} until
* it is extended to list and copy whole directories.
* delSrc indicates if the source should be removed
* @param delSrc whether to delete the src
* @param overwrite whether to overwrite an existing file

15
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
View File

@ -22,23 +22,27 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import org.junit.Ignore;
import org.junit.Test;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathExistsException;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
/**
* Test {@link S3AFileSystem#copyFromLocalFile(boolean, boolean, Path, Path)}.
* Some of the tests have been disabled pending a fix for HADOOP-15932 and
* recursive directory copying; the test cases themselves may be obsolete.
*/
public class ITestS3ACopyFromLocalFile extends AbstractS3ATestBase {
private static final Charset ASCII = Charsets.US_ASCII;
private static final Charset ASCII = StandardCharsets.US_ASCII;
private File file;
@ -80,7 +84,8 @@ public void assertFileTextEquals(Path path, String expected)
public void testCopyFileNoOverwrite() throws Throwable {
file = createTempFile("hello");
Path dest = upload(file, true);
intercept(FileAlreadyExistsException.class,
// HADOOP-15932: the exception type changes here
intercept(PathExistsException.class,
() -> upload(file, false));
}
@ -95,6 +100,7 @@ public void testCopyFileOverwrite() throws Throwable {
}
@Test
@Ignore("HADOOP-15932")
public void testCopyFileNoOverwriteDirectory() throws Throwable {
file = createTempFile("hello");
Path dest = upload(file, true);
@ -110,11 +116,12 @@ public void testCopyMissingFile() throws Throwable {
file = File.createTempFile("test", ".txt");
file.delete();
// first upload to create
intercept(FileNotFoundException.class, "No file",
intercept(FileNotFoundException.class, "",
() -> upload(file, true));
}
@Test
@Ignore("HADOOP-15932")
public void testCopyDirectoryFile() throws Throwable {
file = File.createTempFile("test", ".txt");
// first upload to create