big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-19309: S3A: CopyFromLocalFile operation fails when the source file does not contain file scheme (#7113)

Browse Source 
Contributed by Syed Shameerur Rahman
This commit is contained in:
Syed Shameerur Rahman 2024-10-25 15:41:52 +05:30 committed by GitHub
parent d1ce965645
commit 0b3755347c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CopyFromLocalOperation.java
View File

@ -130,7 +130,7 @@ public CopyFromLocalOperation(
this.callbacks = callbacks; this.callbacks = callbacks;
this.deleteSource = deleteSource; this.deleteSource = deleteSource;
this.overwrite = overwrite; this.overwrite = overwrite;
this.source = source; this.source = source.toUri().getScheme() == null ? new Path("file://", source) : source;
this.destination = destination; this.destination = destination;
// Capacity of 1 is a safe default for now since transfer manager can also // Capacity of 1 is a safe default for now since transfer manager can also

12
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ACopyFromLocalFile.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.fs.s3a; package org.apache.hadoop.fs.s3a;
import java.io.File; import java.io.File;
import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
@ -107,4 +108,15 @@ public void testOnlyFromLocal() throws Throwable {
intercept(IllegalArgumentException.class, intercept(IllegalArgumentException.class,
() -> getFileSystem().copyFromLocalFile(true, true, dest, dest)); () -> getFileSystem().copyFromLocalFile(true, true, dest, dest));
} }
@Test
public void testCopyFromLocalWithNoFileScheme() throws IOException {
describe("Copying from local file with no file scheme to remote s3 destination");
File source = createTempFile("tempData");
Path dest = path(getMethodName());
Path sourcePathWithOutScheme = new Path(source.toURI().getPath());
assertNull(sourcePathWithOutScheme.toUri().getScheme());
getFileSystem().copyFromLocalFile(true, true, sourcePathWithOutScheme, dest);
}
} }