big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17376. Distcp creates Factor 1 replication file on target if Source is EC. (#6540)

Browse Source
This commit is contained in:
Sadanand Shenoy 2024-02-09 22:56:32 +05:30 committed by GitHub
parent 50d256ef3c
commit 0bf439c0f9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 24 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
View File

@ -380,6 +380,9 @@ private static ThrottledInputStream getInputStream(Path path,
private static short getReplicationFactor(
EnumSet<FileAttribute> fileAttributes, CopyListingFileStatus source,
FileSystem targetFS, Path tmpTargetPath) {
if (source.isErasureCoded()) {
return targetFS.getDefaultReplication(tmpTargetPath);
}
return fileAttributes.contains(FileAttribute.REPLICATION)
? source.getReplication()
: targetFS.getDefaultReplication(tmpTargetPath);

24
hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpWithRawXAttrs.java
View File

@ -62,6 +62,7 @@ public class TestDistCpWithRawXAttrs {
private static final Path dir1 = new Path("/src/dir1");
private static final Path subDir1 = new Path(dir1, "subdir1");
private static final Path file1 = new Path("/src/file1");
private static final Path FILE_2 = new Path("/src/dir1/file2");
private static final String rawRootName = "/.reserved/raw";
private static final String rootedDestName = "/dest";
private static final String rootedSrcName = "/src";
@ -73,7 +74,7 @@ public static void init() throws Exception {
conf = new Configuration();
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true);
conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, 2);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true)
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true)
.build();
cluster.waitActive();
fs = cluster.getFileSystem();
@ -178,7 +179,7 @@ private void doTestPreserveRawXAttrs(String src, String dest,
}
@Test
public void testPreserveEC() throws Exception {
public void testPreserveAndNoPreserveEC() throws Exception {
final String src = "/src";
final String dest = "/dest";
@ -190,9 +191,11 @@ public void testPreserveEC() throws Exception {
fs.delete(new Path("/dest"), true);
fs.mkdirs(subDir1);
fs.create(file1).close();
DistributedFileSystem dfs = (DistributedFileSystem) fs;
dfs.enableErasureCodingPolicy("XOR-2-1-1024k");
dfs.setErasureCodingPolicy(dir1, "XOR-2-1-1024k");
fs.create(file1).close();
fs.create(FILE_2).close();
int res = ToolRunner.run(conf, new ECAdmin(conf), args);
assertEquals("Unable to set EC policy on " + subDir1.toString(), res, 0);
@ -203,6 +206,7 @@ public void testPreserveEC() throws Exception {
FileStatus srcStatus = fs.getFileStatus(new Path(src));
FileStatus srcDir1Status = fs.getFileStatus(dir1);
FileStatus srcSubDir1Status = fs.getFileStatus(subDir1);
FileStatus srcFile2Status = fs.getFileStatus(FILE_2);
FileStatus destStatus = fs.getFileStatus(new Path(dest));
FileStatus destDir1Status = fs.getFileStatus(destDir1);
@ -214,12 +218,26 @@ public void testPreserveEC() throws Exception {
destStatus.isErasureCoded());
assertTrue("/src/dir1 is not erasure coded!",
srcDir1Status.isErasureCoded());
assertTrue("/src/dir1/file2 is not erasure coded",
srcFile2Status.isErasureCoded());
assertTrue("/dest/dir1 is not erasure coded!",
destDir1Status.isErasureCoded());
assertTrue("/src/dir1/subdir1 is not erasure coded!",
srcSubDir1Status.isErasureCoded());
assertTrue("/dest/dir1/subdir1 is not erasure coded!",
destSubDir1Status.isErasureCoded());
// test without -p to check if src is EC then target FS default replication
// is obeyed on the target file.
fs.delete(new Path(dest), true);
DistCpTestUtils.assertRunDistCp(DistCpConstants.SUCCESS, src, dest, null,
conf);
FileStatus destFileStatus = fs.getFileStatus(new Path(destDir1, "file2"));
assertFalse(destFileStatus.isErasureCoded());
assertEquals(fs.getDefaultReplication(new Path(destDir1, "file2")),
destFileStatus.getReplication());
dfs.unsetErasureCodingPolicy(dir1);
}
@Test