big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15850. CopyCommitter#concatFileChunks should check that the blocks per chunk is not 0. Contributed by Ted Yu.

Browse Source 
Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
This commit is contained in:
Ted Yu 2018-10-19 13:19:10 -07:00 committed by Wei-Chiu Chuang
parent d7b012e560
commit e2cecb681e
2 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/CopyCommitter.java
View File

@ -73,6 +73,7 @@ public class CopyCommitter extends FileOutputCommitter {
private boolean overwrite = false; private boolean overwrite = false;
private boolean targetPathExists = true; private boolean targetPathExists = true;
private boolean ignoreFailures = false; private boolean ignoreFailures = false;
private int blocksPerChunk = 0;
/** /**
* Create a output committer * Create a output committer
@ -83,6 +84,9 @@ public class CopyCommitter extends FileOutputCommitter {
*/ */
public CopyCommitter(Path outputPath, TaskAttemptContext context) throws IOException { public CopyCommitter(Path outputPath, TaskAttemptContext context) throws IOException {
super(outputPath, context); super(outputPath, context);
blocksPerChunk = context.getConfiguration().getInt(
DistCpOptionSwitch.BLOCKS_PER_CHUNK.getConfigLabel(), 0);
LOG.debug("blocks per chunk {}", blocksPerChunk);
this.taskAttemptContext = context; this.taskAttemptContext = context;
} }
@ -97,7 +101,9 @@ public void commitJob(JobContext jobContext) throws IOException {
ignoreFailures = conf.getBoolean( ignoreFailures = conf.getBoolean(
DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false); DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false);
if (blocksPerChunk > 0) {
concatFileChunks(conf); concatFileChunks(conf);
}
super.commitJob(jobContext); super.commitJob(jobContext);

4
hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyCommitter.java
View File

@ -85,10 +85,6 @@ public static void destroy() {
@Before @Before
public void createMetaFolder() throws IOException { public void createMetaFolder() throws IOException {
config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta"); config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta");
// Unset listing file path since the config is shared by
// multiple tests, and some test doesn't set it, such as
// testNoCommitAction, but the distcp code will check it.
config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, "");
Path meta = new Path("/meta"); Path meta = new Path("/meta");
cluster.getFileSystem().mkdirs(meta); cluster.getFileSystem().mkdirs(meta);
} }