HADOOP-15850. CopyCommitter#concatFileChunks should check that the blocks per chunk is not 0. Contributed by Ted Yu.
Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
This commit is contained in:
parent
d7b012e560
commit
e2cecb681e
@ -73,6 +73,7 @@ public class CopyCommitter extends FileOutputCommitter {
|
|||||||
private boolean overwrite = false;
|
private boolean overwrite = false;
|
||||||
private boolean targetPathExists = true;
|
private boolean targetPathExists = true;
|
||||||
private boolean ignoreFailures = false;
|
private boolean ignoreFailures = false;
|
||||||
|
private int blocksPerChunk = 0;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a output committer
|
* Create a output committer
|
||||||
@ -83,6 +84,9 @@ public class CopyCommitter extends FileOutputCommitter {
|
|||||||
*/
|
*/
|
||||||
public CopyCommitter(Path outputPath, TaskAttemptContext context) throws IOException {
|
public CopyCommitter(Path outputPath, TaskAttemptContext context) throws IOException {
|
||||||
super(outputPath, context);
|
super(outputPath, context);
|
||||||
|
blocksPerChunk = context.getConfiguration().getInt(
|
||||||
|
DistCpOptionSwitch.BLOCKS_PER_CHUNK.getConfigLabel(), 0);
|
||||||
|
LOG.debug("blocks per chunk {}", blocksPerChunk);
|
||||||
this.taskAttemptContext = context;
|
this.taskAttemptContext = context;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -97,7 +101,9 @@ public void commitJob(JobContext jobContext) throws IOException {
|
|||||||
ignoreFailures = conf.getBoolean(
|
ignoreFailures = conf.getBoolean(
|
||||||
DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false);
|
DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false);
|
||||||
|
|
||||||
concatFileChunks(conf);
|
if (blocksPerChunk > 0) {
|
||||||
|
concatFileChunks(conf);
|
||||||
|
}
|
||||||
|
|
||||||
super.commitJob(jobContext);
|
super.commitJob(jobContext);
|
||||||
|
|
||||||
|
@ -85,10 +85,6 @@ public static void destroy() {
|
|||||||
@Before
|
@Before
|
||||||
public void createMetaFolder() throws IOException {
|
public void createMetaFolder() throws IOException {
|
||||||
config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta");
|
config.set(DistCpConstants.CONF_LABEL_META_FOLDER, "/meta");
|
||||||
// Unset listing file path since the config is shared by
|
|
||||||
// multiple tests, and some test doesn't set it, such as
|
|
||||||
// testNoCommitAction, but the distcp code will check it.
|
|
||||||
config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, "");
|
|
||||||
Path meta = new Path("/meta");
|
Path meta = new Path("/meta");
|
||||||
cluster.getFileSystem().mkdirs(meta);
|
cluster.getFileSystem().mkdirs(meta);
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user