big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6780. Add support for HDFS directory with erasure code policy to TeraGen and TeraSort. Contributed by Sammi Chen

Browse Source
This commit is contained in:
Kai Zheng 2016-10-09 15:33:26 +06:00
parent ec0b70716c
commit bea004eaeb
3 changed files with 23 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraGen.java
View File

@ -246,6 +246,9 @@ public void cleanup(Context context) {
private static void usage() throws IOException {
System.err.println("teragen <num rows> <output dir>");
System.err.println("If you want to generate data and store them as " +
"erasure code striping file, just make sure that the parent dir " +
"of <output dir> has erasure code policy set");
}
/**

20
hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java
View File

@ -20,6 +20,8 @@
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@ -40,6 +42,7 @@
* An output format that writes the key and value appended together.
*/
public class TeraOutputFormat extends FileOutputFormat<Text,Text> {
private static final Log LOG = LogFactory.getLog(TeraOutputFormat.class);
private OutputCommitter committer = null;
/**
@ -74,10 +77,22 @@ public synchronized void write(Text key,
out.write(key.getBytes(), 0, key.getLength());
out.write(value.getBytes(), 0, value.getLength());
}
public void close(TaskAttemptContext context) throws IOException {
if (finalSync) {
out.hsync();
try {
out.hsync();
} catch (UnsupportedOperationException e) {
/*
* Currently, hsync operation on striping file with erasure code
* policy is not supported yet. So this is a workaround to make
* teragen and terasort to support directory with striping files. In
* future, if the hsync operation is supported on striping file, this
* workaround should be removed.
*/
LOG.info("Operation hsync is not supported so far on path with " +
"erasure code policy set");
}
}
out.close();
}
@ -135,5 +150,4 @@ public OutputCommitter getOutputCommitter(TaskAttemptContext context)
}
return committer;
}
}

3
hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java
View File

@ -287,6 +287,9 @@ private static void usage() throws IOException {
for (TeraSortConfigKeys teraSortConfigKeys : TeraSortConfigKeys.values()) {
System.err.println(teraSortConfigKeys.toString());
}
System.err.println("If you want to store the output data as " +
"erasure code striping file, just make sure that the parent dir " +
"of <out> has erasure code policy set");
}
public int run(String[] args) throws Exception {