From f35294eb95b67e62cd323703595c7f18e4c9281f Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Tue, 15 May 2012 19:09:59 +0000 Subject: [PATCH] MAPREDUCE-4238. mavenize data_join. (tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1338835 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 + hadoop-project/pom.xml | 5 + hadoop-tools/hadoop-datajoin/pom.xml | 120 ++++++++++++++++++ .../utils/join/ArrayListBackedIterator.java | 0 .../contrib/utils/join/DataJoinJob.java | 0 .../utils/join/DataJoinMapperBase.java | 0 .../utils/join/DataJoinReducerBase.java | 0 .../hadoop/contrib/utils/join/JobBase.java | 6 +- .../contrib/utils/join/ResetableIterator.java | 0 .../contrib/utils/join/TaggedMapOutput.java | 0 .../hadoop-datajoin/src/test/java}/README.txt | 0 .../src/test/java}/SampleDataJoinMapper.java | 0 .../src/test/java}/SampleDataJoinReducer.java | 0 .../src/test/java}/SampleTaggedMapOutput.java | 0 .../contrib/utils/join/TestDataJoin.java | 1 + hadoop-tools/hadoop-tools-dist/pom.xml | 5 + hadoop-tools/pom.xml | 1 + 17 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 hadoop-tools/hadoop-datajoin/pom.xml rename {hadoop-mapreduce-project/src/contrib/data_join/src => hadoop-tools/hadoop-datajoin/src/main}/java/org/apache/hadoop/contrib/utils/join/ArrayListBackedIterator.java (100%) rename {hadoop-mapreduce-project/src/contrib/data_join/src => hadoop-tools/hadoop-datajoin/src/main}/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java (100%) rename {hadoop-mapreduce-project/src/contrib/data_join/src => hadoop-tools/hadoop-datajoin/src/main}/java/org/apache/hadoop/contrib/utils/join/DataJoinMapperBase.java (100%) rename {hadoop-mapreduce-project/src/contrib/data_join/src => hadoop-tools/hadoop-datajoin/src/main}/java/org/apache/hadoop/contrib/utils/join/DataJoinReducerBase.java (100%) rename {hadoop-mapreduce-project/src/contrib/data_join/src => hadoop-tools/hadoop-datajoin/src/main}/java/org/apache/hadoop/contrib/utils/join/JobBase.java (97%) rename {hadoop-mapreduce-project/src/contrib/data_join/src => hadoop-tools/hadoop-datajoin/src/main}/java/org/apache/hadoop/contrib/utils/join/ResetableIterator.java (100%) rename {hadoop-mapreduce-project/src/contrib/data_join/src => hadoop-tools/hadoop-datajoin/src/main}/java/org/apache/hadoop/contrib/utils/join/TaggedMapOutput.java (100%) rename {hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join => hadoop-tools/hadoop-datajoin/src/test/java}/README.txt (100%) rename {hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join => hadoop-tools/hadoop-datajoin/src/test/java}/SampleDataJoinMapper.java (100%) rename {hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join => hadoop-tools/hadoop-datajoin/src/test/java}/SampleDataJoinReducer.java (100%) rename {hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join => hadoop-tools/hadoop-datajoin/src/test/java}/SampleTaggedMapOutput.java (100%) rename {hadoop-mapreduce-project/src/contrib/data_join/src/test => hadoop-tools/hadoop-datajoin/src/test/java}/org/apache/hadoop/contrib/utils/join/TestDataJoin.java (98%) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ff4664675d..3ffaad3e08 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -483,6 +483,8 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4233. NPE can happen in RMNMNodeInfo. (bobby) + MAPREDUCE-4238. mavenize data_join. (tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 06e33cfb52..8fc964b785 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -228,6 +228,11 @@ hadoop-distcp ${project.version} + + org.apache.hadoop + hadoop-datajoin + ${project.version} + org.apache.hadoop hadoop-rumen diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml new file mode 100644 index 0000000000..984682ff5c --- /dev/null +++ b/hadoop-tools/hadoop-datajoin/pom.xml @@ -0,0 +1,120 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-SNAPSHOT + ../../hadoop-project + + org.apache.hadoop + hadoop-datajoin + 3.0.0-SNAPSHOT + Apache Hadoop Data Join + Apache Hadoop Data Join + jar + + + ${project.build.directory}/log + + + + + org.apache.hadoop + hadoop-annotations + provided + + + org.apache.hadoop + hadoop-mapreduce-client-hs + provided + + + org.apache.hadoop + hadoop-mapreduce-client-core + provided + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + provided + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + test + test-jar + + + org.apache.hadoop + hadoop-common + provided + + + org.apache.hadoop + hadoop-hdfs + provided + + + org.apache.hadoop + hadoop-common + test + test-jar + + + org.apache.hadoop + hadoop-hdfs + test + test-jar + + + org.apache.hadoop + hadoop-yarn-server-tests + test-jar + test + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + create-log-dir + process-test-resources + + run + + + + + + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + + diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/ArrayListBackedIterator.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/ArrayListBackedIterator.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/ArrayListBackedIterator.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/ArrayListBackedIterator.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinMapperBase.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinMapperBase.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinMapperBase.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinMapperBase.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinReducerBase.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinReducerBase.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinReducerBase.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/DataJoinReducerBase.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/JobBase.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/JobBase.java similarity index 97% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/JobBase.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/JobBase.java index dd34a4bb56..9ef21b3085 100644 --- a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/JobBase.java +++ b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/JobBase.java @@ -51,7 +51,7 @@ public abstract class JobBase implements Mapper, Reducer { * the value for the counter */ protected void setLongValue(Object name, long value) { - this.longCounters.put(name, new Long(value)); + this.longCounters.put(name, Long.valueOf(value)); } /** @@ -100,9 +100,9 @@ protected Long addLongValue(Object name, long inc) { Long val = this.longCounters.get(name); Long retv = null; if (val == null) { - retv = new Long(inc); + retv = Long.valueOf(inc); } else { - retv = new Long(val.longValue() + inc); + retv = Long.valueOf(val.longValue() + inc); } this.longCounters.put(name, retv); return retv; diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/ResetableIterator.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/ResetableIterator.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/ResetableIterator.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/ResetableIterator.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/TaggedMapOutput.java b/hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/TaggedMapOutput.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/TaggedMapOutput.java rename to hadoop-tools/hadoop-datajoin/src/main/java/org/apache/hadoop/contrib/utils/join/TaggedMapOutput.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/README.txt b/hadoop-tools/hadoop-datajoin/src/test/java/README.txt similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/README.txt rename to hadoop-tools/hadoop-datajoin/src/test/java/README.txt diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/SampleDataJoinMapper.java b/hadoop-tools/hadoop-datajoin/src/test/java/SampleDataJoinMapper.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/SampleDataJoinMapper.java rename to hadoop-tools/hadoop-datajoin/src/test/java/SampleDataJoinMapper.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/SampleDataJoinReducer.java b/hadoop-tools/hadoop-datajoin/src/test/java/SampleDataJoinReducer.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/SampleDataJoinReducer.java rename to hadoop-tools/hadoop-datajoin/src/test/java/SampleDataJoinReducer.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/SampleTaggedMapOutput.java b/hadoop-tools/hadoop-datajoin/src/test/java/SampleTaggedMapOutput.java similarity index 100% rename from hadoop-mapreduce-project/src/contrib/data_join/src/examples/org/apache/hadoop/contrib/utils/join/SampleTaggedMapOutput.java rename to hadoop-tools/hadoop-datajoin/src/test/java/SampleTaggedMapOutput.java diff --git a/hadoop-mapreduce-project/src/contrib/data_join/src/test/org/apache/hadoop/contrib/utils/join/TestDataJoin.java b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/TestDataJoin.java similarity index 98% rename from hadoop-mapreduce-project/src/contrib/data_join/src/test/org/apache/hadoop/contrib/utils/join/TestDataJoin.java rename to hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/TestDataJoin.java index 1eab9593df..37df5b002e 100644 --- a/hadoop-mapreduce-project/src/contrib/data_join/src/test/org/apache/hadoop/contrib/utils/join/TestDataJoin.java +++ b/hadoop-tools/hadoop-datajoin/src/test/java/org/apache/hadoop/contrib/utils/join/TestDataJoin.java @@ -57,6 +57,7 @@ protected void tearDown() throws Exception { public void testDataJoin() throws Exception { final int srcs = 4; JobConf job = new JobConf(); + job.setBoolean("mapreduce.fileoutputcommitter.marksuccessfuljobs", false); Path base = cluster.getFileSystem().makeQualified(new Path("/inner")); Path[] src = writeSimpleSrc(base, job, srcs); job.setInputFormat(SequenceFileInputFormat.class); diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml index 46d1c19530..792f1e6e30 100644 --- a/hadoop-tools/hadoop-tools-dist/pom.xml +++ b/hadoop-tools/hadoop-tools-dist/pom.xml @@ -52,6 +52,11 @@ hadoop-rumen compile + + org.apache.hadoop + hadoop-datajoin + compile + org.apache.hadoop hadoop-extras diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml index dfa9049a80..51013550e5 100644 --- a/hadoop-tools/pom.xml +++ b/hadoop-tools/pom.xml @@ -32,6 +32,7 @@ hadoop-distcp hadoop-archives hadoop-rumen + hadoop-datajoin hadoop-tools-dist hadoop-extras