diff --git a/.gitignore b/.gitignore index d7e321eaed..eb98a3f967 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ .settings target build +dependency-reduced-pom.xml # External tool builders */.externalToolBuilders diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching index 40ccdc71fe..fdbc83cd59 100755 --- a/dev-support/bin/dist-layout-stitching +++ b/dev-support/bin/dist-layout-stitching @@ -137,6 +137,12 @@ run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERS run cp -pr "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${VERSION}"/* . run cp -pr "${ROOT}/hadoop-common-project/hadoop-kms/target/hadoop-kms-${VERSION}"/* . +# copy client jars as-is +run mkdir -p "share/hadoop/client" +run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-${VERSION}.jar" share/hadoop/client/ +run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-${VERSION}.jar" share/hadoop/client/ +run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/ + echo echo "Hadoop dist layout available at: ${BASEDIR}/hadoop-${VERSION}" -echo \ No newline at end of file +echo diff --git a/hadoop-client-modules/hadoop-client-api/pom.xml b/hadoop-client-modules/hadoop-client-api/pom.xml new file mode 100644 index 0000000000..9342d535c5 --- /dev/null +++ b/hadoop-client-modules/hadoop-client-api/pom.xml @@ -0,0 +1,216 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-alpha2-SNAPSHOT + ../../hadoop-project + + hadoop-client-api + 3.0.0-alpha2-SNAPSHOT + jar + + Apache Hadoop Client + Apache Hadoop Client API + + + org.apache.hadoop.shaded + + true + + + + + org.apache.hadoop + hadoop-client + + true + + + + javax.xml.bind + jaxb-api + + + xml-apis + xml-apis + + + + + + org.apache.hadoop + hadoop-annotations + compile + + + jdk.tools + jdk.tools + + + + + + + + + org.apache.maven.plugins + maven-source-plugin + + true + + + + org.apache.maven.plugins + maven-shade-plugin + + + org.apache.hadoop + hadoop-maven-plugins + ${project.version} + + + + + package + + shade + + + + + org.apache.hadoop:* + + + + + + org.apache.hadoop:hadoop-yarn-common + + org/apache/hadoop/yarn/factories/package-info.class + org/apache/hadoop/yarn/util/package-info.class + org/apache/hadoop/yarn/factory/providers/package-info.class + org/apache/hadoop/yarn/client/api/impl/package-info.class + org/apache/hadoop/yarn/client/api/package-info.class + + + + + + org/ + ${shaded.dependency.prefix}.org. + + org/apache/hadoop/* + org/apache/hadoop/**/* + + org/apache/htrace/* + org/apache/htrace/**/* + org/slf4j/* + org/slf4j/**/* + org/apache/commons/logging/* + org/apache/commons/logging/**/* + org/apache/log4j/* + org/apache/log4j/**/* + **/pom.xml + + org/ietf/jgss/* + org/omg/**/* + org/w3c/dom/* + org/w3c/dom/**/* + org/xml/sax/* + org/xml/sax/**/* + + + + com/ + ${shaded.dependency.prefix}.com. + + **/pom.xml + + com/sun/tools/* + com/sun/javadoc/* + com/sun/security/* + com/sun/jndi/* + com/sun/management/* + com/sun/tools/**/* + com/sun/javadoc/**/* + com/sun/security/**/* + com/sun/jndi/**/* + com/sun/management/**/* + + + + io/ + ${shaded.dependency.prefix}.io. + + **/pom.xml + + io/compression/* + io/compression/**/* + io/mapfile/* + io/mapfile/**/* + io/map/index/* + io/seqfile/* + io/seqfile/**/* + io/file/buffer/size + io/skip/checksum/errors + io/sort/* + io/serializations + + + + javax/servlet/ + ${shaded.dependency.prefix}.javax.servlet. + + **/pom.xml + + + + net/ + ${shaded.dependency.prefix}.net. + + **/pom.xml + + net/topology/* + net/topology/**/* + + + + + + + + + false + + + + + + + + org.codehaus.mojo + license-maven-plugin + + + + + diff --git a/hadoop-client-modules/hadoop-client-check-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml new file mode 100644 index 0000000000..f28816f3ad --- /dev/null +++ b/hadoop-client-modules/hadoop-client-check-invariants/pom.xml @@ -0,0 +1,124 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-alpha2-SNAPSHOT + ../../hadoop-project + + hadoop-client-check-invariants + 3.0.0-alpha2-SNAPSHOT + pom + + Enforces our invariants for the api and runtime client modules. + Apache Hadoop Client Packaging Invariants + + + + + + + org.apache.hadoop + hadoop-client-api + + + org.apache.hadoop + hadoop-client-runtime + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 1.4 + + + org.codehaus.mojo + extra-enforcer-rules + 1.0-beta-3 + + + + + enforce-banned-dependencies + + enforce + + + + + + + + org.apache.hadoop:hadoop-annotations + + org.apache.htrace:htrace-core4 + + org.slf4j:slf4j-api + + commons-logging:commons-logging + + log4j:log4j + + + + true + + + org.apache.hadoop + hadoop-annotations + + * + + + + + + + + + + + + + + + + diff --git a/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml new file mode 100644 index 0000000000..23ab2ca583 --- /dev/null +++ b/hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml @@ -0,0 +1,132 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-alpha2-SNAPSHOT + ../../hadoop-project + + hadoop-client-check-test-invariants + 3.0.0-alpha2-SNAPSHOT + pom + + Enforces our invariants for the testing client modules. + Apache Hadoop Client Packaging Invariants for Test + + + + + + + org.apache.hadoop + hadoop-client-api + + + org.apache.hadoop + hadoop-client-runtime + + + org.apache.hadoop + hadoop-client-minicluster + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 1.4 + + + org.codehaus.mojo + extra-enforcer-rules + 1.0-beta-3 + + + + + enforce-banned-dependencies + + enforce + + + + + + + + org.apache.hadoop:hadoop-annotations + + org.apache.htrace:htrace-core4 + + org.slf4j:slf4j-api + + commons-logging:commons-logging + + log4j:log4j + + junit:junit + + org.hamcrest:hamcrest-core + + + + true + + + org.apache.hadoop + hadoop-annotations + + * + + + + + + + + + + + + + + + + diff --git a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml new file mode 100644 index 0000000000..0df6776493 --- /dev/null +++ b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml @@ -0,0 +1,130 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-alpha2-SNAPSHOT + ../../hadoop-project + + hadoop-client-integration-tests + 3.0.0-alpha2-SNAPSHOT + + Checks that we can use the generated artifacts + Apache Hadoop Client Packaging Integration Tests + + + 400 + + + + + org.apache.hadoop + hadoop-client-api + test + + + org.apache.hadoop + hadoop-client-runtime + test + + + org.apache.hadoop + hadoop-client-minicluster + test + + + log4j + log4j + test + + + org.slf4j + slf4j-api + test + + + org.slf4j + slf4j-log4j12 + test + + + junit + junit + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + default-testCompile + test-compile + + + **/IT* + **/*IT + + + + + + compile-integration-tests + pre-integration-test + + testCompile + + + + **/IT* + **/*IT + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + + integration-test + verify + + + + + + + + + diff --git a/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java b/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java new file mode 100644 index 0000000000..3f560d0c83 --- /dev/null +++ b/hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.hadoop.example; + +import java.io.IOException; +import java.net.URISyntaxException; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; + +import org.apache.hadoop.conf.Configuration; + +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; + +import org.apache.hadoop.hdfs.web.WebHdfsTestUtil; +import org.apache.hadoop.hdfs.web.WebHdfsConstants; + +/** + * Ensure that we can perform operations against the shaded minicluster + * given the API and runtime jars by performing some simple smoke tests. + */ +public class ITUseMiniCluster { + + private static final Logger LOG = + LoggerFactory.getLogger(ITUseMiniCluster.class); + + private MiniDFSCluster cluster; + + private static final String TEST_PATH = "/foo/bar/cats/dee"; + private static final String FILENAME = "test.file"; + + private static final String TEXT = "Lorem ipsum dolor sit amet, consectetur " + + "adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore " + + "magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation " + + "ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute " + + "irure dolor in reprehenderit in voluptate velit esse cillum dolore eu " + + "fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident," + + " sunt in culpa qui officia deserunt mollit anim id est laborum."; + + @Before + public void clusterUp() throws IOException { + final Configuration conf = new HdfsConfiguration(); + cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(3) + .build(); + cluster.waitActive(); + } + + @After + public void clusterDown() { + cluster.close(); + } + + @Test + public void useHdfsFileSystem() throws IOException { + try (final FileSystem fs = cluster.getFileSystem()) { + simpleReadAfterWrite(fs); + } + } + + public void simpleReadAfterWrite(final FileSystem fs) throws IOException { + LOG.info("Testing read-after-write with FS implementation: {}", fs); + final Path path = new Path(TEST_PATH, FILENAME); + if (!fs.mkdirs(path.getParent())) { + throw new IOException("Mkdirs failed to create " + + TEST_PATH); + } + try (final FSDataOutputStream out = fs.create(path)) { + out.writeUTF(TEXT); + } + try (final FSDataInputStream in = fs.open(path)) { + final String result = in.readUTF(); + Assert.assertEquals("Didn't read back text we wrote.", TEXT, result); + } + } + + @Test + public void useWebHDFS() throws IOException, URISyntaxException { + try (final FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem( + cluster.getConfiguration(0), WebHdfsConstants.WEBHDFS_SCHEME)) { + simpleReadAfterWrite(fs); + } + } +} diff --git a/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/hdfs-site.xml b/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/hdfs-site.xml new file mode 100644 index 0000000000..cd13532906 --- /dev/null +++ b/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/hdfs-site.xml @@ -0,0 +1,34 @@ + + + + + + + + + + hadoop.security.authentication + simple + + + + dfs.namenode.fs-limits.min-block-size + 0 + + + diff --git a/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/log4j.properties b/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/log4j.properties new file mode 100644 index 0000000000..db66160d85 --- /dev/null +++ b/hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/log4j.properties @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# log4j configuration used during build and unit tests + +log4j.rootLogger=info,stdout +log4j.threshold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n + diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml new file mode 100644 index 0000000000..8ab70d8cfb --- /dev/null +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -0,0 +1,745 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-alpha2-SNAPSHOT + ../../hadoop-project + + hadoop-client-minicluster + 3.0.0-alpha2-SNAPSHOT + jar + + Apache Hadoop Minicluster for Clients + Apache Hadoop Client Test Minicluster + + + org.apache.hadoop.shaded + + true + + + + + org.apache.hadoop + hadoop-client-api + runtime + + + org.apache.hadoop + hadoop-client-runtime + runtime + + + + junit + junit + runtime + + + + org.apache.hadoop + hadoop-annotations + compile + true + + + jdk.tools + jdk.tools + + + + + + + org.apache.hadoop + hadoop-minicluster + true + + + + org.apache.hadoop + hadoop-yarn-server-timelineservice + + + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-hdfs-client + + + org.apache.hadoop + hadoop-mapreduce-client-app + + + org.apache.hadoop + hadoop-yarn-api + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + + + + io.netty + netty + + + javax.servlet + javax.servlet-api + + + javax.xml.bind + jaxb-api + + + org.apache.avro + avro + + + org.apache.curator + curator-client + + + org.apache.hadoop + hadoop-mapreduce-client-common + + + org.apache.hadoop + hadoop-yarn-common + + + org.apache.hadoop + hadoop-yarn-server-common + + + org.apache.zookeeper + zookeeper + + + org.fusesource.leveldbjni + leveldbjni-all + + + org.eclipse.jetty + jetty-util + + + com.google.guava + guava + + + com.google.protobuf + protobuf-java + + + commons-collections + commons-collections + + + commons-io + commons-io + + + commons-lang + commons-lang + + + commons-logging + commons-logging + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + commons-cli + commons-cli + + + commons-codec + commons-codec + + + javax.servlet + javax.servlet-api + + + xmlenc + xmlenc + + + + + + org.apache.hadoop + hadoop-yarn-server-timelineservice + runtime + true + + + * + * + + + + + + org.apache.hadoop + hadoop-common + compile + test-jar + true + + + * + * + + + + + org.apache.hadoop + hadoop-hdfs + compile + test-jar + true + + + * + * + + + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + compile + test-jar + true + + + * + * + + + + + + + + + + com.sun.jersey + jersey-core + true + + + javax.ws.rs + jsr311-api + + + + + com.sun.jersey + jersey-client + true + + + com.sun.jersey + jersey-json + true + + + javax.xml.bind + jaxb-api + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + org.codehaus.jackson + jackson-jaxrs + + + org.codehaus.jackson + jackson-xc + + + + + com.sun.jersey + jersey-server + true + + + com.sun.jersey + jersey-servlet + true + + + org.eclipse.jdt + core + true + + + + net.sf.kosmosfs + kfs + true + + + net.java.dev.jets3t + jets3t + true + + + commons-codec + commons-codec + + + commons-logging + commons-logging + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + + + + + com.jcraft + jsch + true + + + + + org.apache.hadoop + hadoop-yarn-server-nodemanager + true + + + javax.servlet + javax.servlet-api + + + org.apache.hadoop + hadoop-yarn-api + + + org.apache.hadoop + hadoop-yarn-common + + + org.apache.hadoop + hadoop-yarn-server-common + + + org.fusesource.leveldbjni + leveldbjni-all + + + org.eclipse.jetty + jetty-util + + + com.google.guava + guava + + + com.google.protobuf + protobuf-java + + + commons-codec + commons-codec + + + commons-lang + commons-lang + + + commons-logging + commons-logging + + + org.slf4j + slf4j-api + + + javax.xml.bind + jaxb-api + + + + + org.apache.hadoop + hadoop-yarn-server-web-proxy + true + + + javax.servlet + javax.servlet-api + + + org.apache.hadoop + hadoop-yarn-api + + + org.apache.hadoop + hadoop-yarn-common + + + org.apache.hadoop + hadoop-yarn-server-common + + + com.google.guava + guava + + + commons-httpclient + commons-httpclient + + + commons-logging + commons-logging + + + org.eclipse.jetty + jetty-util + + + + + + com.google.inject.extensions + guice-servlet + true + + + + + + + + + com.google.inject + guice + true + + + com.google.guava + guava + + + + + com.sun.jersey.jersey-test-framework + jersey-test-framework-grizzly2 + true + + + + org.glassfish + javax.servlet + + + + + + com.sun.jersey.contribs + jersey-guice + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-source-plugin + + true + + + + org.apache.maven.plugins + maven-shade-plugin + + + org.apache.hadoop + hadoop-maven-plugins + ${project.version} + + + + + package + + shade + + + true + + + + org.apache.hadoop:hadoop-annotations + org.apache.hadoop:hadoop-client-api + org.apache.hadoop:hadoop-client-runtime + + org.apache.htrace:htrace-core4 + org.slf4j:slf4j-api + commons-logging:commons-logging + junit:junit + + org.apache.hadoop:hadoop-yarn-server-timelineservice + log4j:log4j + + + + + + + *:* + + **/*.java + + + + + *:* + + **/Test*.class + + + + + com.sun.jersey:jersey-client + + **/*.class + + + + com.sun.jersey:jersey-core + + **/*.class + + + + com.sun.jersey:jersey-servlet + + **/*.class + + + + + + org/ + ${shaded.dependency.prefix}.org. + + org/apache/hadoop/* + org/apache/hadoop/**/* + + org/apache/htrace/* + org/apache/htrace/**/* + org/slf4j/* + org/slf4j/**/* + org/apache/commons/logging/* + org/apache/commons/logging/**/* + org/apache/log4j/* + org/apache/log4j/**/* + **/pom.xml + + org/junit/* + org/junit/**/* + + org/ietf/jgss/* + org/omg/**/* + org/w3c/dom/* + org/w3c/dom/**/* + org/xml/sax/* + org/xml/sax/**/* + + + + com/ + ${shaded.dependency.prefix}.com. + + **/pom.xml + + com/sun/tools/* + com/sun/javadoc/* + com/sun/security/* + com/sun/jndi/* + com/sun/management/* + com/sun/tools/**/* + com/sun/javadoc/**/* + com/sun/security/**/* + com/sun/jndi/**/* + com/sun/management/**/* + + + + io/ + ${shaded.dependency.prefix}.io. + + **/pom.xml + + io/compression/* + io/compression/**/* + io/mapfile/* + io/mapfile/**/* + io/map/index/* + io/seqfile/* + io/seqfile/**/* + io/file/buffer/size + io/skip/checksum/errors + io/sort/* + io/serializations + + + + javax/el/ + ${shaded.dependency.prefix}.javax.el. + + **/pom.xml + + + + javax/inject/ + ${shaded.dependency.prefix}.javax.inject. + + **/pom.xml + + + + javax/servlet/ + ${shaded.dependency.prefix}.javax.servlet. + + **/pom.xml + + + + net/ + ${shaded.dependency.prefix}.net. + + **/pom.xml + + net/topology/* + net/topology/**/* + + + + + + + + + false + + + + + + + + org.codehaus.mojo + license-maven-plugin + + + + + + diff --git a/hadoop-client-modules/hadoop-client-runtime/pom.xml b/hadoop-client-modules/hadoop-client-runtime/pom.xml new file mode 100644 index 0000000000..aa1a54bb25 --- /dev/null +++ b/hadoop-client-modules/hadoop-client-runtime/pom.xml @@ -0,0 +1,318 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-alpha2-SNAPSHOT + ../../hadoop-project + + hadoop-client-runtime + 3.0.0-alpha2-SNAPSHOT + jar + + Apache Hadoop Client + Apache Hadoop Client Runtime + + + org.apache.hadoop.shaded + + true + + + + + org.apache.hadoop + hadoop-client + + true + + + + javax.xml.bind + jaxb-api + + + xml-apis + xml-apis + + + + + + org.apache.hadoop + hadoop-client-api + runtime + + + + org.apache.hadoop + hadoop-annotations + compile + + + jdk.tools + jdk.tools + + + + + + org.apache.htrace + htrace-core4 + runtime + + + org.slf4j + slf4j-api + runtime + + + commons-logging + commons-logging + runtime + + + + log4j + log4j + runtime + true + + + + + + + org.apache.maven.plugins + maven-source-plugin + + true + + + + org.apache.maven.plugins + maven-shade-plugin + + + org.apache.hadoop + hadoop-maven-plugins + ${project.version} + + + + + package + + shade + + + true + + + + org.apache.hadoop:hadoop-client-api + + org.apache.htrace:htrace-core4 + + org.slf4j:slf4j-api + + commons-logging:commons-logging + + log4j:log4j + + + + + + org.apache.hadoop:* + + **/* + * + + + + + *:* + + **/*.java + + + + + tomcat:jasper-compiler + + org/apache/jasper/compiler/Localizer.class + + + + + xerces:xercesImpl + + META-INF/services/* + + + + + com.sun.jersey:* + + META-INF/services/javax.* + + + + + + org/ + ${shaded.dependency.prefix}.org. + + org/apache/hadoop/* + org/apache/hadoop/**/* + + org/apache/htrace/* + org/apache/htrace/**/* + org/slf4j/* + org/slf4j/**/* + org/apache/commons/logging/* + org/apache/commons/logging/**/* + org/apache/log4j/* + org/apache/log4j/**/* + **/pom.xml + + org/ietf/jgss/* + org/omg/**/* + org/w3c/dom/* + org/w3c/dom/**/* + org/xml/sax/* + org/xml/sax/**/* + + + + com/ + ${shaded.dependency.prefix}.com. + + **/pom.xml + + com/sun/tools/* + com/sun/javadoc/* + com/sun/security/* + com/sun/jndi/* + com/sun/management/* + com/sun/tools/**/* + com/sun/javadoc/**/* + com/sun/security/**/* + com/sun/jndi/**/* + com/sun/management/**/* + + + + io/ + ${shaded.dependency.prefix}.io. + + **/pom.xml + + io/compression/* + io/compression/**/* + io/mapfile/* + io/mapfile/**/* + io/map/index/* + io/seqfile/* + io/seqfile/**/* + io/file/buffer/size + io/skip/checksum/errors + io/sort/* + io/serializations + + + + javax/el/ + ${shaded.dependency.prefix}.javax.el. + + **/pom.xml + + + + javax/servlet/ + ${shaded.dependency.prefix}.javax.servlet. + + **/pom.xml + + + + net/ + ${shaded.dependency.prefix}.net. + + **/pom.xml + + net/topology/* + net/topology/**/* + + + + + + + + + + + false + + + META-INF/jboss-beans.xml + + + + META-INF/mailcap.default + + + META-INF/mimetypes.default + + + + + + + + org.codehaus.mojo + license-maven-plugin + + + + + + diff --git a/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml similarity index 91% rename from hadoop-client/pom.xml rename to hadoop-client-modules/hadoop-client/pom.xml index dbd4facc7b..0394caedb3 100644 --- a/hadoop-client/pom.xml +++ b/hadoop-client-modules/hadoop-client/pom.xml @@ -19,14 +19,14 @@ org.apache.hadoop hadoop-project-dist 3.0.0-alpha2-SNAPSHOT - ../hadoop-project-dist - + ../../hadoop-project-dist + hadoop-client 3.0.0-alpha2-SNAPSHOT - jar + pom - Apache Hadoop Client - Apache Hadoop Client + Apache Hadoop Client aggregation pom with dependencies exposed + Apache Hadoop Client Aggregator client @@ -98,6 +98,11 @@ org.apache.zookeeper zookeeper + + + org.slf4j + slf4j-log4j12 + @@ -170,6 +175,11 @@ io.netty netty + + + org.slf4j + slf4j-log4j12 + org.apache.zookeeper zookeeper @@ -270,6 +280,11 @@ io.netty netty + + + org.slf4j + slf4j-log4j12 + @@ -298,6 +313,11 @@ io.netty netty + + + org.slf4j + slf4j-log4j12 + @@ -312,7 +332,6 @@ - diff --git a/hadoop-client-modules/pom.xml b/hadoop-client-modules/pom.xml new file mode 100644 index 0000000000..eea4c8fb7c --- /dev/null +++ b/hadoop-client-modules/pom.xml @@ -0,0 +1,45 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-alpha2-SNAPSHOT + ../hadoop-project + + hadoop-client-modules + pom + multi-module for Apache Hadoop client artifacts + Apache Hadoop Client Modules + + + + hadoop-client + + hadoop-client-api + + hadoop-client-runtime + + hadoop-client-minicluster + + hadoop-client-check-invariants + hadoop-client-check-test-invariants + + hadoop-client-integration-tests + + + diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index 30e64728f8..ae8c4ceb58 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -51,6 +51,23 @@ hadoop-yarn-api provided + + org.apache.hadoop + hadoop-client-check-invariants + pom + provided + + + org.apache.hadoop + hadoop-client-check-test-invariants + pom + provided + + + org.apache.hadoop + hadoop-client-integration-tests + provided + diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml index eae03a1912..6a2bd73c4d 100644 --- a/hadoop-maven-plugins/pom.xml +++ b/hadoop-maven-plugins/pom.xml @@ -54,6 +54,28 @@ com.fasterxml.jackson.core jackson-databind + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + provided + + + + org.apache.maven.shared + maven-dependency-tree + + + + org.vafer + jdependency + + + diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java new file mode 100644 index 0000000000..a54b5ba58e --- /dev/null +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java @@ -0,0 +1,164 @@ +package org.apache.hadoop.maven.plugin.shade.resource; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +import java.io.BufferedReader; +import org.apache.maven.plugins.shade.relocation.Relocator; +import org.apache.maven.plugins.shade.resource.ResourceTransformer; +import org.codehaus.plexus.util.IOUtil; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; + +/** + * Resources transformer that appends entries in META-INF/services resources + * into a single resource. For example, if there are several + * META-INF/services/org.apache.maven.project.ProjectBuilder resources spread + * across many JARs the individual entries will all be concatenated into a + * single META-INF/services/org.apache.maven.project.ProjectBuilder resource + * packaged into the resultant JAR produced by the shading process. + * + * From following sources, only needed until MSHADE-182 gets released + * * https://s.apache.org/vwjl (source in maven-shade-plugin repo) + * * https://issues.apache.org/jira/secure/attachment/12718938/MSHADE-182.patch + * + * Has been reformatted according to Hadoop checkstyle rules and modified + * to meet Hadoop's threshold for Findbugs problems. + */ +public class ServicesResourceTransformer + implements ResourceTransformer { + + private static final String SERVICES_PATH = "META-INF/services"; + + private Map serviceEntries = new HashMap<>(); + + private List relocators; + + public boolean canTransformResource(String resource) { + if (resource.startsWith(SERVICES_PATH)) { + return true; + } + + return false; + } + + public void processResource(String resource, InputStream is, + List relocatorz) throws IOException { + ServiceStream out = serviceEntries.get(resource); + if (out == null) { + out = new ServiceStream(); + serviceEntries.put(resource, out); + } + + out.append(is); + is.close(); + + if (this.relocators == null) { + this.relocators = relocatorz; + } + } + + public boolean hasTransformedResource() { + return serviceEntries.size() > 0; + } + + public void modifyOutputStream(JarOutputStream jos) + throws IOException { + for (Map.Entry entry : serviceEntries.entrySet()) { + String key = entry.getKey(); + ServiceStream data = entry.getValue(); + + if (relocators != null) { + key = key.substring(SERVICES_PATH.length() + 1); + for (Relocator relocator : relocators) { + if (relocator.canRelocateClass(key)) { + key = relocator.relocateClass(key); + break; + } + } + + key = SERVICES_PATH + '/' + key; + } + + jos.putNextEntry(new JarEntry(key)); + + //read the content of service file for candidate classes for relocation + //presume everything is UTF8, because Findbugs barfs on default + //charset and this seems no worse a choice ¯\_(ツ)_/¯ + PrintWriter writer = new PrintWriter(new OutputStreamWriter(jos, + StandardCharsets.UTF_8)); + InputStreamReader streamReader = + new InputStreamReader(data.toInputStream(), StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(streamReader); + String className; + + while ((className = reader.readLine()) != null) { + + if (relocators != null) { + for (Relocator relocator : relocators) { + //if the class can be relocated then relocate it + if (relocator.canRelocateClass(className)) { + className = relocator.applyToSourceContent(className); + break; + } + } + } + + writer.println(className); + writer.flush(); + } + + reader.close(); + data.reset(); + } + } + + static class ServiceStream extends ByteArrayOutputStream { + + public ServiceStream() { + super(1024); + } + + public void append(InputStream is) + throws IOException { + if (count > 0 && buf[count - 1] != '\n' && buf[count - 1] != '\r') { + write('\n'); + } + + IOUtil.copy(is, this); + } + + public InputStream toInputStream() { + return new ByteArrayInputStream(buf, 0, count); + } + + } + +} diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java new file mode 100644 index 0000000000..571491c035 --- /dev/null +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Resource handling plugins used internal to the Hadoop build. + * IA.Private (build structure encourages not using the actual annotations) + */ +package org.apache.hadoop.maven.plugin.shade.resource; diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 9ec24eaae5..36172f20bf 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -27,6 +27,7 @@ Apache Hadoop Project POM Apache Hadoop Project POM pom + 2008 @@ -114,6 +115,7 @@ 3.1 2.5.1 2.6 + 2.4.3 2.5 2.4 2.3 @@ -148,6 +150,44 @@ hadoop-annotations ${project.version} + + org.apache.hadoop + hadoop-client-modules + ${project.version} + pom + + + org.apache.hadoop + hadoop-client-api + ${project.version} + + + org.apache.hadoop + hadoop-client-check-invariants + ${project.version} + pom + + + org.apache.hadoop + hadoop-client-check-test-invariants + ${project.version} + pom + + + org.apache.hadoop + hadoop-client-integration-tests + ${project.version} + + + org.apache.hadoop + hadoop-client-runtime + ${project.version} + + + org.apache.hadoop + hadoop-client-minicluster + ${project.version} + org.apache.hadoop hadoop-common @@ -1244,6 +1284,40 @@ + + org.codehaus.mojo + build-helper-maven-plugin + ${build-helper-maven-plugin.version} + + + + org.codehaus.mojo + license-maven-plugin + 1.10 + + false + ${project.basedir} + + + + update-pom-license + + update-file-header + + package + + apache_v2 + + dependency-reduced-pom.xml + + + + + maven-clean-plugin ${maven-clean-plugin.version} @@ -1260,13 +1334,8 @@ org.apache.maven.plugins - maven-dependency-plugin - 2.2 - - - org.codehaus.mojo - build-helper-maven-plugin - ${build-helper-maven-plugin.version} + maven-shade-plugin + ${maven-shade-plugin.version} org.apache.maven.plugins @@ -1355,6 +1424,19 @@ + + maven-clean-plugin + + + + ${project.basedir} + + dependency-reduced-pom.xml + + + + + org.codehaus.mojo findbugs-maven-plugin diff --git a/pom.xml b/pom.xml index 8776534e96..9de7b368cf 100644 --- a/pom.xml +++ b/pom.xml @@ -96,7 +96,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 1.5 1.7 2.4 - 2.8 + 2.10 1.4.1 2.9.1 1.5 @@ -125,8 +125,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs hadoop-mapreduce-project hadoop-tools hadoop-dist - hadoop-client hadoop-minicluster + hadoop-client-modules hadoop-build-tools