big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-11804. Shaded Hadoop client artifacts and minicluster. Contributed by Sean Busbey.

Browse Source
This commit is contained in:
Andrew Wang 2016-12-15 11:44:59 -08:00
parent 36947f79ba
commit 70ca1f1e3a
19 changed files with 2231 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -15,6 +15,7 @@
.settings
target
build
dependency-reduced-pom.xml
# External tool builders
*/.externalToolBuilders

8
dev-support/bin/dist-layout-stitching
View File

@ -137,6 +137,12 @@ run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERS
run cp -pr "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/hadoop-hdfs-httpfs-${VERSION}"/* .
run cp -pr "${ROOT}/hadoop-common-project/hadoop-kms/target/hadoop-kms-${VERSION}"/* .
# copy client jars as-is
run mkdir -p "share/hadoop/client"
run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-${VERSION}.jar" share/hadoop/client/
run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-${VERSION}.jar" share/hadoop/client/
run cp -p "${ROOT}/hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-${VERSION}.jar" share/hadoop/client/
echo
echo "Hadoop dist layout available at: ${BASEDIR}/hadoop-${VERSION}"
echo
echo

216
hadoop-client-modules/hadoop-client-api/pom.xml Normal file
View File

@ -0,0 +1,216 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-api</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Apache Hadoop Client</description>
<name>Apache Hadoop Client API</name>
<properties>
<shaded.dependency.prefix>org.apache.hadoop.shaded</shaded.dependency.prefix>
<!-- We contain no source -->
<maven.javadoc.skip>true</maven.javadoc.skip>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<!-- We list this as optional because as a type-pom it won't get included in the shading.
Marking it optional means it doesn't count as a transitive dependency of this artifact.
-->
<optional>true</optional>
<exclusions>
<!-- these APIs are a part of the SE JDK -->
<exclusion>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</exclusion>
<exclusion>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- This comes from our parent pom. If we don't expressly change it here to get included,
downstream will get warnings at compile time. -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
<plugins>
<!-- We contain no source -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<configuration>
<skipSource>true</skipSource>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<artifactSet>
<includes>
<include>org.apache.hadoop:*</include>
</includes>
</artifactSet>
<filters>
<!-- We get these package level classes from various yarn api jars -->
<filter>
<artifact>org.apache.hadoop:hadoop-yarn-common</artifact>
<excludes>
<exclude>org/apache/hadoop/yarn/factories/package-info.class</exclude>
<exclude>org/apache/hadoop/yarn/util/package-info.class</exclude>
<exclude>org/apache/hadoop/yarn/factory/providers/package-info.class</exclude>
<exclude>org/apache/hadoop/yarn/client/api/impl/package-info.class</exclude>
<exclude>org/apache/hadoop/yarn/client/api/package-info.class</exclude>
</excludes>
</filter>
</filters>
<relocations>
<relocation>
<pattern>org/</pattern>
<shadedPattern>${shaded.dependency.prefix}.org.</shadedPattern>
<excludes>
<exclude>org/apache/hadoop/*</exclude>
<exclude>org/apache/hadoop/**/*</exclude>
<!-- Our non-shaded htrace and logging libraries -->
<exclude>org/apache/htrace/*</exclude>
<exclude>org/apache/htrace/**/*</exclude>
<exclude>org/slf4j/*</exclude>
<exclude>org/slf4j/**/*</exclude>
<exclude>org/apache/commons/logging/*</exclude>
<exclude>org/apache/commons/logging/**/*</exclude>
<exclude>org/apache/log4j/*</exclude>
<exclude>org/apache/log4j/**/*</exclude>
<exclude>**/pom.xml</exclude>
<!-- Not the org/ packages that are a part of the jdk -->
<exclude>org/ietf/jgss/*</exclude>
<exclude>org/omg/**/*</exclude>
<exclude>org/w3c/dom/*</exclude>
<exclude>org/w3c/dom/**/*</exclude>
<exclude>org/xml/sax/*</exclude>
<exclude>org/xml/sax/**/*</exclude>
</excludes>
</relocation>
<relocation>
<pattern>com/</pattern>
<shadedPattern>${shaded.dependency.prefix}.com.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
<!-- Not the com/ packages that are a part of particular jdk implementations -->
<exclude>com/sun/tools/*</exclude>
<exclude>com/sun/javadoc/*</exclude>
<exclude>com/sun/security/*</exclude>
<exclude>com/sun/jndi/*</exclude>
<exclude>com/sun/management/*</exclude>
<exclude>com/sun/tools/**/*</exclude>
<exclude>com/sun/javadoc/**/*</exclude>
<exclude>com/sun/security/**/*</exclude>
<exclude>com/sun/jndi/**/*</exclude>
<exclude>com/sun/management/**/*</exclude>
</excludes>
</relocation>
<relocation>
<pattern>io/</pattern>
<shadedPattern>${shaded.dependency.prefix}.io.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
<!-- Exclude config keys for Hadoop that look like package names -->
<exclude>io/compression/*</exclude>
<exclude>io/compression/**/*</exclude>
<exclude>io/mapfile/*</exclude>
<exclude>io/mapfile/**/*</exclude>
<exclude>io/map/index/*</exclude>
<exclude>io/seqfile/*</exclude>
<exclude>io/seqfile/**/*</exclude>
<exclude>io/file/buffer/size</exclude>
<exclude>io/skip/checksum/errors</exclude>
<exclude>io/sort/*</exclude>
<exclude>io/serializations</exclude>
</excludes>
</relocation>
<relocation>
<pattern>javax/servlet/</pattern>
<shadedPattern>${shaded.dependency.prefix}.javax.servlet.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
</excludes>
</relocation>
<relocation>
<pattern>net/</pattern>
<shadedPattern>${shaded.dependency.prefix}.net.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
<!-- Exclude config keys for Hadoop that look like package names -->
<exclude>net/topology/*</exclude>
<exclude>net/topology/**/*</exclude>
</excludes>
</relocation>
</relocations>
<transformers>
<!-- Needed until MSHADE-182 -->
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
<addHeader>false</addHeader>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>license-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

124
hadoop-client-modules/hadoop-client-check-invariants/pom.xml Normal file
View File

@ -0,0 +1,124 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-check-invariants</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>pom</packaging>
<description>Enforces our invariants for the api and runtime client modules.</description>
<name>Apache Hadoop Client Packaging Invariants</name>
<properties>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-runtime</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.4</version>
<dependencies>
<dependency>
<groupId>org.codehaus.mojo</groupId>
<artifactId>extra-enforcer-rules</artifactId>
<version>1.0-beta-3</version>
</dependency>
</dependencies>
<executions>
<execution>
<id>enforce-banned-dependencies</id>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<banTransitiveDependencies>
<!--
<message>
Our client-facing artifacts are not supposed to have additional dependencies
and one or more of them do. The output from the enforcer plugin should give
specifics.
</message>
-->
<excludes>
<!-- annotations is provided, and both artifacts exclude the tools transitive,
but enforcer still sees it.
-->
<exclude>org.apache.hadoop:hadoop-annotations</exclude>
<!-- We leave HTrace as an unshaded dependnecy on purpose so that tracing within a JVM will work -->
<exclude>org.apache.htrace:htrace-core4</exclude>
<!-- Leave slf4j unshaded so downstream users can configure logging. -->
<exclude>org.slf4j:slf4j-api</exclude>
<!-- Leave commons-logging unshaded so downstream users can configure logging. -->
<exclude>commons-logging:commons-logging</exclude>
<!-- Leave log4j unshaded so downstream users can configure logging. -->
<exclude>log4j:log4j</exclude>
</excludes>
</banTransitiveDependencies>
<banDuplicateClasses>
<findAllDuplicates>true</findAllDuplicates>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
<ignoreClasses>
<ignoreClass>*</ignoreClass>
</ignoreClasses>
</dependency>
</dependencies>
</banDuplicateClasses>
</rules>
<!-- TODO we need a rule for "we don't have classes that are outside of the org.apache.hadoop package" -->
<!-- TODO we need a rule for "the constants in this set of classes haven't been shaded / don't have this prefix"
Manually checking the set of Keys that look like packages we relocate:
cat `find . \( -name '*Keys.java' -o -name '*KeysPublic.java' \) -a -path '*/src/main/*'` | grep -E "\"(io\.|org\.|com\.|net\.)" | grep -v "^package" | grep -v "^import" | grep -v "\"org.apache.hadoop"
Manually check the set of shaded artifacts to see if the Keys constants have been relocated:
for clazz in `find . \( -name '*Keys.java' -o -name '*KeysPublic.java' \) -a -path '*/src/main/*'`; do
clazz=${clazz#*src/main/java/}
clazz="${clazz%.java}"
javap -cp hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-3.0.0-alpha2-SNAPSHOT.jar:hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-3.0.0-alpha2-SNAPSHOT.jar:hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-3.0.0-alpha2-SNAPSHOT.jar \
-constants "${clazz//\//.}" | grep "org.apache.hadoop.shaded"
done
-->
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

132
hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml Normal file
View File

@ -0,0 +1,132 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-check-test-invariants</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>pom</packaging>
<description>Enforces our invariants for the testing client modules.</description>
<name>Apache Hadoop Client Packaging Invariants for Test</name>
<properties>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-runtime</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-minicluster</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<version>1.4</version>
<dependencies>
<dependency>
<groupId>org.codehaus.mojo</groupId>
<artifactId>extra-enforcer-rules</artifactId>
<version>1.0-beta-3</version>
</dependency>
</dependencies>
<executions>
<execution>
<id>enforce-banned-dependencies</id>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<banTransitiveDependencies>
<!--
<message>
Our client-facing artifacts are not supposed to have additional dependencies
and one or more of them do. The output from the enforcer plugin should give
specifics.
</message>
-->
<excludes>
<!-- annotations is provided, and both artifacts exclude the tools transitive,
but enforcer still sees it.
-->
<exclude>org.apache.hadoop:hadoop-annotations</exclude>
<!-- We leave HTrace as an unshaded dependnecy on purpose so that tracing within a JVM will work -->
<exclude>org.apache.htrace:htrace-core4</exclude>
<!-- Leave slf4j unshaded so downstream users can configure logging. -->
<exclude>org.slf4j:slf4j-api</exclude>
<!-- Leave commons-logging unshaded so downstream users can configure logging. -->
<exclude>commons-logging:commons-logging</exclude>
<!-- Leave log4j unshaded so downstream users can configure logging. -->
<exclude>log4j:log4j</exclude>
<!-- Leave JUnit unshaded so downstream can use our test helper classes -->
<exclude>junit:junit</exclude>
<!-- JUnit brings in hamcrest -->
<exclude> org.hamcrest:hamcrest-core</exclude>
</excludes>
</banTransitiveDependencies>
<banDuplicateClasses>
<findAllDuplicates>true</findAllDuplicates>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
<ignoreClasses>
<ignoreClass>*</ignoreClass>
</ignoreClasses>
</dependency>
</dependencies>
</banDuplicateClasses>
</rules>
<!-- TODO we need a rule for "we don't have classes that are outside of the org.apache.hadoop package" -->
<!-- TODO we need a rule for "the constants in this set of classes haven't been shaded / don't have this prefix"
Manually checking the set of Keys that look like packages we relocate:
cat `find . \( -name '*Keys.java' -o -name '*KeysPublic.java' \) -a -path '*/src/main/*'` | grep -E "\"(io\.|org\.|com\.|net\.)" | grep -v "^package" | grep -v "^import" | grep -v "\"org.apache.hadoop"
Manually check the set of shaded artifacts to see if the Keys constants have been relocated:
for clazz in `find . \( -name '*Keys.java' -o -name '*KeysPublic.java' \) -a -path '*/src/main/*'`; do
clazz=${clazz#*src/main/java/}
clazz="${clazz%.java}"
javap -cp hadoop-client-modules/hadoop-client-api/target/hadoop-client-api-3.0.0-alpha2-SNAPSHOT.jar:hadoop-client-modules/hadoop-client-runtime/target/hadoop-client-runtime-3.0.0-alpha2-SNAPSHOT.jar:hadoop-client-modules/hadoop-client-minicluster/target/hadoop-client-minicluster-3.0.0-alpha2-SNAPSHOT.jar \
-constants "${clazz//\//.}" | grep "org.apache.hadoop.shaded"
done
-->
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

130
hadoop-client-modules/hadoop-client-integration-tests/pom.xml Normal file
View File

@ -0,0 +1,130 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-integration-tests</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<description>Checks that we can use the generated artifacts</description>
<name>Apache Hadoop Client Packaging Integration Tests</name>
<properties>
<failsafe.timeout>400</failsafe.timeout>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-runtime</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-minicluster</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<!-- Because our tests rely on our shaded artifacts, we can't compile
them until after the package phase has run.
-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<executions>
<!--
First, let's make sure the normal test-compile doesn't try to
compile our integration tests.
-->
<execution>
<id>default-testCompile</id>
<phase>test-compile</phase>
<configuration>
<testExcludes>
<testExclude>**/IT*</testExclude>
<testExclude>**/*IT</testExclude>
</testExcludes>
</configuration>
</execution>
<!--
Finally, let's make a 'just for integration tests'-compile that
fires off prior to our integration tests but after the package
phase has created our shaded artifacts.
-->
<execution>
<id>compile-integration-tests</id>
<phase>pre-integration-test</phase>
<goals>
<goal>testCompile</goal>
</goals>
<configuration>
<testIncludes>
<testInclude>**/IT*</testInclude>
<testInclude>**/*IT</testInclude>
</testIncludes>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

113
hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java Normal file
View File

@ -0,0 +1,113 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
package org.apache.hadoop.example;
import java.io.IOException;
import java.net.URISyntaxException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
/**
* Ensure that we can perform operations against the shaded minicluster
* given the API and runtime jars by performing some simple smoke tests.
*/
public class ITUseMiniCluster {
private static final Logger LOG =
LoggerFactory.getLogger(ITUseMiniCluster.class);
private MiniDFSCluster cluster;
private static final String TEST_PATH = "/foo/bar/cats/dee";
private static final String FILENAME = "test.file";
private static final String TEXT = "Lorem ipsum dolor sit amet, consectetur "
+ "adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore "
+ "magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation "
+ "ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute "
+ "irure dolor in reprehenderit in voluptate velit esse cillum dolore eu "
+ "fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident,"
+ " sunt in culpa qui officia deserunt mollit anim id est laborum.";
@Before
public void clusterUp() throws IOException {
final Configuration conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(3)
.build();
cluster.waitActive();
}
@After
public void clusterDown() {
cluster.close();
}
@Test
public void useHdfsFileSystem() throws IOException {
try (final FileSystem fs = cluster.getFileSystem()) {
simpleReadAfterWrite(fs);
}
}
public void simpleReadAfterWrite(final FileSystem fs) throws IOException {
LOG.info("Testing read-after-write with FS implementation: {}", fs);
final Path path = new Path(TEST_PATH, FILENAME);
if (!fs.mkdirs(path.getParent())) {
throw new IOException("Mkdirs failed to create " +
TEST_PATH);
}
try (final FSDataOutputStream out = fs.create(path)) {
out.writeUTF(TEXT);
}
try (final FSDataInputStream in = fs.open(path)) {
final String result = in.readUTF();
Assert.assertEquals("Didn't read back text we wrote.", TEXT, result);
}
}
@Test
public void useWebHDFS() throws IOException, URISyntaxException {
try (final FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(
cluster.getConfiguration(0), WebHdfsConstants.WEBHDFS_SCHEME)) {
simpleReadAfterWrite(fs);
}
}
}

34
hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/hdfs-site.xml Normal file
View File

@ -0,0 +1,34 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- Put site-specific property overrides in this file. -->
<configuration>
<!-- Turn security off for tests by default -->
<property>
<name>hadoop.security.authentication</name>
<value>simple</value>
</property>
<!-- Disable min block size since most tests use tiny blocks -->
<property>
<name>dfs.namenode.fs-limits.min-block-size</name>
<value>0</value>
</property>
</configuration>

24
hadoop-client-modules/hadoop-client-integration-tests/src/test/resources/log4j.properties Normal file
View File

@ -0,0 +1,24 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# log4j configuration used during build and unit tests
log4j.rootLogger=info,stdout
log4j.threshold=ALL
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n

745
hadoop-client-modules/hadoop-client-minicluster/pom.xml Normal file
View File

@ -0,0 +1,745 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-minicluster</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Apache Hadoop Minicluster for Clients</description>
<name>Apache Hadoop Client Test Minicluster</name>
<properties>
<shaded.dependency.prefix>org.apache.hadoop.shaded</shaded.dependency.prefix>
<!-- We contain no source -->
<maven.javadoc.skip>true</maven.javadoc.skip>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-runtime</artifactId>
<scope>runtime</scope>
</dependency>
<!-- Leave JUnit as a direct dependency -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>runtime</scope>
</dependency>
<!-- Adding hadoop-annotations so we can make it optional to remove from our transitive tree -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
<scope>compile</scope>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- uncomment this dependency if you need to use
`mvn dependency:tree -Dverbose` to determine if a dependency shows up
in both the hadoop-client-* artifacts and something under minicluster.
-->
<!--
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<scope>provided</scope>
</dependency>
-->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<optional>true</optional>
<exclusions>
<!-- Exclude the in-development timeline service and
add it as an optional runtime dependency
-->
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timelineservice</artifactId>
</exclusion>
<!-- exclude everything that comes in via the shaded runtime and api TODO remove once we have a filter for "is in these artifacts" -->
<!-- Skip jersey, since we need it again here. -->
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-app</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
</exclusion>
<!-- exclude things that came in via transitive in shaded runtime and api -->
<exclusion>
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>org.fusesource.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
<exclusion>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
</exclusion>
<exclusion>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</exclusion>
<exclusion>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</exclusion>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>xmlenc</groupId>
<artifactId>xmlenc</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Add optional runtime dependency on the in-development timeline server module
to indicate that downstream folks interested in turning it on need that dep.
-->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timelineservice</artifactId>
<scope>runtime</scope>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Add back in transitive dependencies of hadoop-minicluster that are test-jar artifacts excluded as a side effect of excluding the jar
Note that all of these must be marked "optional" because they won't be removed from the reduced-dependencies pom after they're included.
-->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>compile</scope>
<type>test-jar</type>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<scope>compile</scope>
<type>test-jar</type>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<scope>compile</scope>
<type>test-jar</type>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>*</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Add back in the transitive dependencies excluded from hadoop-common in client TODO remove once we have a filter for "is in these artifacts" -->
<!-- skip javax.servlet:servlet-api because it's in client -->
<!-- Skip commons-logging:commons-logging-api because it looks like nothing actually included it -->
<!-- Skip jetty-util because it's in client -->
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>javax.ws.rs</groupId>
<artifactId>jsr311-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-jaxrs</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-xc</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-servlet</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
<optional>true</optional>
</dependency>
<!-- skip org.apache.avro:avro-ipc because it doesn't look like hadoop-common actually uses it -->
<dependency>
<groupId>net.sf.kosmosfs</groupId>
<artifactId>kfs</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
<optional>true</optional>
</dependency>
<!-- add back in transitive dependencies of hadoop-mapreduce-client-app removed in client -->
<!-- Skipping javax.servlet:servlet-api because it's in client -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.fusesource.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
<exclusion>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</exclusion>
<exclusion>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- skipping hadoop-annotations -->
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
<optional>true</optional>
</dependency>
<!-- skipping junit:junit because it is test scope -->
<!-- skipping avro because it is in client via hadoop-common -->
<!-- skipping jline:jline because it is only present at test scope in the original -->
<!-- skipping io.netty:netty because it's in client -->
<!-- add back in transitive dependencies of hadoop-yarn-api removed in client -->
<!-- skipping hadoop-annotations -->
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
<optional>true</optional>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.sun.jersey.jersey-test-framework</groupId>
<artifactId>jersey-test-framework-grizzly2</artifactId>
<optional>true</optional>
<exclusions>
<!-- excluding because client already has the tomcat version -->
<exclusion>
<groupId>org.glassfish</groupId>
<artifactId>javax.servlet</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- skipping jersey-server because it's above -->
<dependency>
<groupId>com.sun.jersey.contribs</groupId>
<artifactId>jersey-guice</artifactId>
<optional>true</optional>
</dependency>
<!-- skipping guice-servlet because it's above -->
<!-- skipping avro because it is in client via hadoop-common -->
<!-- skipping jersey-core because it's above -->
<!-- skipping jersey-json because it's above. -->
<!-- skipping io.netty:netty because it's in client -->
<!-- Add back in transitive dependencies from hadoop-mapreduce-client-core that were excluded by client -->
<!-- skipping junit:junit because it is test scope -->
<!-- skipping guice because it's above -->
<!-- skipping jersey-test-framework-grizzly2 because it's above -->
<!-- skipping jersey-server because it's above -->
<!-- skipping jersey-guice because it's above -->
<!-- skipping avro because it is in client via hadoop-common -->
<!-- skipping hadoop-annotations -->
<!-- skipping guice-servlet because it's above -->
<!-- skipping jersey-json because it's above. -->
<!-- skipping io.netty:netty because it's in client -->
<!-- add back in transitive dependencies of hadoop-mapreduce-client-jobclient that were excluded from client -->
<!-- skipping junit:junit because it is test scope -->
<!-- skipping avro because it is in client via hadoop-common -->
<!-- skipping hadoop-annotations -->
<!-- skipping guice-servlet because it's above -->
<!-- skipping io.netty:netty because it's in client -->
</dependencies>
<build>
<plugins>
<!-- We contain no source -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<configuration>
<skipSource>true</skipSource>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<createSourceJar>true</createSourceJar>
<artifactSet>
<excludes>
<!-- Fine to expose our already-shaded deps as dependencies -->
<exclude>org.apache.hadoop:hadoop-annotations</exclude>
<exclude>org.apache.hadoop:hadoop-client-api</exclude>
<exclude>org.apache.hadoop:hadoop-client-runtime</exclude>
<!-- Fine to expose our purposefully not-shaded deps as dependencies -->
<exclude>org.apache.htrace:htrace-core4</exclude>
<exclude>org.slf4j:slf4j-api</exclude>
<exclude>commons-logging:commons-logging</exclude>
<exclude>junit:junit</exclude>
<!-- Keep optional runtime deps out of the shading -->
<exclude>org.apache.hadoop:hadoop-yarn-server-timelineservice</exclude>
<exclude>log4j:log4j</exclude>
<!-- We need a filter that matches just those things that are included in the above artiacts -->
</excludes>
</artifactSet>
<filters>
<!-- Some of our dependencies include source, so remove it. -->
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>**/*.java</exclude>
</excludes>
</filter>
<!-- We pull in several test jars; keep out the actual test classes -->
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>**/Test*.class</exclude>
</excludes>
</filter>
<!-- Since runtime has classes for these jars, we exclude them.
We still want the java services api files, since those were excluded in runtime
-->
<filter>
<artifact>com.sun.jersey:jersey-client</artifact>
<excludes>
<exclude>**/*.class</exclude>
</excludes>
</filter>
<filter>
<artifact>com.sun.jersey:jersey-core</artifact>
<excludes>
<exclude>**/*.class</exclude>
</excludes>
</filter>
<filter>
<artifact>com.sun.jersey:jersey-servlet</artifact>
<excludes>
<exclude>**/*.class</exclude>
</excludes>
</filter>
</filters>
<relocations>
<relocation>
<pattern>org/</pattern>
<shadedPattern>${shaded.dependency.prefix}.org.</shadedPattern>
<excludes>
<exclude>org/apache/hadoop/*</exclude>
<exclude>org/apache/hadoop/**/*</exclude>
<!-- Our non-shaded htrace and logging libraries -->
<exclude>org/apache/htrace/*</exclude>
<exclude>org/apache/htrace/**/*</exclude>
<exclude>org/slf4j/*</exclude>
<exclude>org/slf4j/**/*</exclude>
<exclude>org/apache/commons/logging/*</exclude>
<exclude>org/apache/commons/logging/**/*</exclude>
<exclude>org/apache/log4j/*</exclude>
<exclude>org/apache/log4j/**/*</exclude>
<exclude>**/pom.xml</exclude>
<!-- Our non-shaded JUnit library -->
<exclude>org/junit/*</exclude>
<exclude>org/junit/**/*</exclude>
<!-- Not the org/ packages that are a part of the jdk -->
<exclude>org/ietf/jgss/*</exclude>
<exclude>org/omg/**/*</exclude>
<exclude>org/w3c/dom/*</exclude>
<exclude>org/w3c/dom/**/*</exclude>
<exclude>org/xml/sax/*</exclude>
<exclude>org/xml/sax/**/*</exclude>
</excludes>
</relocation>
<relocation>
<pattern>com/</pattern>
<shadedPattern>${shaded.dependency.prefix}.com.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
<!-- Not the com/ packages that are a part of particular jdk implementations -->
<exclude>com/sun/tools/*</exclude>
<exclude>com/sun/javadoc/*</exclude>
<exclude>com/sun/security/*</exclude>
<exclude>com/sun/jndi/*</exclude>
<exclude>com/sun/management/*</exclude>
<exclude>com/sun/tools/**/*</exclude>
<exclude>com/sun/javadoc/**/*</exclude>
<exclude>com/sun/security/**/*</exclude>
<exclude>com/sun/jndi/**/*</exclude>
<exclude>com/sun/management/**/*</exclude>
</excludes>
</relocation>
<relocation>
<pattern>io/</pattern>
<shadedPattern>${shaded.dependency.prefix}.io.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
<!-- Exclude config keys for Hadoop that look like package names -->
<exclude>io/compression/*</exclude>
<exclude>io/compression/**/*</exclude>
<exclude>io/mapfile/*</exclude>
<exclude>io/mapfile/**/*</exclude>
<exclude>io/map/index/*</exclude>
<exclude>io/seqfile/*</exclude>
<exclude>io/seqfile/**/*</exclude>
<exclude>io/file/buffer/size</exclude>
<exclude>io/skip/checksum/errors</exclude>
<exclude>io/sort/*</exclude>
<exclude>io/serializations</exclude>
</excludes>
</relocation>
<relocation>
<pattern>javax/el/</pattern>
<shadedPattern>${shaded.dependency.prefix}.javax.el.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
</excludes>
</relocation>
<relocation>
<pattern>javax/inject/</pattern>
<shadedPattern>${shaded.dependency.prefix}.javax.inject.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
</excludes>
</relocation>
<relocation>
<pattern>javax/servlet/</pattern>
<shadedPattern>${shaded.dependency.prefix}.javax.servlet.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
</excludes>
</relocation>
<relocation>
<pattern>net/</pattern>
<shadedPattern>${shaded.dependency.prefix}.net.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
<!-- Exclude config keys for Hadoop that look like package names -->
<exclude>net/topology/*</exclude>
<exclude>net/topology/**/*</exclude>
</excludes>
</relocation>
</relocations>
<transformers>
<!-- Needed until MSHADE-182 -->
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
<addHeader>false</addHeader>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>license-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

318
hadoop-client-modules/hadoop-client-runtime/pom.xml Normal file
View File

@ -0,0 +1,318 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-runtime</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Apache Hadoop Client</description>
<name>Apache Hadoop Client Runtime</name>
<properties>
<shaded.dependency.prefix>org.apache.hadoop.shaded</shaded.dependency.prefix>
<!-- We contain no source -->
<maven.javadoc.skip>true</maven.javadoc.skip>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<!-- We list this as optional because as a type-pom it won't get included in the shading.
Marking it optional means it doesn't count as a transitive dependency of this artifact.
-->
<optional>true</optional>
<exclusions>
<!-- these APIs are a part of the SE JDK -->
<exclusion>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</exclusion>
<exclusion>
<groupId>xml-apis</groupId>
<artifactId>xml-apis</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- At runtime anyone using us must have the api present -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
<scope>runtime</scope>
</dependency>
<!-- This comes from our parent pom. If we don't expressly change it here to get included,
downstream will get warnings at compile time. -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Since hadoop-client is listed as optional, we have to list transitive
dependencies that we still want to show up.
* HTrace
* Slf4j API
* commons-logging
-->
<dependency>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core4</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<scope>runtime</scope>
</dependency>
<!-- Move log4j to optional, since it is needed for some pieces folks might not use:
* one of the three custom log4j appenders we have
* JobConf (?!) (so essentially any user of MapReduce)
-->
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<scope>runtime</scope>
<optional>true</optional>
</dependency>
</dependencies>
<build>
<plugins>
<!-- We contain no source -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<configuration>
<skipSource>true</skipSource>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<createSourceJar>true</createSourceJar>
<artifactSet>
<excludes>
<!-- We need a filter that matches just those things that aer included in the api jar -->
<exclude>org.apache.hadoop:hadoop-client-api</exclude>
<!-- Leave HTrace as an unshaded dependency on purpose, since a static class member is used to trace within a given JVM instance -->
<exclude>org.apache.htrace:htrace-core4</exclude>
<!-- Leave slf4j unshaded so downstream users can configure logging. -->
<exclude>org.slf4j:slf4j-api</exclude>
<!-- Leave commons-logging unshaded so downstream users can configure logging. -->
<exclude>commons-logging:commons-logging</exclude>
<!-- Leave log4j unshaded so downstream users can configure logging. -->
<exclude>log4j:log4j</exclude>
</excludes>
</artifactSet>
<filters>
<!-- We need a filter that matches just those things that are included in the api jar -->
<filter>
<artifact>org.apache.hadoop:*</artifact>
<excludes>
<exclude>**/*</exclude>
<exclude>*</exclude>
</excludes>
</filter>
<!-- Some of our dependencies include source, so remove it. -->
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>**/*.java</exclude>
</excludes>
</filter>
<!-- We only want one copy of the Localizer class. So long as we keep jasper compiler and runtime on the same version, which one doesn't matter -->
<filter>
<artifact>tomcat:jasper-compiler</artifact>
<excludes>
<exclude>org/apache/jasper/compiler/Localizer.class</exclude>
</excludes>
</filter>
<!-- We only have xerces as a dependency for XML output for the fsimage edits, we don't need anything specific to it for javax xml support -->
<filter>
<artifact>xerces:xercesImpl</artifact>
<excludes>
<exclude>META-INF/services/*</exclude>
</excludes>
</filter>
<!-- We rely on jersey for our web interfaces. We want to use its java services stuff only internal to jersey -->
<filter>
<artifact>com.sun.jersey:*</artifact>
<excludes>
<exclude>META-INF/services/javax.*</exclude>
</excludes>
</filter>
</filters>
<relocations>
<relocation>
<pattern>org/</pattern>
<shadedPattern>${shaded.dependency.prefix}.org.</shadedPattern>
<excludes>
<exclude>org/apache/hadoop/*</exclude>
<exclude>org/apache/hadoop/**/*</exclude>
<!-- Our non-shaded htrace and logging libraries -->
<exclude>org/apache/htrace/*</exclude>
<exclude>org/apache/htrace/**/*</exclude>
<exclude>org/slf4j/*</exclude>
<exclude>org/slf4j/**/*</exclude>
<exclude>org/apache/commons/logging/*</exclude>
<exclude>org/apache/commons/logging/**/*</exclude>
<exclude>org/apache/log4j/*</exclude>
<exclude>org/apache/log4j/**/*</exclude>
<exclude>**/pom.xml</exclude>
<!-- Not the org/ packages that are a part of the jdk -->
<exclude>org/ietf/jgss/*</exclude>
<exclude>org/omg/**/*</exclude>
<exclude>org/w3c/dom/*</exclude>
<exclude>org/w3c/dom/**/*</exclude>
<exclude>org/xml/sax/*</exclude>
<exclude>org/xml/sax/**/*</exclude>
</excludes>
</relocation>
<relocation>
<pattern>com/</pattern>
<shadedPattern>${shaded.dependency.prefix}.com.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
<!-- Not the com/ packages that are a part of particular jdk implementations -->
<exclude>com/sun/tools/*</exclude>
<exclude>com/sun/javadoc/*</exclude>
<exclude>com/sun/security/*</exclude>
<exclude>com/sun/jndi/*</exclude>
<exclude>com/sun/management/*</exclude>
<exclude>com/sun/tools/**/*</exclude>
<exclude>com/sun/javadoc/**/*</exclude>
<exclude>com/sun/security/**/*</exclude>
<exclude>com/sun/jndi/**/*</exclude>
<exclude>com/sun/management/**/*</exclude>
</excludes>
</relocation>
<relocation>
<pattern>io/</pattern>
<shadedPattern>${shaded.dependency.prefix}.io.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
<!-- Exclude config keys for Hadoop that look like package names -->
<exclude>io/compression/*</exclude>
<exclude>io/compression/**/*</exclude>
<exclude>io/mapfile/*</exclude>
<exclude>io/mapfile/**/*</exclude>
<exclude>io/map/index/*</exclude>
<exclude>io/seqfile/*</exclude>
<exclude>io/seqfile/**/*</exclude>
<exclude>io/file/buffer/size</exclude>
<exclude>io/skip/checksum/errors</exclude>
<exclude>io/sort/*</exclude>
<exclude>io/serializations</exclude>
</excludes>
</relocation>
<relocation>
<pattern>javax/el/</pattern>
<shadedPattern>${shaded.dependency.prefix}.javax.el.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
</excludes>
</relocation>
<relocation>
<pattern>javax/servlet/</pattern>
<shadedPattern>${shaded.dependency.prefix}.javax.servlet.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
</excludes>
</relocation>
<relocation>
<pattern>net/</pattern>
<shadedPattern>${shaded.dependency.prefix}.net.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
<!-- Exclude config keys for Hadoop that look like package names -->
<exclude>net/topology/*</exclude>
<exclude>net/topology/**/*</exclude>
</excludes>
</relocation>
<!-- probably not. -->
<!--
<relocation>
<pattern>javax/</pattern>
<shadedPattern>${shaded.dependency.prefix}.javax.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
</excludes>
</relocation>
-->
</relocations>
<transformers>
<!-- Needed until MSHADE-182 -->
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
<addHeader>false</addHeader>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.XmlAppendingTransformer">
<resource>META-INF/jboss-beans.xml</resource>
<!-- Add this to enable loading of DTDs
<ignoreDtd>false</ignoreDtd>
-->
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/mailcap.default</resource>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/mimetypes.default</resource>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>license-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

31
hadoop-client/pom.xml → hadoop-client-modules/hadoop-client/pom.xml
View File

@ -19,14 +19,14 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project-dist</relativePath>
</parent>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<artifactId>hadoop-client</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<packaging>jar</packaging>
<packaging>pom</packaging>
<description>Apache Hadoop Client</description>
<name>Apache Hadoop Client</name>
<description>Apache Hadoop Client aggregation pom with dependencies exposed</description>
<name>Apache Hadoop Client Aggregator</name>
<properties>
<hadoop.component>client</hadoop.component>
@ -98,6 +98,11 @@
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<!-- No slf4j backends for downstream clients -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
@ -170,6 +175,11 @@
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<!-- No slf4j backends for downstream clients -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
@ -270,6 +280,11 @@
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<!-- No slf4j backends for downstream clients -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
@ -298,6 +313,11 @@
<groupId>io.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
<!-- No slf4j backends for downstream clients -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
@ -312,7 +332,6 @@
</exclusion>
</exclusions>
</dependency>
</dependencies>
</project>

45
hadoop-client-modules/pom.xml Normal file
View File

@ -0,0 +1,45 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-client-modules</artifactId>
<packaging>pom</packaging>
<description>multi-module for Apache Hadoop client artifacts</description>
<name>Apache Hadoop Client Modules</name>
<modules>
<!-- Left as an empty artifact w/dep for compat -->
<module>hadoop-client</module>
<!-- Should be used at compile scope for access to IA.Public classes -->
<module>hadoop-client-api</module>
<!-- Should be used at runtime scope for remaining classes necessary for hadoop-client-api to function -->
<module>hadoop-client-runtime</module>
<!-- Should be used at test scope for those that need access to mini cluster that works with above api and runtime -->
<module>hadoop-client-minicluster</module>
<!-- Checks invariants above -->
<module>hadoop-client-check-invariants</module>
<module>hadoop-client-check-test-invariants</module>
<!-- Attempt to use the created libraries -->
<module>hadoop-client-integration-tests</module>
</modules>
</project>

17
hadoop-dist/pom.xml
View File

@ -51,6 +51,23 @@
<artifactId>hadoop-yarn-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-check-invariants</artifactId>
<type>pom</type>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-check-test-invariants</artifactId>
<type>pom</type>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-integration-tests</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
<build>

22
hadoop-maven-plugins/pom.xml
View File

@ -54,6 +54,28 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${maven-shade-plugin.version}</version>
<scope>provided</scope>
<exclusions>
<!-- shade conflicts with itself and our other maven internals
by including a combination of 3.x and 2.x
-->
<exclusion>
<groupId>org.apache.maven.shared</groupId>
<artifactId>maven-dependency-tree</artifactId>
</exclusion>
<!-- again, shade conflicts with itself on the transitives of this
dependency
-->
<exclusion>
<groupId>org.vafer</groupId>
<artifactId>jdependency</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
<plugins>

164
hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java Normal file
View File

@ -0,0 +1,164 @@
package org.apache.hadoop.maven.plugin.shade.resource;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
import java.io.BufferedReader;
import org.apache.maven.plugins.shade.relocation.Relocator;
import org.apache.maven.plugins.shade.resource.ResourceTransformer;
import org.codehaus.plexus.util.IOUtil;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
/**
* Resources transformer that appends entries in META-INF/services resources
* into a single resource. For example, if there are several
* META-INF/services/org.apache.maven.project.ProjectBuilder resources spread
* across many JARs the individual entries will all be concatenated into a
* single META-INF/services/org.apache.maven.project.ProjectBuilder resource
* packaged into the resultant JAR produced by the shading process.
*
* From following sources, only needed until MSHADE-182 gets released
* * https://s.apache.org/vwjl (source in maven-shade-plugin repo)
* * https://issues.apache.org/jira/secure/attachment/12718938/MSHADE-182.patch
*
* Has been reformatted according to Hadoop checkstyle rules and modified
* to meet Hadoop's threshold for Findbugs problems.
*/
public class ServicesResourceTransformer
implements ResourceTransformer {
private static final String SERVICES_PATH = "META-INF/services";
private Map<String, ServiceStream> serviceEntries = new HashMap<>();
private List<Relocator> relocators;
public boolean canTransformResource(String resource) {
if (resource.startsWith(SERVICES_PATH)) {
return true;
}
return false;
}
public void processResource(String resource, InputStream is,
List<Relocator> relocatorz) throws IOException {
ServiceStream out = serviceEntries.get(resource);
if (out == null) {
out = new ServiceStream();
serviceEntries.put(resource, out);
}
out.append(is);
is.close();
if (this.relocators == null) {
this.relocators = relocatorz;
}
}
public boolean hasTransformedResource() {
return serviceEntries.size() > 0;
}
public void modifyOutputStream(JarOutputStream jos)
throws IOException {
for (Map.Entry<String, ServiceStream> entry : serviceEntries.entrySet()) {
String key = entry.getKey();
ServiceStream data = entry.getValue();
if (relocators != null) {
key = key.substring(SERVICES_PATH.length() + 1);
for (Relocator relocator : relocators) {
if (relocator.canRelocateClass(key)) {
key = relocator.relocateClass(key);
break;
}
}
key = SERVICES_PATH + '/' + key;
}
jos.putNextEntry(new JarEntry(key));
//read the content of service file for candidate classes for relocation
//presume everything is UTF8, because Findbugs barfs on default
//charset and this seems no worse a choice ¯\_(ツ)_/¯
PrintWriter writer = new PrintWriter(new OutputStreamWriter(jos,
StandardCharsets.UTF_8));
InputStreamReader streamReader =
new InputStreamReader(data.toInputStream(), StandardCharsets.UTF_8);
BufferedReader reader = new BufferedReader(streamReader);
String className;
while ((className = reader.readLine()) != null) {
if (relocators != null) {
for (Relocator relocator : relocators) {
//if the class can be relocated then relocate it
if (relocator.canRelocateClass(className)) {
className = relocator.applyToSourceContent(className);
break;
}
}
}
writer.println(className);
writer.flush();
}
reader.close();
data.reset();
}
}
static class ServiceStream extends ByteArrayOutputStream {
public ServiceStream() {
super(1024);
}
public void append(InputStream is)
throws IOException {
if (count > 0 && buf[count - 1] != '\n' && buf[count - 1] != '\r') {
write('\n');
}
IOUtil.copy(is, this);
}
public InputStream toInputStream() {
return new ByteArrayInputStream(buf, 0, count);
}
}
}

23
hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Resource handling plugins used internal to the Hadoop build.
* IA.Private (build structure encourages not using the actual annotations)
*/
package org.apache.hadoop.maven.plugin.shade.resource;

96
hadoop-project/pom.xml
View File

@ -27,6 +27,7 @@
<description>Apache Hadoop Project POM</description>
<name>Apache Hadoop Project POM</name>
<packaging>pom</packaging>
<inceptionYear>2008</inceptionYear>
<properties>
<!-- Set the Release year during release -->
@ -114,6 +115,7 @@
<maven-compiler-plugin.version>3.1</maven-compiler-plugin.version>
<maven-install-plugin.version>2.5.1</maven-install-plugin.version>
<maven-resources-plugin.version>2.6</maven-resources-plugin.version>
<maven-shade-plugin.version>2.4.3</maven-shade-plugin.version>
<maven-jar-plugin.version>2.5</maven-jar-plugin.version>
<maven-war-plugin.version>2.4</maven-war-plugin.version>
<maven-source-plugin.version>2.3</maven-source-plugin.version>
@ -148,6 +150,44 @@
<artifactId>hadoop-annotations</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-modules</artifactId>
<version>${project.version}</version>
<type>pom</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-check-invariants</artifactId>
<version>${project.version}</version>
<type>pom</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-check-test-invariants</artifactId>
<version>${project.version}</version>
<type>pom</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-integration-tests</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-runtime</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-minicluster</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
@ -1244,6 +1284,40 @@
<build>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>${build-helper-maven-plugin.version}</version>
</plugin>
<!-- We include the configuration for license-maven-plugin to correct
maven-shade-plugin generated poms because it's always the same. We
can't simply configure the plugin because we must ensure execution
happens in the package phase after the shade plugin runs.
-->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>license-maven-plugin</artifactId>
<version>1.10</version>
<configuration>
<canUpdateCopyright>false</canUpdateCopyright>
<roots><root>${project.basedir}</root></roots>
</configuration>
<executions>
<execution>
<id>update-pom-license</id>
<goals>
<goal>update-file-header</goal>
</goals>
<phase>package</phase>
<configuration>
<licenseName>apache_v2</licenseName>
<includes>
<include>dependency-reduced-pom.xml</include>
</includes>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>${maven-clean-plugin.version}</version>
@ -1260,13 +1334,8 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.2</version>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>${build-helper-maven-plugin.version}</version>
<artifactId>maven-shade-plugin</artifactId>
<version>${maven-shade-plugin.version}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@ -1355,6 +1424,19 @@
</pluginManagement>
<plugins>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<configuration>
<filesets>
<fileset>
<directory>${project.basedir}</directory>
<includes>
<include>dependency-reduced-pom.xml</include>
</includes>
</fileset>
</filesets>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>

4
pom.xml
View File

@ -96,7 +96,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<maven-stylus-skin.version>1.5</maven-stylus-skin.version>
<maven-antrun-plugin.version>1.7</maven-antrun-plugin.version>
<maven-assembly-plugin.version>2.4</maven-assembly-plugin.version>
<maven-dependency-plugin.version>2.8</maven-dependency-plugin.version>
<maven-dependency-plugin.version>2.10</maven-dependency-plugin.version>
<maven-enforcer-plugin.version>1.4.1</maven-enforcer-plugin.version>
<maven-javadoc-plugin.version>2.9.1</maven-javadoc-plugin.version>
<maven-gpg-plugin.version>1.5</maven-gpg-plugin.version>
@ -125,8 +125,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<module>hadoop-mapreduce-project</module>
<module>hadoop-tools</module>
<module>hadoop-dist</module>
<module>hadoop-client</module>
<module>hadoop-minicluster</module>
<module>hadoop-client-modules</module>
<module>hadoop-build-tools</module>
</modules>