big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17324. Don't relocate org.bouncycastle in shaded client jars (#2411)

Browse Source 
Contributed by Chao Sun.
This commit is contained in:
Chao Sun 2020-11-10 08:00:15 -08:00 committed by GitHub
parent 4331c88352
commit 2522bf2f9b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 57 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-client-modules/hadoop-client-api/pom.xml
View File

@ -143,6 +143,8 @@
<exclude>org/w3c/dom/**/*</exclude>
<exclude>org/xml/sax/*</exclude>
<exclude>org/xml/sax/**/*</exclude>
<exclude>org/bouncycastle/*</exclude>
<exclude>org/bouncycastle/**/*</exclude>
</excludes>
</relocation>
<relocation>

4
hadoop-client-modules/hadoop-client-check-test-invariants/src/test/resources/ensure-jars-have-correct-contents.sh
View File

@ -43,6 +43,8 @@ allowed_expr+="|^org/apache/hadoop/"
allowed_expr+="|^META-INF/"
# * whatever under the "webapps" directory; for things shipped by yarn
allowed_expr+="|^webapps/"
# * Resources files used by Hadoop YARN mini cluster
allowed_expr+="|^TERMINAL/"
# * Hadoop's default configuration files, which have the form
# "_module_-default.xml"
allowed_expr+="|^[^-]*-default.xml$"
@ -54,6 +56,8 @@ allowed_expr+="|^org.apache.hadoop.application-classloader.properties$"
# * Used by JavaSandboxLinuxContainerRuntime as a default, loaded
# from root, so can't relocate. :(
allowed_expr+="|^java.policy$"
# * Used by javax.annotation
allowed_expr+="|^jndi.properties$"
# * allowing native libraries from rocksdb. Leaving native libraries as it is.
allowed_expr+="|^librocksdbjni-linux32.so"
allowed_expr+="|^librocksdbjni-linux64.so"

21
hadoop-client-modules/hadoop-client-integration-tests/pom.xml
View File

@ -75,6 +75,27 @@
<artifactId>hadoop-client-minicluster</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpkix-jdk15on</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>javax.activation</groupId>
<artifactId>activation</artifactId>
<version>1.1.1</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>

10
hadoop-client-modules/hadoop-client-integration-tests/src/test/java/org/apache/hadoop/example/ITUseMiniCluster.java
View File

@ -35,6 +35,7 @@
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
@ -43,6 +44,7 @@
import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
/**
* Ensure that we can perform operations against the shaded minicluster
@ -54,6 +56,7 @@ public class ITUseMiniCluster {
LoggerFactory.getLogger(ITUseMiniCluster.class);
private MiniDFSCluster cluster;
private MiniYARNCluster yarnCluster;
private static final String TEST_PATH = "/foo/bar/cats/dee";
private static final String FILENAME = "test.file";
@ -73,6 +76,12 @@ public void clusterUp() throws IOException {
.numDataNodes(3)
.build();
cluster.waitActive();
conf.set("yarn.scheduler.capacity.root.queues", "default");
conf.setInt("yarn.scheduler.capacity.root.default.capacity", 100);
yarnCluster = new MiniYARNCluster(getClass().getName(), 1, 1, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
}
@After
@ -80,6 +89,7 @@ public void clusterDown() {
if (cluster != null) {
cluster.close();
}
IOUtils.cleanupWithLogger(LOG, yarnCluster);
}
@Test

29
hadoop-client-modules/hadoop-client-minicluster/pom.xml
View File

@ -678,10 +678,8 @@
<exclude>junit:junit</exclude>
<exclude>com.google.code.findbugs:jsr305</exclude>
<exclude>log4j:log4j</exclude>
<exclude>org.eclipse.jetty.websocket:*</exclude>
<exclude>javax.websocket:javax.websocket-api</exclude>
<exclude>javax.annotation:javax.annotation-api</exclude>
<exclude>org.eclipse.jetty:jetty-jndi</exclude>
<exclude>org.eclipse.jetty.websocket:websocket-common</exclude>
<exclude>org.eclipse.jetty.websocket:websocket-api</exclude>
<!-- We need a filter that matches just those things that are included in the above artiacts -->
<!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->
<exclude>org.bouncycastle:*</exclude>
@ -731,13 +729,6 @@
<exclude>testdata/*</exclude>
</excludes>
</filter>
<!-- Skip terminal html and javascript -->
<filter>
<artifact>org.apache.hadoop:hadoop-yarn-server-nodemanager:*</artifact>
<excludes>
<exclude>TERMINAL/**/*</exclude>
</excludes>
</filter>
<!-- Mockito tries to include its own unrelocated copy of hamcrest. :( -->
<filter>
@ -784,6 +775,13 @@
<exclude>about.html</exclude>
</excludes>
</filter>
<filter>
<!-- skip jetty license info already incorporated into LICENSE/NOTICE -->
<artifact>org.eclipse.jetty.websocket:*</artifact>
<excludes>
<exclude>about.html</exclude>
</excludes>
</filter>
<filter>
<artifact>org.apache.hadoop:*</artifact>
<excludes>
@ -881,6 +879,8 @@
<exclude>org/w3c/dom/**/*</exclude>
<exclude>org/xml/sax/*</exclude>
<exclude>org/xml/sax/**/*</exclude>
<exclude>org/bouncycastle/*</exclude>
<exclude>org/bouncycastle/**/*</exclude>
</excludes>
</relocation>
<relocation>
@ -977,6 +977,13 @@
<exclude>**/pom.xml</exclude>
</excludes>
</relocation>
<relocation>
<pattern>javax/annotation/</pattern>
<shadedPattern>${shaded.dependency.prefix}.javax.websocket.</shadedPattern>
<excludes>
<exclude>**/pom.xml</exclude>
</excludes>
</relocation>
<relocation>
<pattern>jersey/</pattern>
<shadedPattern>${shaded.dependency.prefix}.jersey.</shadedPattern>

2
hadoop-client-modules/hadoop-client-runtime/pom.xml
View File

@ -267,6 +267,8 @@
<exclude>org/w3c/dom/**/*</exclude>
<exclude>org/xml/sax/*</exclude>
<exclude>org/xml/sax/**/*</exclude>
<exclude>org/bouncycastle/*</exclude>
<exclude>org/bouncycastle/**/*</exclude>
</excludes>
</relocation>
<relocation>