big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18441. Remove hadoop custom ServicesResourceTransformer (#4850). Contributed by PJ Fanning.

Browse Source 
Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>
This commit is contained in:
PJ Fanning 2022-09-07 12:41:12 +01:00 committed by GitHub
parent c664f953c9
commit 42c8f61fec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 3 additions and 266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-client-modules/hadoop-client-api/pom.xml
View File

@ -98,13 +98,6 @@
<createSourcesJar>true</createSourcesJar> <createSourcesJar>true</createSourcesJar>
<shadeSourcesContent>true</shadeSourcesContent> <shadeSourcesContent>true</shadeSourcesContent>
</configuration> </configuration>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<executions> <executions>
<execution> <execution>
<phase>package</phase> <phase>package</phase>
@ -254,8 +247,7 @@
</relocation> </relocation>
</relocations> </relocations>
<transformers> <transformers>
<!-- Needed until MSHADE-182 --> <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/> <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resource>NOTICE.txt</resource> <resource>NOTICE.txt</resource>

10
hadoop-client-modules/hadoop-client-minicluster/pom.xml
View File

@ -671,13 +671,6 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId> <artifactId>maven-shade-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<executions> <executions>
<execution> <execution>
<phase>package</phase> <phase>package</phase>
@ -1052,8 +1045,7 @@
</relocation> </relocation>
</relocations> </relocations>
<transformers> <transformers>
<!-- Needed until MSHADE-182 --> <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/> <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resources> <resources>

10
hadoop-client-modules/hadoop-client-runtime/pom.xml
View File

@ -128,13 +128,6 @@
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId> <artifactId>maven-shade-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-maven-plugins</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<executions> <executions>
<execution> <execution>
<phase>package</phase> <phase>package</phase>
@ -397,8 +390,7 @@
--> -->
</relocations> </relocations>
<transformers> <transformers>
<!-- Needed until MSHADE-182 --> <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.hadoop.maven.plugin.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/> <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resources> <resources>

50
hadoop-maven-plugins/pom.xml
View File

@ -68,56 +68,6 @@
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId> <artifactId>jackson-databind</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${maven-shade-plugin.version}</version>
<scope>provided</scope>
<exclusions>
<!-- shade conflicts with itself and other maven internals so we simply
exclude shade's (older) copies of the dependencies -->
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-artifact</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-compat</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-model</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.vafer</groupId>
<artifactId>jdependency</artifactId>
</exclusion>
<exclusion>
<groupId>org.sonatype.sisu</groupId>
<artifactId>sisu-inject-plexus</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.maven.plugin-tools</groupId>
<artifactId>maven-plugin-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-util</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop.thirdparty</groupId> <groupId>org.apache.hadoop.thirdparty</groupId>
<artifactId>hadoop-shaded-guava</artifactId> <artifactId>hadoop-shaded-guava</artifactId>

166
hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/ServicesResourceTransformer.java
View File

@ -1,166 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
package org.apache.hadoop.maven.plugin.shade.resource;
import java.io.BufferedReader;
import org.apache.maven.plugins.shade.relocation.Relocator;
import org.apache.maven.plugins.shade.resource.ResourceTransformer;
import org.codehaus.plexus.util.IOUtil;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
/**
* Resources transformer that appends entries in META-INF/services resources
* into a single resource. For example, if there are several
* META-INF/services/org.apache.maven.project.ProjectBuilder resources spread
* across many JARs the individual entries will all be concatenated into a
* single META-INF/services/org.apache.maven.project.ProjectBuilder resource
* packaged into the resultant JAR produced by the shading process.
*
* From following sources, only needed until MSHADE-182 gets released
* * https://s.apache.org/vwjl (source in maven-shade-plugin repo)
* * https://issues.apache.org/jira/secure/attachment/12718938/MSHADE-182.patch
*
* Has been reformatted according to Hadoop checkstyle rules and modified
* to meet Hadoop's threshold for Findbugs problems.
*/
public class ServicesResourceTransformer
implements ResourceTransformer {
private static final String SERVICES_PATH = "META-INF/services";
private Map<String, ServiceStream> serviceEntries = new HashMap<>();
private List<Relocator> relocators;
public boolean canTransformResource(String resource) {
if (resource.startsWith(SERVICES_PATH)) {
return true;
}
return false;
}
public void processResource(String resource, InputStream is,
List<Relocator> relocatorz) throws IOException {
ServiceStream out = serviceEntries.get(resource);
if (out == null) {
out = new ServiceStream();
serviceEntries.put(resource, out);
}
out.append(is);
is.close();
if (this.relocators == null) {
this.relocators = relocatorz;
}
}
public boolean hasTransformedResource() {
return serviceEntries.size() > 0;
}
public void modifyOutputStream(JarOutputStream jos)
throws IOException {
for (Map.Entry<String, ServiceStream> entry : serviceEntries.entrySet()) {
String key = entry.getKey();
ServiceStream data = entry.getValue();
if (relocators != null) {
key = key.substring(SERVICES_PATH.length() + 1);
for (Relocator relocator : relocators) {
if (relocator.canRelocateClass(key)) {
key = relocator.relocateClass(key);
break;
}
}
key = SERVICES_PATH + '/' + key;
}
jos.putNextEntry(new JarEntry(key));
//read the content of service file for candidate classes for relocation
//presume everything is UTF8, because Findbugs barfs on default
//charset and this seems no worse a choice ¯\_(ツ)_/¯
PrintWriter writer = new PrintWriter(new OutputStreamWriter(jos,
StandardCharsets.UTF_8));
InputStreamReader streamReader =
new InputStreamReader(data.toInputStream(), StandardCharsets.UTF_8);
BufferedReader reader = new BufferedReader(streamReader);
String className;
while ((className = reader.readLine()) != null) {
if (relocators != null) {
for (Relocator relocator : relocators) {
//if the class can be relocated then relocate it
if (relocator.canRelocateClass(className)) {
className = relocator.applyToSourceContent(className);
break;
}
}
}
writer.println(className);
writer.flush();
}
reader.close();
data.reset();
}
}
static class ServiceStream extends ByteArrayOutputStream {
public ServiceStream() {
super(1024);
}
public void append(InputStream is)
throws IOException {
if (count > 0 && buf[count - 1] != '\n' && buf[count - 1] != '\r') {
write('\n');
}
IOUtil.copy(is, this);
}
public InputStream toInputStream() {
return new ByteArrayInputStream(buf, 0, count);
}
}
}

23
hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/shade/resource/package-info.java
View File

@ -1,23 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Resource handling plugins used internal to the Hadoop build.
* IA.Private (build structure encourages not using the actual annotations)
*/
package org.apache.hadoop.maven.plugin.shade.resource;