big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7190. Ensure only NM classpath in 2.x gets TSv2 related hbase jars, not the user classpath. Contributed by Varun Saxena.

Browse Source
This commit is contained in:
Rohith Sharma K S 2017-12-15 21:50:28 +05:30
parent 890d3d0645
commit 09d996fdd4
7 changed files with 97 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
View File

@ -211,10 +211,37 @@
<include>**/*</include> <include>**/*</include>
</includes> </includes>
</fileSet> </fileSet>
<!-- Copy dependecies from hadoop-yarn-server-timelineservice as well -->
<fileSet>
<directory>hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/target/lib</directory>
<outputDirectory>share/hadoop/${hadoop.component}/timelineservice/lib</outputDirectory>
</fileSet>
</fileSets> </fileSets>
<moduleSets> <moduleSets>
<moduleSet>
<includes>
<include>org.apache.hadoop:hadoop-yarn-server-timelineservice</include>
<include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase</include>
</includes>
<binaries>
<outputDirectory>share/hadoop/${hadoop.component}/timelineservice</outputDirectory>
<includeDependencies>false</includeDependencies>
<unpack>false</unpack>
</binaries>
</moduleSet>
<moduleSet>
<includes>
<include>org.apache.hadoop:hadoop-yarn-server-timelineservice-hbase-tests</include>
</includes>
<binaries>
<outputDirectory>share/hadoop/${hadoop.component}/timelineservice/test</outputDirectory>
<includeDependencies>false</includeDependencies>
<unpack>false</unpack>
</binaries>
</moduleSet>
<moduleSet> <moduleSet>
<excludes> <excludes>
<exclude>org.apache.hadoop:hadoop-yarn-server-timelineservice*</exclude>
<exclude>org.apache.hadoop:hadoop-yarn-ui</exclude> <exclude>org.apache.hadoop:hadoop-yarn-ui</exclude>
</excludes> </excludes>
<binaries> <binaries>

6
hadoop-yarn-project/hadoop-yarn/bin/yarn
View File

@ -121,6 +121,8 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
;; ;;
nodemanager) nodemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*"
hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.nodemanager.NodeManager' HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
# Backwards compatibility # Backwards compatibility
if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then
@ -145,6 +147,8 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
;; ;;
resourcemanager) resourcemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*"
hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
# Backwards compatibility # Backwards compatibility
if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then
@ -178,6 +182,8 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
;; ;;
timelinereader) timelinereader)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/*"
hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
;; ;;
timelineserver) timelineserver)

6
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
View File

@ -219,6 +219,8 @@ goto :eof
:resourcemanager :resourcemanager
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\*
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\*
set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
set YARN_OPTS=%YARN_OPTS% %YARN_RESOURCEMANAGER_OPTS% set YARN_OPTS=%YARN_OPTS% %YARN_RESOURCEMANAGER_OPTS%
if defined YARN_RESOURCEMANAGER_HEAPSIZE ( if defined YARN_RESOURCEMANAGER_HEAPSIZE (
@ -248,6 +250,8 @@ goto :eof
:timelinereader :timelinereader
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\timelineserver-config\log4j.properties set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\timelineserver-config\log4j.properties
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\*
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\*
set CLASS=org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer set CLASS=org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer
set YARN_OPTS=%YARN_OPTS% %YARN_TIMELINEREADER_OPTS% set YARN_OPTS=%YARN_OPTS% %YARN_TIMELINEREADER_OPTS%
goto :eof goto :eof
@ -260,6 +264,8 @@ goto :eof
:nodemanager :nodemanager
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\*
set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\timelineservice\lib\*
set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager
set YARN_OPTS=%YARN_OPTS% -server %HADOOP_NODEMANAGER_OPTS% set YARN_OPTS=%YARN_OPTS% -server %HADOOP_NODEMANAGER_OPTS%
if defined YARN_NODEMANAGER_HEAPSIZE ( if defined YARN_NODEMANAGER_HEAPSIZE (

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
View File

@ -191,6 +191,7 @@
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timelineservice</artifactId> <artifactId>hadoop-yarn-server-timelineservice</artifactId>
<scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/pom.xml
View File

@ -57,11 +57,13 @@
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId> <artifactId>hadoop-annotations</artifactId>
<scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId> <artifactId>hadoop-common</artifactId>
<scope>provided</scope>
</dependency> </dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency --> <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
@ -75,16 +77,19 @@
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId> <artifactId>hadoop-yarn-api</artifactId>
<scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId> <artifactId>hadoop-yarn-common</artifactId>
<scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId> <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
<scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
@ -121,6 +126,7 @@
<dependency> <dependency>
<groupId>org.apache.hbase</groupId> <groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId> <artifactId>hbase-server</artifactId>
<scope>provided</scope>
<exclusions> <exclusions>
<exclusion> <exclusion>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
@ -186,6 +192,23 @@
</additionnalDependencies> </additionnalDependencies>
</configuration> </configuration>
</plugin> </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<includeScope>runtime</includeScope>
<excludeGroupIds>org.slf4j,org.apache.hadoop,com.github.stephenc.findbugs</excludeGroupIds>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins> </plugins>
</build> </build>
</project> </project>

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
View File

@ -38,6 +38,7 @@
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId> <artifactId>hadoop-common</artifactId>
<scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>

38
hadoop-yarn-project/pom.xml
View File

@ -74,10 +74,6 @@
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId> <artifactId>hadoop-yarn-server-web-proxy</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-timelineservice-hbase</artifactId>
</dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-router</artifactId> <artifactId>hadoop-yarn-server-router</artifactId>
@ -167,12 +163,44 @@
</execution> </execution>
</executions> </executions>
</plugin> </plugin>
<!-- Remove duplicate dependencies which exist in both yarn/lib folder and yarn/timelineservice/lib folder -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>dist</id>
<phase>prepare-package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<echo file="${project.build.directory}/remove-duplicate-timelineservice-jars.sh">
yarnjardir="${project.build.directory}/${project.artifactId}-${project.version}/share/hadoop/yarn"
yarnlibdir="${yarnjardir}/lib/"
cd "${yarnjardir}/timelineservice/lib"
for jar in `ls`
do
if [ -f "${yarnlibdir}""${jar}" ];then
rm -rf ${jar}
echo "Removing duplicate jar $jar from share/hadoop/yarn/timelineservice/lib folder"
fi
done
</echo>
<exec executable="${shell-executable}" dir="${project.build.directory}" failonerror="true">
<arg line="./remove-duplicate-timelineservice-jars.sh"/>
</exec>
</target>
</configuration>
</execution>
</executions>
</plugin>
</plugins> </plugins>
</build> </build>
</profile> </profile>
</profiles> </profiles>
<reporting> <reporting>
<plugins> <plugins>
<plugin> <plugin>