big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-9872. Improve protoc version handling and detection. (tucu)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1514068 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2013-08-14 22:15:04 +00:00
parent 6390e64abd
commit 8451ab5c01
18 changed files with 67 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
BUILDING.txt
View File

@ -7,7 +7,7 @@ Requirements:
* JDK 1.6
* Maven 3.0
* Findbugs 1.3.9 (if running findbugs)
* ProtocolBuffer 2.4.1+ (for MapReduce and HDFS)
* ProtocolBuffer 2.5.0
* CMake 2.6 or newer (if compiling native code)
* Internet connection for first build (to fetch all Maven and Hadoop dependencies)
@ -99,6 +99,16 @@ level once; and then work from the submodule. Keep in mind that SNAPSHOTs
time out after a while, using the Maven '-nsu' will stop Maven from trying
to update SNAPSHOTs from external repos.
----------------------------------------------------------------------------------
Protocol Buffer compiler
The version of Protocol Buffer compiler, protoc, must match the version of the
protobuf JAR.
If you have multiple versions of protoc in your system, you can set in your
build shell the HADOOP_PROTOC_PATH environment variable to point to the one you
want to use for the Hadoop build. If you don't define this environment variable,
protoc is looked up in the PATH.
----------------------------------------------------------------------------------
Importing projects to eclipse

6
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -105,8 +105,6 @@ Trunk (Unreleased)
HADOOP-9833 move slf4j to version 1.7.5 (Kousuke Saruta via stevel)
HADOOP-9845. Update protobuf to 2.5 from 2.4.x. (tucu)
BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled.
@ -584,6 +582,10 @@ Release 2.1.0-beta - 2013-08-06
HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs
(todd)
HADOOP-9845. Update protobuf to 2.5 from 2.4.x. (tucu)
HADOOP-9872. Improve protoc version handling and detection. (tucu)
BUG FIXES
HADOOP-9294. GetGroupsTestBase fails on Windows. (Chris Nauroth via suresh)

2
hadoop-common-project/hadoop-common/pom.xml
View File

@ -309,6 +309,7 @@
</goals>
<configuration>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/src/main/proto</param>
</imports>
@ -338,6 +339,7 @@
</goals>
<configuration>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/src/test/proto</param>
</imports>

12
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java
View File

@ -90,6 +90,10 @@ protected String _getBuildVersion(){
" source checksum " + _getSrcChecksum();
}
protected String _getProtocVersion() {
return info.getProperty("protocVersion", "Unknown");
}
private static VersionInfo COMMON_VERSION_INFO = new VersionInfo("common");
/**
* Get the Hadoop version.
@ -154,11 +158,19 @@ public static String getBuildVersion(){
return COMMON_VERSION_INFO._getBuildVersion();
}
/**
* Returns the protoc version used for the build.
*/
public static String getProtocVersion(){
return COMMON_VERSION_INFO._getProtocVersion();
}
public static void main(String[] args) {
LOG.debug("version: "+ getVersion());
System.out.println("Hadoop " + getVersion());
System.out.println("Subversion " + getUrl() + " -r " + getRevision());
System.out.println("Compiled by " + getUser() + " on " + getDate());
System.out.println("Compiled with protoc " + getProtocVersion());
System.out.println("From source with checksum " + getSrcChecksum());
System.out.println("This command was run using " +
ClassUtil.findContainingJar(VersionInfo.class));

1
hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
View File

@ -23,3 +23,4 @@ user=${user.name}
date=${version-info.build.time}
url=${version-info.scm.uri}
srcChecksum=${version-info.source.md5}
protocVersion=${protobuf.version}

2
hadoop-common-project/hadoop-common/src/site/apt/CLIMiniCluster.apt.vm
View File

@ -42,7 +42,7 @@ Hadoop MapReduce Next Generation - CLI MiniCluster.
$ mvn clean install -DskipTests
$ mvn package -Pdist -Dtar -DskipTests -Dmaven.javadoc.skip
+---+
<<NOTE:>> You will need protoc installed of version 2.4.1 or greater.
<<NOTE:>> You will need protoc 2.5.0 installed.
The tarball should be available in <<<hadoop-dist/target/>>> directory.

2
hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm
View File

@ -32,7 +32,7 @@ $ mvn clean install -DskipTests
$ cd hadoop-mapreduce-project
$ mvn clean install assembly:assembly -Pnative
+---+
<<NOTE:>> You will need protoc installed of version 2.4.1 or greater.
<<NOTE:>> You will need protoc 2.5.0 installed.
To ignore the native builds in mapreduce you can omit the <<<-Pnative>>> argument
for maven. The tarball should be available in <<<target/>>> directory.

12
hadoop-hdfs-project/hadoop-hdfs/pom.xml
View File

@ -417,7 +417,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<goal>protoc</goal>
</goals>
<configuration>
<protocVersion>2.5.0</protocVersion>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/src/main/proto</param>
@ -442,7 +443,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<goal>protoc</goal>
</goals>
<configuration>
<protocVersion>2.5.0</protocVersion>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/src/main/proto</param>
@ -464,7 +466,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<goal>protoc</goal>
</goals>
<configuration>
<protocVersion>2.5.0</protocVersion>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/src/main/proto</param>
@ -486,7 +489,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<goal>protoc</goal>
</goals>
<configuration>
<protocVersion>2.5.0</protocVersion>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/src/main/proto</param>

3
hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
View File

@ -103,7 +103,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<goal>protoc</goal>
</goals>
<configuration>
<protocVersion>2.5.0</protocVersion>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/../../../../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/../../../../../hadoop-hdfs-project/hadoop-hdfs/src/main/proto</param>

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
View File

@ -64,7 +64,8 @@
<goal>protoc</goal>
</goals>
<configuration>
<protocVersion>2.5.0</protocVersion>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/../../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/../../../hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto</param>

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
View File

@ -78,7 +78,8 @@
<goal>protoc</goal>
</goals>
<configuration>
<protocVersion>2.5.0</protocVersion>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>${protoc.path}</protocCommand>
<imports>
<param>
${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto

17
hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
View File

@ -45,7 +45,7 @@ public class ProtocMojo extends AbstractMojo {
@Parameter(required=true)
private FileSet source;
@Parameter(defaultValue="protoc")
@Parameter
private String protocCommand;
@Parameter(required=true)
@ -53,21 +53,22 @@ public class ProtocMojo extends AbstractMojo {
public void execute() throws MojoExecutionException {
try {
if (protocCommand == null || protocCommand.trim().isEmpty()) {
protocCommand = "protoc";
}
List<String> command = new ArrayList<String>();
command.add(protocCommand);
command.add("--version");
Exec exec = new Exec(this);
List<String> out = new ArrayList<String>();
if (exec.run(command, out) != 0) {
getLog().error("protoc, could not get version");
for (String s : out) {
getLog().error(s);
}
if (exec.run(command, out) == 127) {
getLog().error("protoc, not found at: " + protocCommand);
throw new MojoExecutionException("protoc failure");
} else {
if (out.size() == 0) {
if (out.isEmpty()) {
getLog().error("stdout: " + out);
throw new MojoExecutionException(
"'protoc -version' did not return a version");
"'protoc --version' did not return a version");
} else {
if (!out.get(0).endsWith(protocVersion)) {
throw new MojoExecutionException(

7
hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java
View File

@ -63,11 +63,10 @@ public int run(List<String> command, List<String> output) {
for (String s : stdErr.getOutput()) {
mojo.getLog().debug(s);
}
} else {
stdOut.join();
stdErr.join();
output.addAll(stdOut.getOutput());
}
stdOut.join();
stdErr.join();
output.addAll(stdOut.getOutput());
} catch (Exception ex) {
mojo.getLog().warn(command + " failed: " + ex.toString());
}

1
hadoop-project/pom.xml
View File

@ -62,6 +62,7 @@
<!-- ProtocolBuffer version, used to verify the protoc version and -->
<!-- define the protobuf JAR version -->
<protobuf.version>2.5.0</protobuf.version>
<protoc.path>${env.HADOOP_PROTOC_PATH}</protoc.path>
</properties>
<dependencyManagement>

8
hadoop-yarn-project/hadoop-yarn/README
View File

@ -8,15 +8,9 @@ Maven: Maven 3
Setup
-----
Install protobuf 2.4.0a or higher (Download from http://code.google.com/p/protobuf/downloads/list)
Install protobuf 2.5.0 (Download from http://code.google.com/p/protobuf/downloads/list)
- install the protoc executable (configure, make, make install)
- install the maven artifact (cd java; mvn install)
Installing protoc requires gcc 4.1.x or higher.
If the make step fails with (Valid until a fix is released for protobuf 2.4.0a)
./google/protobuf/descriptor.h:1152: error:
`google::protobuf::internal::Mutex*google::protobuf::DescriptorPool::mutex_'
is private
Replace descriptor.cc with http://protobuf.googlecode.com/svn-history/r380/trunk/src/google/protobuf/descriptor.cc
Quick Maven Tips

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
View File

@ -45,7 +45,8 @@
<goal>protoc</goal>
</goals>
<configuration>
<protocVersion>2.5.0</protocVersion>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/../../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/src/main/proto</param>

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
View File

@ -73,7 +73,8 @@
<goal>protoc</goal>
</goals>
<configuration>
<protocVersion>2.5.0</protocVersion>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/../../hadoop-yarn-api/src/main/proto</param>

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
View File

@ -165,7 +165,8 @@
<goal>protoc</goal>
</goals>
<configuration>
<protocVersion>2.5.0</protocVersion>
<protocVersion>${protobuf.version}</protocVersion>
<protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/../../hadoop-yarn-api/src/main/proto</param>