From 8451ab5c01fdd290b33881daae8757c5f77a0d0d Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 14 Aug 2013 22:15:04 +0000 Subject: [PATCH] HADOOP-9872. Improve protoc version handling and detection. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1514068 13f79535-47bb-0310-9956-ffa450edef68 --- BUILDING.txt | 12 +++++++++++- hadoop-common-project/hadoop-common/CHANGES.txt | 6 ++++-- hadoop-common-project/hadoop-common/pom.xml | 2 ++ .../org/apache/hadoop/util/VersionInfo.java | 14 +++++++++++++- .../resources/common-version-info.properties | 1 + .../src/site/apt/CLIMiniCluster.apt.vm | 2 +- .../src/site/apt/SingleCluster.apt.vm | 2 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 12 ++++++++---- .../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 3 ++- .../hadoop-mapreduce-client-common/pom.xml | 3 ++- .../hadoop-mapreduce-client-hs/pom.xml | 3 ++- .../hadoop/maven/plugin/protoc/ProtocMojo.java | 17 +++++++++-------- .../apache/hadoop/maven/plugin/util/Exec.java | 7 +++---- hadoop-project/pom.xml | 1 + hadoop-yarn-project/hadoop-yarn/README | 8 +------- .../hadoop-yarn/hadoop-yarn-api/pom.xml | 3 ++- .../hadoop-yarn-server-common/pom.xml | 3 ++- .../hadoop-yarn-server-nodemanager/pom.xml | 3 ++- 18 files changed, 67 insertions(+), 35 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index 7e3d450953..ff6dea26ad 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -7,7 +7,7 @@ Requirements: * JDK 1.6 * Maven 3.0 * Findbugs 1.3.9 (if running findbugs) -* ProtocolBuffer 2.4.1+ (for MapReduce and HDFS) +* ProtocolBuffer 2.5.0 * CMake 2.6 or newer (if compiling native code) * Internet connection for first build (to fetch all Maven and Hadoop dependencies) @@ -99,6 +99,16 @@ level once; and then work from the submodule. Keep in mind that SNAPSHOTs time out after a while, using the Maven '-nsu' will stop Maven from trying to update SNAPSHOTs from external repos. +---------------------------------------------------------------------------------- +Protocol Buffer compiler + +The version of Protocol Buffer compiler, protoc, must match the version of the +protobuf JAR. + +If you have multiple versions of protoc in your system, you can set in your +build shell the HADOOP_PROTOC_PATH environment variable to point to the one you +want to use for the Hadoop build. If you don't define this environment variable, +protoc is looked up in the PATH. ---------------------------------------------------------------------------------- Importing projects to eclipse diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 3ba295873f..77f735504b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -105,8 +105,6 @@ Trunk (Unreleased) HADOOP-9833 move slf4j to version 1.7.5 (Kousuke Saruta via stevel) - HADOOP-9845. Update protobuf to 2.5 from 2.4.x. (tucu) - BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. @@ -584,6 +582,10 @@ Release 2.1.0-beta - 2013-08-06 HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs (todd) + HADOOP-9845. Update protobuf to 2.5 from 2.4.x. (tucu) + + HADOOP-9872. Improve protoc version handling and detection. (tucu) + BUG FIXES HADOOP-9294. GetGroupsTestBase fails on Windows. (Chris Nauroth via suresh) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 36c32607b9..a7462ea5a3 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -309,6 +309,7 @@ ${protobuf.version} + ${protoc.path} ${basedir}/src/main/proto @@ -338,6 +339,7 @@ ${protobuf.version} + ${protoc.path} ${basedir}/src/test/proto diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java index 5d7614f1eb..0f08f15ffa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java @@ -90,6 +90,10 @@ protected String _getBuildVersion(){ " source checksum " + _getSrcChecksum(); } + protected String _getProtocVersion() { + return info.getProperty("protocVersion", "Unknown"); + } + private static VersionInfo COMMON_VERSION_INFO = new VersionInfo("common"); /** * Get the Hadoop version. @@ -153,12 +157,20 @@ public static String getSrcChecksum() { public static String getBuildVersion(){ return COMMON_VERSION_INFO._getBuildVersion(); } - + + /** + * Returns the protoc version used for the build. + */ + public static String getProtocVersion(){ + return COMMON_VERSION_INFO._getProtocVersion(); + } + public static void main(String[] args) { LOG.debug("version: "+ getVersion()); System.out.println("Hadoop " + getVersion()); System.out.println("Subversion " + getUrl() + " -r " + getRevision()); System.out.println("Compiled by " + getUser() + " on " + getDate()); + System.out.println("Compiled with protoc " + getProtocVersion()); System.out.println("From source with checksum " + getSrcChecksum()); System.out.println("This command was run using " + ClassUtil.findContainingJar(VersionInfo.class)); diff --git a/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties index 9a8575c6de..ad9a24d590 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties +++ b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties @@ -23,3 +23,4 @@ user=${user.name} date=${version-info.build.time} url=${version-info.scm.uri} srcChecksum=${version-info.source.md5} +protocVersion=${protobuf.version} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/site/apt/CLIMiniCluster.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/CLIMiniCluster.apt.vm index 957b99463f..51a5a9afac 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/CLIMiniCluster.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/CLIMiniCluster.apt.vm @@ -42,7 +42,7 @@ Hadoop MapReduce Next Generation - CLI MiniCluster. $ mvn clean install -DskipTests $ mvn package -Pdist -Dtar -DskipTests -Dmaven.javadoc.skip +---+ - <> You will need protoc installed of version 2.4.1 or greater. + <> You will need protoc 2.5.0 installed. The tarball should be available in <<>> directory. diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm index 0cec916039..d7058d9491 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm @@ -32,7 +32,7 @@ $ mvn clean install -DskipTests $ cd hadoop-mapreduce-project $ mvn clean install assembly:assembly -Pnative +---+ - <> You will need protoc installed of version 2.4.1 or greater. + <> You will need protoc 2.5.0 installed. To ignore the native builds in mapreduce you can omit the <<<-Pnative>>> argument for maven. The tarball should be available in <<>> directory. diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index dd9b960eec..f0e3ac882b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -417,7 +417,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> protoc - 2.5.0 + ${protobuf.version} + ${protoc.path} ${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto ${basedir}/src/main/proto @@ -442,7 +443,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> protoc - 2.5.0 + ${protobuf.version} + ${protoc.path} ${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto ${basedir}/src/main/proto @@ -464,7 +466,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> protoc - 2.5.0 + ${protobuf.version} + ${protoc.path} ${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto ${basedir}/src/main/proto @@ -486,7 +489,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> protoc - 2.5.0 + ${protobuf.version} + ${protoc.path} ${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto ${basedir}/src/main/proto diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml index a306433776..537dee79c4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml @@ -103,7 +103,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> protoc - 2.5.0 + ${protobuf.version} + ${protoc.path} ${basedir}/../../../../../hadoop-common-project/hadoop-common/src/main/proto ${basedir}/../../../../../hadoop-hdfs-project/hadoop-hdfs/src/main/proto diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml index 39a2ecbe94..7ef7d3b038 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml @@ -64,7 +64,8 @@ protoc - 2.5.0 + ${protobuf.version} + ${protoc.path} ${basedir}/../../../hadoop-common-project/hadoop-common/src/main/proto ${basedir}/../../../hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml index 0d9ccf8c7f..31587782f4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml @@ -78,7 +78,8 @@ protoc - 2.5.0 + ${protobuf.version} + ${protoc.path} ${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java index 51901ff14d..bd62f55d19 100644 --- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java @@ -45,7 +45,7 @@ public class ProtocMojo extends AbstractMojo { @Parameter(required=true) private FileSet source; - @Parameter(defaultValue="protoc") + @Parameter private String protocCommand; @Parameter(required=true) @@ -53,21 +53,22 @@ public class ProtocMojo extends AbstractMojo { public void execute() throws MojoExecutionException { try { + if (protocCommand == null || protocCommand.trim().isEmpty()) { + protocCommand = "protoc"; + } List command = new ArrayList(); command.add(protocCommand); command.add("--version"); Exec exec = new Exec(this); List out = new ArrayList(); - if (exec.run(command, out) != 0) { - getLog().error("protoc, could not get version"); - for (String s : out) { - getLog().error(s); - } + if (exec.run(command, out) == 127) { + getLog().error("protoc, not found at: " + protocCommand); throw new MojoExecutionException("protoc failure"); } else { - if (out.size() == 0) { + if (out.isEmpty()) { + getLog().error("stdout: " + out); throw new MojoExecutionException( - "'protoc -version' did not return a version"); + "'protoc --version' did not return a version"); } else { if (!out.get(0).endsWith(protocVersion)) { throw new MojoExecutionException( diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java index 144ee13562..7dafe817bc 100644 --- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java @@ -63,11 +63,10 @@ public int run(List command, List output) { for (String s : stdErr.getOutput()) { mojo.getLog().debug(s); } - } else { - stdOut.join(); - stdErr.join(); - output.addAll(stdOut.getOutput()); } + stdOut.join(); + stdErr.join(); + output.addAll(stdOut.getOutput()); } catch (Exception ex) { mojo.getLog().warn(command + " failed: " + ex.toString()); } diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 1a923896c1..e1bb833c63 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -62,6 +62,7 @@ 2.5.0 + ${env.HADOOP_PROTOC_PATH} diff --git a/hadoop-yarn-project/hadoop-yarn/README b/hadoop-yarn-project/hadoop-yarn/README index b15870fa83..4e6aaa523e 100644 --- a/hadoop-yarn-project/hadoop-yarn/README +++ b/hadoop-yarn-project/hadoop-yarn/README @@ -8,15 +8,9 @@ Maven: Maven 3 Setup ----- -Install protobuf 2.4.0a or higher (Download from http://code.google.com/p/protobuf/downloads/list) +Install protobuf 2.5.0 (Download from http://code.google.com/p/protobuf/downloads/list) - install the protoc executable (configure, make, make install) - install the maven artifact (cd java; mvn install) -Installing protoc requires gcc 4.1.x or higher. -If the make step fails with (Valid until a fix is released for protobuf 2.4.0a) - ./google/protobuf/descriptor.h:1152: error: - `google::protobuf::internal::Mutex*google::protobuf::DescriptorPool::mutex_' - is private - Replace descriptor.cc with http://protobuf.googlecode.com/svn-history/r380/trunk/src/google/protobuf/descriptor.cc Quick Maven Tips diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml index 9c716cde26..c0df5d8eb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml @@ -45,7 +45,8 @@ protoc - 2.5.0 + ${protobuf.version} + ${protoc.path} ${basedir}/../../../hadoop-common-project/hadoop-common/src/main/proto ${basedir}/src/main/proto diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml index d987f407c0..3d3686e428 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml @@ -73,7 +73,8 @@ protoc - 2.5.0 + ${protobuf.version} + ${protoc.path} ${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto ${basedir}/../../hadoop-yarn-api/src/main/proto diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index f42d567b16..9eacd20f02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -165,7 +165,8 @@ protoc - 2.5.0 + ${protobuf.version} + ${protoc.path} ${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto ${basedir}/../../hadoop-yarn-api/src/main/proto