From 3fc3fa9711d96677f6149e173df0f57cd06ee6b9 Mon Sep 17 00:00:00 2001 From: Chris Douglas Date: Mon, 19 Mar 2018 16:05:55 -0700 Subject: [PATCH] HADOOP-14667. Flexible Visual Studio support. Contributed by Allen Wittenauer --- BUILDING.txt | 30 ++++-------- dev-support/bin/win-vs-upgrade.cmd | 39 +++++++++++++++ dev-support/win-paths-eg.cmd | 49 +++++++++++++++++++ hadoop-common-project/hadoop-common/pom.xml | 28 +++++++++++ .../src/main/native/native.vcxproj | 2 + .../hadoop-hdfs-native-client/pom.xml | 5 +- 6 files changed, 128 insertions(+), 25 deletions(-) create mode 100644 dev-support/bin/win-vs-upgrade.cmd create mode 100644 dev-support/win-paths-eg.cmd diff --git a/BUILDING.txt b/BUILDING.txt index 6c266e5cb3..6d752d400f 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -348,7 +348,7 @@ Requirements: * Maven 3.0 or later * ProtocolBuffer 2.5.0 * CMake 3.1 or newer -* Windows SDK 7.1 or Visual Studio 2010 Professional +* Visual Studio 2010 Professional or Higher * Windows SDK 8.1 (if building CPU rate control for the container executor) * zlib headers (if building native code bindings for zlib) * Internet connection for first build (to fetch all Maven and Hadoop dependencies) @@ -359,18 +359,15 @@ Requirements: Unix command-line tools are also included with the Windows Git package which can be downloaded from http://git-scm.com/downloads -If using Visual Studio, it must be Visual Studio 2010 Professional (not 2012). +If using Visual Studio, it must be Professional level or higher. Do not use Visual Studio Express. It does not support compiling for 64-bit, -which is problematic if running a 64-bit system. The Windows SDK 7.1 is free to -download here: - -http://www.microsoft.com/en-us/download/details.aspx?id=8279 +which is problematic if running a 64-bit system. The Windows SDK 8.1 is available to download at: http://msdn.microsoft.com/en-us/windows/bg162891.aspx -Cygwin is neither required nor supported. +Cygwin is not required. ---------------------------------------------------------------------------------- Building: @@ -378,21 +375,12 @@ Building: Keep the source code tree in a short path to avoid running into problems related to Windows maximum path length limitation (for example, C:\hdc). -Run builds from a Windows SDK Command Prompt. (Start, All Programs, -Microsoft Windows SDK v7.1, Windows SDK 7.1 Command Prompt). +There is one support command file located in dev-support called win-paths-eg.cmd. +It should be copied somewhere convenient and modified to fit your needs. -JAVA_HOME must be set, and the path must not contain spaces. If the full path -would contain spaces, then use the Windows short path instead. - -You must set the Platform environment variable to either x64 or Win32 depending -on whether you're running a 64-bit or 32-bit system. Note that this is -case-sensitive. It must be "Platform", not "PLATFORM" or "platform". -Environment variables on Windows are usually case-insensitive, but Maven treats -them as case-sensitive. Failure to set this environment variable correctly will -cause msbuild to fail while building the native code in hadoop-common. - -set Platform=x64 (when building on a 64-bit system) -set Platform=Win32 (when building on a 32-bit system) +win-paths-eg.cmd sets up the environment for use. You will need to modify this +file. It will put all of the required components in the command path, +configure the bit-ness of the build, and set several optional components. Several tests require that the user must have the Create Symbolic Links privilege. diff --git a/dev-support/bin/win-vs-upgrade.cmd b/dev-support/bin/win-vs-upgrade.cmd new file mode 100644 index 0000000000..d8c9d7385b --- /dev/null +++ b/dev-support/bin/win-vs-upgrade.cmd @@ -0,0 +1,39 @@ +@ECHO OFF +@REM Licensed to the Apache Software Foundation (ASF) under one or more +@REM contributor license agreements. See the NOTICE file distributed with +@REM this work for additional information regarding copyright ownership. +@REM The ASF licenses this file to You under the Apache License, Version 2.0 +@REM (the "License"); you may not use this file except in compliance with +@REM the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. + +@WHERE devenv +IF %ERRORLEVEL% NEQ 0 ( + @ECHO "devenv command was not found. Verify your compiler installation level." + EXIT /b 1 +) + +@REM Need to save output to a file because for loop will just +@REM loop forever... :( + +SET srcdir=%1 +SET workdir=%2 + +IF EXIST %srcdir%\Backup ( + @ECHO "Solution files already upgraded." + EXIT /b 0 +) + +CD %srcdir% +DIR /B *.sln > %workdir%\HADOOP-SLN-UPGRADE.TXT + +FOR /F %%f IN (%workdir%\HADOOP-SLN-UPGRADE.TXT) DO ( + devenv %%f /upgrade +) diff --git a/dev-support/win-paths-eg.cmd b/dev-support/win-paths-eg.cmd new file mode 100644 index 0000000000..f9acae0196 --- /dev/null +++ b/dev-support/win-paths-eg.cmd @@ -0,0 +1,49 @@ +@ECHO OFF +@REM Licensed to the Apache Software Foundation (ASF) under one or more +@REM contributor license agreements. See the NOTICE file distributed with +@REM this work for additional information regarding copyright ownership. +@REM The ASF licenses this file to You under the Apache License, Version 2.0 +@REM (the "License"); you may not use this file except in compliance with +@REM the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. + +@REM ************************************************* +@REM JDK and these settings MUST MATCH +@REM +@REM 64-bit : Platform = x64, VCVARSPLAT = amd64 +@REM +@REM 32-bit : Platform = Win32, VCVARSPLAT = x86 +@REM + +SET Platform=x64 +SET VCVARSPLAT=amd64 + +@REM ****************** +@REM Forcibly move the Maven local repo + +SET MAVEN_OPTS=-Dmaven.repo.local=C:\Tools\m2 + +@REM ******************************************* +@REM +@REM Locations of your bits and pieces +@REM +@REM NOTE: cmake is assumed to already be on the +@REM command path +@REM + +SET MAVEN_HOME=C:\Tools\apache-maven-3.5.0 +SET JAVA_HOME=C:\Tools\jdk +SET MSVS=C:\Program Files (x86)\Microsoft Visual Studio 12.0 +SET PROTO_BIN=C:\Tools\protobuf-2.5.0 +SET GIT_HOME=C:\Program Files\Git + +SET PATH=%JAVA_HOME%\bin;%MAVEN_HOME%\bin;%PROTO_BIN%;%GIT_HOME%\bin;%PATH% + +CALL "%MSVS%\VC\vcvarsall.bat" %VCVARSPLAT% diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 49d3575d3c..1a16dc48fb 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -838,6 +838,20 @@ org.codehaus.mojo exec-maven-plugin + + convert-ms-winutils + generate-sources + + exec + + + ${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd + + ${basedir}\src\main\winutils + ${project.build.directory} + + + compile-ms-winutils compile @@ -857,6 +871,20 @@ + + convert-ms-native-dll + generate-sources + + exec + + + ${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd + + ${basedir}\src\main\native + ${project.build.directory} + + + compile-ms-native-dll compile diff --git a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj index 11190698f8..ac3767b276 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj +++ b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj @@ -71,6 +71,7 @@ $(CustomSnappyPrefix) $(CustomSnappyPrefix)\lib + $(CustomSnappyPrefix)\bin $(CustomSnappyLib) $(CustomSnappyPrefix) $(CustomSnappyPrefix)\include @@ -82,6 +83,7 @@ $(CustomIsalPrefix) $(CustomIsalPrefix)\lib + $(CustomIsalPrefix)\bin $(CustomIsalLib) true diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml index fa0e17479a..d9d98e45b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml @@ -138,13 +138,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> - - - - +