diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml new file mode 100644 index 0000000000..1e3356d337 --- /dev/null +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml @@ -0,0 +1,67 @@ + + + hadoop-tools + + dir + + false + + + ../hadoop-pipes/src/main/native/pipes/api/hadoop + + *.hh + + /include + + + ../hadoop-pipes/src/main/native/utils/api/hadoop + + *.hh + + /include + + + ../hadoop-pipes/target/native + + *.a + + lib/native + + + + + /share/hadoop/${hadoop.component}/lib + false + runtime + false + + + org.apache.hadoop:hadoop-common + org.apache.hadoop:hadoop-hdfs + org.apache.hadoop:hadoop-mapreduce + + org.apache.hadoop:hadoop-pipes + + org.slf4j:slf4j-api + org.slf4j:slf4j-log4j12 + + + + diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 206f9a17c5..29da54d5db 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -394,6 +394,8 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4301. Dedupe some strings in MRAM for memory savings (bobby via tgraves) + MAPREDUCE-4267. mavenize pipes (tgraves via bobby) + OPTIMIZATIONS MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn diff --git a/hadoop-mapreduce-project/src/c++/pipes/.autom4te.cfg b/hadoop-mapreduce-project/src/c++/pipes/.autom4te.cfg deleted file mode 100644 index d21d1c9877..0000000000 --- a/hadoop-mapreduce-project/src/c++/pipes/.autom4te.cfg +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# autom4te configuration for hadoop utils library -# - -begin-language: "Autoheader-preselections" -args: --no-cache -end-language: "Autoheader-preselections" - -begin-language: "Automake-preselections" -args: --no-cache -end-language: "Automake-preselections" - -begin-language: "Autoreconf-preselections" -args: --no-cache -end-language: "Autoreconf-preselections" - -begin-language: "Autoconf-without-aclocal-m4" -args: --no-cache -end-language: "Autoconf-without-aclocal-m4" - -begin-language: "Autoconf" -args: --no-cache -end-language: "Autoconf" - diff --git a/hadoop-mapreduce-project/src/c++/pipes/Makefile.am b/hadoop-mapreduce-project/src/c++/pipes/Makefile.am deleted file mode 100644 index 2c91d7f809..0000000000 --- a/hadoop-mapreduce-project/src/c++/pipes/Makefile.am +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ACLOCAL_AMFLAGS = -I ../utils/m4 -AM_CXXFLAGS=-I$(srcdir)/api -Wall -I$(HADOOP_UTILS_PREFIX)/include - -# List the api header files and where they will be installed -apidir = $(includedir)/hadoop -api_HEADERS = \ - api/hadoop/Pipes.hh \ - api/hadoop/TemplateFactory.hh - -# Define the libaries that need to be built -lib_LIBRARIES = libhadooppipes.a - -# Define the sources for lib -libhadooppipes_a_SOURCES = \ - impl/HadoopPipes.cc - diff --git a/hadoop-mapreduce-project/src/c++/pipes/configure.ac b/hadoop-mapreduce-project/src/c++/pipes/configure.ac deleted file mode 100644 index ac05ece0f5..0000000000 --- a/hadoop-mapreduce-project/src/c++/pipes/configure.ac +++ /dev/null @@ -1,57 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -*- Autoconf -*- -# Process this file with autoconf to produce a configure script. - -AC_PREREQ(2.59) -AC_INIT(hadoop-pipes, 0.13.0, omalley@apache.org) - -AC_CONFIG_AUX_DIR([config]) -AC_CONFIG_MACRO_DIR([../utils/m4]) - -AM_INIT_AUTOMAKE([subdir-objects foreign no-dist]) - -AC_CONFIG_SRCDIR([impl/HadoopPipes.cc]) -AC_CONFIG_HEADER([impl/config.h]) -AC_CONFIG_FILES([Makefile]) - -AC_PREFIX_DEFAULT(`pwd`/../install) - -USE_HADOOP_UTILS -HADOOP_PIPES_SETUP -CHECK_INSTALL_CFLAG - -# Checks for programs. -AC_PROG_CXX -AC_PROG_LIBTOOL - -# Checks for libraries. - -# Checks for header files. -AC_LANG(C++) -AC_CHECK_HEADERS([unistd.h]) - -# Checks for typedefs, structures, and compiler characteristics. -AC_HEADER_STDBOOL -AC_C_CONST -AC_TYPE_OFF_T -AC_TYPE_SIZE_T -AC_FUNC_STRERROR_R - -# Checks for library functions. -AC_CHECK_FUNCS([mkdir uname]) -AC_OUTPUT diff --git a/hadoop-mapreduce-project/src/c++/utils/.autom4te.cfg b/hadoop-mapreduce-project/src/c++/utils/.autom4te.cfg deleted file mode 100644 index d21d1c9877..0000000000 --- a/hadoop-mapreduce-project/src/c++/utils/.autom4te.cfg +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# autom4te configuration for hadoop utils library -# - -begin-language: "Autoheader-preselections" -args: --no-cache -end-language: "Autoheader-preselections" - -begin-language: "Automake-preselections" -args: --no-cache -end-language: "Automake-preselections" - -begin-language: "Autoreconf-preselections" -args: --no-cache -end-language: "Autoreconf-preselections" - -begin-language: "Autoconf-without-aclocal-m4" -args: --no-cache -end-language: "Autoconf-without-aclocal-m4" - -begin-language: "Autoconf" -args: --no-cache -end-language: "Autoconf" - diff --git a/hadoop-mapreduce-project/src/c++/utils/Makefile.am b/hadoop-mapreduce-project/src/c++/utils/Makefile.am deleted file mode 100644 index d99ea14958..0000000000 --- a/hadoop-mapreduce-project/src/c++/utils/Makefile.am +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ACLOCAL_AMFLAGS = -I m4 -AM_CXXFLAGS=-I$(srcdir)/api -Wall - -# List the api header files and where they will be installed -apidir = $(includedir)/hadoop -api_HEADERS = \ - api/hadoop/StringUtils.hh \ - api/hadoop/SerialUtils.hh - - -# Define the libaries that need to be built -lib_LIBRARIES = libhadooputils.a - -# Define the sources for lib -libhadooputils_a_SOURCES = \ - impl/StringUtils.cc \ - impl/SerialUtils.cc - diff --git a/hadoop-mapreduce-project/src/c++/utils/configure.ac b/hadoop-mapreduce-project/src/c++/utils/configure.ac deleted file mode 100644 index 92831816d5..0000000000 --- a/hadoop-mapreduce-project/src/c++/utils/configure.ac +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -*- Autoconf -*- -# Process this file with autoconf to produce a configure script. - -AC_PREREQ(2.59) -AC_INIT(hadoop-utils, 0.13.0, omalley@apache.org) - -AC_CONFIG_AUX_DIR([config]) -AC_CONFIG_MACRO_DIR([m4]) - -AM_INIT_AUTOMAKE([subdir-objects foreign no-dist]) - -AC_CONFIG_SRCDIR([impl/SerialUtils.cc]) -AC_CONFIG_HEADER([impl/config.h]) -AC_CONFIG_FILES([Makefile]) - -AC_PREFIX_DEFAULT(`pwd`/../install) - -CHECK_INSTALL_CFLAG -HADOOP_UTILS_SETUP - -# Checks for programs. -AC_PROG_CXX -AC_PROG_LIBTOOL - -# Checks for libraries. - -# Checks for header files. -AC_LANG(C++) -AC_CHECK_HEADERS([unistd.h]) - -# Checks for typedefs, structures, and compiler characteristics. -AC_HEADER_STDBOOL -AC_C_CONST -AC_TYPE_OFF_T -AC_TYPE_SIZE_T -AC_FUNC_STRERROR_R - -# Checks for library functions. -AC_CHECK_FUNCS([mkdir uname]) -AC_OUTPUT diff --git a/hadoop-mapreduce-project/src/c++/utils/m4/hadoop_utils.m4 b/hadoop-mapreduce-project/src/c++/utils/m4/hadoop_utils.m4 deleted file mode 100644 index d0ed6c4a0d..0000000000 --- a/hadoop-mapreduce-project/src/c++/utils/m4/hadoop_utils.m4 +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# hadoop_utils.m4 - -# Check to see if the install program supports -C -# If so, use "install -C" for the headers. Otherwise, every install -# updates the timestamps on the installed headers, which causes a recompilation -# of any downstream libraries. -AC_DEFUN([CHECK_INSTALL_CFLAG],[ -AC_REQUIRE([AC_PROG_INSTALL]) -touch foo -if $INSTALL -C foo bar; then - INSTALL_DATA="$INSTALL_DATA -C" -fi -rm -f foo bar -]) - -# Set up the things we need for compiling hadoop utils -AC_DEFUN([HADOOP_UTILS_SETUP],[ -AC_REQUIRE([AC_GNU_SOURCE]) -AC_REQUIRE([AC_SYS_LARGEFILE]) -]) - -# define a macro for using hadoop utils -AC_DEFUN([USE_HADOOP_UTILS],[ -AC_REQUIRE([HADOOP_UTILS_SETUP]) -AC_ARG_WITH([hadoop-utils], - AS_HELP_STRING([--with-hadoop-utils=], - [directory to get hadoop_utils from]), - [HADOOP_UTILS_PREFIX="$withval"], - [HADOOP_UTILS_PREFIX="\${prefix}"]) -AC_SUBST(HADOOP_UTILS_PREFIX) -]) - -AC_DEFUN([HADOOP_PIPES_SETUP],[ -AC_CHECK_HEADERS([pthread.h], [], - AC_MSG_ERROR(Please check if you have installed the pthread library)) -AC_CHECK_LIB([pthread], [pthread_create], [], - AC_MSG_ERROR(Cannot find libpthread.so, please check)) -AC_CHECK_LIB([crypto], [HMAC_Init], [], - AC_MSG_ERROR(Cannot find libcrypto.so, please check)) -]) - -# define a macro for using hadoop pipes -AC_DEFUN([USE_HADOOP_PIPES],[ -AC_REQUIRE([USE_HADOOP_UTILS]) -AC_REQUIRE([HADOOP_PIPES_SETUP]) -AC_ARG_WITH([hadoop-pipes], - AS_HELP_STRING([--with-hadoop-pipes=], - [directory to get hadoop pipes from]), - [HADOOP_PIPES_PREFIX="$withval"], - [HADOOP_PIPES_PREFIX="\${prefix}"]) -AC_SUBST(HADOOP_PIPES_PREFIX) -]) diff --git a/hadoop-mapreduce-project/src/examples/pipes/.autom4te.cfg b/hadoop-mapreduce-project/src/examples/pipes/.autom4te.cfg deleted file mode 100644 index d21d1c9877..0000000000 --- a/hadoop-mapreduce-project/src/examples/pipes/.autom4te.cfg +++ /dev/null @@ -1,42 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# autom4te configuration for hadoop utils library -# - -begin-language: "Autoheader-preselections" -args: --no-cache -end-language: "Autoheader-preselections" - -begin-language: "Automake-preselections" -args: --no-cache -end-language: "Automake-preselections" - -begin-language: "Autoreconf-preselections" -args: --no-cache -end-language: "Autoreconf-preselections" - -begin-language: "Autoconf-without-aclocal-m4" -args: --no-cache -end-language: "Autoconf-without-aclocal-m4" - -begin-language: "Autoconf" -args: --no-cache -end-language: "Autoconf" - diff --git a/hadoop-mapreduce-project/src/examples/pipes/Makefile.am b/hadoop-mapreduce-project/src/examples/pipes/Makefile.am deleted file mode 100644 index 731ab1e4b3..0000000000 --- a/hadoop-mapreduce-project/src/examples/pipes/Makefile.am +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -ACLOCAL_AMFLAGS = -I ../../c++/utils/m4 -AM_CXXFLAGS=-Wall -I$(HADOOP_UTILS_PREFIX)/include \ - -I$(HADOOP_PIPES_PREFIX)/include -LDADD=-L$(HADOOP_UTILS_PREFIX)/lib -L$(HADOOP_PIPES_PREFIX)/lib \ - -lhadooppipes -lhadooputils - -bin_PROGRAMS= wordcount-simple wordcount-part wordcount-nopipe pipes-sort - -# Define the sources for each program -wordcount_simple_SOURCES = \ - impl/wordcount-simple.cc - -wordcount_part_SOURCES = \ - impl/wordcount-part.cc - -wordcount_nopipe_SOURCES = \ - impl/wordcount-nopipe.cc - -pipes_sort_SOURCES = \ - impl/sort.cc - diff --git a/hadoop-mapreduce-project/src/examples/pipes/configure.ac b/hadoop-mapreduce-project/src/examples/pipes/configure.ac deleted file mode 100644 index 7959ba64f0..0000000000 --- a/hadoop-mapreduce-project/src/examples/pipes/configure.ac +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -*- Autoconf -*- -# Process this file with autoconf to produce a configure script. - -AC_PREREQ(2.59) -AC_INIT(hadoop-pipes-examples, 0.13.0, omalley@apache.org) - -AC_CONFIG_AUX_DIR([config]) -AC_CONFIG_MACRO_DIR([../../c++/utils/m4]) - -AM_INIT_AUTOMAKE([subdir-objects foreign no-dist]) - -AC_CONFIG_SRCDIR([impl/wordcount-simple.cc]) -AC_CONFIG_HEADER([impl/config.h]) -AC_CONFIG_FILES([Makefile]) - -AC_PREFIX_DEFAULT(`pwd`/../install) - -USE_HADOOP_PIPES - -# Checks for programs. -AC_PROG_CXX -AC_PROG_INSTALL -AC_PROG_LIBTOOL - -# Checks for libraries. - -# Checks for header files. -AC_LANG(C++) -AC_CHECK_HEADERS([unistd.h]) - -# Checks for typedefs, structures, and compiler characteristics. -AC_HEADER_STDBOOL -AC_C_CONST -AC_TYPE_OFF_T -AC_TYPE_SIZE_T -AC_FUNC_STRERROR_R - -# Checks for library functions. -AC_CHECK_FUNCS([mkdir uname]) -AC_CHECK_LIB([socket],[shutdown]) -AC_CHECK_LIB([nsl],[xdr_float]) -AC_OUTPUT diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml new file mode 100644 index 0000000000..70875f26c4 --- /dev/null +++ b/hadoop-tools/hadoop-pipes/pom.xml @@ -0,0 +1,108 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-SNAPSHOT + ../../hadoop-project + + org.apache.hadoop + hadoop-pipes + 3.0.0-SNAPSHOT + Apache Hadoop Pipes + Apache Hadoop Pipes + pom + + + ${project.build.directory}/log + + + + + native + + false + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + make + compile + run + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt new file mode 100644 index 0000000000..8ab7d27812 --- /dev/null +++ b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt @@ -0,0 +1,99 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cmake_minimum_required(VERSION 2.6 FATAL_ERROR) +find_package(OpenSSL REQUIRED) + +set(CMAKE_BUILD_TYPE, Release) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -O2") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64") + +if (JVM_ARCH_DATA_MODEL EQUAL 32) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32") + set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32") + if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") + set(CMAKE_SYSTEM_PROCESSOR "i686") + endif () +endif (JVM_ARCH_DATA_MODEL EQUAL 32) + +function(output_directory TGT DIR) + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") + SET_TARGET_PROPERTIES(${TGT} PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}") +endfunction(output_directory TGT DIR) + +include_directories( + main/native/utils/api + main/native/pipes/api + ${CMAKE_CURRENT_SOURCE_DIR} + ${OPENSSL_INCLUDE_DIR} +) + +# Example programs +add_executable(wordcount-simple main/native/examples/impl/wordcount-simple.cc) +target_link_libraries(wordcount-simple hadooppipes hadooputils) + +add_executable(wordcount-part main/native/examples/impl/wordcount-part.cc) +target_link_libraries(wordcount-part hadooppipes hadooputils) + +add_executable(wordcount-nopipe main/native/examples/impl/wordcount-nopipe.cc) +target_link_libraries(wordcount-nopipe hadooppipes hadooputils) + +add_executable(pipes-sort main/native/examples/impl/sort.cc) +target_link_libraries(pipes-sort hadooppipes hadooputils) + +install(TARGETS wordcount-simple wordcount-part wordcount-nopipe pipes-sort + RUNTIME DESTINATION bin +) + +add_library(hadooputils STATIC + main/native/utils/impl/StringUtils.cc + main/native/utils/impl/SerialUtils.cc +) + +install(FILES + main/native/utils/api/hadoop/SerialUtils.hh + main/native/utils/api/hadoop/StringUtils.hh + DESTINATION api/hadoop + COMPONENT headers +) +install(TARGETS hadooputils DESTINATION lib) + +add_library(hadooppipes STATIC + main/native/pipes/impl/HadoopPipes.cc +) +target_link_libraries(hadooppipes + ${JAVA_JVM_LIBRARY} + ${OPENSSL_LIBRARIES} + pthread +) + +install(FILES + main/native/pipes/api/hadoop/Pipes.hh + main/native/pipes/api/hadoop/TemplateFactory.hh + DESTINATION api/hadoop + COMPONENT headers +) +install(TARGETS hadooppipes DESTINATION lib) diff --git a/hadoop-mapreduce-project/src/examples/pipes/README.txt b/hadoop-tools/hadoop-pipes/src/main/native/examples/README.txt similarity index 50% rename from hadoop-mapreduce-project/src/examples/pipes/README.txt rename to hadoop-tools/hadoop-pipes/src/main/native/examples/README.txt index 4685304332..b9448f558a 100644 --- a/hadoop-mapreduce-project/src/examples/pipes/README.txt +++ b/hadoop-tools/hadoop-pipes/src/main/native/examples/README.txt @@ -1,16 +1,16 @@ To run the examples, first compile them: -% ant -Dcompile.c++=yes examples +% mvn install and then copy the binaries to dfs: -% bin/hadoop fs -put build/c++-examples/Linux-i386-32/bin /examples/bin +% hadoop fs -put target/native/wordcount-simple /examples/bin/ create an input directory with text files: -% bin/hadoop fs -put my-data in-dir +% hadoop fs -put my-data in-dir and run the word count example: -% bin/hadoop pipes -conf src/examples/pipes/conf/word.xml \ +% hadoop pipes -conf src/main/native/examples/conf/word.xml \ -input in-dir -output out-dir diff --git a/hadoop-mapreduce-project/src/examples/pipes/conf/word-part.xml b/hadoop-tools/hadoop-pipes/src/main/native/examples/conf/word-part.xml similarity index 100% rename from hadoop-mapreduce-project/src/examples/pipes/conf/word-part.xml rename to hadoop-tools/hadoop-pipes/src/main/native/examples/conf/word-part.xml diff --git a/hadoop-mapreduce-project/src/examples/pipes/conf/word.xml b/hadoop-tools/hadoop-pipes/src/main/native/examples/conf/word.xml similarity index 100% rename from hadoop-mapreduce-project/src/examples/pipes/conf/word.xml rename to hadoop-tools/hadoop-pipes/src/main/native/examples/conf/word.xml diff --git a/hadoop-mapreduce-project/src/examples/pipes/impl/sort.cc b/hadoop-tools/hadoop-pipes/src/main/native/examples/impl/sort.cc similarity index 100% rename from hadoop-mapreduce-project/src/examples/pipes/impl/sort.cc rename to hadoop-tools/hadoop-pipes/src/main/native/examples/impl/sort.cc diff --git a/hadoop-mapreduce-project/src/examples/pipes/impl/wordcount-nopipe.cc b/hadoop-tools/hadoop-pipes/src/main/native/examples/impl/wordcount-nopipe.cc similarity index 100% rename from hadoop-mapreduce-project/src/examples/pipes/impl/wordcount-nopipe.cc rename to hadoop-tools/hadoop-pipes/src/main/native/examples/impl/wordcount-nopipe.cc diff --git a/hadoop-mapreduce-project/src/examples/pipes/impl/wordcount-part.cc b/hadoop-tools/hadoop-pipes/src/main/native/examples/impl/wordcount-part.cc similarity index 100% rename from hadoop-mapreduce-project/src/examples/pipes/impl/wordcount-part.cc rename to hadoop-tools/hadoop-pipes/src/main/native/examples/impl/wordcount-part.cc diff --git a/hadoop-mapreduce-project/src/examples/pipes/impl/wordcount-simple.cc b/hadoop-tools/hadoop-pipes/src/main/native/examples/impl/wordcount-simple.cc similarity index 100% rename from hadoop-mapreduce-project/src/examples/pipes/impl/wordcount-simple.cc rename to hadoop-tools/hadoop-pipes/src/main/native/examples/impl/wordcount-simple.cc diff --git a/hadoop-mapreduce-project/src/c++/pipes/api/hadoop/Pipes.hh b/hadoop-tools/hadoop-pipes/src/main/native/pipes/api/hadoop/Pipes.hh similarity index 100% rename from hadoop-mapreduce-project/src/c++/pipes/api/hadoop/Pipes.hh rename to hadoop-tools/hadoop-pipes/src/main/native/pipes/api/hadoop/Pipes.hh diff --git a/hadoop-mapreduce-project/src/c++/pipes/api/hadoop/TemplateFactory.hh b/hadoop-tools/hadoop-pipes/src/main/native/pipes/api/hadoop/TemplateFactory.hh similarity index 100% rename from hadoop-mapreduce-project/src/c++/pipes/api/hadoop/TemplateFactory.hh rename to hadoop-tools/hadoop-pipes/src/main/native/pipes/api/hadoop/TemplateFactory.hh diff --git a/hadoop-mapreduce-project/src/c++/pipes/debug/pipes-default-gdb-commands.txt b/hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-gdb-commands.txt similarity index 100% rename from hadoop-mapreduce-project/src/c++/pipes/debug/pipes-default-gdb-commands.txt rename to hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-gdb-commands.txt diff --git a/hadoop-mapreduce-project/src/c++/pipes/debug/pipes-default-script b/hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-script similarity index 100% rename from hadoop-mapreduce-project/src/c++/pipes/debug/pipes-default-script rename to hadoop-tools/hadoop-pipes/src/main/native/pipes/debug/pipes-default-script diff --git a/hadoop-mapreduce-project/src/c++/pipes/impl/HadoopPipes.cc b/hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc similarity index 100% rename from hadoop-mapreduce-project/src/c++/pipes/impl/HadoopPipes.cc rename to hadoop-tools/hadoop-pipes/src/main/native/pipes/impl/HadoopPipes.cc diff --git a/hadoop-mapreduce-project/src/c++/utils/api/hadoop/SerialUtils.hh b/hadoop-tools/hadoop-pipes/src/main/native/utils/api/hadoop/SerialUtils.hh similarity index 100% rename from hadoop-mapreduce-project/src/c++/utils/api/hadoop/SerialUtils.hh rename to hadoop-tools/hadoop-pipes/src/main/native/utils/api/hadoop/SerialUtils.hh diff --git a/hadoop-mapreduce-project/src/c++/utils/api/hadoop/StringUtils.hh b/hadoop-tools/hadoop-pipes/src/main/native/utils/api/hadoop/StringUtils.hh similarity index 100% rename from hadoop-mapreduce-project/src/c++/utils/api/hadoop/StringUtils.hh rename to hadoop-tools/hadoop-pipes/src/main/native/utils/api/hadoop/StringUtils.hh diff --git a/hadoop-mapreduce-project/src/c++/utils/impl/SerialUtils.cc b/hadoop-tools/hadoop-pipes/src/main/native/utils/impl/SerialUtils.cc similarity index 100% rename from hadoop-mapreduce-project/src/c++/utils/impl/SerialUtils.cc rename to hadoop-tools/hadoop-pipes/src/main/native/utils/impl/SerialUtils.cc diff --git a/hadoop-mapreduce-project/src/c++/utils/impl/StringUtils.cc b/hadoop-tools/hadoop-pipes/src/main/native/utils/impl/StringUtils.cc similarity index 100% rename from hadoop-mapreduce-project/src/c++/utils/impl/StringUtils.cc rename to hadoop-tools/hadoop-pipes/src/main/native/utils/impl/StringUtils.cc diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml index 330cfd3731..951eab139c 100644 --- a/hadoop-tools/hadoop-tools-dist/pom.xml +++ b/hadoop-tools/hadoop-tools-dist/pom.xml @@ -70,6 +70,13 @@ hadoop-gridmix compile + + org.apache.hadoop + hadoop-pipes + compile + pom + ${project.version} + @@ -92,4 +99,44 @@ + + + dist + + false + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.hadoop + hadoop-assemblies + ${project.version} + + + + + dist + prepare-package + + single + + + false + false + ${project.artifactId}-${project.version} + + hadoop-tools + + + + + + + + + diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml index 3166b9ef99..bc75e2b653 100644 --- a/hadoop-tools/pom.xml +++ b/hadoop-tools/pom.xml @@ -39,6 +39,7 @@ hadoop-datajoin hadoop-tools-dist hadoop-extras + hadoop-pipes