MAPREDUCE-4267. mavenize pipes (tgraves via bobby)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351818 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans
parent
01b2040810
commit
49feb6a039
@ -0,0 +1,67 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||
<id>hadoop-tools</id>
|
||||
<formats>
|
||||
<format>dir</format>
|
||||
</formats>
|
||||
<includeBaseDirectory>false</includeBaseDirectory>
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory>../hadoop-pipes/src/main/native/pipes/api/hadoop</directory>
|
||||
<includes>
|
||||
<include>*.hh</include>
|
||||
</includes>
|
||||
<outputDirectory>/include</outputDirectory>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>../hadoop-pipes/src/main/native/utils/api/hadoop</directory>
|
||||
<includes>
|
||||
<include>*.hh</include>
|
||||
</includes>
|
||||
<outputDirectory>/include</outputDirectory>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>../hadoop-pipes/target/native</directory>
|
||||
<includes>
|
||||
<include>*.a</include>
|
||||
</includes>
|
||||
<outputDirectory>lib/native</outputDirectory>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
<dependencySets>
|
||||
<dependencySet>
|
||||
<outputDirectory>/share/hadoop/${hadoop.component}/lib</outputDirectory>
|
||||
<unpack>false</unpack>
|
||||
<scope>runtime</scope>
|
||||
<useProjectArtifact>false</useProjectArtifact>
|
||||
<!-- Exclude hadoop artifacts. They will be found via HADOOP* env -->
|
||||
<excludes>
|
||||
<exclude>org.apache.hadoop:hadoop-common</exclude>
|
||||
<exclude>org.apache.hadoop:hadoop-hdfs</exclude>
|
||||
<exclude>org.apache.hadoop:hadoop-mapreduce</exclude>
|
||||
<!-- pipes is native stuff, this just keeps pom from being package-->
|
||||
<exclude>org.apache.hadoop:hadoop-pipes</exclude>
|
||||
<!-- use slf4j from common to avoid multiple binding warnings -->
|
||||
<exclude>org.slf4j:slf4j-api</exclude>
|
||||
<exclude>org.slf4j:slf4j-log4j12</exclude>
|
||||
</excludes>
|
||||
</dependencySet>
|
||||
</dependencySets>
|
||||
</assembly>
|
||||
@ -394,6 +394,8 @@ Release 0.23.3 - UNRELEASED
|
||||
MAPREDUCE-4301. Dedupe some strings in MRAM for memory savings
|
||||
(bobby via tgraves)
|
||||
|
||||
MAPREDUCE-4267. mavenize pipes (tgraves via bobby)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn
|
||||
|
||||
@ -1,42 +0,0 @@
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
#
|
||||
# autom4te configuration for hadoop utils library
|
||||
#
|
||||
|
||||
begin-language: "Autoheader-preselections"
|
||||
args: --no-cache
|
||||
end-language: "Autoheader-preselections"
|
||||
|
||||
begin-language: "Automake-preselections"
|
||||
args: --no-cache
|
||||
end-language: "Automake-preselections"
|
||||
|
||||
begin-language: "Autoreconf-preselections"
|
||||
args: --no-cache
|
||||
end-language: "Autoreconf-preselections"
|
||||
|
||||
begin-language: "Autoconf-without-aclocal-m4"
|
||||
args: --no-cache
|
||||
end-language: "Autoconf-without-aclocal-m4"
|
||||
|
||||
begin-language: "Autoconf"
|
||||
args: --no-cache
|
||||
end-language: "Autoconf"
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
ACLOCAL_AMFLAGS = -I ../utils/m4
|
||||
AM_CXXFLAGS=-I$(srcdir)/api -Wall -I$(HADOOP_UTILS_PREFIX)/include
|
||||
|
||||
# List the api header files and where they will be installed
|
||||
apidir = $(includedir)/hadoop
|
||||
api_HEADERS = \
|
||||
api/hadoop/Pipes.hh \
|
||||
api/hadoop/TemplateFactory.hh
|
||||
|
||||
# Define the libaries that need to be built
|
||||
lib_LIBRARIES = libhadooppipes.a
|
||||
|
||||
# Define the sources for lib
|
||||
libhadooppipes_a_SOURCES = \
|
||||
impl/HadoopPipes.cc
|
||||
|
||||
@ -1,57 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# -*- Autoconf -*-
|
||||
# Process this file with autoconf to produce a configure script.
|
||||
|
||||
AC_PREREQ(2.59)
|
||||
AC_INIT(hadoop-pipes, 0.13.0, omalley@apache.org)
|
||||
|
||||
AC_CONFIG_AUX_DIR([config])
|
||||
AC_CONFIG_MACRO_DIR([../utils/m4])
|
||||
|
||||
AM_INIT_AUTOMAKE([subdir-objects foreign no-dist])
|
||||
|
||||
AC_CONFIG_SRCDIR([impl/HadoopPipes.cc])
|
||||
AC_CONFIG_HEADER([impl/config.h])
|
||||
AC_CONFIG_FILES([Makefile])
|
||||
|
||||
AC_PREFIX_DEFAULT(`pwd`/../install)
|
||||
|
||||
USE_HADOOP_UTILS
|
||||
HADOOP_PIPES_SETUP
|
||||
CHECK_INSTALL_CFLAG
|
||||
|
||||
# Checks for programs.
|
||||
AC_PROG_CXX
|
||||
AC_PROG_LIBTOOL
|
||||
|
||||
# Checks for libraries.
|
||||
|
||||
# Checks for header files.
|
||||
AC_LANG(C++)
|
||||
AC_CHECK_HEADERS([unistd.h])
|
||||
|
||||
# Checks for typedefs, structures, and compiler characteristics.
|
||||
AC_HEADER_STDBOOL
|
||||
AC_C_CONST
|
||||
AC_TYPE_OFF_T
|
||||
AC_TYPE_SIZE_T
|
||||
AC_FUNC_STRERROR_R
|
||||
|
||||
# Checks for library functions.
|
||||
AC_CHECK_FUNCS([mkdir uname])
|
||||
AC_OUTPUT
|
||||
@ -1,42 +0,0 @@
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
#
|
||||
# autom4te configuration for hadoop utils library
|
||||
#
|
||||
|
||||
begin-language: "Autoheader-preselections"
|
||||
args: --no-cache
|
||||
end-language: "Autoheader-preselections"
|
||||
|
||||
begin-language: "Automake-preselections"
|
||||
args: --no-cache
|
||||
end-language: "Automake-preselections"
|
||||
|
||||
begin-language: "Autoreconf-preselections"
|
||||
args: --no-cache
|
||||
end-language: "Autoreconf-preselections"
|
||||
|
||||
begin-language: "Autoconf-without-aclocal-m4"
|
||||
args: --no-cache
|
||||
end-language: "Autoconf-without-aclocal-m4"
|
||||
|
||||
begin-language: "Autoconf"
|
||||
args: --no-cache
|
||||
end-language: "Autoconf"
|
||||
|
||||
@ -1,33 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
ACLOCAL_AMFLAGS = -I m4
|
||||
AM_CXXFLAGS=-I$(srcdir)/api -Wall
|
||||
|
||||
# List the api header files and where they will be installed
|
||||
apidir = $(includedir)/hadoop
|
||||
api_HEADERS = \
|
||||
api/hadoop/StringUtils.hh \
|
||||
api/hadoop/SerialUtils.hh
|
||||
|
||||
|
||||
# Define the libaries that need to be built
|
||||
lib_LIBRARIES = libhadooputils.a
|
||||
|
||||
# Define the sources for lib
|
||||
libhadooputils_a_SOURCES = \
|
||||
impl/StringUtils.cc \
|
||||
impl/SerialUtils.cc
|
||||
|
||||
@ -1,56 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# -*- Autoconf -*-
|
||||
# Process this file with autoconf to produce a configure script.
|
||||
|
||||
AC_PREREQ(2.59)
|
||||
AC_INIT(hadoop-utils, 0.13.0, omalley@apache.org)
|
||||
|
||||
AC_CONFIG_AUX_DIR([config])
|
||||
AC_CONFIG_MACRO_DIR([m4])
|
||||
|
||||
AM_INIT_AUTOMAKE([subdir-objects foreign no-dist])
|
||||
|
||||
AC_CONFIG_SRCDIR([impl/SerialUtils.cc])
|
||||
AC_CONFIG_HEADER([impl/config.h])
|
||||
AC_CONFIG_FILES([Makefile])
|
||||
|
||||
AC_PREFIX_DEFAULT(`pwd`/../install)
|
||||
|
||||
CHECK_INSTALL_CFLAG
|
||||
HADOOP_UTILS_SETUP
|
||||
|
||||
# Checks for programs.
|
||||
AC_PROG_CXX
|
||||
AC_PROG_LIBTOOL
|
||||
|
||||
# Checks for libraries.
|
||||
|
||||
# Checks for header files.
|
||||
AC_LANG(C++)
|
||||
AC_CHECK_HEADERS([unistd.h])
|
||||
|
||||
# Checks for typedefs, structures, and compiler characteristics.
|
||||
AC_HEADER_STDBOOL
|
||||
AC_C_CONST
|
||||
AC_TYPE_OFF_T
|
||||
AC_TYPE_SIZE_T
|
||||
AC_FUNC_STRERROR_R
|
||||
|
||||
# Checks for library functions.
|
||||
AC_CHECK_FUNCS([mkdir uname])
|
||||
AC_OUTPUT
|
||||
@ -1,68 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# hadoop_utils.m4
|
||||
|
||||
# Check to see if the install program supports -C
|
||||
# If so, use "install -C" for the headers. Otherwise, every install
|
||||
# updates the timestamps on the installed headers, which causes a recompilation
|
||||
# of any downstream libraries.
|
||||
AC_DEFUN([CHECK_INSTALL_CFLAG],[
|
||||
AC_REQUIRE([AC_PROG_INSTALL])
|
||||
touch foo
|
||||
if $INSTALL -C foo bar; then
|
||||
INSTALL_DATA="$INSTALL_DATA -C"
|
||||
fi
|
||||
rm -f foo bar
|
||||
])
|
||||
|
||||
# Set up the things we need for compiling hadoop utils
|
||||
AC_DEFUN([HADOOP_UTILS_SETUP],[
|
||||
AC_REQUIRE([AC_GNU_SOURCE])
|
||||
AC_REQUIRE([AC_SYS_LARGEFILE])
|
||||
])
|
||||
|
||||
# define a macro for using hadoop utils
|
||||
AC_DEFUN([USE_HADOOP_UTILS],[
|
||||
AC_REQUIRE([HADOOP_UTILS_SETUP])
|
||||
AC_ARG_WITH([hadoop-utils],
|
||||
AS_HELP_STRING([--with-hadoop-utils=<dir>],
|
||||
[directory to get hadoop_utils from]),
|
||||
[HADOOP_UTILS_PREFIX="$withval"],
|
||||
[HADOOP_UTILS_PREFIX="\${prefix}"])
|
||||
AC_SUBST(HADOOP_UTILS_PREFIX)
|
||||
])
|
||||
|
||||
AC_DEFUN([HADOOP_PIPES_SETUP],[
|
||||
AC_CHECK_HEADERS([pthread.h], [],
|
||||
AC_MSG_ERROR(Please check if you have installed the pthread library))
|
||||
AC_CHECK_LIB([pthread], [pthread_create], [],
|
||||
AC_MSG_ERROR(Cannot find libpthread.so, please check))
|
||||
AC_CHECK_LIB([crypto], [HMAC_Init], [],
|
||||
AC_MSG_ERROR(Cannot find libcrypto.so, please check))
|
||||
])
|
||||
|
||||
# define a macro for using hadoop pipes
|
||||
AC_DEFUN([USE_HADOOP_PIPES],[
|
||||
AC_REQUIRE([USE_HADOOP_UTILS])
|
||||
AC_REQUIRE([HADOOP_PIPES_SETUP])
|
||||
AC_ARG_WITH([hadoop-pipes],
|
||||
AS_HELP_STRING([--with-hadoop-pipes=<dir>],
|
||||
[directory to get hadoop pipes from]),
|
||||
[HADOOP_PIPES_PREFIX="$withval"],
|
||||
[HADOOP_PIPES_PREFIX="\${prefix}"])
|
||||
AC_SUBST(HADOOP_PIPES_PREFIX)
|
||||
])
|
||||
@ -1,42 +0,0 @@
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
#
|
||||
# autom4te configuration for hadoop utils library
|
||||
#
|
||||
|
||||
begin-language: "Autoheader-preselections"
|
||||
args: --no-cache
|
||||
end-language: "Autoheader-preselections"
|
||||
|
||||
begin-language: "Automake-preselections"
|
||||
args: --no-cache
|
||||
end-language: "Automake-preselections"
|
||||
|
||||
begin-language: "Autoreconf-preselections"
|
||||
args: --no-cache
|
||||
end-language: "Autoreconf-preselections"
|
||||
|
||||
begin-language: "Autoconf-without-aclocal-m4"
|
||||
args: --no-cache
|
||||
end-language: "Autoconf-without-aclocal-m4"
|
||||
|
||||
begin-language: "Autoconf"
|
||||
args: --no-cache
|
||||
end-language: "Autoconf"
|
||||
|
||||
@ -1,36 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
ACLOCAL_AMFLAGS = -I ../../c++/utils/m4
|
||||
AM_CXXFLAGS=-Wall -I$(HADOOP_UTILS_PREFIX)/include \
|
||||
-I$(HADOOP_PIPES_PREFIX)/include
|
||||
LDADD=-L$(HADOOP_UTILS_PREFIX)/lib -L$(HADOOP_PIPES_PREFIX)/lib \
|
||||
-lhadooppipes -lhadooputils
|
||||
|
||||
bin_PROGRAMS= wordcount-simple wordcount-part wordcount-nopipe pipes-sort
|
||||
|
||||
# Define the sources for each program
|
||||
wordcount_simple_SOURCES = \
|
||||
impl/wordcount-simple.cc
|
||||
|
||||
wordcount_part_SOURCES = \
|
||||
impl/wordcount-part.cc
|
||||
|
||||
wordcount_nopipe_SOURCES = \
|
||||
impl/wordcount-nopipe.cc
|
||||
|
||||
pipes_sort_SOURCES = \
|
||||
impl/sort.cc
|
||||
|
||||
@ -1,58 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# -*- Autoconf -*-
|
||||
# Process this file with autoconf to produce a configure script.
|
||||
|
||||
AC_PREREQ(2.59)
|
||||
AC_INIT(hadoop-pipes-examples, 0.13.0, omalley@apache.org)
|
||||
|
||||
AC_CONFIG_AUX_DIR([config])
|
||||
AC_CONFIG_MACRO_DIR([../../c++/utils/m4])
|
||||
|
||||
AM_INIT_AUTOMAKE([subdir-objects foreign no-dist])
|
||||
|
||||
AC_CONFIG_SRCDIR([impl/wordcount-simple.cc])
|
||||
AC_CONFIG_HEADER([impl/config.h])
|
||||
AC_CONFIG_FILES([Makefile])
|
||||
|
||||
AC_PREFIX_DEFAULT(`pwd`/../install)
|
||||
|
||||
USE_HADOOP_PIPES
|
||||
|
||||
# Checks for programs.
|
||||
AC_PROG_CXX
|
||||
AC_PROG_INSTALL
|
||||
AC_PROG_LIBTOOL
|
||||
|
||||
# Checks for libraries.
|
||||
|
||||
# Checks for header files.
|
||||
AC_LANG(C++)
|
||||
AC_CHECK_HEADERS([unistd.h])
|
||||
|
||||
# Checks for typedefs, structures, and compiler characteristics.
|
||||
AC_HEADER_STDBOOL
|
||||
AC_C_CONST
|
||||
AC_TYPE_OFF_T
|
||||
AC_TYPE_SIZE_T
|
||||
AC_FUNC_STRERROR_R
|
||||
|
||||
# Checks for library functions.
|
||||
AC_CHECK_FUNCS([mkdir uname])
|
||||
AC_CHECK_LIB([socket],[shutdown])
|
||||
AC_CHECK_LIB([nsl],[xdr_float])
|
||||
AC_OUTPUT
|
||||
108
hadoop-tools/hadoop-pipes/pom.xml
Normal file
108
hadoop-tools/hadoop-pipes/pom.xml
Normal file
@ -0,0 +1,108 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. See accompanying LICENSE file.
|
||||
-->
|
||||
<project>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-project</artifactId>
|
||||
<version>3.0.0-SNAPSHOT</version>
|
||||
<relativePath>../../hadoop-project</relativePath>
|
||||
</parent>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-pipes</artifactId>
|
||||
<version>3.0.0-SNAPSHOT</version>
|
||||
<description>Apache Hadoop Pipes</description>
|
||||
<name>Apache Hadoop Pipes</name>
|
||||
<packaging>pom</packaging>
|
||||
|
||||
<properties>
|
||||
<hadoop.log.dir>${project.build.directory}/log</hadoop.log.dir>
|
||||
</properties>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>native</id>
|
||||
<activation>
|
||||
<activeByDefault>false</activeByDefault>
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>make</id>
|
||||
<phase>compile</phase>
|
||||
<goals><goal>run</goal></goals>
|
||||
<configuration>
|
||||
<target>
|
||||
<mkdir dir="${project.build.directory}/native"/>
|
||||
<exec executable="cmake" dir="${project.build.directory}/native"
|
||||
failonerror="true">
|
||||
<arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
|
||||
</exec>
|
||||
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
|
||||
<arg line="VERBOSE=1"/>
|
||||
</exec>
|
||||
</target>
|
||||
</configuration>
|
||||
</execution>
|
||||
<!-- TODO wire here native testcases
|
||||
<execution>
|
||||
<id>test</id>
|
||||
<phase>test</phase>
|
||||
<goals>
|
||||
<goal>test</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<destDir>${project.build.directory}/native/target</destDir>
|
||||
</configuration>
|
||||
</execution>
|
||||
-->
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
<!--
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>compile</id>
|
||||
<phase>generate-sources</phase>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<target>
|
||||
<mkdir dir="${basedir}/../target/native"/>
|
||||
<copy toDir="${basedir}/../target/native">
|
||||
<fileset dir="${basedir}/src/main/native"/>
|
||||
</copy>
|
||||
</target>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
-->
|
||||
</project>
|
||||
99
hadoop-tools/hadoop-pipes/src/CMakeLists.txt
Normal file
99
hadoop-tools/hadoop-pipes/src/CMakeLists.txt
Normal file
@ -0,0 +1,99 @@
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
|
||||
find_package(OpenSSL REQUIRED)
|
||||
|
||||
set(CMAKE_BUILD_TYPE, Release)
|
||||
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -O2")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64")
|
||||
|
||||
if (JVM_ARCH_DATA_MODEL EQUAL 32)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")
|
||||
set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32")
|
||||
if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
|
||||
set(CMAKE_SYSTEM_PROCESSOR "i686")
|
||||
endif ()
|
||||
endif (JVM_ARCH_DATA_MODEL EQUAL 32)
|
||||
|
||||
function(output_directory TGT DIR)
|
||||
SET_TARGET_PROPERTIES(${TGT} PROPERTIES
|
||||
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
|
||||
SET_TARGET_PROPERTIES(${TGT} PROPERTIES
|
||||
ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
|
||||
SET_TARGET_PROPERTIES(${TGT} PROPERTIES
|
||||
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
|
||||
endfunction(output_directory TGT DIR)
|
||||
|
||||
include_directories(
|
||||
main/native/utils/api
|
||||
main/native/pipes/api
|
||||
${CMAKE_CURRENT_SOURCE_DIR}
|
||||
${OPENSSL_INCLUDE_DIR}
|
||||
)
|
||||
|
||||
# Example programs
|
||||
add_executable(wordcount-simple main/native/examples/impl/wordcount-simple.cc)
|
||||
target_link_libraries(wordcount-simple hadooppipes hadooputils)
|
||||
|
||||
add_executable(wordcount-part main/native/examples/impl/wordcount-part.cc)
|
||||
target_link_libraries(wordcount-part hadooppipes hadooputils)
|
||||
|
||||
add_executable(wordcount-nopipe main/native/examples/impl/wordcount-nopipe.cc)
|
||||
target_link_libraries(wordcount-nopipe hadooppipes hadooputils)
|
||||
|
||||
add_executable(pipes-sort main/native/examples/impl/sort.cc)
|
||||
target_link_libraries(pipes-sort hadooppipes hadooputils)
|
||||
|
||||
install(TARGETS wordcount-simple wordcount-part wordcount-nopipe pipes-sort
|
||||
RUNTIME DESTINATION bin
|
||||
)
|
||||
|
||||
add_library(hadooputils STATIC
|
||||
main/native/utils/impl/StringUtils.cc
|
||||
main/native/utils/impl/SerialUtils.cc
|
||||
)
|
||||
|
||||
install(FILES
|
||||
main/native/utils/api/hadoop/SerialUtils.hh
|
||||
main/native/utils/api/hadoop/StringUtils.hh
|
||||
DESTINATION api/hadoop
|
||||
COMPONENT headers
|
||||
)
|
||||
install(TARGETS hadooputils DESTINATION lib)
|
||||
|
||||
add_library(hadooppipes STATIC
|
||||
main/native/pipes/impl/HadoopPipes.cc
|
||||
)
|
||||
target_link_libraries(hadooppipes
|
||||
${JAVA_JVM_LIBRARY}
|
||||
${OPENSSL_LIBRARIES}
|
||||
pthread
|
||||
)
|
||||
|
||||
install(FILES
|
||||
main/native/pipes/api/hadoop/Pipes.hh
|
||||
main/native/pipes/api/hadoop/TemplateFactory.hh
|
||||
DESTINATION api/hadoop
|
||||
COMPONENT headers
|
||||
)
|
||||
install(TARGETS hadooppipes DESTINATION lib)
|
||||
@ -1,16 +1,16 @@
|
||||
To run the examples, first compile them:
|
||||
|
||||
% ant -Dcompile.c++=yes examples
|
||||
% mvn install
|
||||
|
||||
and then copy the binaries to dfs:
|
||||
|
||||
% bin/hadoop fs -put build/c++-examples/Linux-i386-32/bin /examples/bin
|
||||
% hadoop fs -put target/native/wordcount-simple /examples/bin/
|
||||
|
||||
create an input directory with text files:
|
||||
|
||||
% bin/hadoop fs -put my-data in-dir
|
||||
% hadoop fs -put my-data in-dir
|
||||
|
||||
and run the word count example:
|
||||
|
||||
% bin/hadoop pipes -conf src/examples/pipes/conf/word.xml \
|
||||
% hadoop pipes -conf src/main/native/examples/conf/word.xml \
|
||||
-input in-dir -output out-dir
|
||||
@ -70,6 +70,13 @@
|
||||
<artifactId>hadoop-gridmix</artifactId>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-pipes</artifactId>
|
||||
<scope>compile</scope>
|
||||
<type>pom</type>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
@ -92,4 +99,44 @@
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>dist</id>
|
||||
<activation>
|
||||
<activeByDefault>false</activeByDefault>
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-assemblies</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>dist</id>
|
||||
<phase>prepare-package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<appendAssemblyId>false</appendAssemblyId>
|
||||
<attach>false</attach>
|
||||
<finalName>${project.artifactId}-${project.version}</finalName>
|
||||
<descriptorRefs>
|
||||
<descriptorRef>hadoop-tools</descriptorRef>
|
||||
</descriptorRefs>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
</project>
|
||||
|
||||
@ -39,6 +39,7 @@
|
||||
<module>hadoop-datajoin</module>
|
||||
<module>hadoop-tools-dist</module>
|
||||
<module>hadoop-extras</module>
|
||||
<module>hadoop-pipes</module>
|
||||
</modules>
|
||||
|
||||
<build>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user