MAPREDUCE-4267. mavenize pipes (tgraves via bobby)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1351818 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Joseph Evans
parent
01b2040810
commit
49feb6a039
@ -0,0 +1,67 @@
|
|||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||||
|
<id>hadoop-tools</id>
|
||||||
|
<formats>
|
||||||
|
<format>dir</format>
|
||||||
|
</formats>
|
||||||
|
<includeBaseDirectory>false</includeBaseDirectory>
|
||||||
|
<fileSets>
|
||||||
|
<fileSet>
|
||||||
|
<directory>../hadoop-pipes/src/main/native/pipes/api/hadoop</directory>
|
||||||
|
<includes>
|
||||||
|
<include>*.hh</include>
|
||||||
|
</includes>
|
||||||
|
<outputDirectory>/include</outputDirectory>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>../hadoop-pipes/src/main/native/utils/api/hadoop</directory>
|
||||||
|
<includes>
|
||||||
|
<include>*.hh</include>
|
||||||
|
</includes>
|
||||||
|
<outputDirectory>/include</outputDirectory>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>../hadoop-pipes/target/native</directory>
|
||||||
|
<includes>
|
||||||
|
<include>*.a</include>
|
||||||
|
</includes>
|
||||||
|
<outputDirectory>lib/native</outputDirectory>
|
||||||
|
</fileSet>
|
||||||
|
</fileSets>
|
||||||
|
<dependencySets>
|
||||||
|
<dependencySet>
|
||||||
|
<outputDirectory>/share/hadoop/${hadoop.component}/lib</outputDirectory>
|
||||||
|
<unpack>false</unpack>
|
||||||
|
<scope>runtime</scope>
|
||||||
|
<useProjectArtifact>false</useProjectArtifact>
|
||||||
|
<!-- Exclude hadoop artifacts. They will be found via HADOOP* env -->
|
||||||
|
<excludes>
|
||||||
|
<exclude>org.apache.hadoop:hadoop-common</exclude>
|
||||||
|
<exclude>org.apache.hadoop:hadoop-hdfs</exclude>
|
||||||
|
<exclude>org.apache.hadoop:hadoop-mapreduce</exclude>
|
||||||
|
<!-- pipes is native stuff, this just keeps pom from being package-->
|
||||||
|
<exclude>org.apache.hadoop:hadoop-pipes</exclude>
|
||||||
|
<!-- use slf4j from common to avoid multiple binding warnings -->
|
||||||
|
<exclude>org.slf4j:slf4j-api</exclude>
|
||||||
|
<exclude>org.slf4j:slf4j-log4j12</exclude>
|
||||||
|
</excludes>
|
||||||
|
</dependencySet>
|
||||||
|
</dependencySets>
|
||||||
|
</assembly>
|
||||||
@ -394,6 +394,8 @@ Release 0.23.3 - UNRELEASED
|
|||||||
MAPREDUCE-4301. Dedupe some strings in MRAM for memory savings
|
MAPREDUCE-4301. Dedupe some strings in MRAM for memory savings
|
||||||
(bobby via tgraves)
|
(bobby via tgraves)
|
||||||
|
|
||||||
|
MAPREDUCE-4267. mavenize pipes (tgraves via bobby)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn
|
MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn
|
||||||
|
|||||||
@ -1,42 +0,0 @@
|
|||||||
#
|
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
# or more contributor license agreements. See the NOTICE file
|
|
||||||
# distributed with this work for additional information
|
|
||||||
# regarding copyright ownership. The ASF licenses this file
|
|
||||||
# to you under the Apache License, Version 2.0 (the
|
|
||||||
# "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
#
|
|
||||||
|
|
||||||
#
|
|
||||||
# autom4te configuration for hadoop utils library
|
|
||||||
#
|
|
||||||
|
|
||||||
begin-language: "Autoheader-preselections"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Autoheader-preselections"
|
|
||||||
|
|
||||||
begin-language: "Automake-preselections"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Automake-preselections"
|
|
||||||
|
|
||||||
begin-language: "Autoreconf-preselections"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Autoreconf-preselections"
|
|
||||||
|
|
||||||
begin-language: "Autoconf-without-aclocal-m4"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Autoconf-without-aclocal-m4"
|
|
||||||
|
|
||||||
begin-language: "Autoconf"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Autoconf"
|
|
||||||
|
|
||||||
@ -1,31 +0,0 @@
|
|||||||
# Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
# or more contributor license agreements. See the NOTICE file
|
|
||||||
# distributed with this work for additional information
|
|
||||||
# regarding copyright ownership. The ASF licenses this file
|
|
||||||
# to you under the Apache License, Version 2.0 (the
|
|
||||||
# "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
ACLOCAL_AMFLAGS = -I ../utils/m4
|
|
||||||
AM_CXXFLAGS=-I$(srcdir)/api -Wall -I$(HADOOP_UTILS_PREFIX)/include
|
|
||||||
|
|
||||||
# List the api header files and where they will be installed
|
|
||||||
apidir = $(includedir)/hadoop
|
|
||||||
api_HEADERS = \
|
|
||||||
api/hadoop/Pipes.hh \
|
|
||||||
api/hadoop/TemplateFactory.hh
|
|
||||||
|
|
||||||
# Define the libaries that need to be built
|
|
||||||
lib_LIBRARIES = libhadooppipes.a
|
|
||||||
|
|
||||||
# Define the sources for lib
|
|
||||||
libhadooppipes_a_SOURCES = \
|
|
||||||
impl/HadoopPipes.cc
|
|
||||||
|
|
||||||
@ -1,57 +0,0 @@
|
|||||||
# Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
# or more contributor license agreements. See the NOTICE file
|
|
||||||
# distributed with this work for additional information
|
|
||||||
# regarding copyright ownership. The ASF licenses this file
|
|
||||||
# to you under the Apache License, Version 2.0 (the
|
|
||||||
# "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
#
|
|
||||||
# -*- Autoconf -*-
|
|
||||||
# Process this file with autoconf to produce a configure script.
|
|
||||||
|
|
||||||
AC_PREREQ(2.59)
|
|
||||||
AC_INIT(hadoop-pipes, 0.13.0, omalley@apache.org)
|
|
||||||
|
|
||||||
AC_CONFIG_AUX_DIR([config])
|
|
||||||
AC_CONFIG_MACRO_DIR([../utils/m4])
|
|
||||||
|
|
||||||
AM_INIT_AUTOMAKE([subdir-objects foreign no-dist])
|
|
||||||
|
|
||||||
AC_CONFIG_SRCDIR([impl/HadoopPipes.cc])
|
|
||||||
AC_CONFIG_HEADER([impl/config.h])
|
|
||||||
AC_CONFIG_FILES([Makefile])
|
|
||||||
|
|
||||||
AC_PREFIX_DEFAULT(`pwd`/../install)
|
|
||||||
|
|
||||||
USE_HADOOP_UTILS
|
|
||||||
HADOOP_PIPES_SETUP
|
|
||||||
CHECK_INSTALL_CFLAG
|
|
||||||
|
|
||||||
# Checks for programs.
|
|
||||||
AC_PROG_CXX
|
|
||||||
AC_PROG_LIBTOOL
|
|
||||||
|
|
||||||
# Checks for libraries.
|
|
||||||
|
|
||||||
# Checks for header files.
|
|
||||||
AC_LANG(C++)
|
|
||||||
AC_CHECK_HEADERS([unistd.h])
|
|
||||||
|
|
||||||
# Checks for typedefs, structures, and compiler characteristics.
|
|
||||||
AC_HEADER_STDBOOL
|
|
||||||
AC_C_CONST
|
|
||||||
AC_TYPE_OFF_T
|
|
||||||
AC_TYPE_SIZE_T
|
|
||||||
AC_FUNC_STRERROR_R
|
|
||||||
|
|
||||||
# Checks for library functions.
|
|
||||||
AC_CHECK_FUNCS([mkdir uname])
|
|
||||||
AC_OUTPUT
|
|
||||||
@ -1,42 +0,0 @@
|
|||||||
#
|
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
# or more contributor license agreements. See the NOTICE file
|
|
||||||
# distributed with this work for additional information
|
|
||||||
# regarding copyright ownership. The ASF licenses this file
|
|
||||||
# to you under the Apache License, Version 2.0 (the
|
|
||||||
# "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
#
|
|
||||||
|
|
||||||
#
|
|
||||||
# autom4te configuration for hadoop utils library
|
|
||||||
#
|
|
||||||
|
|
||||||
begin-language: "Autoheader-preselections"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Autoheader-preselections"
|
|
||||||
|
|
||||||
begin-language: "Automake-preselections"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Automake-preselections"
|
|
||||||
|
|
||||||
begin-language: "Autoreconf-preselections"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Autoreconf-preselections"
|
|
||||||
|
|
||||||
begin-language: "Autoconf-without-aclocal-m4"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Autoconf-without-aclocal-m4"
|
|
||||||
|
|
||||||
begin-language: "Autoconf"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Autoconf"
|
|
||||||
|
|
||||||
@ -1,33 +0,0 @@
|
|||||||
# Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
# or more contributor license agreements. See the NOTICE file
|
|
||||||
# distributed with this work for additional information
|
|
||||||
# regarding copyright ownership. The ASF licenses this file
|
|
||||||
# to you under the Apache License, Version 2.0 (the
|
|
||||||
# "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
ACLOCAL_AMFLAGS = -I m4
|
|
||||||
AM_CXXFLAGS=-I$(srcdir)/api -Wall
|
|
||||||
|
|
||||||
# List the api header files and where they will be installed
|
|
||||||
apidir = $(includedir)/hadoop
|
|
||||||
api_HEADERS = \
|
|
||||||
api/hadoop/StringUtils.hh \
|
|
||||||
api/hadoop/SerialUtils.hh
|
|
||||||
|
|
||||||
|
|
||||||
# Define the libaries that need to be built
|
|
||||||
lib_LIBRARIES = libhadooputils.a
|
|
||||||
|
|
||||||
# Define the sources for lib
|
|
||||||
libhadooputils_a_SOURCES = \
|
|
||||||
impl/StringUtils.cc \
|
|
||||||
impl/SerialUtils.cc
|
|
||||||
|
|
||||||
@ -1,56 +0,0 @@
|
|||||||
# Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
# or more contributor license agreements. See the NOTICE file
|
|
||||||
# distributed with this work for additional information
|
|
||||||
# regarding copyright ownership. The ASF licenses this file
|
|
||||||
# to you under the Apache License, Version 2.0 (the
|
|
||||||
# "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
#
|
|
||||||
# -*- Autoconf -*-
|
|
||||||
# Process this file with autoconf to produce a configure script.
|
|
||||||
|
|
||||||
AC_PREREQ(2.59)
|
|
||||||
AC_INIT(hadoop-utils, 0.13.0, omalley@apache.org)
|
|
||||||
|
|
||||||
AC_CONFIG_AUX_DIR([config])
|
|
||||||
AC_CONFIG_MACRO_DIR([m4])
|
|
||||||
|
|
||||||
AM_INIT_AUTOMAKE([subdir-objects foreign no-dist])
|
|
||||||
|
|
||||||
AC_CONFIG_SRCDIR([impl/SerialUtils.cc])
|
|
||||||
AC_CONFIG_HEADER([impl/config.h])
|
|
||||||
AC_CONFIG_FILES([Makefile])
|
|
||||||
|
|
||||||
AC_PREFIX_DEFAULT(`pwd`/../install)
|
|
||||||
|
|
||||||
CHECK_INSTALL_CFLAG
|
|
||||||
HADOOP_UTILS_SETUP
|
|
||||||
|
|
||||||
# Checks for programs.
|
|
||||||
AC_PROG_CXX
|
|
||||||
AC_PROG_LIBTOOL
|
|
||||||
|
|
||||||
# Checks for libraries.
|
|
||||||
|
|
||||||
# Checks for header files.
|
|
||||||
AC_LANG(C++)
|
|
||||||
AC_CHECK_HEADERS([unistd.h])
|
|
||||||
|
|
||||||
# Checks for typedefs, structures, and compiler characteristics.
|
|
||||||
AC_HEADER_STDBOOL
|
|
||||||
AC_C_CONST
|
|
||||||
AC_TYPE_OFF_T
|
|
||||||
AC_TYPE_SIZE_T
|
|
||||||
AC_FUNC_STRERROR_R
|
|
||||||
|
|
||||||
# Checks for library functions.
|
|
||||||
AC_CHECK_FUNCS([mkdir uname])
|
|
||||||
AC_OUTPUT
|
|
||||||
@ -1,68 +0,0 @@
|
|||||||
# Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
# or more contributor license agreements. See the NOTICE file
|
|
||||||
# distributed with this work for additional information
|
|
||||||
# regarding copyright ownership. The ASF licenses this file
|
|
||||||
# to you under the Apache License, Version 2.0 (the
|
|
||||||
# "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
#
|
|
||||||
# hadoop_utils.m4
|
|
||||||
|
|
||||||
# Check to see if the install program supports -C
|
|
||||||
# If so, use "install -C" for the headers. Otherwise, every install
|
|
||||||
# updates the timestamps on the installed headers, which causes a recompilation
|
|
||||||
# of any downstream libraries.
|
|
||||||
AC_DEFUN([CHECK_INSTALL_CFLAG],[
|
|
||||||
AC_REQUIRE([AC_PROG_INSTALL])
|
|
||||||
touch foo
|
|
||||||
if $INSTALL -C foo bar; then
|
|
||||||
INSTALL_DATA="$INSTALL_DATA -C"
|
|
||||||
fi
|
|
||||||
rm -f foo bar
|
|
||||||
])
|
|
||||||
|
|
||||||
# Set up the things we need for compiling hadoop utils
|
|
||||||
AC_DEFUN([HADOOP_UTILS_SETUP],[
|
|
||||||
AC_REQUIRE([AC_GNU_SOURCE])
|
|
||||||
AC_REQUIRE([AC_SYS_LARGEFILE])
|
|
||||||
])
|
|
||||||
|
|
||||||
# define a macro for using hadoop utils
|
|
||||||
AC_DEFUN([USE_HADOOP_UTILS],[
|
|
||||||
AC_REQUIRE([HADOOP_UTILS_SETUP])
|
|
||||||
AC_ARG_WITH([hadoop-utils],
|
|
||||||
AS_HELP_STRING([--with-hadoop-utils=<dir>],
|
|
||||||
[directory to get hadoop_utils from]),
|
|
||||||
[HADOOP_UTILS_PREFIX="$withval"],
|
|
||||||
[HADOOP_UTILS_PREFIX="\${prefix}"])
|
|
||||||
AC_SUBST(HADOOP_UTILS_PREFIX)
|
|
||||||
])
|
|
||||||
|
|
||||||
AC_DEFUN([HADOOP_PIPES_SETUP],[
|
|
||||||
AC_CHECK_HEADERS([pthread.h], [],
|
|
||||||
AC_MSG_ERROR(Please check if you have installed the pthread library))
|
|
||||||
AC_CHECK_LIB([pthread], [pthread_create], [],
|
|
||||||
AC_MSG_ERROR(Cannot find libpthread.so, please check))
|
|
||||||
AC_CHECK_LIB([crypto], [HMAC_Init], [],
|
|
||||||
AC_MSG_ERROR(Cannot find libcrypto.so, please check))
|
|
||||||
])
|
|
||||||
|
|
||||||
# define a macro for using hadoop pipes
|
|
||||||
AC_DEFUN([USE_HADOOP_PIPES],[
|
|
||||||
AC_REQUIRE([USE_HADOOP_UTILS])
|
|
||||||
AC_REQUIRE([HADOOP_PIPES_SETUP])
|
|
||||||
AC_ARG_WITH([hadoop-pipes],
|
|
||||||
AS_HELP_STRING([--with-hadoop-pipes=<dir>],
|
|
||||||
[directory to get hadoop pipes from]),
|
|
||||||
[HADOOP_PIPES_PREFIX="$withval"],
|
|
||||||
[HADOOP_PIPES_PREFIX="\${prefix}"])
|
|
||||||
AC_SUBST(HADOOP_PIPES_PREFIX)
|
|
||||||
])
|
|
||||||
@ -1,42 +0,0 @@
|
|||||||
#
|
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
# or more contributor license agreements. See the NOTICE file
|
|
||||||
# distributed with this work for additional information
|
|
||||||
# regarding copyright ownership. The ASF licenses this file
|
|
||||||
# to you under the Apache License, Version 2.0 (the
|
|
||||||
# "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
#
|
|
||||||
|
|
||||||
#
|
|
||||||
# autom4te configuration for hadoop utils library
|
|
||||||
#
|
|
||||||
|
|
||||||
begin-language: "Autoheader-preselections"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Autoheader-preselections"
|
|
||||||
|
|
||||||
begin-language: "Automake-preselections"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Automake-preselections"
|
|
||||||
|
|
||||||
begin-language: "Autoreconf-preselections"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Autoreconf-preselections"
|
|
||||||
|
|
||||||
begin-language: "Autoconf-without-aclocal-m4"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Autoconf-without-aclocal-m4"
|
|
||||||
|
|
||||||
begin-language: "Autoconf"
|
|
||||||
args: --no-cache
|
|
||||||
end-language: "Autoconf"
|
|
||||||
|
|
||||||
@ -1,36 +0,0 @@
|
|||||||
# Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
# or more contributor license agreements. See the NOTICE file
|
|
||||||
# distributed with this work for additional information
|
|
||||||
# regarding copyright ownership. The ASF licenses this file
|
|
||||||
# to you under the Apache License, Version 2.0 (the
|
|
||||||
# "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
ACLOCAL_AMFLAGS = -I ../../c++/utils/m4
|
|
||||||
AM_CXXFLAGS=-Wall -I$(HADOOP_UTILS_PREFIX)/include \
|
|
||||||
-I$(HADOOP_PIPES_PREFIX)/include
|
|
||||||
LDADD=-L$(HADOOP_UTILS_PREFIX)/lib -L$(HADOOP_PIPES_PREFIX)/lib \
|
|
||||||
-lhadooppipes -lhadooputils
|
|
||||||
|
|
||||||
bin_PROGRAMS= wordcount-simple wordcount-part wordcount-nopipe pipes-sort
|
|
||||||
|
|
||||||
# Define the sources for each program
|
|
||||||
wordcount_simple_SOURCES = \
|
|
||||||
impl/wordcount-simple.cc
|
|
||||||
|
|
||||||
wordcount_part_SOURCES = \
|
|
||||||
impl/wordcount-part.cc
|
|
||||||
|
|
||||||
wordcount_nopipe_SOURCES = \
|
|
||||||
impl/wordcount-nopipe.cc
|
|
||||||
|
|
||||||
pipes_sort_SOURCES = \
|
|
||||||
impl/sort.cc
|
|
||||||
|
|
||||||
@ -1,58 +0,0 @@
|
|||||||
# Licensed to the Apache Software Foundation (ASF) under one
|
|
||||||
# or more contributor license agreements. See the NOTICE file
|
|
||||||
# distributed with this work for additional information
|
|
||||||
# regarding copyright ownership. The ASF licenses this file
|
|
||||||
# to you under the Apache License, Version 2.0 (the
|
|
||||||
# "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
#
|
|
||||||
# -*- Autoconf -*-
|
|
||||||
# Process this file with autoconf to produce a configure script.
|
|
||||||
|
|
||||||
AC_PREREQ(2.59)
|
|
||||||
AC_INIT(hadoop-pipes-examples, 0.13.0, omalley@apache.org)
|
|
||||||
|
|
||||||
AC_CONFIG_AUX_DIR([config])
|
|
||||||
AC_CONFIG_MACRO_DIR([../../c++/utils/m4])
|
|
||||||
|
|
||||||
AM_INIT_AUTOMAKE([subdir-objects foreign no-dist])
|
|
||||||
|
|
||||||
AC_CONFIG_SRCDIR([impl/wordcount-simple.cc])
|
|
||||||
AC_CONFIG_HEADER([impl/config.h])
|
|
||||||
AC_CONFIG_FILES([Makefile])
|
|
||||||
|
|
||||||
AC_PREFIX_DEFAULT(`pwd`/../install)
|
|
||||||
|
|
||||||
USE_HADOOP_PIPES
|
|
||||||
|
|
||||||
# Checks for programs.
|
|
||||||
AC_PROG_CXX
|
|
||||||
AC_PROG_INSTALL
|
|
||||||
AC_PROG_LIBTOOL
|
|
||||||
|
|
||||||
# Checks for libraries.
|
|
||||||
|
|
||||||
# Checks for header files.
|
|
||||||
AC_LANG(C++)
|
|
||||||
AC_CHECK_HEADERS([unistd.h])
|
|
||||||
|
|
||||||
# Checks for typedefs, structures, and compiler characteristics.
|
|
||||||
AC_HEADER_STDBOOL
|
|
||||||
AC_C_CONST
|
|
||||||
AC_TYPE_OFF_T
|
|
||||||
AC_TYPE_SIZE_T
|
|
||||||
AC_FUNC_STRERROR_R
|
|
||||||
|
|
||||||
# Checks for library functions.
|
|
||||||
AC_CHECK_FUNCS([mkdir uname])
|
|
||||||
AC_CHECK_LIB([socket],[shutdown])
|
|
||||||
AC_CHECK_LIB([nsl],[xdr_float])
|
|
||||||
AC_OUTPUT
|
|
||||||
108
hadoop-tools/hadoop-pipes/pom.xml
Normal file
108
hadoop-tools/hadoop-pipes/pom.xml
Normal file
@ -0,0 +1,108 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. See accompanying LICENSE file.
|
||||||
|
-->
|
||||||
|
<project>
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-project</artifactId>
|
||||||
|
<version>3.0.0-SNAPSHOT</version>
|
||||||
|
<relativePath>../../hadoop-project</relativePath>
|
||||||
|
</parent>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-pipes</artifactId>
|
||||||
|
<version>3.0.0-SNAPSHOT</version>
|
||||||
|
<description>Apache Hadoop Pipes</description>
|
||||||
|
<name>Apache Hadoop Pipes</name>
|
||||||
|
<packaging>pom</packaging>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<hadoop.log.dir>${project.build.directory}/log</hadoop.log.dir>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<profiles>
|
||||||
|
<profile>
|
||||||
|
<id>native</id>
|
||||||
|
<activation>
|
||||||
|
<activeByDefault>false</activeByDefault>
|
||||||
|
</activation>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-antrun-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>make</id>
|
||||||
|
<phase>compile</phase>
|
||||||
|
<goals><goal>run</goal></goals>
|
||||||
|
<configuration>
|
||||||
|
<target>
|
||||||
|
<mkdir dir="${project.build.directory}/native"/>
|
||||||
|
<exec executable="cmake" dir="${project.build.directory}/native"
|
||||||
|
failonerror="true">
|
||||||
|
<arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
|
||||||
|
</exec>
|
||||||
|
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
|
||||||
|
<arg line="VERBOSE=1"/>
|
||||||
|
</exec>
|
||||||
|
</target>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
<!-- TODO wire here native testcases
|
||||||
|
<execution>
|
||||||
|
<id>test</id>
|
||||||
|
<phase>test</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>test</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<destDir>${project.build.directory}/native/target</destDir>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
-->
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</profile>
|
||||||
|
</profiles>
|
||||||
|
|
||||||
|
<!--
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-antrun-plugin</artifactId>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>compile</id>
|
||||||
|
<phase>generate-sources</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>run</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<target>
|
||||||
|
<mkdir dir="${basedir}/../target/native"/>
|
||||||
|
<copy toDir="${basedir}/../target/native">
|
||||||
|
<fileset dir="${basedir}/src/main/native"/>
|
||||||
|
</copy>
|
||||||
|
</target>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
-->
|
||||||
|
</project>
|
||||||
99
hadoop-tools/hadoop-pipes/src/CMakeLists.txt
Normal file
99
hadoop-tools/hadoop-pipes/src/CMakeLists.txt
Normal file
@ -0,0 +1,99 @@
|
|||||||
|
#
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
|
||||||
|
find_package(OpenSSL REQUIRED)
|
||||||
|
|
||||||
|
set(CMAKE_BUILD_TYPE, Release)
|
||||||
|
|
||||||
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
|
||||||
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -O2")
|
||||||
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64")
|
||||||
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_REENTRANT -D_FILE_OFFSET_BITS=64")
|
||||||
|
|
||||||
|
if (JVM_ARCH_DATA_MODEL EQUAL 32)
|
||||||
|
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
|
||||||
|
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")
|
||||||
|
set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32")
|
||||||
|
if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
|
||||||
|
set(CMAKE_SYSTEM_PROCESSOR "i686")
|
||||||
|
endif ()
|
||||||
|
endif (JVM_ARCH_DATA_MODEL EQUAL 32)
|
||||||
|
|
||||||
|
function(output_directory TGT DIR)
|
||||||
|
SET_TARGET_PROPERTIES(${TGT} PROPERTIES
|
||||||
|
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
|
||||||
|
SET_TARGET_PROPERTIES(${TGT} PROPERTIES
|
||||||
|
ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
|
||||||
|
SET_TARGET_PROPERTIES(${TGT} PROPERTIES
|
||||||
|
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
|
||||||
|
endfunction(output_directory TGT DIR)
|
||||||
|
|
||||||
|
include_directories(
|
||||||
|
main/native/utils/api
|
||||||
|
main/native/pipes/api
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
${OPENSSL_INCLUDE_DIR}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Example programs
|
||||||
|
add_executable(wordcount-simple main/native/examples/impl/wordcount-simple.cc)
|
||||||
|
target_link_libraries(wordcount-simple hadooppipes hadooputils)
|
||||||
|
|
||||||
|
add_executable(wordcount-part main/native/examples/impl/wordcount-part.cc)
|
||||||
|
target_link_libraries(wordcount-part hadooppipes hadooputils)
|
||||||
|
|
||||||
|
add_executable(wordcount-nopipe main/native/examples/impl/wordcount-nopipe.cc)
|
||||||
|
target_link_libraries(wordcount-nopipe hadooppipes hadooputils)
|
||||||
|
|
||||||
|
add_executable(pipes-sort main/native/examples/impl/sort.cc)
|
||||||
|
target_link_libraries(pipes-sort hadooppipes hadooputils)
|
||||||
|
|
||||||
|
install(TARGETS wordcount-simple wordcount-part wordcount-nopipe pipes-sort
|
||||||
|
RUNTIME DESTINATION bin
|
||||||
|
)
|
||||||
|
|
||||||
|
add_library(hadooputils STATIC
|
||||||
|
main/native/utils/impl/StringUtils.cc
|
||||||
|
main/native/utils/impl/SerialUtils.cc
|
||||||
|
)
|
||||||
|
|
||||||
|
install(FILES
|
||||||
|
main/native/utils/api/hadoop/SerialUtils.hh
|
||||||
|
main/native/utils/api/hadoop/StringUtils.hh
|
||||||
|
DESTINATION api/hadoop
|
||||||
|
COMPONENT headers
|
||||||
|
)
|
||||||
|
install(TARGETS hadooputils DESTINATION lib)
|
||||||
|
|
||||||
|
add_library(hadooppipes STATIC
|
||||||
|
main/native/pipes/impl/HadoopPipes.cc
|
||||||
|
)
|
||||||
|
target_link_libraries(hadooppipes
|
||||||
|
${JAVA_JVM_LIBRARY}
|
||||||
|
${OPENSSL_LIBRARIES}
|
||||||
|
pthread
|
||||||
|
)
|
||||||
|
|
||||||
|
install(FILES
|
||||||
|
main/native/pipes/api/hadoop/Pipes.hh
|
||||||
|
main/native/pipes/api/hadoop/TemplateFactory.hh
|
||||||
|
DESTINATION api/hadoop
|
||||||
|
COMPONENT headers
|
||||||
|
)
|
||||||
|
install(TARGETS hadooppipes DESTINATION lib)
|
||||||
@ -1,16 +1,16 @@
|
|||||||
To run the examples, first compile them:
|
To run the examples, first compile them:
|
||||||
|
|
||||||
% ant -Dcompile.c++=yes examples
|
% mvn install
|
||||||
|
|
||||||
and then copy the binaries to dfs:
|
and then copy the binaries to dfs:
|
||||||
|
|
||||||
% bin/hadoop fs -put build/c++-examples/Linux-i386-32/bin /examples/bin
|
% hadoop fs -put target/native/wordcount-simple /examples/bin/
|
||||||
|
|
||||||
create an input directory with text files:
|
create an input directory with text files:
|
||||||
|
|
||||||
% bin/hadoop fs -put my-data in-dir
|
% hadoop fs -put my-data in-dir
|
||||||
|
|
||||||
and run the word count example:
|
and run the word count example:
|
||||||
|
|
||||||
% bin/hadoop pipes -conf src/examples/pipes/conf/word.xml \
|
% hadoop pipes -conf src/main/native/examples/conf/word.xml \
|
||||||
-input in-dir -output out-dir
|
-input in-dir -output out-dir
|
||||||
@ -70,6 +70,13 @@
|
|||||||
<artifactId>hadoop-gridmix</artifactId>
|
<artifactId>hadoop-gridmix</artifactId>
|
||||||
<scope>compile</scope>
|
<scope>compile</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-pipes</artifactId>
|
||||||
|
<scope>compile</scope>
|
||||||
|
<type>pom</type>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
@ -92,4 +99,44 @@
|
|||||||
</plugins>
|
</plugins>
|
||||||
</build>
|
</build>
|
||||||
|
|
||||||
|
<profiles>
|
||||||
|
<profile>
|
||||||
|
<id>dist</id>
|
||||||
|
<activation>
|
||||||
|
<activeByDefault>false</activeByDefault>
|
||||||
|
</activation>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-assemblies</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>dist</id>
|
||||||
|
<phase>prepare-package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<appendAssemblyId>false</appendAssemblyId>
|
||||||
|
<attach>false</attach>
|
||||||
|
<finalName>${project.artifactId}-${project.version}</finalName>
|
||||||
|
<descriptorRefs>
|
||||||
|
<descriptorRef>hadoop-tools</descriptorRef>
|
||||||
|
</descriptorRefs>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</profile>
|
||||||
|
</profiles>
|
||||||
</project>
|
</project>
|
||||||
|
|||||||
@ -39,6 +39,7 @@
|
|||||||
<module>hadoop-datajoin</module>
|
<module>hadoop-datajoin</module>
|
||||||
<module>hadoop-tools-dist</module>
|
<module>hadoop-tools-dist</module>
|
||||||
<module>hadoop-extras</module>
|
<module>hadoop-extras</module>
|
||||||
|
<module>hadoop-pipes</module>
|
||||||
</modules>
|
</modules>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user