big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-3753. Tests don't run with native libraries. Contributed by Colin Patrick McCabe

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1393113 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eli Collins 2012-10-02 19:13:53 +00:00
parent ccabb684eb
commit f8f79a02aa
6 changed files with 118 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dev-support/test-patch.sh
View File

@ -686,7 +686,7 @@ runTests () {
cd $module
echo " Running tests in $module"
echo " $MVN clean install -fn -Pnative -D${PROJECT_NAME}PatchProcess"
$MVN clean install -fn -Pnative -D${PROJECT_NAME}PatchProcess
$MVN clean install -fn -Pnative -Drequire.test.libhadoop -D${PROJECT_NAME}PatchProcess
module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E "<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-| |g" | sed -e "s|\.xml||g"`
# With -fn mvn always exits with a 0 exit code. Because of this we need to
# find the errors instead of using the exit code. We assume that if the build

49
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;
import org.junit.Test;
import static org.junit.Assert.*;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.NativeCodeLoader;
public class TestNativeCodeLoader {
static final Log LOG = LogFactory.getLog(TestNativeCodeLoader.class);
private static boolean requireTestJni() {
String rtj = System.getProperty("require.test.libhadoop");
if (rtj == null) return false;
if (rtj.compareToIgnoreCase("false") == 0) return false;
return true;
}
@Test
public void testNativeCodeLoaded() {
if (requireTestJni() == false) {
LOG.info("TestNativeCodeLoader: libhadoop.so testing is not required.");
return;
}
if (!NativeCodeLoader.isNativeCodeLoaded()) {
fail("TestNativeCodeLoader: libhadoop.so testing was required, but " +
"libhadoop.so was not loaded.");
}
LOG.info("TestNativeCodeLoader: libhadoop.so is loaded.");
}
}

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -268,6 +268,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-3992. Method org.apache.hadoop.hdfs.TestHftpFileSystem.tearDown()
sometimes throws NPEs. (Ivan A. Veselovsky via atm)
HDFS-3753. Tests don't run with native libraries.
(Colin Patrick McCabe via eli)
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES

51
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestHdfsNativeCodeLoader.java Normal file
View File

@ -0,0 +1,51 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs;
import org.junit.Test;
import static org.junit.Assert.*;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.NativeCodeLoader;
public class TestHdfsNativeCodeLoader {
static final Log LOG = LogFactory.getLog(TestHdfsNativeCodeLoader.class);
private static boolean requireTestJni() {
String rtj = System.getProperty("require.test.libhadoop");
if (rtj == null) return false;
if (rtj.compareToIgnoreCase("false") == 0) return false;
return true;
}
@Test
public void testNativeCodeLoaded() {
if (requireTestJni() == false) {
LOG.info("TestNativeCodeLoader: libhadoop.so testing is not required.");
return;
}
if (!NativeCodeLoader.isNativeCodeLoaded()) {
String LD_LIBRARY_PATH = System.getenv().get("LD_LIBRARY_PATH");
if (LD_LIBRARY_PATH == null) LD_LIBRARY_PATH = "";
fail("TestNativeCodeLoader: libhadoop.so testing was required, but " +
"libhadoop.so was not loaded. LD_LIBRARY_PATH = " + LD_LIBRARY_PATH);
}
LOG.info("TestHdfsNativeCodeLoader: libhadoop.so is loaded.");
}
}

15
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestShortCircuitLocalRead.java
View File

@ -115,6 +115,14 @@ static void checkFileContent(FileSystem fs, Path name, byte[] expected,
stm.close();
}
private static byte [] arrayFromByteBuffer(ByteBuffer buf) {
ByteBuffer alt = buf.duplicate();
alt.clear();
byte[] arr = new byte[alt.remaining()];
alt.get(arr);
return arr;
}
/**
* Verifies that reading a file with the direct read(ByteBuffer) api gives the expected set of bytes.
*/
@ -122,7 +130,7 @@ static void checkFileContentDirect(FileSystem fs, Path name, byte[] expected,
int readOffset) throws IOException {
HdfsDataInputStream stm = (HdfsDataInputStream)fs.open(name);
ByteBuffer actual = ByteBuffer.allocate(expected.length - readOffset);
ByteBuffer actual = ByteBuffer.allocateDirect(expected.length - readOffset);
IOUtils.skipFully(stm, readOffset);
@ -136,7 +144,8 @@ static void checkFileContentDirect(FileSystem fs, Path name, byte[] expected,
// Read across chunk boundary
actual.limit(Math.min(actual.capacity(), nread + 517));
nread += stm.read(actual);
checkData(actual.array(), readOffset, expected, nread, "A few bytes");
checkData(arrayFromByteBuffer(actual), readOffset, expected, nread,
"A few bytes");
//Now read rest of it
actual.limit(actual.capacity());
while (actual.hasRemaining()) {
@ -147,7 +156,7 @@ static void checkFileContentDirect(FileSystem fs, Path name, byte[] expected,
}
nread += nbytes;
}
checkData(actual.array(), readOffset, expected, "Read 3");
checkData(arrayFromByteBuffer(actual), readOffset, expected, "Read 3");
stm.close();
}

3
hadoop-project/pom.xml
View File

@ -836,7 +836,7 @@
<forkedProcessTimeoutInSeconds>900</forkedProcessTimeoutInSeconds>
<argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</argLine>
<environmentVariables>
<LD_LIBRARY_PATH>${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib</LD_LIBRARY_PATH>
<LD_LIBRARY_PATH>${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${basedir}/../../hadoop-common-project/hadoop-common/target/native/target/usr/local/lib/</LD_LIBRARY_PATH>
<MALLOC_ARENA_MAX>4</MALLOC_ARENA_MAX>
</environmentVariables>
<systemPropertyVariables>
@ -854,6 +854,7 @@
<java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
<java.security.krb5.conf>${basedir}/src/test/resources/krb5.conf</java.security.krb5.conf>
<java.security.egd>file:///dev/urandom</java.security.egd>
<require.test.libhadoop>${require.test.libhadoop}</require.test.libhadoop>
</systemPropertyVariables>
<includes>
<include>**/Test*.java</include>