big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-9164. Print paths of loaded native libraries in NativeLibraryChecker. (Binglin Chang via llu)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1504700 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Luke Lu 2013-07-18 22:38:36 +00:00
parent e2efe52ac8
commit ba3d29a2e2
17 changed files with 138 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -378,6 +378,9 @@ Release 2.1.0-beta - 2013-07-02
IMPROVEMENTS
HADOOP-9164. Print paths of loaded native libraries in
NativeLibraryChecker. (Binglin Chang via llu)
HADOOP-9253. Capture ulimit info in the logs at service start time.
(Arpit Gupta via suresh)

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
View File

@ -69,6 +69,10 @@ public static boolean isNativeCodeLoaded() {
return NativeCodeLoader.isNativeCodeLoaded();
}
public static String getLibraryName() {
return Lz4Compressor.getLibraryName();
}
/**
* Create a {@link CompressionOutputStream} that will write to the given
* {@link OutputStream}.

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
View File

@ -79,6 +79,10 @@ public static boolean isNativeCodeLoaded() {
SnappyDecompressor.isNativeCodeLoaded();
}
public static String getLibraryName() {
return SnappyCompressor.getLibraryName();
}
/**
* Create a {@link CompressionOutputStream} that will write to the given
* {@link OutputStream}.

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java
View File

@ -298,4 +298,6 @@ private void checkStream() {
private native static long getBytesRead(long strm);
private native static long getBytesWritten(long strm);
private native static void end(long strm);
public native static String getLibraryName();
}

8
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java
View File

@ -78,6 +78,14 @@ public static boolean isNativeBzip2Loaded(Configuration conf) {
return nativeBzip2Loaded;
}
public static String getLibraryName(Configuration conf) {
if (isNativeBzip2Loaded(conf)) {
return Bzip2Compressor.getLibraryName();
} else {
return bzip2LibraryName;
}
}
/**
* Return the appropriate type of the bzip2 compressor.
*

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java
View File

@ -296,4 +296,6 @@ public synchronized void end() {
private native static void initIDs();
private native int compressBytesDirect();
public native static String getLibraryName();
}

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java
View File

@ -298,4 +298,6 @@ public synchronized void end() {
private native static void initIDs();
private native int compressBytesDirect();
public native static String getLibraryName();
}

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java
View File

@ -444,4 +444,6 @@ private native static void setDictionary(long strm, byte[] b, int off,
private native static long getBytesWritten(long strm);
private native static void reset(long strm);
private native static void end(long strm);
public native static String getLibraryName();
}

4
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java
View File

@ -66,6 +66,10 @@ public static boolean isNativeZlibLoaded(Configuration conf) {
CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT);
}
public static String getLibraryName() {
return ZlibCompressor.getLibraryName();
}
/**
* Return the appropriate type of the zlib compressor.
*

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java
View File

@ -79,6 +79,8 @@ public static boolean isNativeCodeLoaded() {
*/
public static native boolean buildSupportsSnappy();
public static native String getLibraryName();
/**
* Return if native hadoop libraries, if present, can be used for this job.
* @param conf configuration

37
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java
View File

@ -20,7 +20,9 @@
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.compress.Lz4Codec;
import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.io.compress.bzip2.Bzip2Factory;
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@ -35,7 +37,7 @@ public static void main(String[] args) {
String usage = "NativeLibraryChecker [-a|-h]\n"
+ " -a use -a to check all libraries are available\n"
+ " by default just check hadoop library is available\n"
+ " exit with error code if check failed\n"
+ " exit with error code 1 if check failed\n"
+ " -h print this message\n";
if (args.length > 1 ||
(args.length == 1 &&
@ -51,23 +53,44 @@ public static void main(String[] args) {
}
checkAll = true;
}
Configuration conf = new Configuration();
boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded();
boolean zlibLoaded = false;
boolean snappyLoaded = false;
// lz4 is linked within libhadoop
boolean lz4Loaded = nativeHadoopLoaded;
boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
String hadoopLibraryName = "";
String zlibLibraryName = "";
String snappyLibraryName = "";
String lz4LibraryName = "";
String bzip2LibraryName = "";
if (nativeHadoopLoaded) {
zlibLoaded = ZlibFactory.isNativeZlibLoaded(new Configuration());
hadoopLibraryName = NativeCodeLoader.getLibraryName();
zlibLoaded = ZlibFactory.isNativeZlibLoaded(conf);
if (zlibLoaded) {
zlibLibraryName = ZlibFactory.getLibraryName();
}
snappyLoaded = NativeCodeLoader.buildSupportsSnappy() &&
SnappyCodec.isNativeCodeLoaded();
if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) {
snappyLibraryName = SnappyCodec.getLibraryName();
}
if (lz4Loaded) {
lz4LibraryName = Lz4Codec.getLibraryName();
}
if (bzip2Loaded) {
bzip2LibraryName = Bzip2Factory.getLibraryName(conf);
}
}
System.out.println("Native library checking:");
System.out.printf("hadoop: %b\n", nativeHadoopLoaded);
System.out.printf("zlib: %b\n", zlibLoaded);
System.out.printf("snappy: %b\n", snappyLoaded);
System.out.printf("lz4: %b\n", lz4Loaded);
System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
System.out.printf("zlib: %b %s\n", zlibLoaded, zlibLibraryName);
System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName);
System.out.printf("lz4: %b %s\n", lz4Loaded, lz4LibraryName);
System.out.printf("bzip2: %b %s\n", bzip2Loaded, bzip2LibraryName);
if ((!nativeHadoopLoaded) ||
(checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded))) {
(checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
// return 1 to indicated check failed
ExitUtil.terminate(1);
}

15
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c
View File

@ -239,6 +239,21 @@ Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_end(
}
}
JNIEXPORT jstring JNICALL
Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_getLibraryName(JNIEnv *env, jclass class) {
#ifdef UNIX
if (dlsym_BZ2_bzCompress) {
Dl_info dl_info;
if(dladdr(
dlsym_BZ2_bzCompress,
&dl_info)) {
return (*env)->NewStringUTF(env, dl_info.dli_fname);
}
}
#endif
return (*env)->NewStringUTF(env, HADOOP_BZIP2_LIBRARY);
}
/**
* vim: sw=2: ts=2: et:
*/

6
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c
View File

@ -103,3 +103,9 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp
return (jint)compressed_direct_buf_len;
}
JNIEXPORT jstring JNICALL
Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_getLibraryName(
JNIEnv *env, jclass class
) {
return (*env)->NewStringUTF(env, "revision:43");
}

15
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c
View File

@ -117,4 +117,19 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso
return (jint)buf_len;
}
JNIEXPORT jstring JNICALL
Java_org_apache_hadoop_io_compress_snappy_SnappyCompressor_getLibraryName(JNIEnv *env, jclass class) {
#ifdef UNIX
if (dlsym_snappy_compress) {
Dl_info dl_info;
if(dladdr(
dlsym_snappy_compress,
&dl_info)) {
return (*env)->NewStringUTF(env, dl_info.dli_fname);
}
}
#endif
return (*env)->NewStringUTF(env, HADOOP_SNAPPY_LIBRARY);
}
#endif //define HADOOP_SNAPPY_LIBRARY

15
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c
View File

@ -367,6 +367,21 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_end(
}
}
JNIEXPORT jstring JNICALL
Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_getLibraryName(JNIEnv *env, jclass class) {
#ifdef UNIX
if (dlsym_deflateInit2_) {
Dl_info dl_info;
if(dladdr(
dlsym_deflateInit2_,
&dl_info)) {
return (*env)->NewStringUTF(env, dl_info.dli_fname);
}
}
#endif
return (*env)->NewStringUTF(env, HADOOP_ZLIB_LIBRARY);
}
/**
* vim: sw=2: ts=2: et:
*/

11
hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c
View File

@ -19,6 +19,7 @@
#include "org_apache_hadoop.h"
#ifdef UNIX
#include <dlfcn.h>
#include "config.h"
#endif // UNIX
@ -33,3 +34,13 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSup
return JNI_FALSE;
#endif
}
JNIEXPORT jstring JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName
(JNIEnv *env, jclass clazz)
{
Dl_info dl_info;
int ret = dladdr(
Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName,
&dl_info);
return (*env)->NewStringUTF(env, ret==0 ? "Unavailable" : dl_info.dli_fname);
}

11
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java
View File

@ -22,6 +22,9 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.compress.Lz4Codec;
import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.io.compress.zlib.ZlibFactory;
import org.apache.hadoop.util.NativeCodeLoader;
public class TestNativeCodeLoader {
@ -44,6 +47,14 @@ public void testNativeCodeLoaded() {
fail("TestNativeCodeLoader: libhadoop.so testing was required, but " +
"libhadoop.so was not loaded.");
}
assertFalse(NativeCodeLoader.getLibraryName().isEmpty());
// library names are depended on platform and build envs
// so just check names are available
assertFalse(ZlibFactory.getLibraryName().isEmpty());
if (NativeCodeLoader.buildSupportsSnappy()) {
assertFalse(SnappyCodec.getLibraryName().isEmpty());
}
assertFalse(Lz4Codec.getLibraryName().isEmpty());
LOG.info("TestNativeCodeLoader: libhadoop.so is loaded.");
}
}