HADOOP-9164. Print paths of loaded native libraries in NativeLibraryChecker. (Binglin Chang via llu)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1504700 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -378,6 +378,9 @@ Release 2.1.0-beta - 2013-07-02
 
   IMPROVEMENTS
 
+    HADOOP-9164. Print paths of loaded native libraries in
+    NativeLibraryChecker. (Binglin Chang via llu)
+
     HADOOP-9253. Capture ulimit info in the logs at service start time.
     (Arpit Gupta via suresh)
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java

@@ -69,6 +69,10 @@ public static boolean isNativeCodeLoaded() {
     return NativeCodeLoader.isNativeCodeLoaded();
   }
 
+  public static String getLibraryName() {
+    return Lz4Compressor.getLibraryName();
+  }
+
   /**
    * Create a {@link CompressionOutputStream} that will write to the given
    * {@link OutputStream}.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java

@@ -79,6 +79,10 @@ public static boolean isNativeCodeLoaded() {
         SnappyDecompressor.isNativeCodeLoaded();
   }
 
+  public static String getLibraryName() {
+    return SnappyCompressor.getLibraryName();
+  }
+
   /**
    * Create a {@link CompressionOutputStream} that will write to the given
    * {@link OutputStream}.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java

@@ -298,4 +298,6 @@ private void checkStream() {
   private native static long getBytesRead(long strm);
   private native static long getBytesWritten(long strm);
   private native static void end(long strm);
+
+  public native static String getLibraryName();
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java

@@ -78,6 +78,14 @@ public static boolean isNativeBzip2Loaded(Configuration conf) {
     return nativeBzip2Loaded;
   }
 
+  public static String getLibraryName(Configuration conf) {
+    if (isNativeBzip2Loaded(conf)) {
+      return Bzip2Compressor.getLibraryName();
+    } else {
+      return bzip2LibraryName;
+    }
+  }
+
   /**
    * Return the appropriate type of the bzip2 compressor. 
    * 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/lz4/Lz4Compressor.java

@@ -296,4 +296,6 @@ public synchronized void end() {
   private native static void initIDs();
 
   private native int compressBytesDirect();
+
+  public native static String getLibraryName();
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyCompressor.java

@@ -298,4 +298,6 @@ public synchronized void end() {
   private native static void initIDs();
 
   private native int compressBytesDirect();
+
+  public native static String getLibraryName();
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibCompressor.java

@@ -444,4 +444,6 @@ private native static void setDictionary(long strm, byte[] b, int off,
   private native static long getBytesWritten(long strm);
   private native static void reset(long strm);
   private native static void end(long strm);
+
+  public native static String getLibraryName();
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/zlib/ZlibFactory.java

@@ -65,7 +65,11 @@ public static boolean isNativeZlibLoaded(Configuration conf) {
                           CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, 
                           CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT);
   }
-  
+
+  public static String getLibraryName() {
+    return ZlibCompressor.getLibraryName();
+  }
+
   /**
    * Return the appropriate type of the zlib compressor. 
    * 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java

@@ -79,6 +79,8 @@ public static boolean isNativeCodeLoaded() {
    */
   public static native boolean buildSupportsSnappy();
 
+  public static native String getLibraryName();
+
   /**
    * Return if native hadoop libraries, if present, can be used for this job.
    * @param conf configuration

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeLibraryChecker.java

@@ -20,7 +20,9 @@
 
 import org.apache.hadoop.util.NativeCodeLoader;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.compress.Lz4Codec;
 import org.apache.hadoop.io.compress.SnappyCodec;
+import org.apache.hadoop.io.compress.bzip2.Bzip2Factory;
 import org.apache.hadoop.io.compress.zlib.ZlibFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -35,7 +37,7 @@ public static void main(String[] args) {
     String usage = "NativeLibraryChecker [-a|-h]\n"
         + "  -a  use -a to check all libraries are available\n"
         + "      by default just check hadoop library is available\n"
-        + "      exit with error code if check failed\n"
+        + "      exit with error code 1 if check failed\n"
         + "  -h  print this message\n";
     if (args.length > 1 ||
         (args.length == 1 &&
@@ -51,23 +53,44 @@ public static void main(String[] args) {
       }
       checkAll = true;
     }
+    Configuration conf = new Configuration();
     boolean nativeHadoopLoaded = NativeCodeLoader.isNativeCodeLoaded();
     boolean zlibLoaded = false;
     boolean snappyLoaded = false;
     // lz4 is linked within libhadoop
     boolean lz4Loaded = nativeHadoopLoaded;
+    boolean bzip2Loaded = Bzip2Factory.isNativeBzip2Loaded(conf);
+    String hadoopLibraryName = "";
+    String zlibLibraryName = "";
+    String snappyLibraryName = "";
+    String lz4LibraryName = "";
+    String bzip2LibraryName = "";
     if (nativeHadoopLoaded) {
-      zlibLoaded = ZlibFactory.isNativeZlibLoaded(new Configuration());
+      hadoopLibraryName = NativeCodeLoader.getLibraryName();
+      zlibLoaded = ZlibFactory.isNativeZlibLoaded(conf);
+      if (zlibLoaded) {
+        zlibLibraryName = ZlibFactory.getLibraryName();
+      }
       snappyLoaded = NativeCodeLoader.buildSupportsSnappy() &&
           SnappyCodec.isNativeCodeLoaded();
+      if (snappyLoaded && NativeCodeLoader.buildSupportsSnappy()) {
+        snappyLibraryName = SnappyCodec.getLibraryName();
+      }
+      if (lz4Loaded) {
+        lz4LibraryName = Lz4Codec.getLibraryName();
+      }
+      if (bzip2Loaded) {
+        bzip2LibraryName = Bzip2Factory.getLibraryName(conf);
+      }
     }
     System.out.println("Native library checking:");
-    System.out.printf("hadoop: %b\n", nativeHadoopLoaded);
+    System.out.printf("hadoop: %b %s\n", nativeHadoopLoaded, hadoopLibraryName);
-    System.out.printf("zlib:   %b\n", zlibLoaded);
+    System.out.printf("zlib:   %b %s\n", zlibLoaded, zlibLibraryName);
-    System.out.printf("snappy: %b\n", snappyLoaded);
+    System.out.printf("snappy: %b %s\n", snappyLoaded, snappyLibraryName);
-    System.out.printf("lz4:    %b\n", lz4Loaded);
+    System.out.printf("lz4:    %b %s\n", lz4Loaded, lz4LibraryName);
+    System.out.printf("bzip2:  %b %s\n", bzip2Loaded, bzip2LibraryName);
     if ((!nativeHadoopLoaded) ||
-        (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded))) {
+        (checkAll && !(zlibLoaded && snappyLoaded && lz4Loaded && bzip2Loaded))) {
       // return 1 to indicated check failed
       ExitUtil.terminate(1);
     }

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c

@@ -239,6 +239,21 @@ Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_end(
     }
 }
 
+JNIEXPORT jstring JNICALL
+Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_getLibraryName(JNIEnv *env, jclass class) {
+#ifdef UNIX
+  if (dlsym_BZ2_bzCompress) {
+    Dl_info dl_info;
+    if(dladdr(
+        dlsym_BZ2_bzCompress,
+        &dl_info)) {
+      return (*env)->NewStringUTF(env, dl_info.dli_fname);
+    }
+  }
+#endif
+  return (*env)->NewStringUTF(env, HADOOP_BZIP2_LIBRARY);
+}
+
 /**
  * vim: sw=2: ts=2: et:
  */

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c

@@ -103,3 +103,9 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp
   return (jint)compressed_direct_buf_len;
 }
 
+JNIEXPORT jstring JNICALL
+Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_getLibraryName(
+ JNIEnv *env, jclass class
+ ) {
+  return (*env)->NewStringUTF(env, "revision:43");
+}

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c

@@ -117,4 +117,19 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso
   return (jint)buf_len;
 }
 
+JNIEXPORT jstring JNICALL
+Java_org_apache_hadoop_io_compress_snappy_SnappyCompressor_getLibraryName(JNIEnv *env, jclass class) {
+#ifdef UNIX
+  if (dlsym_snappy_compress) {
+    Dl_info dl_info;
+    if(dladdr(
+        dlsym_snappy_compress,
+        &dl_info)) {
+      return (*env)->NewStringUTF(env, dl_info.dli_fname);
+    }
+  }
+#endif
+  return (*env)->NewStringUTF(env, HADOOP_SNAPPY_LIBRARY);
+}
+
 #endif //define HADOOP_SNAPPY_LIBRARY

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c

@@ -367,6 +367,21 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_end(
     }
 }
 
+JNIEXPORT jstring JNICALL
+Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_getLibraryName(JNIEnv *env, jclass class) {
+#ifdef UNIX
+  if (dlsym_deflateInit2_) {
+    Dl_info dl_info;
+    if(dladdr(
+        dlsym_deflateInit2_,
+        &dl_info)) {
+      return (*env)->NewStringUTF(env, dl_info.dli_fname);
+    }
+  }
+#endif
+  return (*env)->NewStringUTF(env, HADOOP_ZLIB_LIBRARY);
+}
+
 /**
  * vim: sw=2: ts=2: et:
  */

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c

@@ -19,6 +19,7 @@
 #include "org_apache_hadoop.h"
 
 #ifdef UNIX
+#include <dlfcn.h>
 #include "config.h"
 #endif // UNIX
 
@@ -32,4 +33,14 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSup
 #else
   return JNI_FALSE;
 #endif
-}
+}
+
+JNIEXPORT jstring JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName
+  (JNIEnv *env, jclass clazz)
+{
+  Dl_info dl_info;
+  int ret = dladdr(
+      Java_org_apache_hadoop_util_NativeCodeLoader_getLibraryName,
+      &dl_info);
+  return (*env)->NewStringUTF(env, ret==0 ? "Unavailable" : dl_info.dli_fname);
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestNativeCodeLoader.java

@@ -22,6 +22,9 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.compress.Lz4Codec;
+import org.apache.hadoop.io.compress.SnappyCodec;
+import org.apache.hadoop.io.compress.zlib.ZlibFactory;
 import org.apache.hadoop.util.NativeCodeLoader;
 
 public class TestNativeCodeLoader {
@@ -44,6 +47,14 @@ public void testNativeCodeLoaded() {
       fail("TestNativeCodeLoader: libhadoop.so testing was required, but " +
           "libhadoop.so was not loaded.");
     }
+    assertFalse(NativeCodeLoader.getLibraryName().isEmpty());
+    // library names are depended on platform and build envs
+    // so just check names are available
+    assertFalse(ZlibFactory.getLibraryName().isEmpty());
+    if (NativeCodeLoader.buildSupportsSnappy()) {
+      assertFalse(SnappyCodec.getLibraryName().isEmpty());
+    }
+    assertFalse(Lz4Codec.getLibraryName().isEmpty());
     LOG.info("TestNativeCodeLoader: libhadoop.so is loaded.");
   }
 }