HDDS-1089. Disable OzoneFSStorageStatistics for hadoop versions older than 2.8.

Contributed by Elek, Marton.

hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterFactory.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.fs.ozone;
 
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
@@ -42,8 +43,35 @@ private OzoneClientAdapterFactory() {
   @SuppressFBWarnings("DP_CREATE_CLASSLOADER_INSIDE_DO_PRIVILEGED")
   public static OzoneClientAdapter createAdapter(
       String volumeStr,
-      String bucketStr, OzoneFSStorageStatistics storageStatistics)
+      String bucketStr) throws IOException {
+    return createAdapter(volumeStr, bucketStr,
+        (aClass) -> (OzoneClientAdapter) aClass
+            .getConstructor(String.class, String.class)
+            .newInstance(
+                volumeStr,
+                bucketStr));
+  }
+
+
+  public static OzoneClientAdapter createAdapter(
+      String volumeStr,
+      String bucketStr,
+      OzoneFSStorageStatistics storageStatistics)
       throws IOException {
+    return createAdapter(volumeStr, bucketStr,
+        (aClass) -> (OzoneClientAdapter) aClass
+            .getConstructor(String.class, String.class,
+                OzoneFSStorageStatistics.class)
+            .newInstance(
+                volumeStr,
+                bucketStr,
+                storageStatistics));
+  }
+
+  public static OzoneClientAdapter createAdapter(
+      String volumeStr,
+      String bucketStr,
+      OzoneClientAdapterCreator creator) throws IOException {
 
     ClassLoader currentClassLoader = OzoneFileSystem.class.getClassLoader();
     List<URL> urls = new ArrayList<>();
@@ -70,13 +98,10 @@ public static OzoneClientAdapter createAdapter(
       reflectionUtils.getMethod("getClassByName", String.class)
           .invoke(null, "org.apache.ratis.grpc.GrpcFactory");
 
-      OzoneClientAdapter ozoneClientAdapter = (OzoneClientAdapter) classLoader
+      Class<?> aClass = classLoader
-          .loadClass("org.apache.hadoop.fs.ozone.OzoneClientAdapterImpl")
+          .loadClass("org.apache.hadoop.fs.ozone.OzoneClientAdapterImpl");
-          .getConstructor(String.class, String.class,
+      OzoneClientAdapter ozoneClientAdapter =
-              OzoneFSStorageStatistics.class)
+          creator.createOzoneClientAdapter(aClass);
-          .newInstance(
-              volumeStr,
-              bucketStr, storageStatistics);
 
       Thread.currentThread().setContextClassLoader(contextClassLoader);
 
@@ -119,4 +144,14 @@ private static void findEmbeddedLibsUrl(List<URL> urls,
 
   }
 
+  /**
+   * Interface to create OzoneClientAdapter implementation with reflection.
+   */
+  @FunctionalInterface
+  interface OzoneClientAdapterCreator {
+    OzoneClientAdapter createOzoneClientAdapter(Class<?> clientAdapter)
+        throws NoSuchMethodException, IllegalAccessException,
+        InvocationTargetException, InstantiationException;
+  }
+
 }

hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterImpl.java

@@ -54,11 +54,33 @@ public class OzoneClientAdapterImpl implements OzoneClientAdapter {
   private ReplicationFactor replicationFactor;
   private OzoneFSStorageStatistics storageStatistics;
 
+  /**
+   * Create new OzoneClientAdapter implementation.
+   *
+   * @param volumeStr         Name of the volume to use.
+   * @param bucketStr         Name of the bucket to use
+   * @param storageStatistics Storage statistic (optional, can be null)
+   * @throws IOException In case of a problem.
+   */
   public OzoneClientAdapterImpl(String volumeStr, String bucketStr,
       OzoneFSStorageStatistics storageStatistics) throws IOException {
     this(createConf(), volumeStr, bucketStr, storageStatistics);
   }
 
+  /**
+   * Create new OzoneClientAdapter implementation.
+   *
+   * @param volumeStr         Name of the volume to use.
+   * @param bucketStr         Name of the bucket to use
+   * @throws IOException In case of a problem.
+   */
+  public OzoneClientAdapterImpl(String volumeStr, String bucketStr)
+      throws IOException {
+    this(createConf(), volumeStr, bucketStr, null);
+  }
+
+
+
   private static OzoneConfiguration createConf() {
     ClassLoader contextClassLoader =
         Thread.currentThread().getContextClassLoader();
@@ -102,13 +124,17 @@ public void close() throws IOException {
 
   @Override
   public InputStream createInputStream(String key) throws IOException {
+    if (storageStatistics != null) {
       storageStatistics.incrementCounter(Statistic.OBJECTS_READ, 1);
+    }
     return bucket.readKey(key).getInputStream();
   }
 
   @Override
   public OzoneFSOutputStream createKey(String key) throws IOException {
+    if (storageStatistics != null) {
       storageStatistics.incrementCounter(Statistic.OBJECTS_CREATED, 1);
+    }
     OzoneOutputStream ozoneOutputStream =
         bucket.createKey(key, 0, replicationType, replicationFactor,
             new HashMap<>());
@@ -117,7 +143,9 @@ public OzoneFSOutputStream createKey(String key) throws IOException {
 
   @Override
   public void renameKey(String key, String newKeyName) throws IOException {
+    if (storageStatistics != null) {
       storageStatistics.incrementCounter(Statistic.OBJECTS_RENAMED, 1);
+    }
     bucket.renameKey(key, newKeyName);
   }
 
@@ -130,7 +158,9 @@ public void renameKey(String key, String newKeyName) throws IOException {
   @Override
   public BasicKeyInfo getKeyInfo(String keyName) {
     try {
+      if (storageStatistics != null) {
         storageStatistics.incrementCounter(Statistic.OBJECTS_QUERY, 1);
+      }
       OzoneKey key = bucket.getKey(keyName);
       return new BasicKeyInfo(
           keyName,
@@ -167,7 +197,9 @@ public boolean isDirectory(BasicKeyInfo key) {
   public boolean createDirectory(String keyName) {
     try {
       LOG.trace("creating dir for key:{}", keyName);
+      if (storageStatistics != null) {
         storageStatistics.incrementCounter(Statistic.OBJECTS_CREATED, 1);
+      }
       bucket.createKey(keyName, 0, replicationType, replicationFactor,
           new HashMap<>()).close();
       return true;
@@ -187,7 +219,9 @@ public boolean createDirectory(String keyName) {
   public boolean deleteObject(String keyName) {
     LOG.trace("issuing delete for key" + keyName);
     try {
+      if (storageStatistics != null) {
         storageStatistics.incrementCounter(Statistic.OBJECTS_DELETED, 1);
+      }
       bucket.deleteKey(keyName);
       return true;
     } catch (IOException ioe) {
@@ -203,13 +237,17 @@ public long getCreationTime() {
 
   @Override
   public boolean hasNextKey(String key) {
+    if (storageStatistics != null) {
       storageStatistics.incrementCounter(Statistic.OBJECTS_LIST, 1);
+    }
     return bucket.listKeys(key).hasNext();
   }
 
   @Override
   public Iterator<BasicKeyInfo> listKeys(String pathKey) {
+    if (storageStatistics != null) {
       storageStatistics.incrementCounter(Statistic.OBJECTS_LIST, 1);
+    }
     return new IteratorAdapter(bucket.listKeys(pathKey));
   }
 

hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java

@@ -125,14 +125,30 @@ public void initialize(URI name, Configuration conf) throws IOException {
       boolean isolatedClassloader =
           conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
 
+      try {
+        //register only to the GlobalStorageStatistics if the class exists.
+        //This is required to support hadoop versions <2.7
+        Class.forName("org.apache.hadoop.fs.GlobalStorageStatistics");
         storageStatistics = (OzoneFSStorageStatistics)
             GlobalStorageStatistics.INSTANCE
                 .put(OzoneFSStorageStatistics.NAME,
                     OzoneFSStorageStatistics::new);
+      } catch (ClassNotFoundException e) {
+        //we don't support storage statistics for hadoop2.7 and older
+      }
+
       if (isolatedClassloader) {
+        try {
+          //register only to the GlobalStorageStatistics if the class exists.
+          //This is required to support hadoop versions <2.7
+          Class.forName("org.apache.hadoop.fs.GlobalStorageStatistics");
           this.adapter =
               OzoneClientAdapterFactory.createAdapter(volumeStr, bucketStr,
                   storageStatistics);
+        } catch (ClassNotFoundException e) {
+          this.adapter =
+              OzoneClientAdapterFactory.createAdapter(volumeStr, bucketStr);
+        }
       } else {
         OzoneConfiguration ozoneConfiguration;
         if (conf instanceof OzoneConfiguration) {
@@ -188,7 +204,9 @@ OzoneFSStorageStatistics getOzoneFSOpsCountStatistics() {
 
   @Override
   public FSDataInputStream open(Path f, int bufferSize) throws IOException {
+    if (storageStatistics != null) {
       storageStatistics.incrementCounter(Statistic.INVOCATION_OPEN, 1);
+    }
     statistics.incrementWriteOps(1);
     LOG.trace("open() path:{}", f);
     final FileStatus fileStatus = getFileStatus(f);
@@ -207,7 +225,9 @@ public FSDataOutputStream create(Path f, FsPermission permission,
                                    short replication, long blockSize,
                                    Progressable progress) throws IOException {
     LOG.trace("create() path:{}", f);
+    if (storageStatistics != null) {
       storageStatistics.incrementCounter(Statistic.INVOCATION_CREATE, 1);
+    }
     statistics.incrementWriteOps(1);
     final String key = pathToKey(f);
     final FileStatus status;
@@ -240,8 +260,10 @@ public FSDataOutputStream createNonRecursive(Path path,
       short replication,
       long blockSize,
       Progressable progress) throws IOException {
+    if (storageStatistics != null) {
       storageStatistics.incrementCounter(
           Statistic.INVOCATION_CREATE_NON_RECURSIVE, 1);
+    }
     statistics.incrementWriteOps(1);
     final Path parent = path.getParent();
     if (parent != null) {
@@ -296,7 +318,9 @@ boolean processKey(String key) throws IOException {
    */
   @Override
   public boolean rename(Path src, Path dst) throws IOException {
+    if (storageStatistics != null) {
       storageStatistics.incrementCounter(Statistic.INVOCATION_RENAME, 1);
+    }
     statistics.incrementWriteOps(1);
     if (src.equals(dst)) {
       return true;
@@ -431,7 +455,9 @@ private boolean innerDelete(Path f, boolean recursive) throws IOException {
 
   @Override
   public boolean delete(Path f, boolean recursive) throws IOException {
+    if (storageStatistics != null) {
       storageStatistics.incrementCounter(Statistic.INVOCATION_DELETE, 1);
+    }
     statistics.incrementWriteOps(1);
     LOG.debug("Delete path {} - recursive {}", f, recursive);
     FileStatus status;
@@ -623,7 +649,9 @@ FileStatus[] getStatuses() {
 
   @Override
   public FileStatus[] listStatus(Path f) throws IOException {
+    if (storageStatistics != null) {
       storageStatistics.incrementCounter(Statistic.INVOCATION_LIST_STATUS, 1);
+    }
     statistics.incrementReadOps(1);
     LOG.trace("listStatus() path:{}", f);
     ListStatusIterator iterator = new ListStatusIterator(f);
@@ -710,7 +738,10 @@ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
 
   @Override
   public FileStatus getFileStatus(Path f) throws IOException {
-    storageStatistics.incrementCounter(Statistic.INVOCATION_GET_FILE_STATUS, 1);
+    if (storageStatistics != null) {
+      storageStatistics
+          .incrementCounter(Statistic.INVOCATION_GET_FILE_STATUS, 1);
+    }
     statistics.incrementReadOps(1);
     LOG.trace("getFileStatus() path:{}", f);
     Path qualifiedPath = f.makeQualified(uri, workingDir);