HDFS-16403. Improve FUSE IO performance by supporting FUSE parameter max_background (#3842)

Reviewed-by: Istvan Fajth <pifta@apache.org>
Reviewed-by: Wei-Chiu Chuang <weichiu@apache.org>

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/doc/README

@@ -88,6 +88,7 @@ rw
 -odebug (do not daemonize - aka -d in fuse speak)
 -obig_writes (use fuse big_writes option so as to allow better performance of writes on kernels >= 2.6.26)
 -initchecks - have fuse-dfs try to connect to hdfs to ensure all is ok upon startup. recommended to have this  on
+-omax_background=%d (maximum number of pending "background" requests - see fuse docs)
 The defaults are:
 
 entry,attribute_timeouts = 60 seconds

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_dfs.c

@@ -81,6 +81,7 @@ int main(int argc, char *argv[])
   options.rdbuffer_size = 10*1024*1024; 
   options.attribute_timeout = 60; 
   options.entry_timeout = 60;
+  options.max_background = 0;
 
   if (-1 == fuse_opt_parse(&args, &options, dfs_opts, dfs_options)) {
     return -1;
@@ -114,6 +115,11 @@ int main(int argc, char *argv[])
 
     snprintf(buf, sizeof buf, "-oentry_timeout=%d",options.entry_timeout);
     fuse_opt_add_arg(&args, buf);
+
+    if (options.max_background > 0) {
+      snprintf(buf, sizeof buf, "-omax_background=%d",options.max_background);
+      fuse_opt_add_arg(&args, buf);
+    }
   }
 
   if (options.nn_uri == NULL) {

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c

@@ -91,11 +91,11 @@ static void dfsPrintOptions(FILE *fp, const struct options *o)
   INFO("Mounting with options: [ protected=%s, nn_uri=%s, nn_port=%d, "
           "debug=%d, read_only=%d, initchecks=%d, "
           "no_permissions=%d, usetrash=%d, entry_timeout=%d, "
-          "attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d ]",
+          "attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d, max_background=%d ]",
           (o->protected ? o->protected : "(NULL)"), o->nn_uri, o->nn_port, 
           o->debug, o->read_only, o->initchecks,
           o->no_permissions, o->usetrash, o->entry_timeout,
-          o->attribute_timeout, o->rdbuffer_size, o->direct_io);
+          o->attribute_timeout, o->rdbuffer_size, o->direct_io, o->max_background);
 }
 
 void *dfs_init(struct fuse_conn_info *conn)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c

@@ -37,11 +37,13 @@ void print_options() {
 	 "\tentry_timeout=%d\n"
 	 "\tattribute_timeout=%d\n"
 	 "\tprivate=%d\n"
-	 "\trdbuffer_size=%d (KBs)\n", 
-	 options.protected, options.nn_uri, options.nn_port, options.debug,
+         "\trdbuffer_size=%d (KBs)\n"
+         "\tmax_background=%d\n",
+         options.protected, options.nn_uri, options.nn_port, options.debug,
 	 options.read_only, options.usetrash, options.entry_timeout, 
 	 options.attribute_timeout, options.private, 
-	 (int)options.rdbuffer_size / 1024);
+         (int)options.rdbuffer_size / 1024,
+         options.max_background);
 }
 
 const char *program;
@@ -56,7 +58,7 @@ void print_usage(const char *pname)
 	 "[-ousetrash] [-obig_writes] [-oprivate (single user)] [ro] "
 	 "[-oserver=<hadoop_servername>] [-oport=<hadoop_port>] "
 	 "[-oentry_timeout=<secs>] [-oattribute_timeout=<secs>] "
-	 "[-odirect_io] [-onopoermissions] [-o<other fuse option>] "
+         "[-odirect_io] [-onopoermissions] [-omax_background=<size>] [-o<other fuse option>] "
 	 "<mntpoint> [fuse options]\n", pname);
   printf("NOTE: debugging option for fuse is -debug\n");
 }
@@ -87,6 +89,7 @@ struct fuse_opt dfs_opts[] =
     DFSFS_OPT_KEY("protected=%s", protected, 0),
     DFSFS_OPT_KEY("port=%d", nn_port, 0),
     DFSFS_OPT_KEY("rdbuffer=%d", rdbuffer_size,0),
+    DFSFS_OPT_KEY("max_background=%d", max_background, 0),
 
     FUSE_OPT_KEY("private", KEY_PRIVATE),
     FUSE_OPT_KEY("ro", KEY_RO),

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h

@@ -34,6 +34,7 @@ struct options {
   int private;
   size_t rdbuffer_size;
   int direct_io;
+  int max_background;
 } options;
 
 extern struct fuse_opt dfs_opts[];

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/test/TestFuseDFS.java

@@ -187,6 +187,7 @@ private static Process establishMount(URI uri) throws IOException  {
       "-ononempty",              // Don't complain about junk in mount point
       "-f",                      // Don't background the process
       "-ordbuffer=32768",        // Read buffer size in kb
+      "-omax_background=100",    // Set fuse max_background=100 (12 by default)
       "rw"
     };