HDFS-3610. fuse_dfs: Provide a way to use the default (configured) NN URI. Contributed by Colin Patrick McCabe

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1361565 13f79535-47bb-0310-9956-ffa450edef68

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -324,6 +324,9 @@ Branch-2 ( Unreleased changes )
 
     HDFS-3539. libhdfs code cleanups. (Colin Patrick McCabe via eli)
 
+    HDFS-3610. fuse_dfs: Provide a way to use the default (configured) NN URI.
+    (Colin Patrick McCabe via eli)
+
   OPTIMIZATIONS
 
     HDFS-2982. Startup performance suffers when there are many edit log

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_connect.c

@@ -172,7 +172,7 @@ done:
  * Connect to the NN as the current user/group.
  * Returns a fs handle on success, or NULL on failure.
  */
-hdfsFS doConnectAsUser(const char *hostname, int port) {
+hdfsFS doConnectAsUser(const char *nn_uri, int nn_port) {
   struct hdfsBuilder *bld;
   uid_t uid = fuse_get_context()->uid;
   char *user = getUsername(uid);
@@ -202,8 +202,10 @@ hdfsFS doConnectAsUser(const char *hostname, int port) {
       goto done;
     }
     hdfsBuilderSetForceNewInstance(bld);
-    hdfsBuilderSetNameNode(bld, hostname);
+    hdfsBuilderSetNameNode(bld, nn_uri);
-    hdfsBuilderSetNameNodePort(bld, port);
+    if (nn_port) {
+        hdfsBuilderSetNameNodePort(bld, nn_port);
+    }
     hdfsBuilderSetUserName(bld, user);
     if (hdfsAuthConf == AUTH_CONF_KERBEROS) {
       findKerbTicketCachePath(kpath, sizeof(kpath));

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_connect.h

@@ -21,7 +21,7 @@
 
 #include "fuse_dfs.h"
 
-hdfsFS doConnectAsUser(const char *hostname, int port);
+hdfsFS doConnectAsUser(const char *nn_uri, int nn_port);
 int doDisconnect(hdfsFS fs);
 int allocFsTable(void);
 

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_context_handle.h

@@ -31,7 +31,7 @@
 //
 typedef struct dfs_context_struct {
   int debug;
-  char *nn_hostname;
+  char *nn_uri;
   int nn_port;
   int read_only;
   int usetrash;

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_dfs.c

@@ -101,14 +101,14 @@ int main(int argc, char *argv[])
     fuse_opt_add_arg(&args, buf);
   }
 
-  if (options.server == NULL || options.port == 0) {
+  if (options.nn_uri == NULL) {
     print_usage(argv[0]);
     exit(0);
   }
 
   // Check connection as root
   if (options.initchecks == 1) {
-    hdfsFS tempFS = hdfsConnectAsUser(options.server, options.port, "root");
+    hdfsFS tempFS = hdfsConnectAsUser(options.nn_uri, options.nn_port, "root");
     if (NULL == tempFS) {
       const char *cp = getenv("CLASSPATH");
       const char *ld = getenv("LD_LIBRARY_PATH");

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_chmod.c

@@ -31,7 +31,7 @@ int dfs_chmod(const char *path, mode_t mode)
   assert(dfs);
   assert('/' == *path);
 
-  hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
+  hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
   if (userFS == NULL) {
     ERROR("Could not connect to HDFS");
     ret = -EIO;

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_chown.c

@@ -54,7 +54,7 @@ int dfs_chown(const char *path, uid_t uid, gid_t gid)
     goto cleanup;
   } 
 
-  userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
+  userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
   if (userFS == NULL) {
     ERROR("Could not connect to HDFS");
     ret = -EIO;

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_getattr.c

@@ -31,9 +31,9 @@ int dfs_getattr(const char *path, struct stat *st)
   assert(path);
   assert(st);
 
-  hdfsFS fs = doConnectAsUser(dfs->nn_hostname,dfs->nn_port);
+  hdfsFS fs = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
   if (NULL == fs) {
-    ERROR("Could not connect to %s:%d", dfs->nn_hostname, dfs->nn_port);
+    ERROR("Could not connect to %s:%d", dfs->nn_uri, dfs->nn_port);
     return -EIO;
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_mkdir.c

@@ -41,7 +41,7 @@ int dfs_mkdir(const char *path, mode_t mode)
     return -EACCES;
   }
   
-  hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
+  hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
   if (userFS == NULL) {
     ERROR("Could not connect");
     return -EIO;

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_open.c

@@ -45,7 +45,7 @@ int dfs_open(const char *path, struct fuse_file_info *fi)
     return -EIO;
   }
 
-  fh->fs = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
+  fh->fs = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
   if (fh->fs == NULL) {
     ERROR("Could not connect to dfs");
     return -EIO;

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_readdir.c

@@ -31,7 +31,7 @@ int dfs_readdir(const char *path, void *buf, fuse_fill_dir_t filler,
   assert(path);
   assert(buf);
 
-  hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
+  hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
   if (userFS == NULL) {
     ERROR("Could not connect");
     return -EIO;

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_rename.c

@@ -46,7 +46,7 @@ int dfs_rename(const char *from, const char *to)
     return -EACCES;
   }
 
-  hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
+  hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
   if (userFS == NULL) {
     ERROR("Could not connect");
     return -EIO;

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_rmdir.c

@@ -43,7 +43,7 @@ int dfs_rmdir(const char *path)
     return -EACCES;
   }
 
-  hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
+  hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
   if (userFS == NULL) {
     ERROR("Could not connect");
     return -EIO;

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_statfs.c

@@ -33,7 +33,7 @@ int dfs_statfs(const char *path, struct statvfs *st)
 
   memset(st,0,sizeof(struct statvfs));
 
-  hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
+  hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
   if (userFS == NULL) {
     ERROR("Could not connect");
     return -EIO;

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_truncate.c

@@ -45,7 +45,7 @@ int dfs_truncate(const char *path, off_t size)
     return ret;
   }
 
-  hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
+  hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
   if (userFS == NULL) {
     ERROR("Could not connect");
     ret = -EIO;

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_unlink.c

@@ -43,7 +43,7 @@ int dfs_unlink(const char *path)
     return -EACCES;
   }
 
-  hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
+  hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
   if (userFS == NULL) {
     ERROR("Could not connect");
     return -EIO;

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_impls_utimens.c

@@ -33,7 +33,7 @@ int dfs_utimens(const char *path, const struct timespec ts[2])
   time_t aTime = ts[0].tv_sec;
   time_t mTime = ts[1].tv_sec;
 
-  hdfsFS userFS = doConnectAsUser(dfs->nn_hostname, dfs->nn_port);
+  hdfsFS userFS = doConnectAsUser(dfs->nn_uri, dfs->nn_port);
   if (userFS == NULL) {
     ERROR("Could not connect");
     return -EIO;

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_init.c

@@ -92,15 +92,15 @@ void *dfs_init(void) {
 
   // initialize the context
   dfs->debug                 = options.debug;
-  dfs->nn_hostname           = options.server;
+  dfs->nn_uri                = options.nn_uri;
-  dfs->nn_port               = options.port;
+  dfs->nn_port               = options.nn_port;
   dfs->read_only             = options.read_only;
   dfs->usetrash              = options.usetrash;
   dfs->protectedpaths        = NULL;
   dfs->rdbuffer_size         = options.rdbuffer_size;
   dfs->direct_io             = options.direct_io;
 
-  INFO("Mounting %s:%d", dfs->nn_hostname, dfs->nn_port);
+  INFO("Mounting.  nn_uri=%s, nn_port=%d", dfs->nn_uri, dfs->nn_port);
 
   init_protectedpaths(dfs);
   assert(dfs->protectedpaths != NULL);

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_options.c

@@ -23,6 +23,9 @@
 #include <getopt.h>
 #include <stdlib.h>
 
+#define OLD_HDFS_URI_LOCATION "dfs://"
+#define NEW_HDFS_URI_LOCATION "hdfs://"
+
 void print_options() {
   printf("options:\n"
 	 "\tprotected=%s\n"
@@ -35,7 +38,7 @@ void print_options() {
 	 "\tattribute_timeout=%d\n"
 	 "\tprivate=%d\n"
 	 "\trdbuffer_size=%d (KBs)\n", 
-	 options.protected, options.server, options.port, options.debug,
+	 options.protected, options.nn_uri, options.nn_port, options.debug,
 	 options.read_only, options.usetrash, options.entry_timeout, 
 	 options.attribute_timeout, options.private, 
 	 (int)options.rdbuffer_size / 1024);
@@ -78,11 +81,11 @@ enum
 
 struct fuse_opt dfs_opts[] =
   {
-    DFSFS_OPT_KEY("server=%s", server, 0),
+    DFSFS_OPT_KEY("server=%s", nn_uri, 0),
     DFSFS_OPT_KEY("entry_timeout=%d", entry_timeout, 0),
     DFSFS_OPT_KEY("attribute_timeout=%d", attribute_timeout, 0),
     DFSFS_OPT_KEY("protected=%s", protected, 0),
-    DFSFS_OPT_KEY("port=%d", port, 0),
+    DFSFS_OPT_KEY("port=%d", nn_port, 0),
     DFSFS_OPT_KEY("rdbuffer=%d", rdbuffer_size,0),
 
     FUSE_OPT_KEY("private", KEY_PRIVATE),
@@ -105,6 +108,7 @@ struct fuse_opt dfs_opts[] =
 int dfs_options(void *data, const char *arg, int key,  struct fuse_args *outargs)
 {
   (void) data;
+  int nn_uri_len;
 
   switch (key) {
   case FUSE_OPT_KEY_OPT:
@@ -150,11 +154,8 @@ int dfs_options(void *data, const char *arg, int key,  struct fuse_args *outargs
 #endif
     break;
   default: {
-    // try and see if the arg is a URI for DFS
+    // try and see if the arg is a URI
-    int tmp_port;
+    if (!strstr(arg, "://")) {
-    char tmp_server[1024];
-
-    if (!sscanf(arg,"dfs://%1024[a-zA-Z0-9_.-]:%d",tmp_server,&tmp_port)) {
       if (strcmp(arg,"ro") == 0) {
         options.read_only = 1;
       } else if (strcmp(arg,"rw") == 0) {
@@ -165,8 +166,21 @@ int dfs_options(void *data, const char *arg, int key,  struct fuse_args *outargs
         return 0;
       }
     } else {
-      options.port = tmp_port;
+      if (options.nn_uri) {
-      options.server = strdup(tmp_server);
+        INFO("Ignoring option %s because '-server' was already "
+          "specified!", arg);
+        return 1;
+      }
+      if (strstr(arg, OLD_HDFS_URI_LOCATION) == arg) {
+        // For historical reasons, we let people refer to hdfs:// as dfs://
+        nn_uri_len = strlen(NEW_HDFS_URI_LOCATION) + 
+                strlen(arg + strlen(OLD_HDFS_URI_LOCATION)) + 1;
+        options.nn_uri = malloc(nn_uri_len);
+        snprintf(options.nn_uri, nn_uri_len, "%s%s", NEW_HDFS_URI_LOCATION, 
+              arg + strlen(OLD_HDFS_URI_LOCATION));
+      } else {
+        options.nn_uri = strdup(arg);
+      }
     }
   }
   }

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/fuse_options.h

@@ -22,8 +22,8 @@
 /** options for fuse_opt.h */
 struct options {
   char* protected;
-  char* server;
+  char* nn_uri;
-  int port;
+  int nn_port;
   int debug;
   int read_only;
   int initchecks;