HADOOP-7054 Change NN LoadGenerator to use FileContext APIs (Sanjay Radia)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1043117 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -23,6 +23,9 @@ Trunk (unreleased changes)
     HADOOP-7049. TestReconfiguration should be junit v4.
     (Patrick Kling via eli)
 
+    HADOOP-7054 Change NN LoadGenerator to use FileContext APIs
+	(Sanjay Radia)
+
   OPTIMIZATIONS
 
   BUG FIXES

src/test/core/org/apache/hadoop/fs/loadGenerator/DataGenerator.java

@@ -22,12 +22,15 @@
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
+import java.util.EnumSet;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Options.CreateOpts;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
@@ -48,7 +51,7 @@
 public class DataGenerator extends Configured implements Tool {
   private File inDir = StructureGenerator.DEFAULT_STRUCTURE_DIRECTORY;
   private Path root = DEFAULT_ROOT;
-  private FileSystem fs;
+  private FileContext fc;
   final static private long BLOCK_SIZE = 10;
   final static private String USAGE = "java DataGenerator " +
   		"-inDir <inDir> " +
@@ -78,7 +81,7 @@ public int run(String[] args) throws Exception {
   /** Parse the command line arguments and initialize the data */
   private int init(String[] args) {
     try { // initialize file system handle
-      fs = FileSystem.get(getConf());
+      fc = FileContext.getFileContext(getConf());
     } catch (IOException ioe) {
       System.err.println("Can not initialize the file system: " + 
           ioe.getLocalizedMessage());
@@ -109,7 +112,7 @@ private void genDirStructure() throws IOException {
             StructureGenerator.DIR_STRUCTURE_FILE_NAME)));
     String line;
     while ((line=in.readLine()) != null) {
-      fs.mkdirs(new Path(root+line));
+      fc.mkdir(new Path(root+line), FileContext.DEFAULT_PERM, true);
     }
   }
 
@@ -137,10 +140,9 @@ private void genFiles() throws IOException {
    * a length of <code>fileSize</code>. The file is filled with character 'a'.
    */
   private void genFile(Path file, long fileSize) throws IOException {
-    FSDataOutputStream out = fs.create(file, true, 
+    FSDataOutputStream out = fc.create(file, EnumSet.of(CreateFlag.OVERWRITE),
-        getConf().getInt("io.file.buffer.size", 4096),
+        CreateOpts.createParent(), CreateOpts.bufferSize(4096),
-        (short)getConf().getInt("dfs.replication", 3),
+        CreateOpts.repFac((short) 3));
-        fs.getDefaultBlockSize());
     for(long i=0; i<fileSize; i++) {
       out.writeByte('a');
     }

src/test/core/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java

@@ -26,16 +26,19 @@
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.Random;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Options.CreateOpts;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
@@ -110,7 +113,7 @@ public class LoadGenerator extends Configured implements Tool {
   
   private volatile boolean shouldRun = true;
   private Path root = DataGenerator.DEFAULT_ROOT;
-  private FileSystem fs;
+  private FileContext fc;
   private int maxDelayBetweenOps = 0;
   private int numOfThreads = 200;
   private long [] durations = {0};
@@ -230,7 +233,7 @@ private void nextOp() throws IOException {
     private void read() throws IOException {
       String fileName = files.get(r.nextInt(files.size()));
       long startTime = System.currentTimeMillis();
-      InputStream in = fs.open(new Path(fileName));
+      InputStream in = fc.open(new Path(fileName));
       executionTime[OPEN] += (System.currentTimeMillis()-startTime);
       totalNumOfOps[OPEN]++;
       while (in.read(buffer) != -1) {}
@@ -252,7 +255,7 @@ private void write() throws IOException {
       while ((fileSize = r.nextGaussian()+2)<=0) {}
       genFile(file, (long)(fileSize*BLOCK_SIZE));
       long startTime = System.currentTimeMillis();
-      fs.delete(file, true);
+      fc.delete(file, true);
       executionTime[DELETE] += (System.currentTimeMillis()-startTime);
       totalNumOfOps[DELETE]++;
     }
@@ -263,7 +266,7 @@ private void write() throws IOException {
     private void list() throws IOException {
       String dirName = dirs.get(r.nextInt(dirs.size()));
       long startTime = System.currentTimeMillis();
-      fs.listStatus(new Path(dirName));
+      fc.listStatus(new Path(dirName));
       executionTime[LIST] += (System.currentTimeMillis()-startTime);
       totalNumOfOps[LIST]++;
     }
@@ -352,7 +355,7 @@ public int run(String[] args) throws Exception {
   /** Parse the command line arguments and initialize the data */
   private int init(String[] args) throws IOException {
     try {
-      fs = FileSystem.get(getConf());
+      fc = FileContext.getFileContext(getConf());
     } catch (IOException ioe) {
       System.err.println("Can not initialize the file system: " + 
           ioe.getLocalizedMessage());
@@ -546,7 +549,7 @@ private int initFileDirTables() {
    * whose name starts with "_file_".
    */
   private void initFileDirTables(Path path) throws IOException {
-    FileStatus[] stats = fs.listStatus(path);
+    FileStatus[] stats = fc.util().listStatus(path);
 
     for (FileStatus stat : stats) {
       if (stat.isDirectory()) {
@@ -581,10 +584,9 @@ private void barrier() {
    */
   private void genFile(Path file, long fileSize) throws IOException {
     long startTime = System.currentTimeMillis();
-    FSDataOutputStream out = fs.create(file, true, 
+    FSDataOutputStream out = fc.create(file, EnumSet.of(CreateFlag.OVERWRITE),
-        getConf().getInt("io.file.buffer.size", 4096),
+        CreateOpts.createParent(), CreateOpts.bufferSize(4096),
-        (short)getConf().getInt("dfs.replication", 3),
+        CreateOpts.repFac((short) 3));
-        fs.getDefaultBlockSize());
     executionTime[CREATE] += (System.currentTimeMillis()-startTime);
     totalNumOfOps[CREATE]++;