HADOOP-7023. Add listCorruptFileBlocks to FileSystem. Contributed by Patrick Kling.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1038003 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -6,6 +6,9 @@ Trunk (unreleased changes)
 
   NEW FEATURES
 
+    HADOOP-7023. Add listCorruptFileBlocks to Filesysem. (Patrick Kling
+    via hairong)
+
   IMPROVEMENTS
 
     HADOOP-7042. Updates to test-patch.sh to include failed test names and

src/java/org/apache/hadoop/fs/AbstractFileSystem.java

@@ -834,6 +834,18 @@ public abstract FileStatus[] listStatus(final Path f)
       throws AccessControlException, FileNotFoundException,
       UnresolvedLinkException, IOException;
 
+  /**
+   * @return a list in which each entry describes a corrupt file/block
+   * @throws IOException
+   */
+  public CorruptFileBlocks listCorruptFileBlocks(String path,
+                                                 String cookie)
+    throws IOException {
+    throw new UnsupportedOperationException(getClass().getCanonicalName() +
+                                            " does not support" +
+                                            " listCorruptFileBlocks");
+  }
+
   /**
    * The specification of this method matches that of
    * {@link FileContext#setVerifyChecksum(boolean, Path)} except that Path f

src/java/org/apache/hadoop/fs/CorruptFileBlocks.java

@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.Text;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Contains a list of paths corresponding to corrupt files and a cookie
+ * used for iterative calls to NameNode.listCorruptFileBlocks.
+ *
+ */
+public class CorruptFileBlocks implements Writable {
+  // used for hashCode
+  private static final int PRIME = 16777619;
+
+  private String[] files;
+  private String cookie;
+
+  public CorruptFileBlocks() {
+    this(new String[0], "");
+  }
+
+  public CorruptFileBlocks(String[] files, String cookie) {
+    this.files = files;
+    this.cookie = cookie;
+  }
+
+  public String[] getFiles() {
+    return files;
+  }
+
+  public String getCookie() {
+    return cookie;
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    int fileCount = in.readInt();
+    files = new String[fileCount];
+    for (int i = 0; i < fileCount; i++) {
+      files[i] = Text.readString(in);
+    }
+    cookie = Text.readString(in);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void write(DataOutput out) throws IOException {
+    out.writeInt(files.length);
+    for (int i = 0; i < files.length; i++) {
+      Text.writeString(out, files[i]);
+    }
+    Text.writeString(out, cookie);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (!(obj instanceof CorruptFileBlocks)) {
+      return false;
+    }
+    CorruptFileBlocks other = (CorruptFileBlocks) obj;
+    return cookie.equals(other.cookie) &&
+      Arrays.equals(files, other.files);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public int hashCode() {
+    int result = cookie.hashCode();
+
+    for (String file : files) {
+      result = PRIME * result + file.hashCode();
+    }
+
+    return result;
+  }
+}

src/java/org/apache/hadoop/fs/FileContext.java

@@ -1296,6 +1296,23 @@ public RemoteIterator<FileStatus> next(
       }
     }.resolve(this, absF);
   }
+
+  /**
+   * @return a list in which each entry describes a corrupt file/block
+   * @throws IOException
+   */
+  public CorruptFileBlocks listCorruptFileBlocks(final String path,
+                                                 final String cookie)
+    throws IOException {
+    final Path absF = fixRelativePart(new Path(path));
+    return new FSLinkResolver<CorruptFileBlocks>() {
+      @Override
+      public CorruptFileBlocks next(final AbstractFileSystem fs, final Path p) 
+        throws IOException, UnresolvedLinkException {
+        return fs.listCorruptFileBlocks(p.toUri().getPath(), cookie);
+      }
+    }.resolve(this, absF);
+  }
   
   /**
    * List the statuses of the files/directories in the given path if the path is

src/java/org/apache/hadoop/fs/FileSystem.java

@@ -1090,6 +1090,18 @@ private void listStatus(ArrayList<FileStatus> results, Path f,
     }
   }
 
+  /**
+   * @return a list in which each entry describes a corrupt file/block
+   * @throws IOException
+   */
+  public CorruptFileBlocks listCorruptFileBlocks(String path,
+                                                 String cookie)
+    throws IOException {
+    throw new UnsupportedOperationException(getClass().getCanonicalName() +
+                                            " does not support" +
+                                            " listCorruptFileBlocks");
+  }
+
   /**
    * Filter files/directories in the given path using the user-supplied path
    * filter.

src/java/org/apache/hadoop/fs/FilterFileSystem.java

@@ -165,7 +165,17 @@ public boolean deleteOnExit(Path f) throws IOException {
   public FileStatus[] listStatus(Path f) throws IOException {
     return fs.listStatus(f);
   }
-  
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public CorruptFileBlocks listCorruptFileBlocks(String path,
+                                                 String cookie)
+    throws IOException {
+    return fs.listCorruptFileBlocks(path, cookie);
+  }
+
   /** List files and its block locations in a directory. */
   public RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f)
   throws IOException {

src/java/org/apache/hadoop/fs/FilterFs.java

@@ -164,6 +164,16 @@ public FileStatus[] listStatus(Path f)
     return myFs.listStatus(f);
   }
 
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public CorruptFileBlocks listCorruptFileBlocks(String path,
+                                                 String cookie)
+    throws IOException {
+    return myFs.listCorruptFileBlocks(path, cookie);
+  }
+
   @Override
   public void mkdir(Path dir, FsPermission permission, boolean createParent)
     throws IOException, UnresolvedLinkException {

src/test/core/org/apache/hadoop/fs/TestCorruptFileBlocks.java

@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.*;
+import org.junit.Test;
+
+import org.apache.hadoop.io.DataOutputBuffer;
+
+public class TestCorruptFileBlocks {
+
+  /**
+   * Serialize the cfb given, deserialize and return the result.
+   */
+  static CorruptFileBlocks serializeAndDeserialize(CorruptFileBlocks cfb) 
+    throws IOException {
+    DataOutputBuffer buf = new DataOutputBuffer();
+    cfb.write(buf);
+
+    byte[] data = buf.getData();
+    DataInputStream input = new DataInputStream(new ByteArrayInputStream(data));
+
+    CorruptFileBlocks result = new CorruptFileBlocks();
+    result.readFields(input);
+
+    return result;
+  }
+
+  /**
+   * Check whether cfb is unchanged after serialization and deserialization.
+   */
+  static boolean checkSerialize(CorruptFileBlocks cfb)
+    throws IOException {
+    return cfb.equals(serializeAndDeserialize(cfb));
+  }
+
+  /**
+   * Test serialization and deserializaton of CorruptFileBlocks.
+   */
+  @Test
+  public void testSerialization() throws IOException {
+    {
+      CorruptFileBlocks cfb = new CorruptFileBlocks();
+      assertTrue(checkSerialize(cfb));
+    }
+
+    {
+      String[] files = new String[0];
+      CorruptFileBlocks cfb = new CorruptFileBlocks(files, "");
+      assertTrue(checkSerialize(cfb));
+    }
+
+    {
+      String[] files = { "a", "bb", "ccc" };
+      CorruptFileBlocks cfb = new CorruptFileBlocks(files, "test");
+      assertTrue(checkSerialize(cfb));
+    }
+  }
+}