HDFS-11009. Add a tool to reconstruct block meta file from CLI.

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java

@@ -284,7 +284,7 @@ public DfsClientConf(Configuration conf) {
     return classes;
   }
 
-  private DataChecksum.Type getChecksumType(Configuration conf) {
+  private static DataChecksum.Type getChecksumType(Configuration conf) {
     final String checksum = conf.get(
         DFS_CHECKSUM_TYPE_KEY,
         DFS_CHECKSUM_TYPE_DEFAULT);
@@ -299,7 +299,7 @@ private DataChecksum.Type getChecksumType(Configuration conf) {
   }
 
   // Construct a checksum option from conf
-  private ChecksumOpt getChecksumOptFromConf(Configuration conf) {
+  public static ChecksumOpt getChecksumOptFromConf(Configuration conf) {
     DataChecksum.Type type = getChecksumType(conf);
     int bytesPerChecksum = conf.getInt(DFS_BYTES_PER_CHECKSUM_KEY,
         DFS_BYTES_PER_CHECKSUM_DEFAULT);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java

@@ -1031,7 +1031,7 @@ public ReplicaInfo moveBlockAcrossVolumes(ExtendedBlock block, FsVolumeSpi
    * @param conf the {@link Configuration}
    * @throws IOException
    */
-  private static void computeChecksum(ReplicaInfo srcReplica, File dstMeta,
+  static void computeChecksum(ReplicaInfo srcReplica, File dstMeta,
       int smallBufferSize, final Configuration conf)
       throws IOException {
     File srcMeta = new File(srcReplica.getMetadataURI());

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetUtil.java

@@ -21,13 +21,19 @@
 import java.io.FileInputStream;
 import java.io.FilenameFilter;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.RandomAccessFile;
+import java.net.URI;
 import java.util.Arrays;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
+import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
 import org.apache.hadoop.io.IOUtils;
 
 /** Utility methods. */
@@ -123,4 +129,30 @@ static long parseGenerationStamp(File blockFile, File metaFile
           + blockFile + ", metaFile=" + metaFile, nfe);
     }
   }
+
+  /**
+   * Compute the checksum for a block file that does not already have
+   * its checksum computed, and save it to dstMeta file.
+   */
+  public static void computeChecksum(File srcMeta, File dstMeta, File blockFile,
+      int smallBufferSize, Configuration conf) throws IOException {
+    Preconditions.checkNotNull(srcMeta);
+    Preconditions.checkNotNull(dstMeta);
+    Preconditions.checkNotNull(blockFile);
+    // Create a dummy ReplicaInfo object pointing to the blockFile.
+    ReplicaInfo wrapper = new FinalizedReplica(0, 0, 0, null, null) {
+      @Override
+      public URI getMetadataURI() {
+        return srcMeta.toURI();
+      }
+
+      @Override
+      public InputStream getDataInputStream(long seekOffset)
+          throws IOException {
+        return new FileInputStream(blockFile);
+      }
+    };
+
+    FsDatasetImpl.computeChecksum(wrapper, dstMeta, smallBufferSize, conf);
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java

@@ -17,7 +17,11 @@
  */
 package org.apache.hadoop.hdfs.tools;
 
+import java.io.BufferedOutputStream;
 import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -30,14 +34,19 @@
 import java.util.concurrent.TimeUnit;
 
 import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetUtil;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.DataChecksum;
 import org.apache.hadoop.util.StringUtils;
@@ -56,9 +65,10 @@ public class DebugAdmin extends Configured implements Tool {
    * All the debug commands we can run.
    */
   private DebugCommand DEBUG_COMMANDS[] = {
-    new VerifyBlockChecksumCommand(),
+      new VerifyMetaCommand(),
-    new RecoverLeaseCommand(),
+      new ComputeMetaCommand(),
-    new HelpCommand()
+      new RecoverLeaseCommand(),
+      new HelpCommand()
   };
 
   /**
@@ -83,10 +93,10 @@ private abstract class DebugCommand {
   /**
    * The command for verifying a block metadata file and possibly block file.
    */
-  private class VerifyBlockChecksumCommand extends DebugCommand {
+  private class VerifyMetaCommand extends DebugCommand {
-    VerifyBlockChecksumCommand() {
+    VerifyMetaCommand() {
-      super("verify",
+      super("verifyMeta",
-"verify -meta <metadata-file> [-block <block-file>]",
+"verifyMeta -meta <metadata-file> [-block <block-file>]",
 "  Verify HDFS metadata and block files.  If a block file is specified, we\n" +
 "  will verify that the checksums in the metadata file match the block\n" +
 "  file.");
@@ -194,6 +204,86 @@ int run(List<String> args) throws IOException {
     }
   }
 
+  /**
+   * The command for verifying a block metadata file and possibly block file.
+   */
+  private class ComputeMetaCommand extends DebugCommand {
+    ComputeMetaCommand() {
+      super("computeMeta",
+          "computeMeta -block <block-file> -out <output-metadata-file>",
+          "  Compute HDFS metadata from the specified block file, and save it"
+              + " to\n  the specified output metadata file.\n\n"
+              + "**NOTE: Use at your own risk!\n If the block file is corrupt"
+              + " and you overwrite it's meta file, \n it will show up"
+              + " as good in HDFS, but you can't read the data.\n"
+              + " Only use as a last measure, and when you are 100% certain"
+              + " the block file is good.");
+    }
+
+    private DataChecksum createChecksum(Options.ChecksumOpt opt) {
+      DataChecksum dataChecksum = DataChecksum
+          .newDataChecksum(opt.getChecksumType(), opt.getBytesPerChecksum());
+      if (dataChecksum == null) {
+        throw new HadoopIllegalArgumentException(
+            "Invalid checksum type: userOpt=" + opt + ", default=" + opt
+                + ", effective=null");
+      }
+      return dataChecksum;
+    }
+
+    int run(List<String> args) throws IOException {
+      if (args.size() == 0) {
+        System.out.println(usageText);
+        System.out.println(helpText + "\n");
+        return 1;
+      }
+      final String name = StringUtils.popOptionWithArgument("-block", args);
+      if (name == null) {
+        System.err.println("You must specify a block file with -block");
+        return 2;
+      }
+      final File blockFile = new File(name);
+      if (!blockFile.exists() || !blockFile.isFile()) {
+        System.err.println("Block file <" + name + "> does not exist "
+            + "or is not a file");
+        return 3;
+      }
+      final String outFile = StringUtils.popOptionWithArgument("-out", args);
+      if (outFile == null) {
+        System.err.println("You must specify a output file with -out");
+        return 4;
+      }
+      final File srcMeta = new File(outFile);
+      if (srcMeta.exists()) {
+        System.err.println("output file already exists!");
+        return 5;
+      }
+
+      DataOutputStream metaOut = null;
+      try {
+        final Configuration conf = new Configuration();
+        final Options.ChecksumOpt checksumOpt =
+            DfsClientConf.getChecksumOptFromConf(conf);
+        final DataChecksum checksum = createChecksum(checksumOpt);
+
+        final int smallBufferSize = DFSUtilClient.getSmallBufferSize(conf);
+        metaOut = new DataOutputStream(
+            new BufferedOutputStream(new FileOutputStream(srcMeta),
+                smallBufferSize));
+        BlockMetadataHeader.writeHeader(metaOut, checksum);
+        metaOut.close();
+        FsDatasetUtil.computeChecksum(
+            srcMeta, srcMeta, blockFile, smallBufferSize, conf);
+        System.out.println(
+            "Checksum calculation succeeded on block file " + name
+                + " saved metadata to meta file " + outFile);
+        return 0;
+      } finally {
+        IOUtils.cleanup(null, metaOut);
+      }
+    }
+  }
+
   /**
    * The command for recovering a file lease.
    */
@@ -353,6 +443,9 @@ public int run(String[] argv) {
 
   private void printUsage() {
     System.out.println("Usage: hdfs debug <command> [arguments]\n");
+    System.out.println("These commands are for advanced users only.\n");
+    System.out.println("Incorrect usages may result in data loss. " +
+        "Use at your own risk.\n");
     for (DebugCommand command : DEBUG_COMMANDS) {
       if (!command.name.equals("help")) {
         System.out.println(command.usageText);

hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md

@@ -49,7 +49,8 @@ HDFS Commands Guide
     * [storagepolicies](#storagepolicies)
     * [zkfc](#zkfc)
 * [Debug Commands](#Debug_Commands)
-    * [verify](#verify)
+    * [verifyMeta](#verifyMeta)
+    * [computeMeta](#computeMeta)
     * [recoverLease](#recoverLease)
 
 Overview
@@ -606,11 +607,11 @@ This comamnd starts a Zookeeper Failover Controller process for use with [HDFS H
 Debug Commands
 --------------
 
-Useful commands to help administrators debug HDFS issues, like validating block files and calling recoverLease.
+Useful commands to help administrators debug HDFS issues. These commands are for advanced users only.
 
-### `verify`
+### `verifyMeta`
 
-Usage: `hdfs debug verify -meta <metadata-file> [-block <block-file>]`
+Usage: `hdfs debug verifyMeta -meta <metadata-file> [-block <block-file>]`
 
 | COMMAND\_OPTION | Description |
 |:---- |:---- |
@@ -619,6 +620,19 @@ Usage: `hdfs debug verify -meta <metadata-file> [-block <block-file>]`
 
 Verify HDFS metadata and block files. If a block file is specified, we will verify that the checksums in the metadata file match the block file.
 
+### `computeMeta`
+
+Usage: `hdfs debug computeMeta -block <block-file> -out <output-metadata-file>`
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+| `-block` *block-file* | Absolute path for the block file on the local file system of the data node. |
+| `-out` *output-metadata-file* | Absolute path for the output metadata file to store the checksum computation result from the block file. |
+
+Compute HDFS metadata from block files. If a block file is specified, we will compute the checksums from the block file, and save it to the specified output metadata file.
+
+**NOTE**: Use at your own risk! If the block file is corrupt and you overwrite it's meta file, it will show up as 'good' in HDFS, but you can't read the data. Only use as a last measure, and when you are 100% certain the block file is good.
+
 ### `recoverLease`
 
 Usage: `hdfs debug recoverLease -path <path> [-retries <num-retries>]`

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDebugAdmin.java

@@ -40,6 +40,11 @@
 import static org.junit.Assert.assertTrue;
 
 public class TestDebugAdmin {
+
+  static private final String TEST_ROOT_DIR =
+      new File(System.getProperty("test.build.data", "/tmp"),
+          TestDebugAdmin.class.getSimpleName()).getAbsolutePath();
+
   private MiniDFSCluster cluster;
   private DistributedFileSystem fs;
   private DebugAdmin admin;
@@ -47,6 +52,9 @@ public class TestDebugAdmin {
 
   @Before
   public void setUp() throws Exception {
+    final File testRoot = new File(TEST_ROOT_DIR);
+    testRoot.delete();
+    testRoot.mkdirs();
     Configuration conf = new Configuration();
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
     cluster.waitActive();
@@ -94,30 +102,68 @@ public void testRecoverLease() throws Exception {
   }
 
   @Test(timeout = 60000)
-  public void testVerifyBlockChecksumCommand() throws Exception {
+  public void testVerifyMetaCommand() throws Exception {
     DFSTestUtil.createFile(fs, new Path("/bar"), 1234, (short) 1, 0xdeadbeef);
     FsDatasetSpi<?> fsd = datanode.getFSDataset();
     ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, new Path("/bar"));
     File blockFile = getBlockFile(fsd,
         block.getBlockPoolId(), block.getLocalBlock());
-    assertEquals("ret: 1, You must specify a meta file with -meta",
+    assertEquals("ret: 1, You must specify a meta file with -meta", runCmd(
-        runCmd(new String[]{"verify", "-block", blockFile.getAbsolutePath()}));
+        new String[] {"verifyMeta", "-block", blockFile.getAbsolutePath()}));
     File metaFile = getMetaFile(fsd,
         block.getBlockPoolId(), block.getLocalBlock());
     assertEquals("ret: 0, Checksum type: " +
           "DataChecksum(type=CRC32C, chunkSize=512)",
-        runCmd(new String[]{"verify",
+        runCmd(new String[]{"verifyMeta",
             "-meta", metaFile.getAbsolutePath()}));
     assertEquals("ret: 0, Checksum type: " +
           "DataChecksum(type=CRC32C, chunkSize=512)" +
           "Checksum verification succeeded on block file " +
           blockFile.getAbsolutePath(),
-        runCmd(new String[]{"verify",
+        runCmd(new String[]{"verifyMeta",
             "-meta", metaFile.getAbsolutePath(),
             "-block", blockFile.getAbsolutePath()})
     );
   }
 
+  @Test(timeout = 60000)
+  public void testComputeMetaCommand() throws Exception {
+    DFSTestUtil.createFile(fs, new Path("/bar"), 1234, (short) 1, 0xdeadbeef);
+    FsDatasetSpi<?> fsd = datanode.getFSDataset();
+    ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, new Path("/bar"));
+    File blockFile = getBlockFile(fsd,
+        block.getBlockPoolId(), block.getLocalBlock());
+
+    assertEquals("ret: 1, computeMeta -block <block-file> -out "
+            + "<output-metadata-file>  Compute HDFS metadata from the specified"
+            + " block file, and save it to  the specified output metadata file."
+            + "**NOTE: Use at your own risk! If the block file is corrupt"
+            + " and you overwrite it's meta file,  it will show up"
+            + " as good in HDFS, but you can't read the data."
+            + " Only use as a last measure, and when you are 100% certain"
+            + " the block file is good.",
+        runCmd(new String[] {"computeMeta"}));
+    assertEquals("ret: 2, You must specify a block file with -block",
+        runCmd(new String[] {"computeMeta", "-whatever"}));
+    assertEquals("ret: 3, Block file <bla> does not exist or is not a file",
+        runCmd(new String[] {"computeMeta", "-block", "bla"}));
+    assertEquals("ret: 4, You must specify a output file with -out", runCmd(
+        new String[] {"computeMeta", "-block", blockFile.getAbsolutePath()}));
+    assertEquals("ret: 5, output file already exists!", runCmd(
+        new String[] {"computeMeta", "-block", blockFile.getAbsolutePath(),
+            "-out", blockFile.getAbsolutePath()}));
+
+    File outFile = new File(TEST_ROOT_DIR, "out.meta");
+    outFile.delete();
+    assertEquals("ret: 0, Checksum calculation succeeded on block file " +
+        blockFile.getAbsolutePath() + " saved metadata to meta file " +
+        outFile.getAbsolutePath(), runCmd(new String[] {"computeMeta", "-block",
+        blockFile.getAbsolutePath(), "-out", outFile.getAbsolutePath()}));
+
+    assertTrue(outFile.exists());
+    assertTrue(outFile.length() > 0);
+  }
+
   @Test(timeout = 60000)
   public void testRecoverLeaseforFileNotFound() throws Exception {
     assertTrue(runCmd(new String[] {