From cbb492578ef09300821b7199de54c6508f9d7fe8 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Thu, 12 Feb 2015 04:32:43 -0800 Subject: [PATCH] HDFS-7789. DFSck should resolve the path to support cross-FS symlinks. (gera) --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../org/apache/hadoop/hdfs/tools/DFSck.java | 31 ++++++++++++------- .../hadoop/hdfs/server/namenode/TestFsck.java | 14 ++++++--- .../TestFsckWithMultipleNameNodes.java | 20 ++++++++++++ 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5ca16afc8c..d5208da764 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -697,6 +697,9 @@ Release 2.7.0 - UNRELEASED HDFS-7439. Add BlockOpResponseProto's message to the exception messages. (Takanobu Asanuma via szetszwo) + HDFS-7789. DFSck should resolve the path to support cross-FS symlinks. + (gera) + OPTIMIZATIONS HDFS-7454. Reduce memory footprint for AclEntries in NameNode. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java index ec83a9029e..dc6d9d4726 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java @@ -225,6 +225,14 @@ private Integer listCorruptFileBlocks(String dir, String baseUrl) return errCode; } + + private Path getResolvedPath(String dir) throws IOException { + Configuration conf = getConf(); + Path dirPath = new Path(dir); + FileSystem fs = dirPath.getFileSystem(conf); + return fs.resolvePath(dirPath); + } + /** * Derive the namenode http address from the current file system, * either default or as set by "-fs" in the generic options. @@ -236,19 +244,12 @@ private URI getCurrentNamenodeAddress(Path target) throws IOException { Configuration conf = getConf(); //get the filesystem object to verify it is an HDFS system - final FileSystem fs; - try { - fs = target.getFileSystem(conf); - } catch (IOException ioe) { - System.err.println("FileSystem is inaccessible due to:\n" - + StringUtils.stringifyException(ioe)); - return null; - } + final FileSystem fs = target.getFileSystem(conf); if (!(fs instanceof DistributedFileSystem)) { System.err.println("FileSystem is " + fs.getUri()); return null; } - + return DFSUtil.getInfoServer(HAUtil.getAddressOfActive(fs), conf, DFSUtil.getHttpClientScheme(conf)); } @@ -303,8 +304,16 @@ else if (args[idx].equals("-list-corruptfileblocks")) { dir = "/"; } - final Path dirpath = new Path(dir); - final URI namenodeAddress = getCurrentNamenodeAddress(dirpath); + Path dirpath = null; + URI namenodeAddress = null; + try { + dirpath = getResolvedPath(dir); + namenodeAddress = getCurrentNamenodeAddress(dirpath); + } catch (IOException ioe) { + System.err.println("FileSystem is inaccessible due to:\n" + + StringUtils.stringifyException(ioe)); + } + if (namenodeAddress == null) { //Error message already output in {@link #getCurrentNamenodeAddress()} System.err.println("DFSck exiting."); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index 1053b5f970..409fffcd2d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -211,10 +211,16 @@ private void verifyAuditLogs() throws IOException { try { // Audit log should contain one getfileinfo and one fsck reader = new BufferedReader(new FileReader(auditLogFile)); - String line = reader.readLine(); - assertNotNull(line); - assertTrue("Expected getfileinfo event not found in audit log", - getfileinfoPattern.matcher(line).matches()); + String line; + + // one extra getfileinfo stems from resolving the path + // + for (int i = 0; i < 2; i++) { + line = reader.readLine(); + assertNotNull(line); + assertTrue("Expected getfileinfo event not found in audit log", + getfileinfoPattern.matcher(line).matches()); + } line = reader.readLine(); assertNotNull(line); assertTrue("Expected fsck event not found in audit log", fsckPattern diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java index f4cb624995..124b301cb9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsckWithMultipleNameNodes.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.IOException; +import java.net.URI; import java.util.Random; import java.util.concurrent.TimeoutException; @@ -26,6 +27,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.viewfs.ConfigUtil; +import org.apache.hadoop.fs.viewfs.ViewFileSystem; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -119,6 +122,23 @@ private void runTest(final int nNameNodes, final int nDataNodes, LOG.info("result=" + result); Assert.assertTrue(result.contains("Status: HEALTHY")); } + + // Test viewfs + // + LOG.info("RUN_TEST 3"); + final String[] vurls = new String[nNameNodes]; + for (int i = 0; i < vurls.length; i++) { + String link = "/mount/nn_" + i + FILE_NAME; + ConfigUtil.addLink(conf, link, new URI(urls[i])); + vurls[i] = "viewfs:" + link; + } + + for(int i = 0; i < vurls.length; i++) { + LOG.info("vurls[" + i + "]=" + vurls[i]); + final String result = TestFsck.runFsck(conf, 0, false, vurls[i]); + LOG.info("result=" + result); + Assert.assertTrue(result.contains("Status: HEALTHY")); + } } finally { cluster.shutdown(); }