From 7b23ad1ef76739bc04374aefe6e8e88c23449653 Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Tue, 21 Jun 2016 17:34:58 -0700 Subject: [PATCH] HDFS-10550. DiskBalancer: fix issue of order dependency in iteration in ReportCommand test. Contributed by Xiaobing Zhou. --- .../diskbalancer/command/ReportCommand.java | 17 +++-- .../command/TestDiskBalancerCommand.java | 74 ++++++++----------- 2 files changed, 44 insertions(+), 47 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java index eb6afccd7c..40729f8979 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java @@ -19,6 +19,7 @@ import java.io.PrintStream; import java.util.Collections; +import java.util.List; import java.util.ListIterator; import org.apache.commons.cli.CommandLine; @@ -32,6 +33,7 @@ import org.apache.hadoop.hdfs.tools.DiskBalancer; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; /** * Executes the report command. @@ -164,9 +166,10 @@ private void handleNodeReport(final CommandLine cmd, StrBuilder result, dbdn.getVolumeCount(), dbdn.getNodeDataDensity())); + List volumeList = Lists.newArrayList(); for (DiskBalancerVolumeSet vset : dbdn.getVolumeSets().values()) { for (DiskBalancerVolume vol : vset.getVolumes()) { - result.appendln(String.format(volumeFormat, + volumeList.add(String.format(volumeFormat, vol.getStorageType(), vol.getPath(), vol.getUsedRatio(), @@ -181,6 +184,10 @@ private void handleNodeReport(final CommandLine cmd, StrBuilder result, vol.isTransient() ? trueStr : falseStr)); } } + + Collections.sort(volumeList); + result.appendln( + StringUtils.join(volumeList.toArray(), System.lineSeparator())); } } } @@ -194,13 +201,13 @@ public void printHelp() { " datanode, or prints out the list of nodes that will benefit from " + "running disk balancer. Top defaults to " + getDefaultTop(); String footer = ". E.g.:\n" - + "hdfs diskbalancer -uri http://namenode.uri -report\n" - + "hdfs diskbalancer -uri http://namenode.uri -report -top 5\n" - + "hdfs diskbalancer -uri http://namenode.uri -report " + + "hdfs diskbalancer -fs http://namenode.uri -report\n" + + "hdfs diskbalancer -fs http://namenode.uri -report -top 5\n" + + "hdfs diskbalancer -fs http://namenode.uri -report " + "-node {DataNodeID | IP | Hostname}"; HelpFormatter helpFormatter = new HelpFormatter(); - helpFormatter.printHelp("hdfs diskbalancer -uri http://namenode.uri " + + helpFormatter.printHelp("hdfs diskbalancer -fs http://namenode.uri " + "-report [options]", header, DiskBalancer.getReportOptions(), footer); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java index 57e59f6844..3accbc209a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java @@ -71,8 +71,10 @@ public void tearDown() throws Exception { } } - private void testReportSimple() throws Exception { - final String cmdLine = String.format("hdfs diskbalancer -uri %s -report", + /* test basic report */ + @Test(timeout=60000) + public void testReportSimple() throws Exception { + final String cmdLine = String.format("hdfs diskbalancer -fs %s -report", clusterJson.toString()); final List outputs = runCommand(cmdLine); @@ -98,9 +100,11 @@ private void testReportSimple() throws Exception { } - private void testReportLessThanTotal() throws Exception { + /* test less than 64 DataNode(s) as total, e.g., -report -top 32 */ + @Test(timeout=60000) + public void testReportLessThanTotal() throws Exception { final String cmdLine = String.format( - "hdfs diskbalancer -uri %s -report -top 32", clusterJson.toString()); + "hdfs diskbalancer -fs %s -report -top 32", clusterJson.toString()); final List outputs = runCommand(cmdLine); assertThat( @@ -120,9 +124,11 @@ private void testReportLessThanTotal() throws Exception { containsString("9 volumes with node data density 1.97")))); } - private void testReportMoreThanTotal() throws Exception { + /* test more than 64 DataNode(s) as total, e.g., -report -top 128 */ + @Test(timeout=60000) + public void testReportMoreThanTotal() throws Exception { final String cmdLine = String.format( - "hdfs diskbalancer -uri %s -report -top 128", clusterJson.toString()); + "hdfs diskbalancer -fs %s -report -top 128", clusterJson.toString()); final List outputs = runCommand(cmdLine); assertThat( @@ -143,9 +149,11 @@ private void testReportMoreThanTotal() throws Exception { } - private void testReportInvalidTopLimit() throws Exception { + /* test invalid top limit, e.g., -report -top xx */ + @Test(timeout=60000) + public void testReportInvalidTopLimit() throws Exception { final String cmdLine = String.format( - "hdfs diskbalancer -uri %s -report -top xx", clusterJson.toString()); + "hdfs diskbalancer -fs %s -report -top xx", clusterJson.toString()); final List outputs = runCommand(cmdLine); assertThat( @@ -169,10 +177,12 @@ private void testReportInvalidTopLimit() throws Exception { containsString("9 volumes with node data density 1.97")))); } - private void testReportNode() throws Exception { + /* test -report -node DataNodeID */ + @Test(timeout=60000) + public void testReportNode() throws Exception { final String cmdLine = String .format( - "hdfs diskbalancer -uri %s -report -node " + "hdfs diskbalancer -fs %s -report -node " + "a87654a9-54c7-4693-8dd9-c9c7021dc340", clusterJson.toString()); final List outputs = runCommand(cmdLine); @@ -192,9 +202,9 @@ private void testReportNode() throws Exception { assertThat( outputs.get(3), is(allOf(containsString("DISK"), - containsString("/tmp/disk/xx3j3ph3zd"), - containsString("0.72 used: 289544224916/400000000000"), - containsString("0.28 free: 110455775084/400000000000")))); + containsString("/tmp/disk/KmHefYNURo"), + containsString("0.20 used: 39160240782/200000000000"), + containsString("0.80 free: 160839759218/200000000000")))); assertThat( outputs.get(4), is(allOf(containsString("DISK"), @@ -204,16 +214,15 @@ private void testReportNode() throws Exception { assertThat( outputs.get(5), is(allOf(containsString("DISK"), - containsString("DISK"), - containsString("/tmp/disk/KmHefYNURo"), - containsString("0.20 used: 39160240782/200000000000"), - containsString("0.80 free: 160839759218/200000000000")))); + containsString("/tmp/disk/xx3j3ph3zd"), + containsString("0.72 used: 289544224916/400000000000"), + containsString("0.28 free: 110455775084/400000000000")))); assertThat( outputs.get(6), is(allOf(containsString("RAM_DISK"), - containsString("/tmp/disk/MXRyYsCz3U"), - containsString("0.55 used: 438102096853/800000000000"), - containsString("0.45 free: 361897903147/800000000000")))); + containsString("/tmp/disk/BoBlQFxhfw"), + containsString("0.60 used: 477590453390/800000000000"), + containsString("0.40 free: 322409546610/800000000000")))); assertThat( outputs.get(7), is(allOf(containsString("RAM_DISK"), @@ -223,9 +232,9 @@ private void testReportNode() throws Exception { assertThat( outputs.get(8), is(allOf(containsString("RAM_DISK"), - containsString("/tmp/disk/BoBlQFxhfw"), - containsString("0.60 used: 477590453390/800000000000"), - containsString("0.40 free: 322409546610/800000000000")))); + containsString("/tmp/disk/MXRyYsCz3U"), + containsString("0.55 used: 438102096853/800000000000"), + containsString("0.45 free: 361897903147/800000000000")))); assertThat( outputs.get(9), is(allOf(containsString("SSD"), @@ -247,25 +256,6 @@ private void testReportNode() throws Exception { } @Test(timeout=60000) - public void testReportCommmand() throws Exception { - - /* test basic report */ - testReportSimple(); - - /* test less than 64 DataNode(s) as total, e.g., -report -top 32 */ - testReportLessThanTotal(); - - /* test more than 64 DataNode(s) as total, e.g., -report -top 128 */ - testReportMoreThanTotal(); - - /* test invalid top limit, e.g., -report -top xx */ - testReportInvalidTopLimit(); - - /* test -report -node DataNodeID */ - testReportNode(); - } - - @Test public void testReadClusterFromJson() throws Exception { Configuration conf = new HdfsConfiguration(); conf.setBoolean(DFSConfigKeys.DFS_DISK_BALANCER_ENABLED, true);