From 9afb8025d6549f0ade0ae7d36f5e67cd20c500f4 Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Sat, 13 Jan 2018 13:16:43 +0800 Subject: [PATCH] HDFS-12972. RBF: Display mount table quota info in Web UI and admin command. Contributed by Yiqun Lin. --- .../federation/router/RouterQuotaUsage.java | 31 +++++++++++++++++++ .../federation/store/records/MountTable.java | 10 ++++-- .../records/impl/pb/MountTablePBImpl.java | 21 ++++++++----- .../hdfs/tools/federation/RouterAdmin.java | 16 ++++++---- .../main/webapps/router/federationhealth.html | 2 ++ .../metrics/TestFederationMetrics.java | 1 + .../federation/router/TestRouterAdminCLI.java | 19 ++++++++++-- 7 files changed, 83 insertions(+), 17 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java index 55bfc483dc..eedd80f774 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java @@ -19,14 +19,20 @@ import org.apache.hadoop.fs.QuotaUsage; import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; import org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature; import org.apache.hadoop.hdfs.server.namenode.Quota; +import org.apache.hadoop.util.StringUtils; /** * The subclass of {@link QuotaUsage} used in Router-based federation. */ public final class RouterQuotaUsage extends QuotaUsage { + + /** Default quota usage count. */ + public static final long QUOTA_USAGE_COUNT_DEFAULT = 0; + private RouterQuotaUsage(Builder builder) { super(builder); } @@ -85,4 +91,29 @@ public void verifyStoragespaceQuota() throws DSQuotaExceededException { throw new DSQuotaExceededException(getSpaceQuota(), getSpaceConsumed()); } } + + @Override + public String toString() { + String nsQuota = String.valueOf(getQuota()); + String nsCount = String.valueOf(getFileAndDirectoryCount()); + if (getQuota() == HdfsConstants.QUOTA_DONT_SET) { + nsQuota = "-"; + nsCount = "-"; + } + + String ssQuota = StringUtils.byteDesc(getSpaceQuota()); + String ssCount = StringUtils.byteDesc(getSpaceConsumed()); + if (getSpaceQuota() == HdfsConstants.QUOTA_DONT_SET) { + ssQuota = "-"; + ssCount = "-"; + } + + StringBuilder str = new StringBuilder(); + str.append("[NsQuota: ").append(nsQuota).append("/") + .append(nsCount); + str.append(", SsQuota: ").append(ssQuota) + .append("/").append(ssCount) + .append("]"); + return str.toString(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java index 53ad1e1919..3a99f144ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java @@ -144,8 +144,10 @@ public static MountTable newInstance(final String src, // Set quota for mount table RouterQuotaUsage quota = new RouterQuotaUsage.Builder() - .fileAndDirectoryCount(0).quota(HdfsConstants.QUOTA_DONT_SET) - .spaceConsumed(0).spaceQuota(HdfsConstants.QUOTA_DONT_SET).build(); + .fileAndDirectoryCount(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT) + .quota(HdfsConstants.QUOTA_DONT_SET) + .spaceConsumed(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT) + .spaceQuota(HdfsConstants.QUOTA_DONT_SET).build(); record.setQuota(quota); // Validate @@ -325,6 +327,10 @@ public String toString() { sb.append("[mode:").append(this.getMode()).append("]"); } + if (this.getQuota() != null) { + sb.append("[quota:").append(this.getQuota()).append("]"); + } + return sb.toString(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java index d9e95550bc..3beeca320d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto.DestOrder; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProtoOrBuilder; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoteLocationProto; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.QuotaUsageProto; import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation; import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder; @@ -255,16 +256,22 @@ public void setMode(FsPermission mode) { @Override public RouterQuotaUsage getQuota() { MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder(); - if (!proto.hasQuota()) { - return null; + + long nsQuota = HdfsConstants.QUOTA_DONT_SET; + long nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT; + long ssQuota = HdfsConstants.QUOTA_DONT_SET; + long ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT; + if (proto.hasQuota()) { + QuotaUsageProto quotaProto = proto.getQuota(); + nsQuota = quotaProto.getQuota(); + nsCount = quotaProto.getFileAndDirectoryCount(); + ssQuota = quotaProto.getSpaceQuota(); + ssCount = quotaProto.getSpaceConsumed(); } - QuotaUsageProto quotaProto = proto.getQuota(); RouterQuotaUsage.Builder builder = new RouterQuotaUsage.Builder() - .fileAndDirectoryCount(quotaProto.getFileAndDirectoryCount()) - .quota(quotaProto.getQuota()) - .spaceConsumed(quotaProto.getSpaceConsumed()) - .spaceQuota(quotaProto.getSpaceQuota()); + .quota(nsQuota).fileAndDirectoryCount(nsCount).spaceQuota(ssQuota) + .spaceConsumed(ssCount); return builder.build(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java index 0681ed5278..d5a2d77a3e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java @@ -47,6 +47,7 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.slf4j.Logger; @@ -395,8 +396,8 @@ public void listMounts(String path) throws IOException { private static void printMounts(List entries) { System.out.println("Mount Table Entries:"); System.out.println(String.format( - "%-25s %-25s %-25s %-25s %-25s", - "Source", "Destinations", "Owner", "Group", "Mode")); + "%-25s %-25s %-25s %-25s %-25s %-25s", + "Source", "Destinations", "Owner", "Group", "Mode", "Quota/Usage")); for (MountTable entry : entries) { StringBuilder destBuilder = new StringBuilder(); for (RemoteLocation location : entry.getDestinations()) { @@ -409,8 +410,10 @@ private static void printMounts(List entries) { System.out.print(String.format("%-25s %-25s", entry.getSourcePath(), destBuilder.toString())); - System.out.println(String.format(" %-25s %-25s %-25s", + System.out.print(String.format(" %-25s %-25s %-25s", entry.getOwnerName(), entry.getGroupName(), entry.getMode())); + + System.out.println(String.format(" %-25s", entry.getQuota())); } } @@ -436,7 +439,8 @@ private boolean setQuota(String[] parameters, int i) throws IOException { } else if (parameters[i].equals("-ssQuota")) { i++; try { - ssQuota = Long.parseLong(parameters[i]); + ssQuota = StringUtils.TraditionalBinaryPrefix + .string2long(parameters[i]); } catch (Exception e) { System.err.println("Cannot parse ssQuota: " + parameters[i]); } @@ -499,11 +503,11 @@ private boolean updateQuota(String mount, long nsQuota, long ssQuota) // If nsQuota or ssQuota was unset, reset corresponding usage // value to zero. if (nsQuota == HdfsConstants.QUOTA_DONT_SET) { - nsCount = 0; + nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT; } if (nsQuota == HdfsConstants.QUOTA_DONT_SET) { - ssCount = 0; + ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT; } RouterQuotaUsage updatedQuota = new RouterQuotaUsage.Builder() diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html index 9c28975b14..ac5bb49812 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html @@ -338,6 +338,7 @@ Owner Group Permission + Quota/Usage Date Modified Date Created @@ -353,6 +354,7 @@ {ownerName} {groupName} {mode} + {quota} {dateModified} {dateCreated} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java index b445df5c35..06d1890653 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java @@ -87,6 +87,7 @@ public void testMountTableStatsDataSource() assertEquals(entry.getOwnerName(), json.getString("ownerName")); assertEquals(entry.getGroupName(), json.getString("groupName")); assertEquals(entry.getMode().toString(), json.getString("mode")); + assertEquals(entry.getQuota().toString(), json.getString("quota")); assertNotNullAndNotEmpty(json.getString("dateCreated")); assertNotNullAndNotEmpty(json.getString("dateModified")); match++; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java index ec47a41d47..265fcde437 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java @@ -313,9 +313,11 @@ public void testSetAndClearQuota() throws Exception { RouterQuotaUsage quotaUsage = mountTable.getQuota(); // verify the default quota set - assertEquals(0, quotaUsage.getFileAndDirectoryCount()); + assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT, + quotaUsage.getFileAndDirectoryCount()); assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getQuota()); - assertEquals(0, quotaUsage.getSpaceConsumed()); + assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT, + quotaUsage.getSpaceConsumed()); assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getSpaceQuota()); long nsQuota = 50; @@ -334,6 +336,19 @@ public void testSetAndClearQuota() throws Exception { assertEquals(nsQuota, quotaUsage.getQuota()); assertEquals(ssQuota, quotaUsage.getSpaceQuota()); + // use quota string for setting ss quota + String newSsQuota = "2m"; + argv = new String[] {"-setQuota", src, "-ssQuota", newSsQuota}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + mountTable = getResponse.getEntries().get(0); + quotaUsage = mountTable.getQuota(); + // verify if ss quota is correctly set + assertEquals(2 * 1024 * 1024, quotaUsage.getSpaceQuota()); + // test clrQuota command argv = new String[] {"-clrQuota", src}; assertEquals(0, ToolRunner.run(admin, argv));