big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12972. RBF: Display mount table quota info in Web UI and admin command. Contributed by Yiqun Lin.

Browse Source
This commit is contained in:
Yiqun Lin 2018-01-13 13:16:43 +08:00
parent 2dcfc1876e
commit 9afb8025d6
7 changed files with 83 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterQuotaUsage.java
View File

@ -19,14 +19,20 @@
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.server.namenode.DirectoryWithQuotaFeature;
import org.apache.hadoop.hdfs.server.namenode.Quota;
import org.apache.hadoop.util.StringUtils;
/**
* The subclass of {@link QuotaUsage} used in Router-based federation.
*/
public final class RouterQuotaUsage extends QuotaUsage {
/** Default quota usage count. */
public static final long QUOTA_USAGE_COUNT_DEFAULT = 0;
private RouterQuotaUsage(Builder builder) {
super(builder);
}
@ -85,4 +91,29 @@ public void verifyStoragespaceQuota() throws DSQuotaExceededException {
throw new DSQuotaExceededException(getSpaceQuota(), getSpaceConsumed());
}
}
@Override
public String toString() {
String nsQuota = String.valueOf(getQuota());
String nsCount = String.valueOf(getFileAndDirectoryCount());
if (getQuota() == HdfsConstants.QUOTA_DONT_SET) {
nsQuota = "-";
nsCount = "-";
}
String ssQuota = StringUtils.byteDesc(getSpaceQuota());
String ssCount = StringUtils.byteDesc(getSpaceConsumed());
if (getSpaceQuota() == HdfsConstants.QUOTA_DONT_SET) {
ssQuota = "-";
ssCount = "-";
}
StringBuilder str = new StringBuilder();
str.append("[NsQuota: ").append(nsQuota).append("/")
.append(nsCount);
str.append(", SsQuota: ").append(ssQuota)
.append("/").append(ssCount)
.append("]");
return str.toString();
}
}

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
View File

@ -144,8 +144,10 @@ public static MountTable newInstance(final String src,
// Set quota for mount table
RouterQuotaUsage quota = new RouterQuotaUsage.Builder()
.fileAndDirectoryCount(0).quota(HdfsConstants.QUOTA_DONT_SET)
.spaceConsumed(0).spaceQuota(HdfsConstants.QUOTA_DONT_SET).build();
.fileAndDirectoryCount(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
.quota(HdfsConstants.QUOTA_DONT_SET)
.spaceConsumed(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT)
.spaceQuota(HdfsConstants.QUOTA_DONT_SET).build();
record.setQuota(quota);
// Validate
@ -325,6 +327,10 @@ public String toString() {
sb.append("[mode:").append(this.getMode()).append("]");
}
if (this.getQuota() != null) {
sb.append("[quota:").append(this.getQuota()).append("]");
}
return sb.toString();
}

21
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/impl/pb/MountTablePBImpl.java
View File

@ -27,6 +27,7 @@
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProto.DestOrder;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.MountTableRecordProtoOrBuilder;
import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoteLocationProto;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.QuotaUsageProto;
import org.apache.hadoop.hdfs.server.federation.resolver.RemoteLocation;
import org.apache.hadoop.hdfs.server.federation.resolver.order.DestinationOrder;
@ -255,16 +256,22 @@ public void setMode(FsPermission mode) {
@Override
public RouterQuotaUsage getQuota() {
MountTableRecordProtoOrBuilder proto = this.translator.getProtoOrBuilder();
if (!proto.hasQuota()) {
return null;
long nsQuota = HdfsConstants.QUOTA_DONT_SET;
long nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
long ssQuota = HdfsConstants.QUOTA_DONT_SET;
long ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
if (proto.hasQuota()) {
QuotaUsageProto quotaProto = proto.getQuota();
nsQuota = quotaProto.getQuota();
nsCount = quotaProto.getFileAndDirectoryCount();
ssQuota = quotaProto.getSpaceQuota();
ssCount = quotaProto.getSpaceConsumed();
}
QuotaUsageProto quotaProto = proto.getQuota();
RouterQuotaUsage.Builder builder = new RouterQuotaUsage.Builder()
.fileAndDirectoryCount(quotaProto.getFileAndDirectoryCount())
.quota(quotaProto.getQuota())
.spaceConsumed(quotaProto.getSpaceConsumed())
.spaceQuota(quotaProto.getSpaceQuota());
.quota(nsQuota).fileAndDirectoryCount(nsCount).spaceQuota(ssQuota)
.spaceConsumed(ssCount);
return builder.build();
}

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
View File

@ -47,6 +47,7 @@
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.slf4j.Logger;
@ -395,8 +396,8 @@ public void listMounts(String path) throws IOException {
private static void printMounts(List<MountTable> entries) {
System.out.println("Mount Table Entries:");
System.out.println(String.format(
"%-25s %-25s %-25s %-25s %-25s",
"Source", "Destinations", "Owner", "Group", "Mode"));
"%-25s %-25s %-25s %-25s %-25s %-25s",
"Source", "Destinations", "Owner", "Group", "Mode", "Quota/Usage"));
for (MountTable entry : entries) {
StringBuilder destBuilder = new StringBuilder();
for (RemoteLocation location : entry.getDestinations()) {
@ -409,8 +410,10 @@ private static void printMounts(List<MountTable> entries) {
System.out.print(String.format("%-25s %-25s", entry.getSourcePath(),
destBuilder.toString()));
System.out.println(String.format(" %-25s %-25s %-25s",
System.out.print(String.format(" %-25s %-25s %-25s",
entry.getOwnerName(), entry.getGroupName(), entry.getMode()));
System.out.println(String.format(" %-25s", entry.getQuota()));
}
}
@ -436,7 +439,8 @@ private boolean setQuota(String[] parameters, int i) throws IOException {
} else if (parameters[i].equals("-ssQuota")) {
i++;
try {
ssQuota = Long.parseLong(parameters[i]);
ssQuota = StringUtils.TraditionalBinaryPrefix
.string2long(parameters[i]);
} catch (Exception e) {
System.err.println("Cannot parse ssQuota: " + parameters[i]);
}
@ -499,11 +503,11 @@ private boolean updateQuota(String mount, long nsQuota, long ssQuota)
// If nsQuota or ssQuota was unset, reset corresponding usage
// value to zero.
if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
nsCount = 0;
nsCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
}
if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
ssCount = 0;
ssCount = RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT;
}
RouterQuotaUsage updatedQuota = new RouterQuotaUsage.Builder()

2
hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/router/federationhealth.html
View File

@ -338,6 +338,7 @@
<th>Owner</th>
<th>Group</th>
<th>Permission</th>
<th>Quota/Usage</th>
<th>Date Modified</th>
<th>Date Created</th>
</tr>
@ -353,6 +354,7 @@
<td>{ownerName}</td>
<td>{groupName}</td>
<td>{mode}</td>
<td>{quota}</td>
<td>{dateModified}</td>
<td>{dateCreated}</td>
</tr>

1
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestFederationMetrics.java
View File

@ -87,6 +87,7 @@ public void testMountTableStatsDataSource()
assertEquals(entry.getOwnerName(), json.getString("ownerName"));
assertEquals(entry.getGroupName(), json.getString("groupName"));
assertEquals(entry.getMode().toString(), json.getString("mode"));
assertEquals(entry.getQuota().toString(), json.getString("quota"));
assertNotNullAndNotEmpty(json.getString("dateCreated"));
assertNotNullAndNotEmpty(json.getString("dateModified"));
match++;

19
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
View File

@ -313,9 +313,11 @@ public void testSetAndClearQuota() throws Exception {
RouterQuotaUsage quotaUsage = mountTable.getQuota();
// verify the default quota set
assertEquals(0, quotaUsage.getFileAndDirectoryCount());
assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
quotaUsage.getFileAndDirectoryCount());
assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getQuota());
assertEquals(0, quotaUsage.getSpaceConsumed());
assertEquals(RouterQuotaUsage.QUOTA_USAGE_COUNT_DEFAULT,
quotaUsage.getSpaceConsumed());
assertEquals(HdfsConstants.QUOTA_DONT_SET, quotaUsage.getSpaceQuota());
long nsQuota = 50;
@ -334,6 +336,19 @@ public void testSetAndClearQuota() throws Exception {
assertEquals(nsQuota, quotaUsage.getQuota());
assertEquals(ssQuota, quotaUsage.getSpaceQuota());
// use quota string for setting ss quota
String newSsQuota = "2m";
argv = new String[] {"-setQuota", src, "-ssQuota", newSsQuota};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
getResponse = client.getMountTableManager()
.getMountTableEntries(getRequest);
mountTable = getResponse.getEntries().get(0);
quotaUsage = mountTable.getQuota();
// verify if ss quota is correctly set
assertEquals(2 * 1024 * 1024, quotaUsage.getSpaceQuota());
// test clrQuota command
argv = new String[] {"-clrQuota", src};
assertEquals(0, ToolRunner.run(admin, argv));