big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15536. RBF: Clear Quota in Router was not consistent.

Browse Source 
(cherry picked from commit 6e618b6a7e)
This commit is contained in:
hemanthboyina 2020-08-26 13:03:08 +05:30
parent ba0eca6a2c
commit ac0a377779
2 changed files with 54 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/records/MountTable.java
View File

@ -430,6 +430,8 @@ public int hashCode() {
.append(this.isReadOnly())
.append(this.getDestOrder())
.append(this.isFaultTolerant())
.append(this.getQuota().getQuota())
.append(this.getQuota().getSpaceQuota())
.toHashCode();
}
@ -443,6 +445,9 @@ public boolean equals(Object obj) {
.append(this.isReadOnly(), other.isReadOnly())
.append(this.getDestOrder(), other.getDestOrder())
.append(this.isFaultTolerant(), other.isFaultTolerant())
.append(this.getQuota().getQuota(), other.getQuota().getQuota())
.append(this.getQuota().getSpaceQuota(),
other.getQuota().getSpaceQuota())
.isEquals();
}
return false;

49
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java
View File

@ -25,6 +25,7 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@ -34,6 +35,7 @@
import java.util.TreeSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Options.Rename;
@ -56,7 +58,9 @@
import org.apache.hadoop.hdfs.server.federation.store.protocol.GetDestinationResponse;
import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest;
import org.apache.hadoop.hdfs.server.federation.store.records.MountTable;
import org.apache.hadoop.hdfs.tools.federation.RouterAdmin;
import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.util.ToolRunner;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@ -509,6 +513,43 @@ public void testRenameMultipleDestDirectories() throws Exception {
verifyRenameOnMultiDestDirectories(DestinationOrder.SPACE, true);
}
@Test
public void testClearQuota() throws Exception {
long nsQuota = 5;
long ssQuota = 100;
Path path = new Path("/router_test");
nnFs0.mkdirs(path);
nnFs1.mkdirs(path);
MountTable addEntry = MountTable.newInstance("/router_test",
Collections.singletonMap("ns0", "/router_test"));
addEntry.setQuota(new RouterQuotaUsage.Builder().build());
assertTrue(addMountTable(addEntry));
RouterQuotaUpdateService updateService =
routerContext.getRouter().getQuotaCacheUpdateService();
updateService.periodicInvoke();
//set quota and validate the quota
RouterAdmin admin = getRouterAdmin();
String[] argv = new String[] {"-setQuota", path.toString(), "-nsQuota",
String.valueOf(nsQuota), "-ssQuota", String.valueOf(ssQuota)};
assertEquals(0, ToolRunner.run(admin, argv));
updateService.periodicInvoke();
resolver.loadCache(true);
ContentSummary cs = routerFs.getContentSummary(path);
assertEquals(nsQuota, cs.getQuota());
assertEquals(ssQuota, cs.getSpaceQuota());
//clear quota and validate the quota
argv = new String[] {"-clrQuota", path.toString()};
assertEquals(0, ToolRunner.run(admin, argv));
updateService.periodicInvoke();
resolver.loadCache(true);
//quota should be cleared
ContentSummary cs1 = routerFs.getContentSummary(path);
assertEquals(-1, cs1.getQuota());
assertEquals(-1, cs1.getSpaceQuota());
}
/**
* Test to verify rename operation on directories in case of multiple
* destinations.
@ -690,4 +731,12 @@ private static FileSystem getFileSystem(final String nsId) {
return null;
}
private RouterAdmin getRouterAdmin() {
Router router = routerContext.getRouter();
Configuration configuration = routerContext.getConf();
InetSocketAddress routerSocket = router.getAdminServerAddress();
configuration.setSocketAddr(RBFConfigKeys.DFS_ROUTER_ADMIN_ADDRESS_KEY,
routerSocket);
return new RouterAdmin(configuration);
}
}