big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13380. RBF: mv/rm fail after the directory exceeded the quota limit. Contributed by Yiqun Lin.

Browse Source
This commit is contained in:
Inigo Goiri 2018-04-09 10:09:25 -07:00
parent ac32b3576d
commit e9b9f48dad
2 changed files with 28 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
View File

@ -900,7 +900,8 @@ public boolean rename(final String src, final String dst)
throws IOException { throws IOException {
checkOperation(OperationCategory.WRITE); checkOperation(OperationCategory.WRITE);
final List<RemoteLocation> srcLocations = getLocationsForPath(src, true); final List<RemoteLocation> srcLocations =
getLocationsForPath(src, true, false);
// srcLocations may be trimmed by getRenameDestinations() // srcLocations may be trimmed by getRenameDestinations()
final List<RemoteLocation> locs = new LinkedList<>(srcLocations); final List<RemoteLocation> locs = new LinkedList<>(srcLocations);
RemoteParam dstParam = getRenameDestinations(locs, dst); RemoteParam dstParam = getRenameDestinations(locs, dst);
@ -921,7 +922,8 @@ public void rename2(final String src, final String dst,
final Options.Rename... options) throws IOException { final Options.Rename... options) throws IOException {
checkOperation(OperationCategory.WRITE); checkOperation(OperationCategory.WRITE);
final List<RemoteLocation> srcLocations = getLocationsForPath(src, true); final List<RemoteLocation> srcLocations =
getLocationsForPath(src, true, false);
// srcLocations may be trimmed by getRenameDestinations() // srcLocations may be trimmed by getRenameDestinations()
final List<RemoteLocation> locs = new LinkedList<>(srcLocations); final List<RemoteLocation> locs = new LinkedList<>(srcLocations);
RemoteParam dstParam = getRenameDestinations(locs, dst); RemoteParam dstParam = getRenameDestinations(locs, dst);
@ -998,7 +1000,8 @@ public boolean truncate(String src, long newLength, String clientName)
public boolean delete(String src, boolean recursive) throws IOException { public boolean delete(String src, boolean recursive) throws IOException {
checkOperation(OperationCategory.WRITE); checkOperation(OperationCategory.WRITE);
final List<RemoteLocation> locations = getLocationsForPath(src, true); final List<RemoteLocation> locations =
getLocationsForPath(src, true, false);
RemoteMethod method = new RemoteMethod("delete", RemoteMethod method = new RemoteMethod("delete",
new Class<?>[] {String.class, boolean.class}, new RemoteParam(), new Class<?>[] {String.class, boolean.class}, new RemoteParam(),
recursive); recursive);
@ -2213,14 +2216,29 @@ private RemoteLocation getLocationForPath(
/** /**
* Get the possible locations of a path in the federated cluster. * Get the possible locations of a path in the federated cluster.
* During the get operation, it will do the quota verification.
* *
* @param path Path to check. * @param path Path to check.
* @param failIfLocked Fail the request if locked (top mount point). * @param failIfLocked Fail the request if locked (top mount point).
* @return Prioritized list of locations in the federated cluster. * @return Prioritized list of locations in the federated cluster.
* @throws IOException If the location for this path cannot be determined. * @throws IOException If the location for this path cannot be determined.
*/ */
protected List<RemoteLocation> getLocationsForPath( protected List<RemoteLocation> getLocationsForPath(String path,
String path, boolean failIfLocked) throws IOException { boolean failIfLocked) throws IOException {
return getLocationsForPath(path, failIfLocked, true);
}
/**
* Get the possible locations of a path in the federated cluster.
*
* @param path Path to check.
* @param failIfLocked Fail the request if locked (top mount point).
* @param needQuotaVerify If need to do the quota verification.
* @return Prioritized list of locations in the federated cluster.
* @throws IOException If the location for this path cannot be determined.
*/
protected List<RemoteLocation> getLocationsForPath(String path,
boolean failIfLocked, boolean needQuotaVerify) throws IOException {
try { try {
// Check the location for this path // Check the location for this path
final PathLocation location = final PathLocation location =
@ -2241,7 +2259,7 @@ protected List<RemoteLocation> getLocationsForPath(
} }
// Check quota // Check quota
if (this.router.isQuotaEnabled()) { if (this.router.isQuotaEnabled() && needQuotaVerify) {
RouterQuotaUsage quotaUsage = this.router.getQuotaManager() RouterQuotaUsage quotaUsage = this.router.getQuotaManager()
.getQuotaUsage(path); .getQuotaUsage(path);
if (quotaUsage != null) { if (quotaUsage != null) {

4
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterQuota.java
View File

@ -151,6 +151,10 @@ public Boolean get() {
// mkdir in real FileSystem should be okay // mkdir in real FileSystem should be okay
nnFs1.mkdirs(new Path("/testdir1/" + UUID.randomUUID())); nnFs1.mkdirs(new Path("/testdir1/" + UUID.randomUUID()));
nnFs2.mkdirs(new Path("/testdir2/" + UUID.randomUUID())); nnFs2.mkdirs(new Path("/testdir2/" + UUID.randomUUID()));
// delete/rename call should be still okay
routerFs.delete(new Path("/nsquota"), true);
routerFs.rename(new Path("/nsquota/subdir"), new Path("/nsquota/subdir"));
} }
@Test @Test