big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14810. Review FSNameSystem editlog sync. Contributed by Xiaoqiao He.

Browse Source
This commit is contained in:
Ayush Saxena 2019-10-17 21:56:30 +05:30
parent 3990ffa083
commit 5527d79adb
1 changed files with 54 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -2114,9 +2114,8 @@ void concat(String target, String [] srcs, boolean logRetryCache)
logAuditEvent(false, operationName, Arrays.toString(srcs), logAuditEvent(false, operationName, Arrays.toString(srcs),
target, stat); target, stat);
throw ace; throw ace;
} finally {
getEditLog().logSync();
} }
getEditLog().logSync();
logAuditEvent(true, operationName, Arrays.toString(srcs), target, stat); logAuditEvent(true, operationName, Arrays.toString(srcs), target, stat);
} }
@ -3076,7 +3075,7 @@ boolean renameTo(String src, String dst, boolean logRetryCache)
boolean success = ret.success; boolean success = ret.success;
if (success) { if (success) {
getEditLog().logSync(); getEditLog().logSync();
logAuditEvent(success, operationName, src, dst, ret.auditStat); logAuditEvent(true, operationName, src, dst, ret.auditStat);
} }
return success; return success;
} }
@ -3259,11 +3258,12 @@ boolean isFileClosed(final String src) throws IOException {
final String operationName = "isFileClosed"; final String operationName = "isFileClosed";
checkOperation(OperationCategory.READ); checkOperation(OperationCategory.READ);
final FSPermissionChecker pc = getPermissionChecker(); final FSPermissionChecker pc = getPermissionChecker();
boolean success = false;
try { try {
readLock(); readLock();
try { try {
checkOperation(OperationCategory.READ); checkOperation(OperationCategory.READ);
return FSDirStatAndListingOp.isFileClosed(dir, pc, src); success = FSDirStatAndListingOp.isFileClosed(dir, pc, src);
} finally { } finally {
readUnlock(operationName); readUnlock(operationName);
} }
@ -3271,6 +3271,10 @@ boolean isFileClosed(final String src) throws IOException {
logAuditEvent(false, operationName, src); logAuditEvent(false, operationName, src);
throw e; throw e;
} }
if (success) {
logAuditEvent(true, operationName, src);
}
return success;
} }
/** /**
@ -3398,9 +3402,8 @@ void setQuota(String src, long nsQuota, long ssQuota, StorageType type)
} catch (AccessControlException ace) { } catch (AccessControlException ace) {
logAuditEvent(false, operationName, src); logAuditEvent(false, operationName, src);
throw ace; throw ace;
} finally {
getEditLog().logSync();
} }
getEditLog().logSync();
logAuditEvent(true, operationName, src); logAuditEvent(true, operationName, src);
} }
@ -5745,7 +5748,6 @@ DelegationTokenSecretManager getDelegationTokenSecretManager() {
Token<DelegationTokenIdentifier> getDelegationToken(Text renewer) Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
throws IOException { throws IOException {
final String operationName = "getDelegationToken"; final String operationName = "getDelegationToken";
final boolean success;
final String tokenId; final String tokenId;
Token<DelegationTokenIdentifier> token; Token<DelegationTokenIdentifier> token;
checkOperation(OperationCategory.WRITE); checkOperation(OperationCategory.WRITE);
@ -5776,12 +5778,11 @@ Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
long expiryTime = dtSecretManager.getTokenExpiryTime(dtId); long expiryTime = dtSecretManager.getTokenExpiryTime(dtId);
getEditLog().logGetDelegationToken(dtId, expiryTime); getEditLog().logGetDelegationToken(dtId, expiryTime);
tokenId = dtId.toStringStable(); tokenId = dtId.toStringStable();
success = true;
} finally { } finally {
writeUnlock(operationName); writeUnlock(operationName);
} }
getEditLog().logSync(); getEditLog().logSync();
logAuditEvent(success, operationName, tokenId); logAuditEvent(true, operationName, tokenId);
return token; return token;
} }
@ -6555,19 +6556,17 @@ public SnapshotManager getSnapshotManager() {
void allowSnapshot(String path) throws IOException { void allowSnapshot(String path) throws IOException {
checkOperation(OperationCategory.WRITE); checkOperation(OperationCategory.WRITE);
final String operationName = "allowSnapshot"; final String operationName = "allowSnapshot";
boolean success = false;
checkSuperuserPrivilege(operationName); checkSuperuserPrivilege(operationName);
writeLock(); writeLock();
try { try {
checkOperation(OperationCategory.WRITE); checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot allow snapshot for " + path); checkNameNodeSafeMode("Cannot allow snapshot for " + path);
FSDirSnapshotOp.allowSnapshot(dir, snapshotManager, path); FSDirSnapshotOp.allowSnapshot(dir, snapshotManager, path);
success = true;
} finally { } finally {
writeUnlock(operationName); writeUnlock(operationName);
} }
getEditLog().logSync(); getEditLog().logSync();
logAuditEvent(success, operationName, path, null, null); logAuditEvent(true, operationName, path, null, null);
} }
/** Disallow snapshot on a directory. */ /** Disallow snapshot on a directory. */
@ -6609,8 +6608,7 @@ String createSnapshot(String snapshotRoot, String snapshotName,
writeUnlock(operationName); writeUnlock(operationName);
} }
} catch (AccessControlException ace) { } catch (AccessControlException ace) {
logAuditEvent(false, operationName, snapshotRoot, logAuditEvent(false, operationName, snapshotRoot);
snapshotPath, null);
throw ace; throw ace;
} }
getEditLog().logSync(); getEditLog().logSync();
@ -7124,15 +7122,12 @@ long addCacheDirective(CacheDirectiveInfo directive,
writeUnlock(operationName); writeUnlock(operationName);
} }
} catch (AccessControlException ace) { } catch (AccessControlException ace) {
logAuditEvent(false, operationName, null, logAuditEvent(false, operationName, null);
null, null);
throw ace; throw ace;
} finally {
getEditLog().logSync();
} }
getEditLog().logSync();
effectiveDirectiveStr = effectiveDirective.toString(); effectiveDirectiveStr = effectiveDirective.toString();
logAuditEvent(true, operationName, effectiveDirectiveStr, logAuditEvent(true, operationName, effectiveDirectiveStr);
null, null);
return effectiveDirective.getId(); return effectiveDirective.getId();
} }
@ -7158,9 +7153,8 @@ void modifyCacheDirective(CacheDirectiveInfo directive,
logAuditEvent(false, operationName, idStr, logAuditEvent(false, operationName, idStr,
directive.toString(), null); directive.toString(), null);
throw ace; throw ace;
} finally {
getEditLog().logSync();
} }
getEditLog().logSync();
logAuditEvent(true, operationName, idStr, logAuditEvent(true, operationName, idStr,
directive.toString(), null); directive.toString(), null);
} }
@ -7183,8 +7177,8 @@ void removeCacheDirective(long id, boolean logRetryCache) throws IOException {
logAuditEvent(false, operationName, idStr, null, null); logAuditEvent(false, operationName, idStr, null, null);
throw ace; throw ace;
} }
logAuditEvent(true, operationName, idStr, null, null);
getEditLog().logSync(); getEditLog().logSync();
logAuditEvent(true, operationName, idStr, null, null);
} }
BatchedListEntries<CacheDirectiveEntry> listCacheDirectives( BatchedListEntries<CacheDirectiveEntry> listCacheDirectives(
@ -7203,12 +7197,10 @@ BatchedListEntries<CacheDirectiveEntry> listCacheDirectives(
readUnlock(operationName); readUnlock(operationName);
} }
} catch (AccessControlException ace) { } catch (AccessControlException ace) {
logAuditEvent(false, operationName, filter.toString(), null, logAuditEvent(false, operationName, filter.toString());
null);
throw ace; throw ace;
} }
logAuditEvent(true, operationName, filter.toString(), null, logAuditEvent(true, operationName, filter.toString());
null);
return results; return results;
} }
@ -7230,11 +7222,11 @@ void addCachePool(CachePoolInfo req, boolean logRetryCache)
writeUnlock(operationName); writeUnlock(operationName);
} }
} catch (AccessControlException ace) { } catch (AccessControlException ace) {
logAuditEvent(false, operationName, poolInfoStr, null, null); logAuditEvent(false, operationName, poolInfoStr);
throw ace; throw ace;
} }
logAuditEvent(true, operationName, poolInfoStr, null, null);
getEditLog().logSync(); getEditLog().logSync();
logAuditEvent(true, operationName, poolInfoStr);
} }
void modifyCachePool(CachePoolInfo req, boolean logRetryCache) void modifyCachePool(CachePoolInfo req, boolean logRetryCache)
@ -7258,10 +7250,9 @@ void modifyCachePool(CachePoolInfo req, boolean logRetryCache)
req == null ? null : req.toString(), null); req == null ? null : req.toString(), null);
throw ace; throw ace;
} }
getEditLog().logSync();
logAuditEvent(true, operationName, poolNameStr, logAuditEvent(true, operationName, poolNameStr,
req == null ? null : req.toString(), null); req == null ? null : req.toString(), null);
getEditLog().logSync();
} }
void removeCachePool(String cachePoolName, boolean logRetryCache) void removeCachePool(String cachePoolName, boolean logRetryCache)
@ -7280,11 +7271,11 @@ void removeCachePool(String cachePoolName, boolean logRetryCache)
writeUnlock(operationName); writeUnlock(operationName);
} }
} catch (AccessControlException ace) { } catch (AccessControlException ace) {
logAuditEvent(false, operationName, poolNameStr, null, null); logAuditEvent(false, operationName, poolNameStr);
throw ace; throw ace;
} }
logAuditEvent(true, operationName, poolNameStr, null, null);
getEditLog().logSync(); getEditLog().logSync();
logAuditEvent(true, operationName, poolNameStr);
} }
BatchedListEntries<CachePoolEntry> listCachePools(String prevKey) BatchedListEntries<CachePoolEntry> listCachePools(String prevKey)
@ -7302,10 +7293,10 @@ BatchedListEntries<CachePoolEntry> listCachePools(String prevKey)
readUnlock(operationName); readUnlock(operationName);
} }
} catch (AccessControlException ace) { } catch (AccessControlException ace) {
logAuditEvent(false, operationName, null, null, null); logAuditEvent(false, operationName, null);
throw ace; throw ace;
} }
logAuditEvent(true, operationName, null, null, null); logAuditEvent(true, operationName, null);
return results; return results;
} }
@ -7457,12 +7448,12 @@ void createEncryptionZone(final String src, final String keyName,
boolean logRetryCache) throws IOException, UnresolvedLinkException, boolean logRetryCache) throws IOException, UnresolvedLinkException,
SafeModeException, AccessControlException { SafeModeException, AccessControlException {
final String operationName = "createEncryptionZone"; final String operationName = "createEncryptionZone";
final FileStatus resultingStat;
try { try {
Metadata metadata = FSDirEncryptionZoneOp.ensureKeyIsInitialized(dir, Metadata metadata = FSDirEncryptionZoneOp.ensureKeyIsInitialized(dir,
keyName, src); keyName, src);
final FSPermissionChecker pc = getPermissionChecker(); final FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE); checkOperation(OperationCategory.WRITE);
final FileStatus resultingStat;
writeLock(); writeLock();
try { try {
checkSuperuserPrivilege(pc); checkSuperuserPrivilege(pc);
@ -7473,13 +7464,12 @@ void createEncryptionZone(final String src, final String keyName,
} finally { } finally {
writeUnlock(operationName); writeUnlock(operationName);
} }
getEditLog().logSync();
logAuditEvent(true, operationName, src, null, resultingStat);
} catch (AccessControlException e) { } catch (AccessControlException e) {
logAuditEvent(false, operationName, src); logAuditEvent(false, operationName, src);
throw e; throw e;
} }
getEditLog().logSync();
logAuditEvent(true, operationName, src, null, resultingStat);
} }
/** /**
@ -7646,21 +7636,20 @@ void setErasureCodingPolicy(final String srcArg, final String ecPolicyName,
checkErasureCodingSupported(operationName); checkErasureCodingSupported(operationName);
FileStatus resultingStat = null; FileStatus resultingStat = null;
final FSPermissionChecker pc = getPermissionChecker(); final FSPermissionChecker pc = getPermissionChecker();
boolean success = false;
writeLock(); writeLock();
try { try {
checkOperation(OperationCategory.WRITE); checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot set erasure coding policy on " + srcArg); checkNameNodeSafeMode("Cannot set erasure coding policy on " + srcArg);
resultingStat = FSDirErasureCodingOp.setErasureCodingPolicy(this, resultingStat = FSDirErasureCodingOp.setErasureCodingPolicy(this,
srcArg, ecPolicyName, pc, logRetryCache); srcArg, ecPolicyName, pc, logRetryCache);
success = true; } catch (AccessControlException ace) {
logAuditEvent(false, operationName, srcArg);
throw ace;
} finally { } finally {
writeUnlock(operationName); writeUnlock(operationName);
if (success) { }
getEditLog().logSync(); getEditLog().logSync();
} logAuditEvent(true, operationName, srcArg, null, resultingStat);
logAuditEvent(success, operationName, srcArg, null, resultingStat);
}
} }
/** /**
@ -7679,7 +7668,6 @@ AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
checkErasureCodingSupported(operationName); checkErasureCodingSupported(operationName);
List<AddErasureCodingPolicyResponse> responses = List<AddErasureCodingPolicyResponse> responses =
new ArrayList<>(policies.length); new ArrayList<>(policies.length);
boolean success = false;
writeLock(); writeLock();
try { try {
checkOperation(OperationCategory.WRITE); checkOperation(OperationCategory.WRITE);
@ -7695,16 +7683,12 @@ AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
responses.add(new AddErasureCodingPolicyResponse(policy, e)); responses.add(new AddErasureCodingPolicyResponse(policy, e));
} }
} }
success = true;
return responses.toArray(new AddErasureCodingPolicyResponse[0]);
} finally { } finally {
writeUnlock(operationName); writeUnlock(operationName);
if (success) { }
getEditLog().logSync(); getEditLog().logSync();
} logAuditEvent(true, operationName, addECPolicyNames.toString());
logAuditEvent(success, operationName, addECPolicyNames.toString(), return responses.toArray(new AddErasureCodingPolicyResponse[0]);
null, null);
}
} }
/** /**
@ -7719,7 +7703,6 @@ void removeErasureCodingPolicy(String ecPolicyName,
final String operationName = "removeErasureCodingPolicy"; final String operationName = "removeErasureCodingPolicy";
checkOperation(OperationCategory.WRITE); checkOperation(OperationCategory.WRITE);
checkErasureCodingSupported(operationName); checkErasureCodingSupported(operationName);
boolean success = false;
writeLock(); writeLock();
try { try {
checkOperation(OperationCategory.WRITE); checkOperation(OperationCategory.WRITE);
@ -7727,14 +7710,11 @@ void removeErasureCodingPolicy(String ecPolicyName,
+ ecPolicyName); + ecPolicyName);
FSDirErasureCodingOp.removeErasureCodingPolicy(this, ecPolicyName, FSDirErasureCodingOp.removeErasureCodingPolicy(this, ecPolicyName,
logRetryCache); logRetryCache);
success = true;
} finally { } finally {
writeUnlock(operationName); writeUnlock(operationName);
if (success) { }
getEditLog().logSync(); getEditLog().logSync();
} logAuditEvent(true, operationName, ecPolicyName, null, null);
logAuditEvent(success, operationName, ecPolicyName, null, null);
}
} }
/** /**
@ -7763,12 +7743,12 @@ boolean enableErasureCodingPolicy(String ecPolicyName,
writeUnlock(operationName); writeUnlock(operationName);
} }
} catch (AccessControlException ace) { } catch (AccessControlException ace) {
logAuditEvent(false, operationName, ecPolicyName, null, null); logAuditEvent(false, operationName, ecPolicyName);
} finally { throw ace;
}
if (success) { if (success) {
getEditLog().logSync(); getEditLog().logSync();
logAuditEvent(success, operationName, ecPolicyName, null, null); logAuditEvent(true, operationName, ecPolicyName);
}
} }
return success; return success;
} }
@ -7799,12 +7779,12 @@ boolean disableErasureCodingPolicy(String ecPolicyName,
writeUnlock(operationName); writeUnlock(operationName);
} }
} catch (AccessControlException ace) { } catch (AccessControlException ace) {
logAuditEvent(false, operationName, ecPolicyName, null, null); logAuditEvent(false, operationName, ecPolicyName);
} finally { throw ace;
}
if (success) { if (success) {
getEditLog().logSync(); getEditLog().logSync();
logAuditEvent(success, operationName, ecPolicyName, null, null); logAuditEvent(true, operationName, ecPolicyName);
}
} }
return success; return success;
} }
@ -7824,21 +7804,17 @@ void unsetErasureCodingPolicy(final String srcArg,
checkErasureCodingSupported(operationName); checkErasureCodingSupported(operationName);
FileStatus resultingStat = null; FileStatus resultingStat = null;
final FSPermissionChecker pc = getPermissionChecker(); final FSPermissionChecker pc = getPermissionChecker();
boolean success = false;
writeLock(); writeLock();
try { try {
checkOperation(OperationCategory.WRITE); checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot unset erasure coding policy on " + srcArg); checkNameNodeSafeMode("Cannot unset erasure coding policy on " + srcArg);
resultingStat = FSDirErasureCodingOp.unsetErasureCodingPolicy(this, resultingStat = FSDirErasureCodingOp.unsetErasureCodingPolicy(this,
srcArg, pc, logRetryCache); srcArg, pc, logRetryCache);
success = true;
} finally { } finally {
writeUnlock(operationName); writeUnlock(operationName);
if (success) { }
getEditLog().logSync(); getEditLog().logSync();
} logAuditEvent(true, operationName, srcArg, null, resultingStat);
logAuditEvent(success, operationName, srcArg, null, resultingStat);
}
} }
/** /**
@ -8045,6 +8021,7 @@ void checkAccess(String src, FsAction mode) throws IOException {
logAuditEvent(false, operationName, src); logAuditEvent(false, operationName, src);
throw e; throw e;
} }
logAuditEvent(true, operationName, src);
} }
/** /**