big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11239. Optimize FederationClientInterceptor audit log. (#5127)

Browse Source
This commit is contained in:
slfan1989 2023-04-15 04:09:18 +08:00 committed by GitHub
parent 0185afafea
commit 0bcdea7912
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 484 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterAuditLogger.java
View File

@ -30,7 +30,7 @@
* Manages Router audit logs.
* Audit log format is written as key=value pairs. Tab separated.
*/
public class RouterAuditLogger {
public final class RouterAuditLogger {
private static final Logger LOG =
LoggerFactory.getLogger(RouterAuditLogger.class);
@ -51,6 +51,43 @@ public static class AuditConstants {
public static final String GET_APP_REPORT = "Get Application Report";
public static final String TARGET_CLIENT_RM_SERVICE = "RouterClientRMService";
public static final String UNKNOWN = "UNKNOWN";
public static final String GET_APPLICATIONS = "Get Applications";
public static final String GET_CLUSTERMETRICS = "Get ClusterMetrics";
public static final String GET_CLUSTERNODES = "Get ClusterNodes";
public static final String GET_QUEUEINFO = "Get QueueInfo";
public static final String GET_QUEUE_USER_ACLS = "Get QueueUserAcls";
public static final String MOVE_APPLICATION_ACROSS_QUEUES = "Move ApplicationAcrossQueues";
public static final String GET_NEW_RESERVATION = "Get NewReservation";
public static final String SUBMIT_RESERVATION = "Submit Reservation";
public static final String LIST_RESERVATIONS = "List Reservations";
public static final String UPDATE_RESERVATION = "Update Reservation";
public static final String DELETE_RESERVATION = "Delete Reservation";
public static final String GET_NODETOLABELS = "Get NodeToLabels";
public static final String GET_LABELSTONODES = "Get LabelsToNodes";
public static final String GET_CLUSTERNODELABELS = "Get ClusterNodeLabels";
public static final String GET_APPLICATION_ATTEMPT_REPORT = "Get ApplicationAttemptReport";
public static final String GET_APPLICATION_ATTEMPTS = "Get ApplicationAttempts";
public static final String GET_CONTAINERREPORT = "Get ContainerReport";
public static final String GET_CONTAINERS = "Get Containers";
public static final String GET_DELEGATIONTOKEN = "Get DelegationToken";
public static final String RENEW_DELEGATIONTOKEN = "Renew DelegationToken";
public static final String CANCEL_DELEGATIONTOKEN = "Cancel DelegationToken";
public static final String FAIL_APPLICATIONATTEMPT = "Fail ApplicationAttempt";
public static final String UPDATE_APPLICATIONPRIORITY = "Update ApplicationPriority";
public static final String SIGNAL_TOCONTAINER = "Signal ToContainer";
public static final String UPDATE_APPLICATIONTIMEOUTS = "Update ApplicationTimeouts";
public static final String GET_RESOURCEPROFILES = "Get ResourceProfiles";
public static final String GET_RESOURCEPROFILE = "Get ResourceProfile";
public static final String GET_RESOURCETYPEINFO = "Get ResourceTypeInfo";
public static final String GET_ATTRIBUTESTONODES = "Get AttributesToNodes";
public static final String GET_CLUSTERNODEATTRIBUTES = "Get ClusterNodeAttributes";
public static final String GET_NODESTOATTRIBUTES = "Get NodesToAttributes";
}
public static void logSuccess(String user, String operation, String target) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, null, null));
}
}
/**
@ -146,6 +183,28 @@ public static void logFailure(String user, String operation, String perm,
}
}
/**
* Create a readable and parseable audit log string for a failed event.
*
* @param user User who made the service request.
* @param operation Operation requested by the user.
* @param perm Target permissions.
* @param target The target on which the operation is being performed.
* @param descriptionFormat the description message format string.
* @param args format parameter.
*
* <br><br>
* Note that the {@link RouterAuditLogger} uses tabs ('\t') as a key-val
* delimiter and hence the value fields should not contains tabs ('\t').
*/
public static void logFailure(String user, String operation, String perm,
String target, String descriptionFormat, Object... args) {
if (LOG.isInfoEnabled()) {
String description = String.format(descriptionFormat, args);
LOG.info(createFailureLog(user, operation, perm, target, description, null, null));
}
}
/**
* Create a readable and parseable audit log string for a failed event.
*

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java
View File

@ -131,6 +131,19 @@ public static void logAndThrowException(String errMsg, Throwable t)
}
}
/**
* Throws an exception due to an error.
*
* @param errMsg the error message
* @throws YarnException on failure
*/
@Public
@Unstable
public static void logAndThrowException(String errMsg) throws YarnException {
LOG.error(errMsg);
throw new YarnException(errMsg);
}
public static <R> R createRequestInterceptorChain(Configuration conf, String pipeLineClassName,
String interceptorClassName, Class<R> clazz) {

526
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
View File

File diff suppressed because it is too large Load Diff

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterAuditLogger.java
View File

@ -65,7 +65,7 @@ public class TestRouterAuditLogger {
* Test the AuditLog format with key-val pair.
*/
@Test
public void testKeyValLogFormat() throws Exception {
public void testKeyValLogFormat() {
StringBuilder actLog = new StringBuilder();
StringBuilder expLog = new StringBuilder();
@ -80,7 +80,7 @@ public void testKeyValLogFormat() throws Exception {
assertEquals(expLog.toString(), actLog.toString());
// append another k1=null pair and test
RouterAuditLogger.add(RouterAuditLogger.Keys.APPID, (String) null, actLog);
RouterAuditLogger.add(RouterAuditLogger.Keys.APPID, null, actLog);
expLog.append("\tAPPID=null");
assertEquals(expLog.toString(), actLog.toString());
@ -102,7 +102,10 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
expLog.append("USER=test\t");
if (checkIP) {
InetAddress ip = Server.getRemoteIp();
expLog.append(RouterAuditLogger.Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
if (ip != null && ip.getHostAddress() != null) {
expLog.append(RouterAuditLogger.Keys.IP.name())
.append("=").append(ip.getHostAddress()).append("\t");
}
}
expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");
if (appId != null) {
@ -149,7 +152,11 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
expLog.append("USER=test\t");
if (checkIP) {
InetAddress ip = Server.getRemoteIp();
expLog.append(RouterAuditLogger.Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
if (ip != null && ip.getHostAddress() != null) {
expLog.append(RouterAuditLogger.Keys.IP.name())
.append("=")
.append(ip.getHostAddress()).append("\t");
}
}
expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t");
expLog.append("DESCRIPTION=description of an audit log");
@ -179,7 +186,7 @@ private void testFailureLogFormat(boolean checkIP) {
* Test {@link RouterAuditLogger}.
*/
@Test
public void testRouterAuditLoggerWithOutIP() throws Exception {
public void testRouterAuditLoggerWithOutIP() {
testSuccessLogFormat(false);
testFailureLogFormat(false);
}