From 3d78b1223d3fdc29d500803cefd2931b54f44928 Mon Sep 17 00:00:00 2001 From: Jonathan Hung Date: Mon, 23 Sep 2019 11:42:41 -0700 Subject: [PATCH] YARN-9762. Add submission context label to audit logs. Contributed by Manoj Kumar --- .../resourcemanager/ClientRMService.java | 6 ++- .../server/resourcemanager/RMAuditLogger.java | 53 +++++++++++++++++-- .../resourcemanager/TestRMAuditLogger.java | 21 ++++++-- 3 files changed, 69 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 2b93ca777a..f9681e0856 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -695,13 +695,15 @@ public SubmitApplicationResponse submitApplication( " submitted by user " + user); RMAuditLogger.logSuccess(user, AuditConstants.SUBMIT_APP_REQUEST, "ClientRMService", applicationId, callerContext, - submissionContext.getQueue()); + submissionContext.getQueue(), + submissionContext.getNodeLabelExpression()); } catch (YarnException e) { LOG.info("Exception in submitting " + applicationId, e); RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, e.getMessage(), "ClientRMService", "Exception in submitting application", applicationId, callerContext, - submissionContext.getQueue()); + submissionContext.getQueue(), + submissionContext.getNodeLabelExpression()); throw e; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index b24cac91dc..854b6ca64e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -271,6 +271,16 @@ public static void logSuccess(String user, String operation, String target, } } + public static void logSuccess(String user, String operation, String target, + ApplicationId appId, CallerContext callerContext, String queueName, + String partition) { + if (LOG.isInfoEnabled()) { + LOG.info( + createSuccessLog(user, operation, target, appId, null, null, null, + callerContext, Server.getRemoteIp(), queueName, partition)); + } + } + /** * Create a readable and parseable audit log string for a successful event. * @@ -391,7 +401,8 @@ private static StringBuilder createStringBuilderForFailureLog(String user, static String createFailureLog(String user, String operation, String perm, String target, String description, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, - Resource resource, CallerContext callerContext, String queueName) { + Resource resource, CallerContext callerContext, String queueName, + String partition) { StringBuilder b = createStringBuilderForFailureLog(user, operation, target, description, perm); if (appId != null) { @@ -410,6 +421,10 @@ static String createFailureLog(String user, String operation, String perm, if (queueName != null) { add(Keys.QUEUENAME, queueName, b); } + if (partition != null) { + add(Keys.NODELABEL, partition, b); + } + return b.toString(); } @@ -420,7 +435,7 @@ static String createFailureLog(String user, String operation, String perm, String target, String description, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, Resource resource) { return createFailureLog(user, operation, perm, target, description, appId, - attemptId, containerId, resource, null, null); + attemptId, containerId, resource, null, null, null); } /** @@ -492,7 +507,7 @@ public static void logFailure(String user, String operation, String perm, CallerContext callerContext) { if (LOG.isWarnEnabled()) { LOG.warn(createFailureLog(user, operation, perm, target, description, - appId, null, null, null, callerContext, null)); + appId, null, null, null, callerContext, null, null)); } } @@ -501,7 +516,7 @@ public static void logFailure(String user, String operation, String perm, CallerContext callerContext, String queueName) { if (LOG.isWarnEnabled()) { LOG.warn(createFailureLog(user, operation, perm, target, description, - appId, null, null, null, callerContext, queueName)); + appId, null, null, null, callerContext, queueName, null)); } } @@ -533,7 +548,7 @@ public static void logFailure(String user, String operation, String perm, String queueName) { if (LOG.isWarnEnabled()) { LOG.warn(createFailureLog(user, operation, perm, target, description, - appId, null, null, null, null, queueName)); + appId, null, null, null, null, queueName, null)); } } @@ -581,6 +596,34 @@ public static void logFailure(String user, String operation, String perm, } } + /** + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param perm Target permissions. + * @param target The target on which the operation is being performed. + * @param description Some additional information as to why the operation + * failed. + * @param appId ApplicationId in which operation was performed. + * @param callerContext Caller context + * @param queueName Name of queue. + * @param partition Name of labeled partition. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, String perm, + String target, String description, ApplicationId appId, + CallerContext callerContext, String queueName, String partition) { + if (LOG.isWarnEnabled()) { + LOG.warn( + createFailureLog(user, operation, perm, target, description, appId, + null, null, null, callerContext, queueName, partition)); + } + } + /** * A helper api to add remote IP address. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java index 7d8eeab18e..84ea2d633e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java @@ -293,16 +293,17 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, CallerContext callerContext, Resource resource) { testFailureLogFormatHelper(checkIP, appId, attemptId, containerId, - callerContext, resource, null); + callerContext, resource, null, null, null); } private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, CallerContext callerContext, Resource resource, - RMAuditLogger.ArgsBuilder args) { + String queueName, String partition, RMAuditLogger.ArgsBuilder args) { String fLog = args == null ? RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, - appId, attemptId, containerId, resource, callerContext, null) : + appId, attemptId, containerId, resource, callerContext, + queueName, partition) : RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, args); StringBuilder expLog = new StringBuilder(); @@ -334,6 +335,12 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, expLog.append("\tCALLERSIGNATURE=signature"); } } + if (queueName != null) { + expLog.append("\tQUEUENAME=" + QUEUE); + } + if (partition != null) { + expLog.append("\tNODELABEL=" + PARTITION); + } if (args != null) { expLog.append("\tQUEUENAME=root"); expLog.append("\tRECURSIVE=true"); @@ -364,10 +371,16 @@ private void testFailureLogFormat(boolean checkIP) { testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) .build(), RESOURCE); + testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, + new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) + .build(), RESOURCE, QUEUE, null, null); + testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, + new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) + .build(), RESOURCE, QUEUE, PARTITION, null); RMAuditLogger.ArgsBuilder args = new RMAuditLogger.ArgsBuilder() .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true"); testFailureLogFormatHelper(checkIP, null, null, null, null, null, - args); + null, null, args); } /**