big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-4163: Audit getQueueInfo and getApplications calls

Browse Source
This commit is contained in:
Eric Payne 2017-10-23 11:43:41 -05:00
parent 248d9b6fff
commit 921338cd86
3 changed files with 217 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -150,6 +150,7 @@
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
@ -925,6 +926,8 @@ public void remove() {
callerUGI.getUserName(), allowAccess));
}
RMAuditLogger.logSuccess(callerUGI.getUserName(),
AuditConstants.GET_APPLICATIONS_REQUEST, "ClientRMService");
GetApplicationsResponse response =
recordFactory.newRecordInstance(GetApplicationsResponse.class);
response.setApplicationList(reports);
@ -975,6 +978,13 @@ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request)
GetQueueInfoResponse response =
recordFactory.newRecordInstance(GetQueueInfoResponse.class);
RMAuditLogger.ArgsBuilder arguments = new RMAuditLogger.ArgsBuilder()
.append(Keys.QUEUENAME, request.getQueueName())
.append(Keys.INCLUDEAPPS,
String.valueOf(request.getIncludeApplications()))
.append(Keys.INCLUDECHILDQUEUES,
String.valueOf(request.getIncludeChildQueues()))
.append(Keys.RECURSIVE, String.valueOf(request.getRecursive()));
try {
QueueInfo queueInfo =
scheduler.getQueueInfo(request.getQueueName(),
@ -1001,8 +1011,14 @@ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request)
}
queueInfo.setApplications(appReports);
response.setQueueInfo(queueInfo);
RMAuditLogger.logSuccess(callerUGI.getUserName(),
AuditConstants.GET_QUEUE_INFO_REQUEST,
"ClientRMService", arguments);
} catch (IOException ioe) {
LOG.info("Failed to getQueueInfo for " + request.getQueueName(), ioe);
RMAuditLogger.logFailure(callerUGI.getUserName(),
AuditConstants.GET_QUEUE_INFO_REQUEST, "UNKNOWN", "ClientRMService",
ioe.getMessage(), arguments);
}
return response;

172
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
View File

@ -39,7 +39,8 @@ public class RMAuditLogger {
enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS,
DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID,
CALLERCONTEXT, CALLERSIGNATURE, RESOURCE}
CALLERCONTEXT, CALLERSIGNATURE, RESOURCE, QUEUENAME,
INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE}
public static class AuditConstants {
static final String SUCCESS = "SUCCESS";
@ -59,6 +60,10 @@ public static class AuditConstants {
= "Get Application Attempt Report";
public static final String GET_CONTAINERS = "Get Containers";
public static final String GET_CONTAINER_REPORT = "Get Container Report";
public static final String GET_QUEUE_INFO_REQUEST =
"Get Queue Info Request";
public static final String GET_APPLICATIONS_REQUEST =
"Get Applications Request";
public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded";
public static final String FINISH_FAILED_APP = "Application Finished - Failed";
public static final String FINISH_KILLED_APP = "Application Finished - Killed";
@ -97,12 +102,11 @@ static String createSuccessLog(String user, String operation, String target,
}
/**
* A helper api for creating an audit log for a successful event.
* A helper function for creating the common portion of a successful
* log message.
*/
static String createSuccessLog(String user, String operation, String target,
ApplicationId appId, ApplicationAttemptId attemptId,
ContainerId containerId, Resource resource, CallerContext callerContext,
InetAddress ip) {
private static StringBuilder createStringBuilderForSuccessEvent(String user,
String operation, String target, InetAddress ip) {
StringBuilder b = new StringBuilder();
start(Keys.USER, user, b);
if (ip != null) {
@ -111,6 +115,18 @@ static String createSuccessLog(String user, String operation, String target,
add(Keys.OPERATION, operation, b);
add(Keys.TARGET, target ,b);
add(Keys.RESULT, AuditConstants.SUCCESS, b);
return b;
}
/**
* A helper api for creating an audit log for a successful event.
*/
static String createSuccessLog(String user, String operation, String target,
ApplicationId appId, ApplicationAttemptId attemptId,
ContainerId containerId, Resource resource, CallerContext callerContext,
InetAddress ip) {
StringBuilder b =
createStringBuilderForSuccessEvent(user, operation, target, ip);
if (appId != null) {
add(Keys.APPID, appId.toString(), b);
}
@ -150,6 +166,20 @@ private static void appendCallerContext(StringBuilder sb, CallerContext callerCo
}
}
/**
* A general helper api for creating an audit log for a successful event.
*/
@SuppressWarnings("rawtypes")
static String createSuccessLog(String user, String operation, String target,
InetAddress ip, ArgsBuilder args) {
StringBuilder b =
createStringBuilderForSuccessEvent(user, operation, target, ip);
if(args != null) {
add(args, b);
}
return b.toString();
}
/**
* Create a readable and parseable audit log string for a successful event.
*
@ -172,12 +202,53 @@ public static void logSuccess(String user, String operation, String target,
}
}
/**
* Create a general readable and parseable audit log string for a successful
* event.
*
* @param user User who made the service request to the ResourceManager.
* @param operation Operation requested by the user.
* @param target The target on which the operation is being performed.
* @param args The ArgsBuilder arguments for the operation request.
*
* <br><br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
* <br>
* This method will attempt to retrieve the remote IP
*/
public static void logSuccess(String user, String operation, String target,
ArgsBuilder args) {
logSuccess(user, operation, target, Server.getRemoteIp(), args);
}
/**
* Create a general readable and parseable audit log string for a successful
* event.
*
* @param user User who made the service request to the ResourceManager.
* @param operation Operation requested by the user.
* @param target The target on which the operation is being performed.
* @param ip The ip address of the caller.
* @param args The ArgsBuilder arguments for the operation request.
*
* <br><br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logSuccess(String user, String operation, String target,
InetAddress ip, ArgsBuilder args) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, ip, args));
}
}
/**
* Create a readable and parseable audit log string for a successful event.
*
* @param user User who made the service request to the ResourceManager.
* @param operation Operation requested by the user.
* @param target The target on which the operation is being performed.
* @param target The target on which the operation is being performed.
* @param appId Application Id in which operation was performed.
* @param attemptId Application Attempt Id in which operation was performed.
*
@ -265,10 +336,8 @@ public static void logSuccess(String user, String operation, String target) {
}
}
static String createFailureLog(String user, String operation, String perm,
String target, String description, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
Resource resource, CallerContext callerContext) {
private static StringBuilder createStringBuilderForFailureLog(String user,
String operation, String target, String description, String perm) {
StringBuilder b = new StringBuilder();
start(Keys.USER, user, b);
addRemoteIP(b);
@ -277,6 +346,18 @@ static String createFailureLog(String user, String operation, String perm,
add(Keys.RESULT, AuditConstants.FAILURE, b);
add(Keys.DESCRIPTION, description, b);
add(Keys.PERMISSIONS, perm, b);
return b;
}
/**
* A helper api for creating an audit log for a failure event.
*/
static String createFailureLog(String user, String operation, String perm,
String target, String description, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
Resource resource, CallerContext callerContext) {
StringBuilder b = createStringBuilderForFailureLog(user,
operation, target, description, perm);
if (appId != null) {
add(Keys.APPID, appId.toString(), b);
}
@ -303,6 +384,20 @@ static String createFailureLog(String user, String operation, String perm,
attemptId, containerId, resource, null);
}
/**
* A helper api for creating an audit log for a failure event.
*/
@SuppressWarnings("rawtypes")
static String createFailureLog(String user, String operation, String perm,
String target, String description, ArgsBuilder args) {
StringBuilder b = createStringBuilderForFailureLog(user,
operation, target, description, perm);
if(args != null) {
add(args, b);
}
return b.toString();
}
/**
* Create a readable and parseable audit log string for a failed event.
*
@ -408,7 +503,29 @@ public static void logFailure(String user, String operation, String perm,
}
/**
* A helper api to add remote IP address
* Create a readable and parseable audit log string for a failed event.
*
* @param user User who made the service request.
* @param operation Operation requested by the user.
* @param perm Target permissions.
* @param target The target on which the operation is being performed.
* @param description The failure description
* @param args The arguments for the operation request.
*
* <br><br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logFailure(String user, String operation, String perm,
String target, String description, ArgsBuilder args) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
args));
}
}
/**
* A helper api to add remote IP address.
*/
static void addRemoteIP(StringBuilder b) {
InetAddress ip = Server.getRemoteIp();
@ -434,4 +551,35 @@ static void add(Keys key, String value, StringBuilder b) {
b.append(AuditConstants.PAIR_SEPARATOR).append(key.name())
.append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
}
/**
* Appends the key-val pair to the passed builder in the following format
* <pair-delim>key=value
*/
static void add(ArgsBuilder args, StringBuilder b) {
b.append(AuditConstants.PAIR_SEPARATOR).append(args.getArgs());
}
/**
* Builder to create and pass a list of arbitrary key value pairs for logging.
*/
public static class ArgsBuilder {
private StringBuilder b;
public ArgsBuilder() {
b = new StringBuilder();
}
public ArgsBuilder append(Keys key, String value) {
if (b.length() != 0) {
b.append(AuditConstants.PAIR_SEPARATOR);
}
b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
return this;
}
public StringBuilder getArgs() {
return b;
}
}
}

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
View File

@ -60,6 +60,7 @@ public class TestRMAuditLogger {
private static final String TARGET = "tgt";
private static final String PERM = "admin group";
private static final String DESC = "description of an audit log";
private static final String QUEUE = "root";
private static final ApplicationId APPID = mock(ApplicationId.class);
private static final ApplicationAttemptId ATTEMPTID = mock(ApplicationAttemptId.class);
private static final ContainerId CONTAINERID = mock(ContainerId.class);
@ -123,14 +124,22 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
CallerContext callerContext, Resource resource, InetAddress remoteIp) {
testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId,
callerContext, resource, remoteIp, null);
}
private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
CallerContext callerContext, Resource resource, InetAddress remoteIp,
RMAuditLogger.ArgsBuilder args) {
String sLog;
if (checkIP) {
sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
attemptId, containerId, resource, callerContext, remoteIp);
InetAddress tmpIp = checkIP ? remoteIp : null;
if (args != null) {
sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
tmpIp, args);
} else {
sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
attemptId, containerId, resource, callerContext, null);
attemptId, containerId, resource, callerContext, tmpIp);
}
StringBuilder expLog = new StringBuilder();
expLog.append("USER=test\t");
@ -165,6 +174,10 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
expLog.append("\tCALLERSIGNATURE=signature");
}
}
if (args != null) {
expLog.append("\tQUEUENAME=root");
expLog.append("\tRECURSIVE=true");
}
assertEquals(expLog.toString(), sLog);
}
@ -241,6 +254,10 @@ private void testSuccessLogFormat(boolean checkIP) {
testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE)
.build(), RESOURCE);
RMAuditLogger.ArgsBuilder args = new RMAuditLogger.ArgsBuilder()
.append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true");
testSuccessLogFormatHelper(checkIP, null, null, null, null, null,
Server.getRemoteIp(), args);
testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID);
testSuccessLogNulls(checkIP);
}
@ -256,9 +273,19 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
CallerContext callerContext, Resource resource) {
String fLog =
testFailureLogFormatHelper(checkIP, appId, attemptId, containerId,
callerContext, resource, null);
}
private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
CallerContext callerContext, Resource resource,
RMAuditLogger.ArgsBuilder args) {
String fLog = args == null ?
RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
appId, attemptId, containerId, resource, callerContext);
appId, attemptId, containerId, resource, callerContext) :
RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
args);
StringBuilder expLog = new StringBuilder();
expLog.append("USER=test\t");
if (checkIP) {
@ -288,6 +315,10 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
expLog.append("\tCALLERSIGNATURE=signature");
}
}
if (args != null) {
expLog.append("\tQUEUENAME=root");
expLog.append("\tRECURSIVE=true");
}
assertEquals(expLog.toString(), fLog);
}
@ -314,6 +345,10 @@ private void testFailureLogFormat(boolean checkIP) {
testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE)
.build(), RESOURCE);
RMAuditLogger.ArgsBuilder args = new RMAuditLogger.ArgsBuilder()
.append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true");
testFailureLogFormatHelper(checkIP, null, null, null, null, null,
args);
}
/**