diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java new file mode 100644 index 0000000000..cb4021f8ee --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java @@ -0,0 +1,201 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager; + +import java.net.InetAddress; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; + +/** + * Manages NodeManager audit logs. + * + * Audit log format is written as key=value pairs. Tab separated. + */ +public class NMAuditLogger { + private static final Log LOG = LogFactory.getLog(NMAuditLogger.class); + + static enum Keys {USER, OPERATION, TARGET, RESULT, IP, + DESCRIPTION, APPID, CONTAINERID} + + public static class AuditConstants { + static final String SUCCESS = "SUCCESS"; + static final String FAILURE = "FAILURE"; + static final String KEY_VAL_SEPARATOR = "="; + static final char PAIR_SEPARATOR = '\t'; + + // Some commonly used descriptions + public static final String START_CONTAINER = "Start Container Request"; + public static final String STOP_CONTAINER = "Stop Container Request"; + public static final String FINISH_SUCCESS_CONTAINER = "Container Finished - Succeeded"; + public static final String FINISH_FAILED_CONTAINER = "Container Finished - Failed"; + public static final String FINISH_KILLED_CONTAINER = "Container Finished - Killed"; + } + + /** + * A helper api for creating an audit log for a successful event. + */ + static String createSuccessLog(String user, String operation, String target, + ApplicationId appId, ContainerId containerId) { + StringBuilder b = new StringBuilder(); + start(Keys.USER, user, b); + addRemoteIP(b); + add(Keys.OPERATION, operation, b); + add(Keys.TARGET, target ,b); + add(Keys.RESULT, AuditConstants.SUCCESS, b); + if (appId != null) { + add(Keys.APPID, appId.toString(), b); + } + if (containerId != null) { + add(Keys.CONTAINERID, containerId.toString(), b); + } + return b.toString(); + } + + /** + * Create a readable and parseable audit log string for a successful event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user + * @param target The target on which the operation is being performed. + * @param appId Application Id in which operation was performed. + * @param containerId Container Id in which operation was performed. + * + *

+ * Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + ApplicationId appId, ContainerId containerId) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, appId, containerId)); + } + } + + /** + * Create a readable and parseable audit log string for a successful event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user + * @param target The target on which the operation is being performed. + * + *

+ * Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, null, null)); + } + } + + /** + * A helper api for creating an audit log for a failure event. + * This is factored out for testing purpose. + */ + static String createFailureLog(String user, String operation, String target, + String description, ApplicationId appId, ContainerId containerId) { + StringBuilder b = new StringBuilder(); + start(Keys.USER, user, b); + addRemoteIP(b); + add(Keys.OPERATION, operation, b); + add(Keys.TARGET, target ,b); + add(Keys.RESULT, AuditConstants.FAILURE, b); + add(Keys.DESCRIPTION, description, b); + if (appId != null) { + add(Keys.APPID, appId.toString(), b); + } + if (containerId != null) { + add(Keys.CONTAINERID, containerId.toString(), b); + } + return b.toString(); + } + + /** + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param description Some additional information as to why the operation + * failed. + * @param appId ApplicationId in which operation was performed. + * @param containerId Container Id in which operation was performed. + * + *

+ * Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, String target, + String description, ApplicationId appId, ContainerId containerId) { + if (LOG.isWarnEnabled()) { + LOG.warn(createFailureLog(user, operation, target, description, appId, containerId)); + } + } + + /** + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param description Some additional information as to why the operation + * failed. + * + *

+ * Note that the {@link NMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, + String target, String description) { + if (LOG.isWarnEnabled()) { + LOG.warn(createFailureLog(user, operation, target, description, null, null)); + } + } + + /** + * A helper api to add remote IP address + */ + static void addRemoteIP(StringBuilder b) { + InetAddress ip = Server.getRemoteIp(); + // ip address can be null for testcases + if (ip != null) { + add(Keys.IP, ip.getHostAddress(), b); + } + } + + /** + * Adds the first key-val pair to the passed builder in the following format + * key=value + */ + static void start(Keys key, String value, StringBuilder b) { + b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value); + } + + /** + * Appends the key-val pair to the passed builder in the following format + * key=value + */ + static void add(Keys key, String value, StringBuilder b) { + b.append(AuditConstants.PAIR_SEPARATOR).append(key.name()) + .append(AuditConstants.KEY_VAL_SEPARATOR).append(value); + } +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java new file mode 100644 index 0000000000..b642279206 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java @@ -0,0 +1,227 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager; + +import java.net.InetAddress; +import java.net.InetSocketAddress; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.ipc.TestRPC.TestImpl; +import org.apache.hadoop.ipc.TestRPC.TestProtocol; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger; +import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants; +import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.Keys; + +import org.apache.hadoop.net.NetUtils; + +import static org.mockito.Mockito.*; +import static junit.framework.Assert.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + + +/** + * Tests {@link NMAuditLogger}. + */ +public class TestNMAuditLogger { + private static final Log LOG = LogFactory.getLog(TestNMAuditLogger.class); + private static final String USER = "test"; + private static final String OPERATION = "oper"; + private static final String TARGET = "tgt"; + private static final String DESC = "description of an audit log"; + + private static final ApplicationId APPID = mock(ApplicationId.class); + private static final ContainerId CONTAINERID = mock(ContainerId.class); + + @Before + public void setUp() throws Exception { + when(APPID.toString()).thenReturn("app_1"); + when(CONTAINERID.toString()).thenReturn("container_1"); + } + + + /** + * Test the AuditLog format with key-val pair. + */ + @Test + public void testKeyValLogFormat() throws Exception { + StringBuilder actLog = new StringBuilder(); + StringBuilder expLog = new StringBuilder(); + // add the first k=v pair and check + NMAuditLogger.start(Keys.USER, USER, actLog); + expLog.append("USER=test"); + assertEquals(expLog.toString(), actLog.toString()); + + // append another k1=v1 pair to already added k=v and test + NMAuditLogger.add(Keys.OPERATION, OPERATION, actLog); + expLog.append("\tOPERATION=oper"); + assertEquals(expLog.toString(), actLog.toString()); + + // append another k1=null pair and test + NMAuditLogger.add(Keys.APPID, (String)null, actLog); + expLog.append("\tAPPID=null"); + assertEquals(expLog.toString(), actLog.toString()); + + // now add the target and check of the final string + NMAuditLogger.add(Keys.TARGET, TARGET, actLog); + expLog.append("\tTARGET=tgt"); + assertEquals(expLog.toString(), actLog.toString()); + } + + + /** + * Test the AuditLog format for successful events. + */ + private void testSuccessLogFormatHelper(boolean checkIP, + ApplicationId appId, ContainerId containerId) { + // check without the IP + String sLog = NMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, + appId, containerId); + StringBuilder expLog = new StringBuilder(); + expLog.append("USER=test\t"); + if (checkIP) { + InetAddress ip = Server.getRemoteIp(); + expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t"); + } + expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS"); + if (appId != null) { + expLog.append("\tAPPID=app_1"); + } + if (containerId != null) { + expLog.append("\tCONTAINERID=container_1"); + } + assertEquals(expLog.toString(), sLog); + } + + /** + * Test the AuditLog format for successful events passing nulls. + */ + private void testSuccessLogNulls(boolean checkIP) { + String sLog = NMAuditLogger.createSuccessLog(null, null, null, + null, null); + StringBuilder expLog = new StringBuilder(); + expLog.append("USER=null\t"); + if (checkIP) { + InetAddress ip = Server.getRemoteIp(); + expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t"); + } + expLog.append("OPERATION=null\tTARGET=null\tRESULT=SUCCESS"); + assertEquals(expLog.toString(), sLog); + } + + /** + * Test the AuditLog format for successful events with the various + * parameters. + */ + private void testSuccessLogFormat(boolean checkIP) { + testSuccessLogFormatHelper(checkIP, null, null); + testSuccessLogFormatHelper(checkIP, APPID, null); + testSuccessLogFormatHelper(checkIP, null, CONTAINERID); + testSuccessLogFormatHelper(checkIP, APPID, CONTAINERID); + testSuccessLogNulls(checkIP); + } + + + /** + * Test the AuditLog format for failure events. + */ + private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, + ContainerId containerId) { + String fLog = + NMAuditLogger.createFailureLog(USER, OPERATION, TARGET, DESC, appId, + containerId); + StringBuilder expLog = new StringBuilder(); + expLog.append("USER=test\t"); + if (checkIP) { + InetAddress ip = Server.getRemoteIp(); + expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t"); + } + expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t"); + expLog.append("DESCRIPTION=description of an audit log"); + + if (appId != null) { + expLog.append("\tAPPID=app_1"); + } + if (containerId != null) { + expLog.append("\tCONTAINERID=container_1"); + } + assertEquals(expLog.toString(), fLog); + } + + /** + * Test the AuditLog format for failure events with the various + * parameters. + */ + private void testFailureLogFormat(boolean checkIP) { + testFailureLogFormatHelper(checkIP, null, null); + testFailureLogFormatHelper(checkIP, APPID, null); + testFailureLogFormatHelper(checkIP, null, CONTAINERID); + testFailureLogFormatHelper(checkIP, APPID, CONTAINERID); + } + + /** + * Test {@link NMAuditLogger} without IP set. + */ + @Test + public void testNMAuditLoggerWithoutIP() throws Exception { + // test without ip + testSuccessLogFormat(false); + testFailureLogFormat(false); + } + + /** + * A special extension of {@link TestImpl} RPC server with + * {@link TestImpl#ping()} testing the audit logs. + */ + private class MyTestRPCServer extends TestImpl { + @Override + public void ping() { + // test with ip set + testSuccessLogFormat(true); + testFailureLogFormat(true); + } + } + + /** + * Test {@link NMAuditLogger} with IP set. + */ + @Test + public void testNMAuditLoggerWithIP() throws Exception { + Configuration conf = new Configuration(); + // start the IPC server + Server server = RPC.getServer(new MyTestRPCServer(), "0.0.0.0", 0, conf); + server.start(); + + InetSocketAddress addr = NetUtils.getConnectAddress(server); + + // Make a client connection and test the audit log + TestProtocol proxy = (TestProtocol)RPC.getProxy(TestProtocol.class, + TestProtocol.versionID, addr, conf); + // Start the testcase + proxy.ping(); + + server.stop(); + } +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java new file mode 100644 index 0000000000..b9261cac20 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -0,0 +1,309 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager; + +import java.net.InetAddress; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; + +/** + * Manages ResourceManager audit logs. + * + * Audit log format is written as key=value pairs. Tab separated. + */ +public class RMAuditLogger { + private static final Log LOG = LogFactory.getLog(RMAuditLogger.class); + + static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS, + DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID} + + public static class AuditConstants { + static final String SUCCESS = "SUCCESS"; + static final String FAILURE = "FAILURE"; + static final String KEY_VAL_SEPARATOR = "="; + static final char PAIR_SEPARATOR = '\t'; + + public static final String KILL_APP_REQUEST = "Kill Application Request"; + public static final String SUBMIT_APP_REQUEST = "Submit Application Request"; + public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded"; + public static final String FINISH_FAILED_APP = "Application Finished - Failed"; + public static final String FINISH_KILLED_APP = "Application Finished - Killed"; + public static final String REGISTER_AM = "Register App Master"; + public static final String ALLOC_CONTAINER = "AM Allocated Container"; + public static final String RELEASE_CONTAINER = "AM Released Container"; + + // Some commonly used descriptions + public static final String UNAUTHORIZED_USER = "Unauthorized user"; + } + + /** + * A helper api for creating an audit log for a successful event. + */ + static String createSuccessLog(String user, String operation, String target, + ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) { + StringBuilder b = new StringBuilder(); + start(Keys.USER, user, b); + addRemoteIP(b); + add(Keys.OPERATION, operation, b); + add(Keys.TARGET, target ,b); + add(Keys.RESULT, AuditConstants.SUCCESS, b); + if (appId != null) { + add(Keys.APPID, appId.toString(), b); + } + if (attemptId != null) { + add(Keys.APPATTEMPTID, attemptId.toString(), b); + } + if (containerId != null) { + add(Keys.CONTAINERID, containerId.toString(), b); + } + return b.toString(); + } + + /** + * Create a readable and parseable audit log string for a successful event. + * + * @param user User who made the service request to the ResourceManager + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param appId Application Id in which operation was performed. + * @param containerId Container Id in which operation was performed. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + ApplicationId appId, ContainerId containerId) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, appId, null, + containerId)); + } + } + + /** + * Create a readable and parseable audit log string for a successful event. + * + * @param user User who made the service request to the ResourceManager. + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param appId Application Id in which operation was performed. + * @param attemptId Application Attempt Id in which operation was performed. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + ApplicationId appId, ApplicationAttemptId attemptId) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, appId, attemptId, + null)); + } + } + + + /** + * Create a readable and parseable audit log string for a successful event. + * + * @param user User who made the service request to the ResourceManager. + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param appId Application Id in which operation was performed. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + ApplicationId appId) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, appId, null, null)); + } + } + + /** + * Create a readable and parseable audit log string for a successful event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, null, null, null)); + } + } + + /** + * A helper api for creating an audit log for a failure event. + */ + static String createFailureLog(String user, String operation, String perm, + String target, String description, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId) { + StringBuilder b = new StringBuilder(); + start(Keys.USER, user, b); + addRemoteIP(b); + add(Keys.OPERATION, operation, b); + add(Keys.TARGET, target ,b); + add(Keys.RESULT, AuditConstants.FAILURE, b); + add(Keys.DESCRIPTION, description, b); + add(Keys.PERMISSIONS, perm, b); + if (appId != null) { + add(Keys.APPID, appId.toString(), b); + } + if (attemptId != null) { + add(Keys.APPATTEMPTID, attemptId.toString(), b); + } + if (containerId != null) { + add(Keys.CONTAINERID, containerId.toString(), b); + } + return b.toString(); + } + + /** + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param perm Target permissions. + * @param target The target on which the operation is being performed. + * @param description Some additional information as to why the operation + * failed. + * @param appId Application Id in which operation was performed. + * @param containerId Container Id in which operation was performed. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, String perm, + String target, String description, ApplicationId appId, + ContainerId containerId) { + if (LOG.isWarnEnabled()) { + LOG.warn(createFailureLog(user, operation, perm, target, description, + appId, null, containerId)); + } + } + + /** + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param perm Target permissions. + * @param target The target on which the operation is being performed. + * @param description Some additional information as to why the operation + * failed. + * @param appId ApplicationId in which operation was performed. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, String perm, + String target, String description, ApplicationId appId, + ApplicationAttemptId attemptId) { + if (LOG.isWarnEnabled()) { + LOG.warn(createFailureLog(user, operation, perm, target, description, + appId, attemptId, null)); + } + } + + + /** + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param perm Target permissions. + * @param target The target on which the operation is being performed. + * @param description Some additional information as to why the operation + * failed. + * @param appId ApplicationId in which operation was performed. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, String perm, + String target, String description, ApplicationId appId) { + if (LOG.isWarnEnabled()) { + LOG.warn(createFailureLog(user, operation, perm, target, description, + appId, null, null)); + } + } + + /** + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param perm Target permissions. + * @param target The target on which the operation is being performed. + * @param description Some additional information as to why the operation + * failed. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, String perm, + String target, String description) { + if (LOG.isWarnEnabled()) { + LOG.warn(createFailureLog(user, operation, perm, target, description, + null, null, null)); + } + } + + /** + * A helper api to add remote IP address + */ + static void addRemoteIP(StringBuilder b) { + InetAddress ip = Server.getRemoteIp(); + // ip address can be null for testcases + if (ip != null) { + add(Keys.IP, ip.getHostAddress(), b); + } + } + + /** + * Adds the first key-val pair to the passed builder in the following format + * key=value + */ + static void start(Keys key, String value, StringBuilder b) { + b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value); + } + + /** + * Appends the key-val pair to the passed builder in the following format + * key=value + */ + static void add(Keys key, String value, StringBuilder b) { + b.append(AuditConstants.PAIR_SEPARATOR).append(key.name()) + .append(AuditConstants.KEY_VAL_SEPARATOR).append(value); + } +} diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java new file mode 100644 index 0000000000..9291b49aba --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java @@ -0,0 +1,244 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager; + +import java.net.InetAddress; +import java.net.InetSocketAddress; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.ipc.TestRPC.TestImpl; +import org.apache.hadoop.ipc.TestRPC.TestProtocol; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys; + +import org.apache.hadoop.net.NetUtils; + +import static org.mockito.Mockito.*; +import static junit.framework.Assert.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + + +/** + * Tests {@link RMAuditLogger}. + */ +public class TestRMAuditLogger { + private static final Log LOG = LogFactory.getLog(TestRMAuditLogger.class); + private static final String USER = "test"; + private static final String OPERATION = "oper"; + private static final String TARGET = "tgt"; + private static final String PERM = "admin group"; + private static final String DESC = "description of an audit log"; + private static final ApplicationId APPID = mock(ApplicationId.class); + private static final ApplicationAttemptId ATTEMPTID = mock(ApplicationAttemptId.class); + private static final ContainerId CONTAINERID = mock(ContainerId.class); + + @Before + public void setUp() throws Exception { + when(APPID.toString()).thenReturn("app_1"); + when(ATTEMPTID.toString()).thenReturn("app_attempt_1"); + when(CONTAINERID.toString()).thenReturn("container_1"); + } + + + /** + * Test the AuditLog format with key-val pair. + */ + @Test + public void testKeyValLogFormat() throws Exception { + StringBuilder actLog = new StringBuilder(); + StringBuilder expLog = new StringBuilder(); + // add the first k=v pair and check + RMAuditLogger.start(Keys.USER, USER, actLog); + expLog.append("USER=test"); + assertEquals(expLog.toString(), actLog.toString()); + + // append another k1=v1 pair to already added k=v and test + RMAuditLogger.add(Keys.OPERATION, OPERATION, actLog); + expLog.append("\tOPERATION=oper"); + assertEquals(expLog.toString(), actLog.toString()); + + // append another k1=null pair and test + RMAuditLogger.add(Keys.APPID, (String)null, actLog); + expLog.append("\tAPPID=null"); + assertEquals(expLog.toString(), actLog.toString()); + + // now add the target and check of the final string + RMAuditLogger.add(Keys.TARGET, TARGET, actLog); + expLog.append("\tTARGET=tgt"); + assertEquals(expLog.toString(), actLog.toString()); + } + + + /** + * Test the AuditLog format for successful events. + */ + private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId) { + String sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, + appId, attemptId, containerId); + StringBuilder expLog = new StringBuilder(); + expLog.append("USER=test\t"); + if (checkIP) { + InetAddress ip = Server.getRemoteIp(); + expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t"); + } + expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS"); + + if (appId != null) { + expLog.append("\tAPPID=app_1"); + } + if (attemptId != null) { + expLog.append("\tAPPATTEMPTID=app_attempt_1"); + } + if (containerId != null) { + expLog.append("\tCONTAINERID=container_1"); + } + assertEquals(expLog.toString(), sLog); + } + + /** + * Test the AuditLog format for successful events passing nulls. + */ + private void testSuccessLogNulls(boolean checkIP) { + String sLog = RMAuditLogger.createSuccessLog(null, null, null, null, + null, null); + StringBuilder expLog = new StringBuilder(); + expLog.append("USER=null\t"); + if (checkIP) { + InetAddress ip = Server.getRemoteIp(); + expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t"); + } + expLog.append("OPERATION=null\tTARGET=null\tRESULT=SUCCESS"); + assertEquals(expLog.toString(), sLog); + } + + /** + * Test the AuditLog format for successful events with the various + * parameters. + */ + private void testSuccessLogFormat(boolean checkIP) { + testSuccessLogFormatHelper(checkIP, null, null, null); + testSuccessLogFormatHelper(checkIP, APPID, null, null); + testSuccessLogFormatHelper(checkIP, null, null, CONTAINERID); + testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, null); + testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, null); + testSuccessLogFormatHelper(checkIP, APPID, null, CONTAINERID); + testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID); + testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID); + testSuccessLogNulls(checkIP); + } + + + /** + * Test the AuditLog format for failure events. + */ + private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId) { + String fLog = + RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, + appId, attemptId, containerId); + StringBuilder expLog = new StringBuilder(); + expLog.append("USER=test\t"); + if (checkIP) { + InetAddress ip = Server.getRemoteIp(); + expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t"); + } + expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=FAILURE\t"); + expLog.append("DESCRIPTION=description of an audit log"); + expLog.append("\tPERMISSIONS=admin group"); + if (appId != null) { + expLog.append("\tAPPID=app_1"); + } + if (attemptId != null) { + expLog.append("\tAPPATTEMPTID=app_attempt_1"); + } + if (containerId != null) { + expLog.append("\tCONTAINERID=container_1"); + } + assertEquals(expLog.toString(), fLog); + } + + /** + * Test the AuditLog format for failure events with the various + * parameters. + */ + private void testFailureLogFormat(boolean checkIP) { + testFailureLogFormatHelper(checkIP, null, null, null); + testFailureLogFormatHelper(checkIP, APPID, null, null); + testFailureLogFormatHelper(checkIP, null, null, CONTAINERID); + testFailureLogFormatHelper(checkIP, null, ATTEMPTID, null); + testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, null); + testFailureLogFormatHelper(checkIP, APPID, null, CONTAINERID); + testFailureLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID); + testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID); + } + + /** + * Test {@link RMAuditLogger} without IP set. + */ + @Test + public void testRMAuditLoggerWithoutIP() throws Exception { + // test without ip + testSuccessLogFormat(false); + testFailureLogFormat(false); + } + + /** + * A special extension of {@link TestImpl} RPC server with + * {@link TestImpl#ping()} testing the audit logs. + */ + private class MyTestRPCServer extends TestImpl { + @Override + public void ping() { + // test with ip set + testSuccessLogFormat(true); + testFailureLogFormat(true); + } + } + + /** + * Test {@link RMAuditLogger} with IP set. + */ + @Test + public void testRMAuditLoggerWithIP() throws Exception { + Configuration conf = new Configuration(); + // start the IPC server + Server server = RPC.getServer(new MyTestRPCServer(), "0.0.0.0", 0, conf); + server.start(); + + InetSocketAddress addr = NetUtils.getConnectAddress(server); + + // Make a client connection and test the audit log + TestProtocol proxy = (TestProtocol)RPC.getProxy(TestProtocol.class, + TestProtocol.versionID, addr, conf); + // Start the testcase + proxy.ping(); + + server.stop(); + } +}