big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-376. Create custom message structure for use in AuditLogging

Browse Source 
Contributed by Dinesh Chitlangia.
This commit is contained in:
Anu Engineer 2018-08-28 12:59:08 -07:00
parent cb9d371ae2
commit ac515d22d8
4 changed files with 179 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditLogger.java
View File

@ -21,10 +21,8 @@
import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Marker; import org.apache.logging.log4j.Marker;
import org.apache.logging.log4j.message.StructuredDataMessage;
import org.apache.logging.log4j.spi.ExtendedLogger; import org.apache.logging.log4j.spi.ExtendedLogger;
import java.util.Map;
/** /**
* Class to define Audit Logger for Ozone. * Class to define Audit Logger for Ozone.
@ -32,16 +30,13 @@
public class AuditLogger { public class AuditLogger {
private ExtendedLogger logger; private ExtendedLogger logger;
private static final String SUCCESS = AuditEventStatus.SUCCESS.getStatus();
private static final String FAILURE = AuditEventStatus.FAILURE.getStatus();
private static final String FQCN = AuditLogger.class.getName(); private static final String FQCN = AuditLogger.class.getName();
private static final Marker WRITE_MARKER = AuditMarker.WRITE.getMarker(); private static final Marker WRITE_MARKER = AuditMarker.WRITE.getMarker();
private static final Marker READ_MARKER = AuditMarker.READ.getMarker(); private static final Marker READ_MARKER = AuditMarker.READ.getMarker();
/** /**
* Parametrized Constructor to initialize logger. * Parametrized Constructor to initialize logger.
* @param type * @param type Audit Logger Type
*/ */
public AuditLogger(AuditLoggerType type){ public AuditLogger(AuditLoggerType type){
initializeLogger(type); initializeLogger(type);
@ -60,68 +55,53 @@ public ExtendedLogger getLogger() {
return logger; return logger;
} }
public void logWriteSuccess(AuditAction type, Map<String, String> data) { public void logWriteSuccess(AuditMessage msg) {
logWriteSuccess(type, data, Level.INFO); logWriteSuccess(Level.INFO, msg);
} }
public void logWriteSuccess(AuditAction type, Map<String, String> data, Level public void logWriteSuccess(Level level, AuditMessage msg) {
level) {
StructuredDataMessage msg = new StructuredDataMessage("", SUCCESS,
type.getAction(), data);
this.logger.logIfEnabled(FQCN, level, WRITE_MARKER, msg, null); this.logger.logIfEnabled(FQCN, level, WRITE_MARKER, msg, null);
} }
public void logWriteFailure(AuditMessage msg) {
public void logWriteFailure(AuditAction type, Map<String, String> data) { logWriteFailure(Level.ERROR, msg);
logWriteFailure(type, data, Level.INFO, null);
} }
public void logWriteFailure(AuditAction type, Map<String, String> data, Level public void logWriteFailure(Level level, AuditMessage msg) {
level) { logWriteFailure(level, msg, null);
logWriteFailure(type, data, level, null);
} }
public void logWriteFailure(AuditAction type, Map<String, String> data, public void logWriteFailure(AuditMessage msg, Throwable exception) {
logWriteFailure(Level.ERROR, msg, exception);
}
public void logWriteFailure(Level level, AuditMessage msg,
Throwable exception) { Throwable exception) {
logWriteFailure(type, data, Level.INFO, exception);
}
public void logWriteFailure(AuditAction type, Map<String, String> data, Level
level, Throwable exception) {
StructuredDataMessage msg = new StructuredDataMessage("", FAILURE,
type.getAction(), data);
this.logger.logIfEnabled(FQCN, level, WRITE_MARKER, msg, exception); this.logger.logIfEnabled(FQCN, level, WRITE_MARKER, msg, exception);
} }
public void logReadSuccess(AuditAction type, Map<String, String> data) { public void logReadSuccess(AuditMessage msg) {
logReadSuccess(type, data, Level.INFO); logReadSuccess(Level.INFO, msg);
} }
public void logReadSuccess(AuditAction type, Map<String, String> data, Level public void logReadSuccess(Level level, AuditMessage msg) {
level) {
StructuredDataMessage msg = new StructuredDataMessage("", SUCCESS,
type.getAction(), data);
this.logger.logIfEnabled(FQCN, level, READ_MARKER, msg, null); this.logger.logIfEnabled(FQCN, level, READ_MARKER, msg, null);
} }
public void logReadFailure(AuditAction type, Map<String, String> data) { public void logReadFailure(AuditMessage msg) {
logReadFailure(type, data, Level.INFO, null); logReadFailure(Level.ERROR, msg);
} }
public void logReadFailure(AuditAction type, Map<String, String> data, Level public void logReadFailure(Level level, AuditMessage msg) {
level) { logReadFailure(level, msg, null);
logReadFailure(type, data, level, null);
} }
public void logReadFailure(AuditAction type, Map<String, String> data, public void logReadFailure(AuditMessage msg, Throwable exception) {
logReadFailure(Level.ERROR, msg, exception);
}
public void logReadFailure(Level level, AuditMessage msg,
Throwable exception) { Throwable exception) {
logReadFailure(type, data, Level.INFO, exception);
}
public void logReadFailure(AuditAction type, Map<String, String> data, Level
level, Throwable exception) {
StructuredDataMessage msg = new StructuredDataMessage("", FAILURE,
type.getAction(), data);
this.logger.logIfEnabled(FQCN, level, READ_MARKER, msg, exception); this.logger.logIfEnabled(FQCN, level, READ_MARKER, msg, exception);
} }

64
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/AuditMessage.java Normal file
View File

@ -0,0 +1,64 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.audit;
import org.apache.logging.log4j.message.Message;
import java.util.Map;
/**
* Defines audit message structure.
*/
public class AuditMessage implements Message {
private String message;
public AuditMessage(String user, String ip, String op,
Map<String, String> params, String ret){
this.message = String.format("user=%s ip=%s op=%s %s ret=%s",
user, ip, op, params, ret);
}
@Override
public String getFormattedMessage() {
return message;
}
@Override
public String getFormat() {
return null;
}
@Override
public Object[] getParameters() {
return new Object[0];
}
@Override
public Throwable getThrowable() {
return null;
}
/**
* Use when there are custom string to be added to default msg.
* @param customMessage custom string
*/
private void appendMessage(String customMessage) {
this.message += customMessage;
}
}

19
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/audit/package-info.java
View File

@ -46,7 +46,7 @@
* **** Auditable *** * **** Auditable ***
* This is an interface to mark an entity as auditable. * This is an interface to mark an entity as auditable.
* This interface must be implemented by entities requiring audit logging. * This interface must be implemented by entities requiring audit logging.
* For example - KSMVolumeArgs, KSMBucketArgs. * For example - OMVolumeArgs, OMBucketArgs.
* The implementing class must override toAuditMap() to return an * The implementing class must override toAuditMap() to return an
* instance of Map<Key, Value> where both Key and Value are String. * instance of Map<Key, Value> where both Key and Value are String.
* *
@ -81,6 +81,11 @@
* *** AuditMarker *** * *** AuditMarker ***
* Enum to define various Audit Markers used in AuditLogging. * Enum to define various Audit Markers used in AuditLogging.
* *
* *** AuditMessage ***
* Entity to define an audit message to be logged
* It will generate a message formatted as:
* user=xxx ip=xxx op=XXXX_XXXX {key=val, key1=val1..} ret=XXXXXX
*
* **************************************************************************** * ****************************************************************************
* Usage * Usage
* **************************************************************************** * ****************************************************************************
@ -88,14 +93,16 @@
* 1. Get a logger by specifying the appropriate logger type * 1. Get a logger by specifying the appropriate logger type
* Example: ExtendedLogger AUDIT = new AuditLogger(AuditLoggerType.OMLogger) * Example: ExtendedLogger AUDIT = new AuditLogger(AuditLoggerType.OMLogger)
* *
* 2. Log Read/Write and Success/Failure event as needed. * 2. Construct an instance of AuditMessage
*
* 3. Log Read/Write and Success/Failure event as needed.
* Example * Example
* AUDIT.logWriteSuccess(AuditAction type, Map<String, String> data, Level * AUDIT.logWriteSuccess(Level level, AuditMessage msg)
* level)
* *
* If logging is done without specifying Level, then Level implicitly * If logging is done without specifying Level, then Level implicitly
* defaults to INFO * defaults to INFO for xxxxSuccess() and ERROR for xxxxFailure()
* AUDIT.logWriteSuccess(AuditAction type, Map<String, String> data) * AUDIT.logWriteSuccess(AuditMessage msg)
* AUDIT.logWriteFailure(AuditMessage msg)
* *
* See sample invocations in src/test in the following class: * See sample invocations in src/test in the following class:
* org.apache.hadoop.ozone.audit.TestOzoneAuditLogger * org.apache.hadoop.ozone.audit.TestOzoneAuditLogger

124
hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/audit/TestOzoneAuditLogger.java
View File

@ -28,6 +28,7 @@
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@ -36,10 +37,29 @@
*/ */
public class TestOzoneAuditLogger { public class TestOzoneAuditLogger {
private static final Logger LOG = LoggerFactory.getLogger private static final Logger LOG =
(TestOzoneAuditLogger.class.getName()); LoggerFactory.getLogger(TestOzoneAuditLogger.class.getName());
private static AuditLogger AUDIT = new AuditLogger(AuditLoggerType.OMLOGGER);
public DummyEntity auditableObj = new DummyEntity(); private static final AuditLogger AUDIT =
new AuditLogger(AuditLoggerType.OMLOGGER);
private static final String SUCCESS = AuditEventStatus.SUCCESS.name();
private static final String FAILURE = AuditEventStatus.FAILURE.name();
private static final Map<String, String> PARAMS =
new DummyEntity().toAuditMap();
private static final AuditMessage WRITE_FAIL_MSG = new AuditMessage("john",
"192.168.0.1", DummyAction.CREATE_VOLUME.name(), PARAMS, FAILURE);
private static final AuditMessage WRITE_SUCCESS_MSG = new AuditMessage("john",
"192.168.0.1", DummyAction.CREATE_VOLUME.name(), PARAMS, SUCCESS);
private static final AuditMessage READ_FAIL_MSG = new AuditMessage("john",
"192.168.0.1", DummyAction.READ_VOLUME.name(), PARAMS, FAILURE);
private static final AuditMessage READ_SUCCESS_MSG = new AuditMessage("john",
"192.168.0.1", DummyAction.READ_VOLUME.name(), PARAMS, SUCCESS);
@BeforeClass @BeforeClass
public static void setUp(){ public static void setUp(){
@ -48,13 +68,13 @@ public static void setUp(){
@AfterClass @AfterClass
public static void tearDown() { public static void tearDown() {
File file = new File("audit.log"); File file = new File("audit.log");
if (FileUtils.deleteQuietly(file)) { if (FileUtils.deleteQuietly(file)) {
LOG.info(file.getName() + LOG.info(file.getName() +
" has been deleted as all tests have completed."); " has been deleted as all tests have completed.");
} else { } else {
LOG.info("audit.log could not be deleted."); LOG.info("audit.log could not be deleted.");
} }
} }
/** /**
@ -62,20 +82,31 @@ public static void tearDown() {
*/ */
@Test @Test
public void logInfoWriteSuccess() throws IOException { public void logInfoWriteSuccess() throws IOException {
AUDIT.logWriteSuccess(DummyAction.CREATE_VOLUME, auditableObj.toAuditMap(), Level.INFO); AUDIT.logWriteSuccess(Level.INFO, WRITE_SUCCESS_MSG);
String expected = "[INFO ] OMAudit - CREATE_VOLUME [ key1=\"value1\" " + String expected =
"key2=\"value2\"] SUCCESS"; "[INFO ] OMAudit - " + WRITE_SUCCESS_MSG.getFormattedMessage();
verifyLog(expected); verifyLog(expected);
} }
/** /**
* Test to verify default log level is INFO * Test to verify default log level is INFO when logging success events.
*/ */
@Test @Test
public void verifyDefaultLogLevel() throws IOException { public void verifyDefaultLogLevelForSuccess() throws IOException {
AUDIT.logWriteSuccess(DummyAction.CREATE_VOLUME, auditableObj.toAuditMap()); AUDIT.logWriteSuccess(WRITE_SUCCESS_MSG);
String expected = "[INFO ] OMAudit - CREATE_VOLUME [ key1=\"value1\" " + String expected =
"key2=\"value2\"] SUCCESS"; "[INFO ] OMAudit - " + WRITE_SUCCESS_MSG.getFormattedMessage();
verifyLog(expected);
}
/**
* Test to verify default log level is ERROR when logging failure events.
*/
@Test
public void verifyDefaultLogLevelForFailure() throws IOException {
AUDIT.logWriteFailure(WRITE_FAIL_MSG);
String expected =
"[ERROR] OMAudit - " + WRITE_FAIL_MSG.getFormattedMessage();
verifyLog(expected); verifyLog(expected);
} }
@ -84,9 +115,9 @@ public void verifyDefaultLogLevel() throws IOException {
*/ */
@Test @Test
public void logErrorWriteFailure() throws IOException { public void logErrorWriteFailure() throws IOException {
AUDIT.logWriteFailure(DummyAction.CREATE_VOLUME, auditableObj.toAuditMap(), Level.ERROR); AUDIT.logWriteFailure(Level.ERROR, WRITE_FAIL_MSG);
String expected = "[ERROR] OMAudit - CREATE_VOLUME [ key1=\"value1\" " + String expected =
"key2=\"value2\"] FAILURE"; "[ERROR] OMAudit - " + WRITE_FAIL_MSG.getFormattedMessage();
verifyLog(expected); verifyLog(expected);
} }
@ -95,11 +126,10 @@ public void logErrorWriteFailure() throws IOException {
*/ */
@Test @Test
public void notLogReadEvents() throws IOException { public void notLogReadEvents() throws IOException {
AUDIT.logReadSuccess(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.INFO); AUDIT.logReadSuccess(Level.INFO, READ_SUCCESS_MSG);
AUDIT.logReadFailure(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.INFO); AUDIT.logReadFailure(Level.INFO, READ_FAIL_MSG);
AUDIT.logReadFailure(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.ERROR); AUDIT.logReadFailure(Level.ERROR, READ_FAIL_MSG);
AUDIT.logReadFailure(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.ERROR, AUDIT.logReadFailure(Level.ERROR, READ_FAIL_MSG, new Exception("test"));
new Exception("test"));
verifyNoLog(); verifyNoLog();
} }
@ -108,34 +138,34 @@ public void notLogReadEvents() throws IOException {
*/ */
@Test @Test
public void notLogDebugEvents() throws IOException { public void notLogDebugEvents() throws IOException {
AUDIT.logWriteSuccess(DummyAction.CREATE_VOLUME, auditableObj.toAuditMap(), Level.DEBUG); AUDIT.logWriteSuccess(Level.DEBUG, WRITE_SUCCESS_MSG);
AUDIT.logReadSuccess(DummyAction.READ_VOLUME, auditableObj.toAuditMap(), Level.DEBUG); AUDIT.logReadSuccess(Level.DEBUG, READ_SUCCESS_MSG);
verifyNoLog(); verifyNoLog();
} }
private void verifyLog(String expected) throws IOException { private void verifyLog(String expected) throws IOException {
File file = new File("audit.log"); File file = new File("audit.log");
List<String> lines = FileUtils.readLines(file, (String)null); List<String> lines = FileUtils.readLines(file, (String)null);
final int retry = 5; final int retry = 5;
int i = 0; int i = 0;
while (lines.isEmpty() && i < retry) { while (lines.isEmpty() && i < retry) {
lines = FileUtils.readLines(file, (String)null); lines = FileUtils.readLines(file, (String)null);
try { try {
Thread.sleep( 500 * (i + 1)); Thread.sleep(500 * (i + 1));
} catch(InterruptedException ie) { } catch(InterruptedException ie) {
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
break; break;
}
i++;
} }
i++;
}
// When log entry is expected, the log file will contain one line and // When log entry is expected, the log file will contain one line and
// that must be equal to the expected string // that must be equal to the expected string
assertTrue(lines.size() != 0); assertTrue(lines.size() != 0);
assertTrue(expected.equalsIgnoreCase(lines.get(0))); assertTrue(expected.equalsIgnoreCase(lines.get(0)));
//empty the file //empty the file
lines.remove(0); lines.remove(0);
FileUtils.writeLines(file, lines, false); FileUtils.writeLines(file, lines, false);
} }
private void verifyNoLog() throws IOException { private void verifyNoLog() throws IOException {