big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18567. LogThrottlingHelper: properly trigger dependent recorders in cases of infrequent logging (#5215)

Browse Source 
Signed-off-by: Erik Krogen <xkrogen@apache.org>
Co-authored-by: Chengbing Liu <liuchengbing@qiyi.com>
This commit is contained in:
Chengbing Liu 2022-12-17 01:15:11 +08:00 committed by GitHub
parent f7bdf6c667
commit ca3526da92
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 25 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogThrottlingHelper.java
View File

@ -262,9 +262,15 @@ public LogAction record(String recorderName, long currentTimeMs,
if (primaryRecorderName.equals(recorderName) && if (primaryRecorderName.equals(recorderName) &&
currentTimeMs - minLogPeriodMs >= lastLogTimestampMs) { currentTimeMs - minLogPeriodMs >= lastLogTimestampMs) {
lastLogTimestampMs = currentTimeMs; lastLogTimestampMs = currentTimeMs;
for (LoggingAction log : currentLogs.values()) { currentLogs.replaceAll((key, log) -> {
log.setShouldLog(); LoggingAction newLog = log;
} if (log.hasLogged()) {
// create a fresh log since the old one has already been logged
newLog = new LoggingAction(log.getValueCount());
}
newLog.setShouldLog();
return newLog;
});
} }
if (currentLog.shouldLog()) { if (currentLog.shouldLog()) {
currentLog.setHasLogged(); currentLog.setHasLogged();
@ -357,6 +363,10 @@ private void setHasLogged() {
hasLogged = true; hasLogged = true;
} }
private int getValueCount() {
return stats.length;
}
private void recordValues(double... values) { private void recordValues(double... values) {
if (values.length != stats.length) { if (values.length != stats.length) {
throw new IllegalArgumentException("received " + values.length + throw new IllegalArgumentException("received " + values.length +

12
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogThrottlingHelper.java
View File

@ -142,6 +142,18 @@ public void testPrimaryAndDependentLoggers() {
assertTrue(helper.record("bar", 0).shouldLog()); assertTrue(helper.record("bar", 0).shouldLog());
} }
@Test
public void testInfrequentPrimaryAndDependentLoggers() {
helper = new LogThrottlingHelper(LOG_PERIOD, "foo", timer);
assertTrue(helper.record("foo", 0).shouldLog());
assertTrue(helper.record("bar", 0).shouldLog());
// Both should log once the period has elapsed
assertTrue(helper.record("foo", LOG_PERIOD).shouldLog());
assertTrue(helper.record("bar", LOG_PERIOD).shouldLog());
}
@Test @Test
public void testMultipleLoggersWithValues() { public void testMultipleLoggersWithValues() {
helper = new LogThrottlingHelper(LOG_PERIOD, "foo", timer); helper = new LogThrottlingHelper(LOG_PERIOD, "foo", timer);