big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-16531. Log more timing information for slow RPCs. Contributed by Chen Zhang.

Browse Source
This commit is contained in:
Erik Krogen 2019-09-06 10:28:21 -07:00
parent 9637097ef9
commit a23417533e
1 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
View File

@ -508,7 +508,7 @@ protected void setLogSlowRPC(boolean logSlowRPCFlag) {
* Logs a Slow RPC Request. * Logs a Slow RPC Request.
* *
* @param methodName - RPC Request method name * @param methodName - RPC Request method name
* @param processingTime - Processing Time. * @param details - Processing Detail.
* *
* if this request took too much time relative to other requests * if this request took too much time relative to other requests
* we consider that as a slow RPC. 3 is a magic number that comes * we consider that as a slow RPC. 3 is a magic number that comes
@ -517,7 +517,8 @@ protected void setLogSlowRPC(boolean logSlowRPCFlag) {
* if and only if it falls above 99.7% of requests. We start this logic * if and only if it falls above 99.7% of requests. We start this logic
* only once we have enough sample size. * only once we have enough sample size.
*/ */
void logSlowRpcCalls(String methodName, Call call, long processingTime) { void logSlowRpcCalls(String methodName, Call call,
ProcessingDetails details) {
final int deviation = 3; final int deviation = 3;
// 1024 for minSampleSize just a guess -- not a number computed based on // 1024 for minSampleSize just a guess -- not a number computed based on
@ -528,10 +529,15 @@ void logSlowRpcCalls(String methodName, Call call, long processingTime) {
final double threeSigma = rpcMetrics.getProcessingMean() + final double threeSigma = rpcMetrics.getProcessingMean() +
(rpcMetrics.getProcessingStdDev() * deviation); (rpcMetrics.getProcessingStdDev() * deviation);
long processingTime =
details.get(Timing.PROCESSING, RpcMetrics.TIMEUNIT);
if ((rpcMetrics.getProcessingSampleCount() > minSampleSize) && if ((rpcMetrics.getProcessingSampleCount() > minSampleSize) &&
(processingTime > threeSigma)) { (processingTime > threeSigma)) {
LOG.warn("Slow RPC : {} took {} {} to process from client {}", LOG.warn(
methodName, processingTime, RpcMetrics.TIMEUNIT, call); "Slow RPC : {} took {} {} to process from client {},"
+ " the processing detail is {}",
methodName, processingTime, RpcMetrics.TIMEUNIT, call,
details.toString());
rpcMetrics.incrSlowRpc(); rpcMetrics.incrSlowRpc();
} }
} }
@ -570,7 +576,7 @@ void updateMetrics(Call call, long startTime, boolean connDropped) {
rpcDetailedMetrics.addProcessingTime(name, processingTime); rpcDetailedMetrics.addProcessingTime(name, processingTime);
callQueue.addResponseTime(name, call, details); callQueue.addResponseTime(name, call, details);
if (isLogSlowRPC()) { if (isLogSlowRPC()) {
logSlowRpcCalls(name, call, processingTime); logSlowRpcCalls(name, call, details);
} }
} }