big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-6860. BlockStateChange logs are too noisy. Contributed by Chang Li and Xiaoyu Yao.

Browse Source
This commit is contained in:
Xiaoyu Yao 2015-07-31 16:15:50 -07:00
parent 454da959c7
commit d311a38a6b
5 changed files with 26 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -768,6 +768,8 @@ Release 2.8.0 - UNRELEASED
HDFS-7192. DN should ignore lazyPersist hint if the writer is
not local. (Arpit Agarwal)
HDFS-6860. BlockStateChange logs are too noisy. (Chang Li and xyao via xyao)
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.blockmanagement;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@ -237,7 +236,7 @@ public void setGenerationStampAndVerifyReplicas(long genStamp) {
for (ReplicaUnderConstruction r : replicas) {
if (genStamp != r.getGenerationStamp()) {
r.getExpectedStorageLocation().removeBlock(this);
NameNode.blockStateChangeLog.info("BLOCK* Removing stale replica "
NameNode.blockStateChangeLog.debug("BLOCK* Removing stale replica "
+ "from location: {}", r.getExpectedStorageLocation());
}
}
@ -310,7 +309,7 @@ public void initializeBlockRecovery(long recoveryId) {
primary.getExpectedStorageLocation().
getDatanodeDescriptor().addBlockToBeRecovered(this);
primary.setChosenAsPrimary(true);
NameNode.blockStateChangeLog.info(
NameNode.blockStateChangeLog.debug(
"BLOCK* {} recovery started, primary={}", this, primary);
}
}

38
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -1151,7 +1151,7 @@ private void addToInvalidates(BlockInfo storedBlock) {
datanodes.append(node).append(" ");
}
if (datanodes.length() != 0) {
blockLog.info("BLOCK* addToInvalidates: {} {}", storedBlock,
blockLog.debug("BLOCK* addToInvalidates: {} {}", storedBlock,
datanodes.toString());
}
}
@ -1186,7 +1186,7 @@ public void findAndMarkBlockAsCorrupt(final ExtendedBlock blk,
// ignore the request for now. This could happen when BlockScanner
// thread of Datanode reports bad block before Block reports are sent
// by the Datanode on startup
blockLog.info("BLOCK* findAndMarkBlockAsCorrupt: {} not found", blk);
blockLog.debug("BLOCK* findAndMarkBlockAsCorrupt: {} not found", blk);
return;
}
@ -1214,7 +1214,7 @@ private void markBlockAsCorrupt(BlockToMarkCorrupt b,
DatanodeDescriptor node) throws IOException {
if (b.stored.isDeleted()) {
blockLog.info("BLOCK markBlockAsCorrupt: {} cannot be marked as" +
blockLog.debug("BLOCK markBlockAsCorrupt: {} cannot be marked as" +
" corrupt as it does not belong to any file", b);
addToInvalidates(b.corrupted, node);
return;
@ -1264,7 +1264,7 @@ private void markBlockAsCorrupt(BlockToMarkCorrupt b,
*/
private boolean invalidateBlock(BlockToMarkCorrupt b, DatanodeInfo dn
) throws IOException {
blockLog.info("BLOCK* invalidateBlock: {} on {}", b, dn);
blockLog.debug("BLOCK* invalidateBlock: {} on {}", b, dn);
DatanodeDescriptor node = getDatanodeManager().getDatanode(dn);
if (node == null) {
throw new IOException("Cannot invalidate " + b
@ -1274,7 +1274,7 @@ private boolean invalidateBlock(BlockToMarkCorrupt b, DatanodeInfo dn
// Check how many copies we have of the block
NumberReplicas nr = countNodes(b.stored);
if (nr.replicasOnStaleNodes() > 0) {
blockLog.info("BLOCK* invalidateBlocks: postponing " +
blockLog.debug("BLOCK* invalidateBlocks: postponing " +
"invalidation of {} on {} because {} replica(s) are located on " +
"nodes with potentially out-of-date block reports", b, dn,
nr.replicasOnStaleNodes());
@ -1288,7 +1288,7 @@ private boolean invalidateBlock(BlockToMarkCorrupt b, DatanodeInfo dn
b, dn);
return true;
} else {
blockLog.info("BLOCK* invalidateBlocks: {} on {} is the only copy and" +
blockLog.debug("BLOCK* invalidateBlocks: {} on {} is the only copy and" +
" was not deleted", b, dn);
return false;
}
@ -1421,7 +1421,7 @@ int computeReplicationWorkForBlocks(List<List<BlockInfo>> blocksToReplicate) {
if ( (pendingReplications.getNumReplicas(block) > 0) ||
(blockHasEnoughRacks(block, requiredReplication)) ) {
neededReplications.remove(block, priority); // remove from neededReplications
blockLog.info("BLOCK* Removing {} from neededReplications as" +
blockLog.debug("BLOCK* Removing {} from neededReplications as" +
" it has enough replicas", block);
continue;
}
@ -1491,7 +1491,7 @@ int computeReplicationWorkForBlocks(List<List<BlockInfo>> blocksToReplicate) {
(blockHasEnoughRacks(block, requiredReplication)) ) {
neededReplications.remove(block, priority); // remove from neededReplications
rw.targets = null;
blockLog.info("BLOCK* Removing {} from neededReplications as" +
blockLog.debug("BLOCK* Removing {} from neededReplications as" +
" it has enough replicas", block);
continue;
}
@ -1539,7 +1539,7 @@ int computeReplicationWorkForBlocks(List<List<BlockInfo>> blocksToReplicate) {
targetList.append(' ');
targetList.append(targets[k].getDatanodeDescriptor());
}
blockLog.info("BLOCK* ask {} to replicate {} to {}", rw.srcNode,
blockLog.debug("BLOCK* ask {} to replicate {} to {}", rw.srcNode,
rw.block, targetList);
}
}
@ -2141,7 +2141,7 @@ public void markBlockReplicasAsCorrupt(Block oldBlock, BlockInfo block,
}
}
if (isCorrupt) {
blockLog.info("BLOCK* markBlockReplicasAsCorrupt: mark block replica" +
blockLog.debug("BLOCK* markBlockReplicasAsCorrupt: mark block replica" +
" {} on {} as corrupt because the dn is not in the new committed " +
"storage list.", b, storage.getDatanodeDescriptor());
markBlockAsCorrupt(b, storage, storage.getDatanodeDescriptor());
@ -2636,7 +2636,7 @@ private Block addStoredBlock(final BlockInfo block,
}
if (storedBlock == null || storedBlock.isDeleted()) {
// If this block does not belong to anyfile, then we are done.
blockLog.info("BLOCK* addStoredBlock: {} on {} size {} but it does not" +
blockLog.debug("BLOCK* addStoredBlock: {} on {} size {} but it does not" +
" belong to any file", block, node, block.getNumBytes());
// we could add this block to invalidate set of this datanode.
@ -2726,10 +2726,10 @@ private Block addStoredBlock(final BlockInfo block,
private void logAddStoredBlock(BlockInfo storedBlock,
DatanodeDescriptor node) {
if (!blockLog.isInfoEnabled()) {
if (!blockLog.isDebugEnabled()) {
return;
}
StringBuilder sb = new StringBuilder(500);
sb.append("BLOCK* addStoredBlock: blockMap updated: ")
.append(node)
@ -2737,7 +2737,7 @@ private void logAddStoredBlock(BlockInfo storedBlock,
storedBlock.appendStringTo(sb);
sb.append(" size " )
.append(storedBlock.getNumBytes());
blockLog.info(sb.toString());
blockLog.debug(sb.toString());
}
/**
* Invalidate corrupt replicas.
@ -2767,7 +2767,7 @@ private void invalidateCorruptReplicas(BlockInfo blk, Block reported) {
removedFromBlocksMap = false;
}
} catch (IOException e) {
blockLog.info("invalidateCorruptReplicas error in deleting bad block"
blockLog.debug("invalidateCorruptReplicas error in deleting bad block"
+ " {} on {}", blk, node, e);
removedFromBlocksMap = false;
}
@ -3106,7 +3106,7 @@ private void processChosenExcessReplica(
// upon giving instructions to the datanodes.
//
addToInvalidates(storedBlock, chosen.getDatanodeDescriptor());
blockLog.info("BLOCK* chooseExcessReplicates: "
blockLog.debug("BLOCK* chooseExcessReplicates: "
+"({}, {}) is added to invalidated blocks set", chosen, storedBlock);
}
@ -3304,11 +3304,11 @@ private void processAndHandleReportedBlock(
numBlocksLogged++;
}
if (numBlocksLogged > maxNumBlocksToLog) {
blockLog.info("BLOCK* addBlock: logged info for {} of {} reported.",
blockLog.debug("BLOCK* addBlock: logged info for {} of {} reported.",
maxNumBlocksToLog, numBlocksLogged);
}
for (Block b : toInvalidate) {
blockLog.info("BLOCK* addBlock: block {} on node {} size {} does not " +
blockLog.debug("BLOCK* addBlock: block {} on node {} size {} does not " +
"belong to any file", b, node, b.getNumBytes());
addToInvalidates(b, node);
}
@ -3654,7 +3654,7 @@ private int invalidateWorkForOneNode(DatanodeInfo dn) {
} finally {
namesystem.writeUnlock();
}
blockLog.info("BLOCK* {}: ask {} to delete {}", getClass().getSimpleName(),
blockLog.debug("BLOCK* {}: ask {} to delete {}", getClass().getSimpleName(),
dn, toInvalidate);
return toInvalidate.size();
}

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
View File

@ -80,12 +80,12 @@ void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn,
}
if (!nodes.keySet().contains(dn)) {
NameNode.blockStateChangeLog.info(
NameNode.blockStateChangeLog.debug(
"BLOCK NameSystem.addToCorruptReplicasMap: {} added as corrupt on "
+ "{} by {} {}", blk.getBlockName(), dn, Server.getRemoteIp(),
reasonText);
} else {
NameNode.blockStateChangeLog.info(
NameNode.blockStateChangeLog.debug(
"BLOCK NameSystem.addToCorruptReplicasMap: duplicate requested for" +
" {} to add as corrupt on {} by {} {}", blk.getBlockName(), dn,
Server.getRemoteIp(), reasonText);

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/InvalidateBlocks.java
View File

@ -113,7 +113,7 @@ synchronized void add(final Block block, final DatanodeInfo datanode,
if (set.add(block)) {
numBlocks++;
if (log) {
NameNode.blockStateChangeLog.info("BLOCK* {}: add {} to {}",
NameNode.blockStateChangeLog.debug("BLOCK* {}: add {} to {}",
getClass().getSimpleName(), block, datanode);
}
}