big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13658. Expose HighestPriorityLowRedundancy blocks statistics. Contributed by Kitti Nanasi.

Browse Source
This commit is contained in:
Xiao Chen 2018-08-08 10:36:44 -07:00
parent ff06bd1be8
commit 9499df7b81
15 changed files with 247 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
View File

@ -244,6 +244,8 @@ Each metrics record contains tags such as HAState and Hostname as additional inf
| `StaleDataNodes` | Current number of DataNodes marked stale due to delayed heartbeat |
| `NumStaleStorages` | Number of storages marked as content stale (after NameNode restart/failover before first block report is received) |
| `MissingReplOneBlocks` | Current number of missing blocks with replication factor 1 |
| `HighestPriorityLowRedundancyReplicatedBlocks` | Current number of non-corrupt, low redundancy replicated blocks with the highest risk of loss (have 0 or 1 replica). Will be recovered with the highest priority. |
| `HighestPriorityLowRedundancyECBlocks` | Current number of non-corrupt, low redundancy EC blocks with the highest risk of loss. Will be recovered with the highest priority. |
| `NumFilesUnderConstruction` | Current number of files under construction |
| `NumActiveClients` | Current number of active clients holding lease |
| `HAState` | (HA-only) Current state of the NameNode: initializing or active or standby or stopping state |

27
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ECBlockGroupStats.java
View File

@ -34,15 +34,26 @@ public final class ECBlockGroupStats {
private final long missingBlockGroups;
private final long bytesInFutureBlockGroups;
private final long pendingDeletionBlocks;
private final Long highestPriorityLowRedundancyBlocks;
public ECBlockGroupStats(long lowRedundancyBlockGroups,
long corruptBlockGroups, long missingBlockGroups,
long bytesInFutureBlockGroups, long pendingDeletionBlocks) {
this(lowRedundancyBlockGroups, corruptBlockGroups, missingBlockGroups,
bytesInFutureBlockGroups, pendingDeletionBlocks, null);
}
public ECBlockGroupStats(long lowRedundancyBlockGroups,
long corruptBlockGroups, long missingBlockGroups,
long bytesInFutureBlockGroups, long pendingDeletionBlocks,
Long highestPriorityLowRedundancyBlocks) {
this.lowRedundancyBlockGroups = lowRedundancyBlockGroups;
this.corruptBlockGroups = corruptBlockGroups;
this.missingBlockGroups = missingBlockGroups;
this.bytesInFutureBlockGroups = bytesInFutureBlockGroups;
this.pendingDeletionBlocks = pendingDeletionBlocks;
this.highestPriorityLowRedundancyBlocks
= highestPriorityLowRedundancyBlocks;
}
public long getBytesInFutureBlockGroups() {
@ -65,6 +76,14 @@ public long getPendingDeletionBlocks() {
return pendingDeletionBlocks;
}
public boolean hasHighestPriorityLowRedundancyBlocks() {
return getHighestPriorityLowRedundancyBlocks() != null;
}
public Long getHighestPriorityLowRedundancyBlocks() {
return highestPriorityLowRedundancyBlocks;
}
@Override
public String toString() {
StringBuilder statsBuilder = new StringBuilder();
@ -76,8 +95,12 @@ public String toString() {
.append(", BytesInFutureBlockGroups=").append(
getBytesInFutureBlockGroups())
.append(", PendingDeletionBlocks=").append(
getPendingDeletionBlocks())
.append("]");
getPendingDeletionBlocks());
if (hasHighestPriorityLowRedundancyBlocks()) {
statsBuilder.append(", HighestPriorityLowRedundancyBlocks=")
.append(getHighestPriorityLowRedundancyBlocks());
}
statsBuilder.append("]");
return statsBuilder.toString();
}
}

28
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReplicatedBlockStats.java
View File

@ -35,17 +35,29 @@ public final class ReplicatedBlockStats {
private final long missingReplicationOneBlocks;
private final long bytesInFutureBlocks;
private final long pendingDeletionBlocks;
private final Long highestPriorityLowRedundancyBlocks;
public ReplicatedBlockStats(long lowRedundancyBlocks,
long corruptBlocks, long missingBlocks,
long missingReplicationOneBlocks, long bytesInFutureBlocks,
long pendingDeletionBlocks) {
this(lowRedundancyBlocks, corruptBlocks, missingBlocks,
missingReplicationOneBlocks, bytesInFutureBlocks, pendingDeletionBlocks,
null);
}
public ReplicatedBlockStats(long lowRedundancyBlocks,
long corruptBlocks, long missingBlocks,
long missingReplicationOneBlocks, long bytesInFutureBlocks,
long pendingDeletionBlocks, Long highestPriorityLowRedundancyBlocks) {
this.lowRedundancyBlocks = lowRedundancyBlocks;
this.corruptBlocks = corruptBlocks;
this.missingBlocks = missingBlocks;
this.missingReplicationOneBlocks = missingReplicationOneBlocks;
this.bytesInFutureBlocks = bytesInFutureBlocks;
this.pendingDeletionBlocks = pendingDeletionBlocks;
this.highestPriorityLowRedundancyBlocks
= highestPriorityLowRedundancyBlocks;
}
public long getLowRedundancyBlocks() {
@ -72,6 +84,14 @@ public long getPendingDeletionBlocks() {
return pendingDeletionBlocks;
}
public boolean hasHighestPriorityLowRedundancyBlocks() {
return getHighestPriorityLowRedundancyBlocks() != null;
}
public Long getHighestPriorityLowRedundancyBlocks(){
return highestPriorityLowRedundancyBlocks;
}
@Override
public String toString() {
StringBuilder statsBuilder = new StringBuilder();
@ -83,8 +103,12 @@ public String toString() {
getMissingReplicationOneBlocks())
.append(", BytesInFutureBlocks=").append(getBytesInFutureBlocks())
.append(", PendingDeletionBlocks=").append(
getPendingDeletionBlocks())
.append("]");
getPendingDeletionBlocks());
if (hasHighestPriorityLowRedundancyBlocks()) {
statsBuilder.append(", HighestPriorityLowRedundancyBlocks=").append(
getHighestPriorityLowRedundancyBlocks());
}
statsBuilder.append("]");
return statsBuilder.toString();
}
}

21
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
View File

@ -1990,6 +1990,13 @@ public static long[] convert(GetFsStatsResponseProto res) {
public static ReplicatedBlockStats convert(
GetFsReplicatedBlockStatsResponseProto res) {
if (res.hasHighestPrioLowRedundancyBlocks()) {
return new ReplicatedBlockStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
res.getPendingDeletionBlocks(),
res.getHighestPrioLowRedundancyBlocks());
}
return new ReplicatedBlockStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getMissingReplOneBlocks(), res.getBlocksInFuture(),
@ -1998,6 +2005,12 @@ public static ReplicatedBlockStats convert(
public static ECBlockGroupStats convert(
GetFsECBlockGroupStatsResponseProto res) {
if (res.hasHighestPrioLowRedundancyBlocks()) {
return new ECBlockGroupStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getBlocksInFuture(), res.getPendingDeletionBlocks(),
res.getHighestPrioLowRedundancyBlocks());
}
return new ECBlockGroupStats(res.getLowRedundancy(),
res.getCorruptBlocks(), res.getMissingBlocks(),
res.getBlocksInFuture(), res.getPendingDeletionBlocks());
@ -2432,6 +2445,10 @@ public static GetFsReplicatedBlockStatsResponseProto convert(
replicatedBlockStats.getBytesInFutureBlocks());
result.setPendingDeletionBlocks(
replicatedBlockStats.getPendingDeletionBlocks());
if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
result.setHighestPrioLowRedundancyBlocks(
replicatedBlockStats.getHighestPriorityLowRedundancyBlocks());
}
return result.build();
}
@ -2447,6 +2464,10 @@ public static GetFsECBlockGroupStatsResponseProto convert(
ecBlockGroupStats.getBytesInFutureBlockGroups());
result.setPendingDeletionBlocks(
ecBlockGroupStats.getPendingDeletionBlocks());
if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
result.setHighestPrioLowRedundancyBlocks(
ecBlockGroupStats.getHighestPriorityLowRedundancyBlocks());
}
return result.build();
}

3
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
View File

@ -348,6 +348,8 @@ message GetFsReplicatedBlockStatsResponseProto {
required uint64 missing_repl_one_blocks = 4;
required uint64 blocks_in_future = 5;
required uint64 pending_deletion_blocks = 6;
optional uint64 highest_prio_low_redundancy_blocks = 7;
}
message GetFsECBlockGroupStatsRequestProto { // no input paramters
@ -359,6 +361,7 @@ message GetFsECBlockGroupStatsResponseProto {
required uint64 missing_blocks = 3;
required uint64 blocks_in_future = 4;
required uint64 pending_deletion_blocks = 5;
optional uint64 highest_prio_low_redundancy_blocks = 6;
}
enum DatanodeReportTypeProto { // type of the datanode report

10
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
View File

@ -320,6 +320,16 @@ public long getNumberOfMissingBlocksWithReplicationFactorOne() {
return 0;
}
@Override
public long getHighestPriorityLowRedundancyReplicatedBlocks() {
return 0;
}
@Override
public long getHighestPriorityLowRedundancyECBlocks() {
return 0;
}
@Override
public String getCorruptFiles() {
return "N/A";

13
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/ErasureCoding.java
View File

@ -185,12 +185,25 @@ public ECBlockGroupStats getECBlockGroupStats() throws IOException {
long missingBlockGroups = 0;
long bytesInFutureBlockGroups = 0;
long pendingDeletionBlocks = 0;
long highestPriorityLowRedundancyBlocks = 0;
boolean hasHighestPriorityLowRedundancyBlocks = false;
for (ECBlockGroupStats stats : allStats.values()) {
lowRedundancyBlockGroups += stats.getLowRedundancyBlockGroups();
corruptBlockGroups += stats.getCorruptBlockGroups();
missingBlockGroups += stats.getMissingBlockGroups();
bytesInFutureBlockGroups += stats.getBytesInFutureBlockGroups();
pendingDeletionBlocks += stats.getPendingDeletionBlocks();
if (stats.hasHighestPriorityLowRedundancyBlocks()) {
hasHighestPriorityLowRedundancyBlocks = true;
highestPriorityLowRedundancyBlocks +=
stats.getHighestPriorityLowRedundancyBlocks();
}
}
if (hasHighestPriorityLowRedundancyBlocks) {
return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks,
highestPriorityLowRedundancyBlocks);
}
return new ECBlockGroupStats(lowRedundancyBlockGroups, corruptBlockGroups,
missingBlockGroups, bytesInFutureBlockGroups, pendingDeletionBlocks);

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -4428,6 +4428,14 @@ public long getMissingReplOneBlocksCount() {
return this.neededReconstruction.getCorruptReplicationOneBlockSize();
}
public long getHighestPriorityReplicatedBlockCount(){
return this.neededReconstruction.getHighestPriorityReplicatedBlockCount();
}
public long getHighestPriorityECBlockCount(){
return this.neededReconstruction.getHighestPriorityECBlockCount();
}
public BlockInfo addBlockCollection(BlockInfo block,
BlockCollection bc) {
return blocksMap.addBlockCollection(block, bc);

28
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/LowRedundancyBlocks.java
View File

@ -92,6 +92,10 @@ class LowRedundancyBlocks implements Iterable<BlockInfo> {
private final LongAdder corruptReplicationOneBlocks = new LongAdder();
private final LongAdder lowRedundancyECBlockGroups = new LongAdder();
private final LongAdder corruptECBlockGroups = new LongAdder();
private final LongAdder highestPriorityLowRedundancyReplicatedBlocks
= new LongAdder();
private final LongAdder highestPriorityLowRedundancyECBlocks
= new LongAdder();
/** Create an object. */
LowRedundancyBlocks() {
@ -162,6 +166,18 @@ long getCorruptReplicationOneBlocks() {
return corruptReplicationOneBlocks.longValue();
}
/** Return the number of under replicated blocks
* with the highest priority to recover */
long getHighestPriorityReplicatedBlockCount() {
return highestPriorityLowRedundancyReplicatedBlocks.longValue();
}
/** Return the number of under replicated EC blocks
* with the highest priority to recover */
long getHighestPriorityECBlockCount() {
return highestPriorityLowRedundancyECBlocks.longValue();
}
/**
* Return low redundancy striped blocks excluding corrupt blocks.
*/
@ -300,6 +316,9 @@ private void incrementBlockStat(BlockInfo blockInfo, int priLevel,
if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS) {
corruptECBlockGroups.increment();
}
if (priLevel == QUEUE_HIGHEST_PRIORITY) {
highestPriorityLowRedundancyECBlocks.increment();
}
} else {
lowRedundancyBlocks.increment();
if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS) {
@ -308,6 +327,9 @@ private void incrementBlockStat(BlockInfo blockInfo, int priLevel,
corruptReplicationOneBlocks.increment();
}
}
if (priLevel == QUEUE_HIGHEST_PRIORITY) {
highestPriorityLowRedundancyReplicatedBlocks.increment();
}
}
}
@ -380,6 +402,9 @@ private void decrementBlockStat(BlockInfo blockInfo, int priLevel,
if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS) {
corruptECBlockGroups.decrement();
}
if (priLevel == QUEUE_HIGHEST_PRIORITY) {
highestPriorityLowRedundancyECBlocks.decrement();
}
} else {
lowRedundancyBlocks.decrement();
if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS) {
@ -391,6 +416,9 @@ private void decrementBlockStat(BlockInfo blockInfo, int priLevel,
"should be non-negative";
}
}
if (priLevel == QUEUE_HIGHEST_PRIORITY) {
highestPriorityLowRedundancyReplicatedBlocks.decrement();
}
}
}

20
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -4218,7 +4218,8 @@ ReplicatedBlockStats getReplicatedBlockStats() {
return new ReplicatedBlockStats(getLowRedundancyReplicatedBlocks(),
getCorruptReplicatedBlocks(), getMissingReplicatedBlocks(),
getMissingReplicationOneBlocks(), getBytesInFutureReplicatedBlocks(),
getPendingDeletionReplicatedBlocks());
getPendingDeletionReplicatedBlocks(),
getHighestPriorityLowRedundancyReplicatedBlocks());
}
/**
@ -4230,7 +4231,8 @@ ReplicatedBlockStats getReplicatedBlockStats() {
ECBlockGroupStats getECBlockGroupStats() {
return new ECBlockGroupStats(getLowRedundancyECBlockGroups(),
getCorruptECBlockGroups(), getMissingECBlockGroups(),
getBytesInFutureECBlockGroups(), getPendingDeletionECBlocks());
getBytesInFutureECBlockGroups(), getPendingDeletionECBlocks(),
getHighestPriorityLowRedundancyECBlocks());
}
@Override // FSNamesystemMBean
@ -4837,6 +4839,20 @@ public long getMissingReplicationOneBlocks() {
return blockManager.getMissingReplicationOneBlocks();
}
@Override // ReplicatedBlocksMBean
@Metric({"HighestPriorityLowRedundancyReplicatedBlocks", "Number of " +
"replicated blocks which have the highest risk of loss."})
public long getHighestPriorityLowRedundancyReplicatedBlocks() {
return blockManager.getHighestPriorityReplicatedBlockCount();
}
@Override // ReplicatedBlocksMBean
@Metric({"HighestPriorityLowRedundancyECBlocks", "Number of erasure coded " +
"blocks which have the highest risk of loss."})
public long getHighestPriorityLowRedundancyECBlocks() {
return blockManager.getHighestPriorityECBlockCount();
}
@Override // ReplicatedBlocksMBean
@Metric({"BytesInFutureReplicatedBlocks", "Total bytes in replicated " +
"blocks with future generation stamp"})

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
View File

@ -162,6 +162,24 @@ public interface NameNodeMXBean {
*/
public long getNumberOfMissingBlocksWithReplicationFactorOne();
/**
* Gets the total number of replicated low redundancy blocks on the cluster
* with the highest risk of loss.
*
* @return the total number of low redundancy blocks on the cluster
* with the highest risk of loss.
*/
public long getHighestPriorityLowRedundancyReplicatedBlocks();
/**
* Gets the total number of erasure coded low redundancy blocks on the cluster
* with the highest risk of loss
*
* @return the total number of low redundancy blocks on the cluster
* with the highest risk of loss
*/
public long getHighestPriorityLowRedundancyECBlocks();
/**
* Gets the total number of snapshottable dirs in the system.
*

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
View File

@ -549,6 +549,11 @@ public void report(String[] argv, int i) throws IOException {
replicatedBlockStats.getMissingReplicaBlocks());
System.out.println("\tMissing blocks (with replication factor 1): " +
replicatedBlockStats.getMissingReplicationOneBlocks());
if (replicatedBlockStats.hasHighestPriorityLowRedundancyBlocks()) {
System.out.println("\tLow redundancy blocks with highest priority " +
"to recover: " +
replicatedBlockStats.getHighestPriorityLowRedundancyBlocks());
}
System.out.println("\tPending deletion blocks: " +
replicatedBlockStats.getPendingDeletionBlocks());
@ -561,6 +566,11 @@ public void report(String[] argv, int i) throws IOException {
ecBlockGroupStats.getCorruptBlockGroups());
System.out.println("\tMissing block groups: " +
ecBlockGroupStats.getMissingBlockGroups());
if (ecBlockGroupStats.hasHighestPriorityLowRedundancyBlocks()) {
System.out.println("\tLow redundancy blocks with highest priority " +
"to recover: " +
ecBlockGroupStats.getHighestPriorityLowRedundancyBlocks());
}
System.out.println("\tPending deletion blocks: " +
ecBlockGroupStats.getPendingDeletionBlocks());

43
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestLowRedundancyBlockQueues.java
View File

@ -63,7 +63,8 @@ private BlockInfo genStripedBlockInfo(long id, long numBytes) {
private void verifyBlockStats(LowRedundancyBlocks queues,
int lowRedundancyReplicaCount, int corruptReplicaCount,
int corruptReplicationOneCount, int lowRedundancyStripedCount,
int corruptStripedCount) {
int corruptStripedCount, int highestPriorityReplicatedBlockCount,
int highestPriorityECBlockCount) {
assertEquals("Low redundancy replica count incorrect!",
lowRedundancyReplicaCount, queues.getLowRedundancyBlocks());
assertEquals("Corrupt replica count incorrect!",
@ -81,6 +82,14 @@ private void verifyBlockStats(LowRedundancyBlocks queues,
assertEquals("LowRedundancyBlocks queue size incorrect!",
(lowRedundancyReplicaCount + corruptReplicaCount +
lowRedundancyStripedCount + corruptStripedCount), queues.size());
assertEquals("Highest priority replicated low redundancy " +
"blocks count is incorrect!",
highestPriorityReplicatedBlockCount,
queues.getHighestPriorityReplicatedBlockCount());
assertEquals("Highest priority erasure coded low redundancy " +
"blocks count is incorrect!",
highestPriorityECBlockCount,
queues.getHighestPriorityECBlockCount());
}
/**
@ -100,42 +109,46 @@ public void testBlockPriorities() throws Throwable {
// Add a block with a single entry
assertAdded(queues, block1, 1, 0, 3);
assertInLevel(queues, block1, LowRedundancyBlocks.QUEUE_HIGHEST_PRIORITY);
verifyBlockStats(queues, 1, 0, 0, 0, 0);
verifyBlockStats(queues, 1, 0, 0, 0, 0, 1, 0);
// Repeated additions fail
assertFalse(queues.add(block1, 1, 0, 0, 3));
verifyBlockStats(queues, 1, 0, 0, 0, 0);
verifyBlockStats(queues, 1, 0, 0, 0, 0, 1, 0);
// Add a second block with two replicas
assertAdded(queues, block2, 2, 0, 3);
assertInLevel(queues, block2, LowRedundancyBlocks.QUEUE_LOW_REDUNDANCY);
verifyBlockStats(queues, 2, 0, 0, 0, 0);
verifyBlockStats(queues, 2, 0, 0, 0, 0, 1, 0);
// Now try to add a block that is corrupt
assertAdded(queues, block_corrupt, 0, 0, 3);
assertInLevel(queues, block_corrupt,
LowRedundancyBlocks.QUEUE_WITH_CORRUPT_BLOCKS);
verifyBlockStats(queues, 2, 1, 0, 0, 0);
verifyBlockStats(queues, 2, 1, 0, 0, 0, 1, 0);
// Insert a very insufficiently redundancy block
assertAdded(queues, block_very_low_redundancy, 4, 0, 25);
assertInLevel(queues, block_very_low_redundancy,
LowRedundancyBlocks.QUEUE_VERY_LOW_REDUNDANCY);
verifyBlockStats(queues, 3, 1, 0, 0, 0);
verifyBlockStats(queues, 3, 1, 0, 0, 0, 1, 0);
// Insert a corrupt block with replication factor 1
assertAdded(queues, block_corrupt_repl_one, 0, 0, 1);
verifyBlockStats(queues, 3, 2, 1, 0, 0);
verifyBlockStats(queues, 3, 2, 1, 0, 0, 1, 0);
// Bump up the expected count for corrupt replica one block from 1 to 3
queues.update(block_corrupt_repl_one, 0, 0, 0, 3, 0, 2);
verifyBlockStats(queues, 3, 2, 0, 0, 0);
verifyBlockStats(queues, 3, 2, 0, 0, 0, 1, 0);
// Reduce the expected replicas to 1
queues.update(block_corrupt, 0, 0, 0, 1, 0, -2);
verifyBlockStats(queues, 3, 2, 1, 0, 0);
verifyBlockStats(queues, 3, 2, 1, 0, 0, 1, 0);
queues.update(block_very_low_redundancy, 0, 0, 0, 1, -4, -24);
verifyBlockStats(queues, 2, 3, 2, 0, 0);
verifyBlockStats(queues, 2, 3, 2, 0, 0, 1, 0);
// Reduce the expected replicas to 1 for block1
queues.update(block1, 1, 0, 0, 1, 0, 0);
verifyBlockStats(queues, 2, 3, 2, 0, 0, 0, 0);
}
@Test
@ -145,12 +158,12 @@ public void testRemoveWithWrongPriority() {
assertAdded(queues, corruptBlock, 0, 0, 3);
assertInLevel(queues, corruptBlock,
LowRedundancyBlocks.QUEUE_WITH_CORRUPT_BLOCKS);
verifyBlockStats(queues, 0, 1, 0, 0, 0);
verifyBlockStats(queues, 0, 1, 0, 0, 0, 0, 0);
// Remove with wrong priority
queues.remove(corruptBlock, LowRedundancyBlocks.QUEUE_LOW_REDUNDANCY);
// Verify the number of corrupt block is decremented
verifyBlockStats(queues, 0, 0, 0, 0, 0);
verifyBlockStats(queues, 0, 0, 0, 0, 0, 0, 0);
}
@Test
@ -186,17 +199,17 @@ private void doTestStripedBlockPriorities(int dataBlkNum, int parityBlkNum)
assertInLevel(queues, block,
LowRedundancyBlocks.QUEUE_LOW_REDUNDANCY);
}
verifyBlockStats(queues, 0, 0, 0, numUR, 0);
verifyBlockStats(queues, 0, 0, 0, numUR, 0, 0, 1);
}
// add a corrupted block
BlockInfo block_corrupt = genStripedBlockInfo(-10, numBytes);
assertEquals(numCorrupt, queues.getCorruptBlockSize());
verifyBlockStats(queues, 0, 0, 0, numUR, numCorrupt);
verifyBlockStats(queues, 0, 0, 0, numUR, numCorrupt, 0, 1);
assertAdded(queues, block_corrupt, dataBlkNum - 1, 0, groupSize);
numCorrupt++;
verifyBlockStats(queues, 0, 0, 0, numUR, numCorrupt);
verifyBlockStats(queues, 0, 0, 0, numUR, numCorrupt, 0, 1);
assertInLevel(queues, block_corrupt,
LowRedundancyBlocks.QUEUE_WITH_CORRUPT_BLOCKS);

12
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
View File

@ -412,10 +412,12 @@ private void verifyZeroMetrics() throws Exception {
// Verify replica metrics
assertGauge("LowRedundancyReplicatedBlocks", 0L, rb);
assertGauge("CorruptReplicatedBlocks", 0L, rb);
assertGauge("HighestPriorityLowRedundancyReplicatedBlocks", 0L, rb);
// Verify striped block groups metrics
assertGauge("LowRedundancyECBlockGroups", 0L, rb);
assertGauge("CorruptECBlockGroups", 0L, rb);
assertGauge("HighestPriorityLowRedundancyECBlocks", 0L, rb);
}
/**
@ -492,9 +494,11 @@ public void testCorruptBlock() throws Exception {
// Verify replicated blocks metrics
assertGauge("LowRedundancyReplicatedBlocks", 1L, rb);
assertGauge("CorruptReplicatedBlocks", 1L, rb);
assertGauge("HighestPriorityLowRedundancyReplicatedBlocks", 1L, rb);
// Verify striped blocks metrics
assertGauge("LowRedundancyECBlockGroups", 0L, rb);
assertGauge("CorruptECBlockGroups", 0L, rb);
assertGauge("HighestPriorityLowRedundancyECBlocks", 0L, rb);
verifyAggregatedMetricsTally();
@ -517,9 +521,11 @@ public void testCorruptBlock() throws Exception {
// Verify replicated blocks metrics
assertGauge("LowRedundancyReplicatedBlocks", 0L, rb);
assertGauge("CorruptReplicatedBlocks", 0L, rb);
assertGauge("HighestPriorityLowRedundancyReplicatedBlocks", 0L, rb);
// Verify striped blocks metrics
assertGauge("LowRedundancyECBlockGroups", 0L, rb);
assertGauge("CorruptECBlockGroups", 0L, rb);
assertGauge("HighestPriorityLowRedundancyECBlocks", 0L, rb);
verifyAggregatedMetricsTally();
@ -580,9 +586,11 @@ public void testStripedFileCorruptBlocks() throws Exception {
// Verify replica metrics
assertGauge("LowRedundancyReplicatedBlocks", 0L, rb);
assertGauge("CorruptReplicatedBlocks", 0L, rb);
assertGauge("HighestPriorityLowRedundancyReplicatedBlocks", 0L, rb);
// Verify striped block groups metrics
assertGauge("LowRedundancyECBlockGroups", 1L, rb);
assertGauge("CorruptECBlockGroups", 1L, rb);
assertGauge("HighestPriorityLowRedundancyECBlocks", 1L, rb);
verifyAggregatedMetricsTally();
@ -602,9 +610,11 @@ public void testStripedFileCorruptBlocks() throws Exception {
// Verify replicated blocks metrics
assertGauge("LowRedundancyReplicatedBlocks", 0L, rb);
assertGauge("CorruptReplicatedBlocks", 0L, rb);
assertGauge("HighestPriorityLowRedundancyReplicatedBlocks", 0L, rb);
// Verify striped blocks metrics
assertGauge("LowRedundancyECBlockGroups", 0L, rb);
assertGauge("CorruptECBlockGroups", 0L, rb);
assertGauge("HighestPriorityLowRedundancyECBlocks", 0L, rb);
verifyAggregatedMetricsTally();
@ -666,6 +676,8 @@ public void testMissingBlock() throws Exception {
assertGauge("UnderReplicatedBlocks", 1L, rb);
assertGauge("MissingBlocks", 1L, rb);
assertGauge("MissingReplOneBlocks", 1L, rb);
assertGauge("HighestPriorityLowRedundancyReplicatedBlocks", 0L, rb);
assertGauge("HighestPriorityLowRedundancyECBlocks", 0L, rb);
fs.delete(file, true);
waitForDnMetricValue(NS_METRICS, "UnderReplicatedBlocks", 0L);
}

32
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
View File

@ -579,7 +579,7 @@ public void testReportCommand() throws Exception {
// Verify report command for all counts to be zero
resetStream();
assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"}));
verifyNodesAndCorruptBlocks(numDn, numDn, 0, 0, client);
verifyNodesAndCorruptBlocks(numDn, numDn, 0, 0, client, 0L, 0L);
final short replFactor = 1;
final long fileLength = 512L;
@ -614,7 +614,7 @@ public void testReportCommand() throws Exception {
// Verify report command for all counts to be zero
resetStream();
assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"}));
verifyNodesAndCorruptBlocks(numDn, numDn, 0, 0, client);
verifyNodesAndCorruptBlocks(numDn, numDn, 0, 0, client, 0L, 0L);
// Choose a DataNode to shutdown
final List<DataNode> datanodes = miniCluster.getDataNodes();
@ -636,7 +636,7 @@ public void testReportCommand() throws Exception {
// Verify report command to show dead DataNode
assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"}));
verifyNodesAndCorruptBlocks(numDn, numDn - 1, 0, 0, client);
verifyNodesAndCorruptBlocks(numDn, numDn - 1, 0, 0, client, 0L, 1L);
// Corrupt the replicated block
final int blockFilesCorrupted = miniCluster
@ -664,7 +664,7 @@ public void testReportCommand() throws Exception {
// verify report command for corrupt replicated block
resetStream();
assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"}));
verifyNodesAndCorruptBlocks(numDn, numDn - 1, 1, 0, client);
verifyNodesAndCorruptBlocks(numDn, numDn - 1, 1, 0, client, 0L, 1L);
lbs = miniCluster.getFileSystem().getClient().
getNamenode().getBlockLocations(
@ -689,7 +689,7 @@ public void testReportCommand() throws Exception {
// and EC block group
resetStream();
assertEquals(0, ToolRunner.run(dfsAdmin, new String[] {"-report"}));
verifyNodesAndCorruptBlocks(numDn, numDn - 1, 1, 1, client);
verifyNodesAndCorruptBlocks(numDn, numDn - 1, 1, 1, client, 0L, 0L);
}
}
@ -834,7 +834,10 @@ private void verifyNodesAndCorruptBlocks(
final int numLiveDn,
final int numCorruptBlocks,
final int numCorruptECBlockGroups,
final DFSClient client) throws IOException {
final DFSClient client,
final Long highestPriorityLowRedundancyReplicatedBlocks,
final Long highestPriorityLowRedundancyECBlocks)
throws IOException {
/* init vars */
final String outStr = scanIntoString(out);
@ -847,12 +850,23 @@ private void verifyNodesAndCorruptBlocks(
final String expectedCorruptedECBlockGroupsStr = String.format(
"Block groups with corrupt internal blocks: %d",
numCorruptECBlockGroups);
final String highestPriorityLowRedundancyReplicatedBlocksStr
= String.format(
"\tLow redundancy blocks with highest priority " +
"to recover: %d",
highestPriorityLowRedundancyReplicatedBlocks);
final String highestPriorityLowRedundancyECBlocksStr = String.format(
"\tLow redundancy blocks with highest priority " +
"to recover: %d",
highestPriorityLowRedundancyReplicatedBlocks);
// verify nodes and corrupt blocks
assertThat(outStr, is(allOf(
containsString(expectedLiveNodesStr),
containsString(expectedCorruptedBlocksStr),
containsString(expectedCorruptedECBlockGroupsStr))));
containsString(expectedCorruptedECBlockGroupsStr),
containsString(highestPriorityLowRedundancyReplicatedBlocksStr),
containsString(highestPriorityLowRedundancyECBlocksStr))));
assertEquals(
numDn,
@ -867,8 +881,12 @@ private void verifyNodesAndCorruptBlocks(
client.getCorruptBlocksCount());
assertEquals(numCorruptBlocks, client.getNamenode()
.getReplicatedBlockStats().getCorruptBlocks());
assertEquals(highestPriorityLowRedundancyReplicatedBlocks, client.getNamenode()
.getReplicatedBlockStats().getHighestPriorityLowRedundancyBlocks());
assertEquals(numCorruptECBlockGroups, client.getNamenode()
.getECBlockGroupStats().getCorruptBlockGroups());
assertEquals(highestPriorityLowRedundancyECBlocks, client.getNamenode()
.getECBlockGroupStats().getHighestPriorityLowRedundancyBlocks());
}
@Test