big-data/hadoop
big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

HDFS-15911 : Provide blocks moved count in Balancer iteration result (#2794)

Browse Source 
Contributed by Viraj Jasani.

Signed-off-by: Mingliang Liu <liuml07@apache.org>
Signed-off-by: Ayush Saxena <ayushsaxena@apache.org>
This commit is contained in:
Viraj Jasani 2021-03-24 11:17:45 +05:30 committed by GitHub
parent 03cfc85279
commit 4b4ccce02f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 30 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
View File

@ -38,6 +38,7 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.DFSUtilClient;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -638,13 +639,15 @@ public class Balancer {
private final long bytesLeftToMove; private final long bytesLeftToMove;
private final long bytesBeingMoved; private final long bytesBeingMoved;
private final long bytesAlreadyMoved; private final long bytesAlreadyMoved;
private final long blocksMoved;
Result(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved, Result(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved,
long bytesAlreadyMoved) { long bytesAlreadyMoved, long blocksMoved) {
this.exitStatus = exitStatus; this.exitStatus = exitStatus;
this.bytesLeftToMove = bytesLeftToMove; this.bytesLeftToMove = bytesLeftToMove;
this.bytesBeingMoved = bytesBeingMoved; this.bytesBeingMoved = bytesBeingMoved;
this.bytesAlreadyMoved = bytesAlreadyMoved; this.bytesAlreadyMoved = bytesAlreadyMoved;
this.blocksMoved = blocksMoved;
} }
public ExitStatus getExitStatus() { public ExitStatus getExitStatus() {
@ -663,23 +666,40 @@ public class Balancer {
return bytesAlreadyMoved; return bytesAlreadyMoved;
} }
public long getBlocksMoved() {
return blocksMoved;
}
void print(int iteration, NameNodeConnector nnc, PrintStream out) { void print(int iteration, NameNodeConnector nnc, PrintStream out) {
out.printf("%-24s %10d %19s %18s %17s %s%n", out.printf("%-24s %10d %19s %18s %17s %17s %s%n",
DateFormat.getDateTimeInstance().format(new Date()), iteration, DateFormat.getDateTimeInstance().format(new Date()), iteration,
StringUtils.byteDesc(bytesAlreadyMoved), StringUtils.byteDesc(bytesAlreadyMoved),
StringUtils.byteDesc(bytesLeftToMove), StringUtils.byteDesc(bytesLeftToMove),
StringUtils.byteDesc(bytesBeingMoved), StringUtils.byteDesc(bytesBeingMoved),
blocksMoved,
nnc.getNameNodeUri()); nnc.getNameNodeUri());
} }
@Override
public String toString() {
return new ToStringBuilder(this)
.append("exitStatus", exitStatus)
.append("bytesLeftToMove", bytesLeftToMove)
.append("bytesBeingMoved", bytesBeingMoved)
.append("bytesAlreadyMoved", bytesAlreadyMoved)
.append("blocksMoved", blocksMoved)
.toString();
}
} }
Result newResult(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved) { Result newResult(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved) {
return new Result(exitStatus, bytesLeftToMove, bytesBeingMoved, return new Result(exitStatus, bytesLeftToMove, bytesBeingMoved,
dispatcher.getBytesMoved()); dispatcher.getBytesMoved(), dispatcher.getBblocksMoved());
} }
Result newResult(ExitStatus exitStatus) { Result newResult(ExitStatus exitStatus) {
return new Result(exitStatus, -1, -1, dispatcher.getBytesMoved()); return new Result(exitStatus, -1, -1, dispatcher.getBytesMoved(),
dispatcher.getBblocksMoved());
} }
/** Run an iteration for all datanodes. */ /** Run an iteration for all datanodes. */

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
View File

@ -1658,6 +1658,7 @@ public class TestBalancer {
// a block is moved unexpectedly, IN_PROGRESS will be reported. // a block is moved unexpectedly, IN_PROGRESS will be reported.
assertEquals("We expect ExitStatus.NO_MOVE_PROGRESS to be reported.", assertEquals("We expect ExitStatus.NO_MOVE_PROGRESS to be reported.",
ExitStatus.NO_MOVE_PROGRESS, r.getExitStatus()); ExitStatus.NO_MOVE_PROGRESS, r.getExitStatus());
assertEquals(0, r.getBlocksMoved());
} }
} finally { } finally {
for (NameNodeConnector nnc : connectors) { for (NameNodeConnector nnc : connectors) {
@ -2309,8 +2310,11 @@ public class TestBalancer {
// Hence, overall total blocks moved by HDFS balancer would be either of these 2 options: // Hence, overall total blocks moved by HDFS balancer would be either of these 2 options:
// a) 2 blocks of total size (100B + 100B) // a) 2 blocks of total size (100B + 100B)
// b) 3 blocks of total size (50B + 100B + 100B) // b) 3 blocks of total size (50B + 100B + 100B)
assertTrue(balancerResult.getBytesAlreadyMoved() == 200 assertTrue("BalancerResult is not as expected. " + balancerResult,
|| balancerResult.getBytesAlreadyMoved() == 250); (balancerResult.getBytesAlreadyMoved() == 200
&& balancerResult.getBlocksMoved() == 2)
|| (balancerResult.getBytesAlreadyMoved() == 250
&& balancerResult.getBlocksMoved() == 3));
// 100% and 95% used nodes will be balanced, so top used will be 900 // 100% and 95% used nodes will be balanced, so top used will be 900
assertEquals(900, maxUsage); assertEquals(900, maxUsage);
} }