big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13728. Disk Balancer should not fail if volume usage is greater than capacity. Contributed by Stephen O'Donnell.

Browse Source
This commit is contained in:
Xiao Chen 2018-08-07 22:04:41 -07:00
parent 2b0f977241
commit 6677717c68
2 changed files with 28 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java
View File

@ -21,9 +21,10 @@
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectReader;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.web.JsonUtil; import org.apache.hadoop.hdfs.web.JsonUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException; import java.io.IOException;
@ -35,6 +36,9 @@ public class DiskBalancerVolume {
private static final ObjectReader READER = private static final ObjectReader READER =
new ObjectMapper().readerFor(DiskBalancerVolume.class); new ObjectMapper().readerFor(DiskBalancerVolume.class);
private static final Logger LOG =
LoggerFactory.getLogger(DiskBalancerVolume.class);
private String path; private String path;
private long capacity; private long capacity;
private String storageType; private String storageType;
@ -269,10 +273,13 @@ public long getUsed() {
* @param dfsUsedSpace - dfsUsedSpace for this volume. * @param dfsUsedSpace - dfsUsedSpace for this volume.
*/ */
public void setUsed(long dfsUsedSpace) { public void setUsed(long dfsUsedSpace) {
Preconditions.checkArgument(dfsUsedSpace < this.getCapacity(), if (dfsUsedSpace > this.getCapacity()) {
"DiskBalancerVolume.setUsed: dfsUsedSpace(%s) < capacity(%s)", LOG.warn("Volume usage ("+dfsUsedSpace+") is greater than capacity ("+
dfsUsedSpace, getCapacity()); this.getCapacity()+"). Setting volume usage to the capacity");
this.used = dfsUsedSpace; this.used = this.getCapacity();
} else {
this.used = dfsUsedSpace;
}
} }
/** /**

16
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/TestDataModels.java
View File

@ -224,4 +224,20 @@ public void testClusterSerialize() throws Exception {
Assert Assert
.assertEquals(cluster.getNodes().size(), newCluster.getNodes().size()); .assertEquals(cluster.getNodes().size(), newCluster.getNodes().size());
} }
@Test
public void testUsageLimitedToCapacity() throws Exception {
DiskBalancerTestUtil util = new DiskBalancerTestUtil();
// If usage is greater than capacity, then it should be set to capacity
DiskBalancerVolume v1 = util.createRandomVolume(StorageType.DISK);
v1.setCapacity(DiskBalancerTestUtil.GB);
v1.setUsed(2 * DiskBalancerTestUtil.GB);
Assert.assertEquals(v1.getUsed(),v1.getCapacity());
// If usage is less than capacity, usage should be set to the real usage
DiskBalancerVolume v2 = util.createRandomVolume(StorageType.DISK);
v2.setCapacity(2*DiskBalancerTestUtil.GB);
v2.setUsed(DiskBalancerTestUtil.GB);
Assert.assertEquals(v1.getUsed(),DiskBalancerTestUtil.GB);
}
} }