big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-1559. Include committedBytes to determine Out of Space in VolumeChoosingPolicy. Contributed by Supratim Deka (#841)

Browse Source
This commit is contained in:
supratimdeka 2019-05-29 09:18:14 +05:30 committed by Arpit Agarwal
parent 7f2e87a419
commit 346c2b7980
2 changed files with 38 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/RoundRobinVolumeChoosingPolicy.java
View File

@ -58,7 +58,9 @@ public HddsVolume chooseVolume(List<HddsVolume> volumes,
while (true) {
final HddsVolume volume = volumes.get(currentVolumeIndex);
long availableVolumeSize = volume.getAvailable();
// adjust for remaining capacity in Open containers
long availableVolumeSize = volume.getAvailable()
- volume.getCommittedBytes();
currentVolumeIndex = (currentVolumeIndex + 1) % volumes.size();

35
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
View File

@ -28,6 +28,7 @@
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
@ -52,6 +53,7 @@
import java.util.List;
import java.util.ArrayList;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.DISK_OUT_OF_SPACE;
import static org.junit.Assert.assertEquals;
/**
@ -135,6 +137,39 @@ public void testBuildContainerMap() throws Exception {
verifyCommittedSpace(ozoneContainer);
}
@Test
public void testContainerCreateDiskFull() throws Exception {
volumeSet = new VolumeSet(datanodeDetails.getUuidString(), conf);
volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy();
long containerSize = (long) StorageUnit.MB.toBytes(100);
boolean diskSpaceException = false;
// Format the volumes
for (HddsVolume volume : volumeSet.getVolumesList()) {
volume.format(UUID.randomUUID().toString());
// eat up all available space except size of 1 container
volume.incCommittedBytes(volume.getAvailable() - containerSize);
// eat up 10 bytes more, now available space is less than 1 container
volume.incCommittedBytes(10);
}
keyValueContainerData = new KeyValueContainerData(99, containerSize,
UUID.randomUUID().toString(), datanodeDetails.getUuidString());
keyValueContainer = new KeyValueContainer(keyValueContainerData, conf);
// we expect an out of space Exception
try {
keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId);
} catch (StorageContainerException e) {
if (e.getResult() == DISK_OUT_OF_SPACE) {
diskSpaceException = true;
}
}
// Test failed if there was no exception
assertEquals(true, diskSpaceException);
}
//verify committed space on each volume
private void verifyCommittedSpace(OzoneContainer oc) {
for (HddsVolume dnVol : oc.getVolumeSet().getVolumesList()) {