big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-8822. Add SSD storagepolicy tests in TestBlockStoragePolicy#testDefaultPolicies (Contributed by Vinayakumar B)

Browse Source
This commit is contained in:
Vinayakumar B 2015-07-29 11:50:48 +05:30
parent 50887e5b07
commit 975e138df3
2 changed files with 75 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -760,6 +760,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8811. Move BlockStoragePolicy name's constants from HDFS-8811. Move BlockStoragePolicy name's constants from
HdfsServerConstants.java to HdfsConstants.java (vinayakumarb) HdfsServerConstants.java to HdfsConstants.java (vinayakumarb)
HDFS-8822. Add SSD storagepolicy tests in TestBlockStoragePolicy#
testDefaultPolicies (vinayakumarb)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

88
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
View File

@ -67,7 +67,11 @@ public class TestBlockStoragePolicy {
static final EnumSet<StorageType> none = EnumSet.noneOf(StorageType.class); static final EnumSet<StorageType> none = EnumSet.noneOf(StorageType.class);
static final EnumSet<StorageType> archive = EnumSet.of(StorageType.ARCHIVE); static final EnumSet<StorageType> archive = EnumSet.of(StorageType.ARCHIVE);
static final EnumSet<StorageType> disk = EnumSet.of(StorageType.DISK); static final EnumSet<StorageType> disk = EnumSet.of(StorageType.DISK);
static final EnumSet<StorageType> both = EnumSet.of(StorageType.DISK, StorageType.ARCHIVE); static final EnumSet<StorageType> ssd = EnumSet.of(StorageType.SSD);
static final EnumSet<StorageType> disk_archive = EnumSet.of(StorageType.DISK,
StorageType.ARCHIVE);
static final EnumSet<StorageType> all = EnumSet.of(StorageType.SSD,
StorageType.DISK, StorageType.ARCHIVE);
static final long FILE_LEN = 1024; static final long FILE_LEN = 1024;
static final short REPLICATION = 3; static final short REPLICATION = 3;
@ -153,8 +157,8 @@ public void testDefaultPolicies() {
final List<StorageType> computed = cold.chooseStorageTypes(replication); final List<StorageType> computed = cold.chooseStorageTypes(replication);
assertStorageType(computed, replication, StorageType.ARCHIVE); assertStorageType(computed, replication, StorageType.ARCHIVE);
} }
assertCreationFallback(cold, null, null, null); assertCreationFallback(cold, null, null, null, null, null);
assertReplicationFallback(cold, null, null, null); assertReplicationFallback(cold, null, null, null, null);
} }
{ // check Warm policy { // check Warm policy
@ -163,8 +167,10 @@ public void testDefaultPolicies() {
final List<StorageType> computed = warm.chooseStorageTypes(replication); final List<StorageType> computed = warm.chooseStorageTypes(replication);
assertStorageType(computed, replication, StorageType.DISK, StorageType.ARCHIVE); assertStorageType(computed, replication, StorageType.DISK, StorageType.ARCHIVE);
} }
assertCreationFallback(warm, StorageType.DISK, StorageType.DISK, StorageType.ARCHIVE); assertCreationFallback(warm, StorageType.DISK, StorageType.DISK,
assertReplicationFallback(warm, StorageType.DISK, StorageType.DISK, StorageType.ARCHIVE); StorageType.ARCHIVE, StorageType.DISK, null);
assertReplicationFallback(warm, StorageType.DISK, StorageType.DISK,
StorageType.ARCHIVE, StorageType.DISK);
} }
{ // check Hot policy { // check Hot policy
@ -173,8 +179,50 @@ public void testDefaultPolicies() {
final List<StorageType> computed = hot.chooseStorageTypes(replication); final List<StorageType> computed = hot.chooseStorageTypes(replication);
assertStorageType(computed, replication, StorageType.DISK); assertStorageType(computed, replication, StorageType.DISK);
} }
assertCreationFallback(hot, null, null, null); assertCreationFallback(hot, null, null, null, null, null);
assertReplicationFallback(hot, StorageType.ARCHIVE, null, StorageType.ARCHIVE); assertReplicationFallback(hot, StorageType.ARCHIVE, null,
StorageType.ARCHIVE, StorageType.ARCHIVE);
}
{ // check ONE_SSD policy
final BlockStoragePolicy onessd = POLICY_SUITE.getPolicy(ONESSD);
for (short replication = 1; replication < 6; replication++) {
final List<StorageType> computed = onessd
.chooseStorageTypes(replication);
assertStorageType(computed, replication, StorageType.SSD,
StorageType.DISK);
}
assertCreationFallback(onessd, StorageType.SSD, StorageType.SSD,
StorageType.SSD, StorageType.DISK, StorageType.SSD);
assertReplicationFallback(onessd, StorageType.SSD, StorageType.SSD,
StorageType.SSD, StorageType.DISK);
}
{ // check ALL_SSD policy
final BlockStoragePolicy allssd = POLICY_SUITE.getPolicy(ALLSSD);
for (short replication = 1; replication < 6; replication++) {
final List<StorageType> computed = allssd
.chooseStorageTypes(replication);
assertStorageType(computed, replication, StorageType.SSD);
}
assertCreationFallback(allssd, StorageType.DISK, StorageType.DISK, null,
StorageType.DISK, null);
assertReplicationFallback(allssd, StorageType.DISK, StorageType.DISK,
null, StorageType.DISK);
}
{ // check LAZY_PERSIST policy
final BlockStoragePolicy lazyPersist = POLICY_SUITE
.getPolicy(LAZY_PERSIST);
for (short replication = 1; replication < 6; replication++) {
final List<StorageType> computed = lazyPersist
.chooseStorageTypes(replication);
assertStorageType(computed, replication, StorageType.DISK);
}
assertCreationFallback(lazyPersist, StorageType.DISK, StorageType.DISK,
null, StorageType.DISK, null);
assertReplicationFallback(lazyPersist, StorageType.DISK,
StorageType.DISK, null, StorageType.DISK);
} }
} }
@ -199,20 +247,28 @@ static void assertStorageType(List<StorageType> computed, short replication,
} }
} }
static void assertCreationFallback(BlockStoragePolicy policy, StorageType noneExpected, static void assertCreationFallback(BlockStoragePolicy policy,
StorageType archiveExpected, StorageType diskExpected) { StorageType noneExpected, StorageType archiveExpected,
StorageType diskExpected, StorageType ssdExpected,
StorageType disk_archiveExpected) {
Assert.assertEquals(noneExpected, policy.getCreationFallback(none)); Assert.assertEquals(noneExpected, policy.getCreationFallback(none));
Assert.assertEquals(archiveExpected, policy.getCreationFallback(archive)); Assert.assertEquals(archiveExpected, policy.getCreationFallback(archive));
Assert.assertEquals(diskExpected, policy.getCreationFallback(disk)); Assert.assertEquals(diskExpected, policy.getCreationFallback(disk));
Assert.assertEquals(null, policy.getCreationFallback(both)); Assert.assertEquals(ssdExpected, policy.getCreationFallback(ssd));
Assert.assertEquals(disk_archiveExpected,
policy.getCreationFallback(disk_archive));
Assert.assertEquals(null, policy.getCreationFallback(all));
} }
static void assertReplicationFallback(BlockStoragePolicy policy, StorageType noneExpected, static void assertReplicationFallback(BlockStoragePolicy policy,
StorageType archiveExpected, StorageType diskExpected) { StorageType noneExpected, StorageType archiveExpected,
StorageType diskExpected, StorageType ssdExpected) {
Assert.assertEquals(noneExpected, policy.getReplicationFallback(none)); Assert.assertEquals(noneExpected, policy.getReplicationFallback(none));
Assert.assertEquals(archiveExpected, policy.getReplicationFallback(archive)); Assert
.assertEquals(archiveExpected, policy.getReplicationFallback(archive));
Assert.assertEquals(diskExpected, policy.getReplicationFallback(disk)); Assert.assertEquals(diskExpected, policy.getReplicationFallback(disk));
Assert.assertEquals(null, policy.getReplicationFallback(both)); Assert.assertEquals(ssdExpected, policy.getReplicationFallback(ssd));
Assert.assertEquals(null, policy.getReplicationFallback(all));
} }
private static interface CheckChooseStorageTypes { private static interface CheckChooseStorageTypes {
@ -260,7 +316,7 @@ public void checkChooseStorageTypes(BlockStoragePolicy p,
public void checkChooseStorageTypes(BlockStoragePolicy p, public void checkChooseStorageTypes(BlockStoragePolicy p,
short replication, List<StorageType> chosen, StorageType... expected) { short replication, List<StorageType> chosen, StorageType... expected) {
final List<StorageType> types = p.chooseStorageTypes(replication, final List<StorageType> types = p.chooseStorageTypes(replication,
chosen, both, true); chosen, disk_archive, true);
assertStorageTypes(types, expected); assertStorageTypes(types, expected);
} }
}; };
@ -272,7 +328,7 @@ public void checkChooseStorageTypes(BlockStoragePolicy p,
public void checkChooseStorageTypes(BlockStoragePolicy p, public void checkChooseStorageTypes(BlockStoragePolicy p,
short replication, List<StorageType> chosen, StorageType... expected) { short replication, List<StorageType> chosen, StorageType... expected) {
final List<StorageType> types = p.chooseStorageTypes(replication, final List<StorageType> types = p.chooseStorageTypes(replication,
chosen, both, false); chosen, disk_archive, false);
assertStorageTypes(types, expected); assertStorageTypes(types, expected);
} }
}; };