big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12750. Ozone: Fix TestStorageContainerManager#testBlockDeletionTransactions. Contributed by Xiaoyu Yao.

Browse Source
This commit is contained in:
Weiwei Yang 2017-11-01 17:17:03 +08:00 committed by Owen O'Malley
parent 0760418bec
commit 0559265822
1 changed files with 82 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
View File

@ -175,91 +175,97 @@ public void testBlockDeletionTransactions() throws Exception {
new MiniOzoneCluster.Builder(conf).numDataNodes(1) new MiniOzoneCluster.Builder(conf).numDataNodes(1)
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build(); .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
DeletedBlockLog delLog = cluster.getStorageContainerManager() try {
.getScmBlockManager().getDeletedBlockLog(); DeletedBlockLog delLog = cluster.getStorageContainerManager()
Assert.assertEquals(0, delLog.getNumOfValidTransactions()); .getScmBlockManager().getDeletedBlockLog();
Assert.assertEquals(0, delLog.getNumOfValidTransactions());
// Create 20 random names keys. // Create 20 random names keys.
TestStorageContainerManagerHelper helper = TestStorageContainerManagerHelper helper =
new TestStorageContainerManagerHelper(cluster, conf); new TestStorageContainerManagerHelper(cluster, conf);
Map<String, KsmKeyInfo> keyLocations = helper.createKeys(20, 4096); Map<String, KsmKeyInfo> keyLocations = helper.createKeys(20, 4096);
// These keys will be written into a bunch of containers, // These keys will be written into a bunch of containers,
// gets a set of container names, verify container containerBlocks // gets a set of container names, verify container containerBlocks
// on datanodes. // on datanodes.
Set<String> containerNames = new HashSet<>(); Set<String> containerNames = new HashSet<>();
for (Map.Entry<String, KsmKeyInfo> entry : keyLocations.entrySet()) { for (Map.Entry<String, KsmKeyInfo> entry : keyLocations.entrySet()) {
entry.getValue().getKeyLocationList() entry.getValue().getKeyLocationList()
.forEach(loc -> containerNames.add(loc.getContainerName())); .forEach(loc -> containerNames.add(loc.getContainerName()));
} }
// Total number of containerBlocks of these containers should be equal to // Total number of containerBlocks of these containers should be equal to
// total number of containerBlocks via creation call. // total number of containerBlocks via creation call.
int totalCreatedBlocks = 0; int totalCreatedBlocks = 0;
for (KsmKeyInfo info : keyLocations.values()) { for (KsmKeyInfo info : keyLocations.values()) {
totalCreatedBlocks += info.getKeyLocationList().size(); totalCreatedBlocks += info.getKeyLocationList().size();
} }
Assert.assertTrue(totalCreatedBlocks > 0); Assert.assertTrue(totalCreatedBlocks > 0);
Assert.assertEquals(totalCreatedBlocks, Assert.assertEquals(totalCreatedBlocks,
helper.getAllBlocks(containerNames).size()); helper.getAllBlocks(containerNames).size());
// Create a deletion TX for each key. // Create a deletion TX for each key.
Map<String, List<String>> containerBlocks = Maps.newHashMap(); Map<String, List<String>> containerBlocks = Maps.newHashMap();
for (KsmKeyInfo info : keyLocations.values()) { for (KsmKeyInfo info : keyLocations.values()) {
List<KsmKeyLocationInfo> list = info.getKeyLocationList(); List<KsmKeyLocationInfo> list = info.getKeyLocationList();
list.forEach(location -> { list.forEach(location -> {
if (containerBlocks.containsKey(location.getContainerName())) { if (containerBlocks.containsKey(location.getContainerName())) {
containerBlocks.get(location.getContainerName()) containerBlocks.get(location.getContainerName())
.add(location.getBlockID()); .add(location.getBlockID());
} else { } else {
List<String> blks = Lists.newArrayList(); List<String> blks = Lists.newArrayList();
blks.add(location.getBlockID()); blks.add(location.getBlockID());
containerBlocks.put(location.getContainerName(), blks); containerBlocks.put(location.getContainerName(), blks);
}
});
}
for (Map.Entry<String, List<String>> tx : containerBlocks.entrySet()) {
delLog.addTransaction(tx.getKey(), tx.getValue());
}
// Verify a few TX gets created in the TX log.
Assert.assertTrue(delLog.getNumOfValidTransactions() > 0);
// Once TXs are written into the log, SCM starts to fetch TX
// entries from the log and schedule block deletions in HB interval,
// after sometime, all the TX should be proceed and by then
// the number of containerBlocks of all known containers will be
// empty again.
GenericTestUtils.waitFor(() -> {
try {
return delLog.getNumOfValidTransactions() == 0;
} catch (IOException e) {
return false;
} }
}); }, 1000, 10000);
} Assert.assertTrue(helper.getAllBlocks(containerNames).isEmpty());
for (Map.Entry<String, List<String>> tx : containerBlocks.entrySet()) {
delLog.addTransaction(tx.getKey(), tx.getValue());
}
// Verify a few TX gets created in the TX log. // Continue the work, add some TXs that with known container names,
Assert.assertTrue(delLog.getNumOfValidTransactions() > 0); // but unknown block IDs.
for (String containerName : containerBlocks.keySet()) {
// Once TXs are written into the log, SCM starts to fetch TX // Add 2 TXs per container.
// entries from the log and schedule block deletions in HB interval, delLog.addTransaction(containerName,
// after sometime, all the TX should be proceed and by then Collections.singletonList(RandomStringUtils.randomAlphabetic(5)));
// the number of containerBlocks of all known containers will be delLog.addTransaction(containerName,
// empty again. Collections.singletonList(RandomStringUtils.randomAlphabetic(5)));
GenericTestUtils.waitFor(() -> {
try {
return delLog.getNumOfValidTransactions() == 0;
} catch (IOException e) {
return false;
} }
}, 1000, 10000);
Assert.assertTrue(helper.getAllBlocks(containerNames).isEmpty());
// Continue the work, add some TXs that with known container names, // Verify a few TX gets created in the TX log.
// but unknown block IDs. Assert.assertTrue(delLog.getNumOfValidTransactions() > 0);
for (String containerName : containerBlocks.keySet()) {
// Add 2 TXs per container.
delLog.addTransaction(containerName,
Collections.singletonList(RandomStringUtils.randomAlphabetic(5)));
delLog.addTransaction(containerName,
Collections.singletonList(RandomStringUtils.randomAlphabetic(5)));
}
// Verify a few TX gets created in the TX log. // These blocks cannot be found in the container, skip deleting them
Assert.assertTrue(delLog.getNumOfValidTransactions() > 0); // eventually these TX will success.
GenericTestUtils.waitFor(() -> {
// These blocks cannot be found in the container, skip deleting them try {
// eventually these TX will success. return delLog.getFailedTransactions().size() == 0;
GenericTestUtils.waitFor(() -> { } catch (IOException e) {
try { return false;
return delLog.getFailedTransactions().size() == 0; }
} catch (IOException e) { }, 1000, 10000);
return false; } finally {
if (cluster != null) {
cluster.shutdown();
} }
}, 1000, 10000); }
} }
} }