HDDS-1949. Missing or error-prone test cleanup.

Contributed by Doroszlai, Attila.
This commit is contained in:
Anu Engineer 2019-09-20 09:55:57 -07:00
parent dd0834696a
commit 5553887d95
9 changed files with 261 additions and 156 deletions

View File

@ -240,7 +240,7 @@ abstract class Builder {
protected static final int ACTIVE_OMS_NOT_SET = -1;
protected final OzoneConfiguration conf;
protected final String path;
protected String path;
protected String clusterId;
protected String omServiceId;
@ -269,9 +269,7 @@ abstract class Builder {
protected Builder(OzoneConfiguration conf) {
this.conf = conf;
this.clusterId = UUID.randomUUID().toString();
this.path = GenericTestUtils.getTempPath(
MiniOzoneClusterImpl.class.getSimpleName() + "-" + clusterId);
setClusterId(UUID.randomUUID().toString());
}
/**
@ -283,6 +281,8 @@ protected Builder(OzoneConfiguration conf) {
*/
public Builder setClusterId(String id) {
clusterId = id;
path = GenericTestUtils.getTempPath(
MiniOzoneClusterImpl.class.getSimpleName() + "-" + clusterId);
return this;
}

View File

@ -19,6 +19,8 @@
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import org.apache.commons.io.FileUtils;
@ -317,6 +319,7 @@ public void shutdown() {
stop();
FileUtils.deleteDirectory(baseDir);
ContainerCache.getInstance(conf).shutdownCache();
DefaultMetricsSystem.shutdown();
} catch (IOException e) {
LOG.error("Exception while shutting down the cluster.", e);
}
@ -325,26 +328,9 @@ public void shutdown() {
@Override
public void stop() {
LOG.info("Stopping the Mini Ozone Cluster");
if (ozoneManager != null) {
LOG.info("Stopping the OzoneManager");
ozoneManager.stop();
ozoneManager.join();
}
if (!hddsDatanodes.isEmpty()) {
LOG.info("Shutting the HddsDatanodes");
hddsDatanodes.parallelStream()
.forEach(dn -> {
dn.stop();
dn.join();
});
}
if (scm != null) {
LOG.info("Stopping the StorageContainerManager");
scm.stop();
scm.join();
}
stopOM(ozoneManager);
stopDatanodes(hddsDatanodes);
stopSCM(scm);
}
/**
@ -385,6 +371,37 @@ private void setCAClient(CertificateClient client) {
this.caClient = client;
}
private static void stopDatanodes(
Collection<HddsDatanodeService> hddsDatanodes) {
if (!hddsDatanodes.isEmpty()) {
LOG.info("Stopping the HddsDatanodes");
hddsDatanodes.parallelStream()
.forEach(MiniOzoneClusterImpl::stopDatanode);
}
}
private static void stopDatanode(HddsDatanodeService dn) {
if (dn != null) {
dn.stop();
dn.join();
}
}
private static void stopSCM(StorageContainerManager scm) {
if (scm != null) {
LOG.info("Stopping the StorageContainerManager");
scm.stop();
scm.join();
}
}
private static void stopOM(OzoneManager om) {
if (om != null) {
LOG.info("Stopping the OzoneManager");
om.stop();
om.join();
}
}
/**
* Builder for configuring the MiniOzoneCluster to run.
@ -404,8 +421,9 @@ public Builder(OzoneConfiguration conf) {
public MiniOzoneCluster build() throws IOException {
DefaultMetricsSystem.setMiniClusterMode(true);
initializeConfiguration();
StorageContainerManager scm;
OzoneManager om;
StorageContainerManager scm = null;
OzoneManager om = null;
List<HddsDatanodeService> hddsDatanodes = Collections.emptyList();
try {
scm = createSCM();
scm.start();
@ -413,19 +431,32 @@ public MiniOzoneCluster build() throws IOException {
if(certClient != null) {
om.setCertClient(certClient);
}
} catch (AuthenticationException ex) {
om.start();
hddsDatanodes = createHddsDatanodes(scm);
MiniOzoneClusterImpl cluster = new MiniOzoneClusterImpl(conf, om, scm,
hddsDatanodes);
cluster.setCAClient(certClient);
if (startDataNodes) {
cluster.startHddsDatanodes();
}
return cluster;
} catch (Exception ex) {
stopOM(om);
if (startDataNodes) {
stopDatanodes(hddsDatanodes);
}
stopSCM(scm);
removeConfiguration();
if (ex instanceof IOException) {
throw (IOException) ex;
}
if (ex instanceof RuntimeException) {
throw (RuntimeException) ex;
}
throw new IOException("Unable to build MiniOzoneCluster. ", ex);
}
om.start();
final List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(scm);
MiniOzoneClusterImpl cluster = new MiniOzoneClusterImpl(conf, om, scm,
hddsDatanodes);
cluster.setCAClient(certClient);
if (startDataNodes) {
cluster.startHddsDatanodes();
}
return cluster;
}
/**
@ -466,6 +497,10 @@ void initializeConfiguration() throws IOException {
configureTrace();
}
void removeConfiguration() {
FileUtils.deleteQuietly(new File(path));
}
/**
* Creates a new StorageContainerManager instance.
*

View File

@ -35,6 +35,7 @@
import java.util.HashSet;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsConfigKeys;
@ -53,6 +54,7 @@
import org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer;
import org.apache.hadoop.test.PathUtils;
import org.apache.hadoop.test.TestGenericTestUtils;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@ -64,7 +66,7 @@
*/
public class TestMiniOzoneCluster {
private static MiniOzoneCluster cluster;
private MiniOzoneCluster cluster;
private static OzoneConfiguration conf;
private final static File TEST_ROOT = TestGenericTestUtils.getTestDir();
@ -78,17 +80,21 @@ public static void setup() {
conf.setBoolean(DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, true);
WRITE_TMP.mkdirs();
READ_TMP.mkdirs();
WRITE_TMP.deleteOnExit();
READ_TMP.deleteOnExit();
}
@AfterClass
public static void cleanup() {
@After
public void cleanup() {
if (cluster != null) {
cluster.shutdown();
}
}
@AfterClass
public static void afterClass() {
FileUtils.deleteQuietly(WRITE_TMP);
FileUtils.deleteQuietly(READ_TMP);
}
@Test(timeout = 30000)
public void testStartMultipleDatanodes() throws Exception {
final int numberOfNodes = 3;

View File

@ -28,6 +28,7 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
@ -41,6 +42,7 @@
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.HddsUtils;
@ -92,6 +94,8 @@
import org.junit.rules.Timeout;
import org.mockito.ArgumentMatcher;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@ -101,6 +105,9 @@
*/
public class TestStorageContainerManager {
private static XceiverClientManager xceiverClientManager;
private static final Logger LOG = LoggerFactory.getLogger(
TestStorageContainerManager.class);
/**
* Set the timeout for every test.
*/
@ -306,9 +313,7 @@ public void testBlockDeletionTransactions() throws Exception {
}
}, 1000, 10000);
} finally {
if (cluster != null) {
cluster.shutdown();
}
cluster.shutdown();
}
}
@ -329,50 +334,54 @@ public void testBlockDeletingThrottling() throws Exception {
.build();
cluster.waitForClusterToBeReady();
DeletedBlockLog delLog = cluster.getStorageContainerManager()
.getScmBlockManager().getDeletedBlockLog();
Assert.assertEquals(0, delLog.getNumOfValidTransactions());
try {
DeletedBlockLog delLog = cluster.getStorageContainerManager()
.getScmBlockManager().getDeletedBlockLog();
Assert.assertEquals(0, delLog.getNumOfValidTransactions());
int limitSize = 1;
// Reset limit value to 1, so that we only allow one TX is dealt per
// datanode.
SCMBlockDeletingService delService = cluster.getStorageContainerManager()
.getScmBlockManager().getSCMBlockDeletingService();
delService.setBlockDeleteTXNum(limitSize);
int limitSize = 1;
// Reset limit value to 1, so that we only allow one TX is dealt per
// datanode.
SCMBlockDeletingService delService = cluster.getStorageContainerManager()
.getScmBlockManager().getSCMBlockDeletingService();
delService.setBlockDeleteTXNum(limitSize);
// Create {numKeys} random names keys.
TestStorageContainerManagerHelper helper =
new TestStorageContainerManagerHelper(cluster, conf);
Map<String, OmKeyInfo> keyLocations = helper.createKeys(numKeys, 4096);
// Wait for container report
Thread.sleep(5000);
for (OmKeyInfo keyInfo : keyLocations.values()) {
OzoneTestUtils.closeContainers(keyInfo.getKeyLocationVersions(),
cluster.getStorageContainerManager());
}
// Create {numKeys} random names keys.
TestStorageContainerManagerHelper helper =
new TestStorageContainerManagerHelper(cluster, conf);
Map<String, OmKeyInfo> keyLocations = helper.createKeys(numKeys, 4096);
// Wait for container report
Thread.sleep(5000);
for (OmKeyInfo keyInfo : keyLocations.values()) {
OzoneTestUtils.closeContainers(keyInfo.getKeyLocationVersions(),
cluster.getStorageContainerManager());
}
createDeleteTXLog(delLog, keyLocations, helper);
// Verify a few TX gets created in the TX log.
Assert.assertTrue(delLog.getNumOfValidTransactions() > 0);
createDeleteTXLog(delLog, keyLocations, helper);
// Verify a few TX gets created in the TX log.
Assert.assertTrue(delLog.getNumOfValidTransactions() > 0);
// Verify the size in delete commands is expected.
GenericTestUtils.waitFor(() -> {
NodeManager nodeManager = cluster.getStorageContainerManager()
.getScmNodeManager();
List<SCMCommand> commands = nodeManager.processHeartbeat(
nodeManager.getNodes(NodeState.HEALTHY).get(0));
// Verify the size in delete commands is expected.
GenericTestUtils.waitFor(() -> {
NodeManager nodeManager = cluster.getStorageContainerManager()
.getScmNodeManager();
List<SCMCommand> commands = nodeManager.processHeartbeat(
nodeManager.getNodes(NodeState.HEALTHY).get(0));
if (commands != null) {
for (SCMCommand cmd : commands) {
if (cmd.getType() == SCMCommandProto.Type.deleteBlocksCommand) {
List<DeletedBlocksTransaction> deletedTXs =
((DeleteBlocksCommand) cmd).blocksTobeDeleted();
return deletedTXs != null && deletedTXs.size() == limitSize;
if (commands != null) {
for (SCMCommand cmd : commands) {
if (cmd.getType() == SCMCommandProto.Type.deleteBlocksCommand) {
List<DeletedBlocksTransaction> deletedTXs =
((DeleteBlocksCommand) cmd).blocksTobeDeleted();
return deletedTXs != null && deletedTXs.size() == limitSize;
}
}
}
}
return false;
}, 500, 10000);
return false;
}, 500, 10000);
} finally {
cluster.shutdown();
}
}
private Map<Long, List<Long>> createDeleteTXLog(DeletedBlockLog delLog,
@ -450,12 +459,15 @@ public void testSCMReinitialization() throws Exception {
MiniOzoneCluster cluster =
MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build();
cluster.waitForClusterToBeReady();
// This will initialize SCM
StorageContainerManager.scmInit(conf, "testClusterId");
SCMStorageConfig scmStore = new SCMStorageConfig(conf);
Assert.assertEquals(NodeType.SCM, scmStore.getNodeType());
Assert.assertNotEquals("testClusterId", scmStore.getClusterID());
cluster.shutdown();
try {
// This will initialize SCM
StorageContainerManager.scmInit(conf, "testClusterId");
SCMStorageConfig scmStore = new SCMStorageConfig(conf);
Assert.assertEquals(NodeType.SCM, scmStore.getNodeType());
Assert.assertNotEquals("testClusterId", scmStore.getClusterID());
} finally {
cluster.shutdown();
}
}
@Test
@ -478,25 +490,29 @@ public void testScmInfo() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
final String path =
GenericTestUtils.getTempPath(UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
SCMStorageConfig scmStore = new SCMStorageConfig(conf);
String clusterId = UUID.randomUUID().toString();
String scmId = UUID.randomUUID().toString();
scmStore.setClusterId(clusterId);
scmStore.setScmId(scmId);
// writes the version file properties
scmStore.initialize();
StorageContainerManager scm = StorageContainerManager.createSCM(conf);
//Reads the SCM Info from SCM instance
ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
Assert.assertEquals(clusterId, scmInfo.getClusterId());
Assert.assertEquals(scmId, scmInfo.getScmId());
try {
Path scmPath = Paths.get(path, "scm-meta");
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
SCMStorageConfig scmStore = new SCMStorageConfig(conf);
String clusterId = UUID.randomUUID().toString();
String scmId = UUID.randomUUID().toString();
scmStore.setClusterId(clusterId);
scmStore.setScmId(scmId);
// writes the version file properties
scmStore.initialize();
StorageContainerManager scm = StorageContainerManager.createSCM(conf);
//Reads the SCM Info from SCM instance
ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
Assert.assertEquals(clusterId, scmInfo.getClusterId());
Assert.assertEquals(scmId, scmInfo.getScmId());
String expectedVersion = HddsVersionInfo.HDDS_VERSION_INFO.getVersion();
String actualVersion = scm.getSoftwareVersion();
Assert.assertEquals(expectedVersion, actualVersion);
String expectedVersion = HddsVersionInfo.HDDS_VERSION_INFO.getVersion();
String actualVersion = scm.getSoftwareVersion();
Assert.assertEquals(expectedVersion, actualVersion);
} finally {
FileUtils.deleteQuietly(new File(path));
}
}
/**
@ -564,48 +580,52 @@ public void testCloseContainerCommandOnRestart() throws Exception {
.build();
cluster.waitForClusterToBeReady();
TestStorageContainerManagerHelper helper =
new TestStorageContainerManagerHelper(cluster, conf);
try {
TestStorageContainerManagerHelper helper =
new TestStorageContainerManagerHelper(cluster, conf);
helper.createKeys(10, 4096);
Thread.sleep(5000);
helper.createKeys(10, 4096);
Thread.sleep(5000);
StorageContainerManager scm = cluster.getStorageContainerManager();
List<ContainerInfo> containers = cluster.getStorageContainerManager()
.getContainerManager().getContainers();
Assert.assertNotNull(containers);
ContainerInfo selectedContainer = containers.iterator().next();
StorageContainerManager scm = cluster.getStorageContainerManager();
List<ContainerInfo> containers = cluster.getStorageContainerManager()
.getContainerManager().getContainers();
Assert.assertNotNull(containers);
ContainerInfo selectedContainer = containers.iterator().next();
// Stop processing HB
scm.getDatanodeProtocolServer().stop();
// Stop processing HB
scm.getDatanodeProtocolServer().stop();
scm.getContainerManager().updateContainerState(selectedContainer
.containerID(), HddsProtos.LifeCycleEvent.FINALIZE);
cluster.restartStorageContainerManager(true);
scm = cluster.getStorageContainerManager();
EventPublisher publisher = mock(EventPublisher.class);
ReplicationManager replicationManager = scm.getReplicationManager();
Field f = replicationManager.getClass().getDeclaredField("eventPublisher");
f.setAccessible(true);
Field modifiersField = Field.class.getDeclaredField("modifiers");
modifiersField.setAccessible(true);
modifiersField.setInt(f, f.getModifiers() & ~Modifier.FINAL);
f.set(replicationManager, publisher);
scm.getReplicationManager().start();
Thread.sleep(2000);
scm.getContainerManager().updateContainerState(selectedContainer
.containerID(), HddsProtos.LifeCycleEvent.FINALIZE);
cluster.restartStorageContainerManager(true);
scm = cluster.getStorageContainerManager();
EventPublisher publisher = mock(EventPublisher.class);
ReplicationManager replicationManager = scm.getReplicationManager();
Field f = ReplicationManager.class.getDeclaredField("eventPublisher");
f.setAccessible(true);
Field modifiersField = Field.class.getDeclaredField("modifiers");
modifiersField.setAccessible(true);
modifiersField.setInt(f, f.getModifiers() & ~Modifier.FINAL);
f.set(replicationManager, publisher);
scm.getReplicationManager().start();
Thread.sleep(2000);
UUID dnUuid = cluster.getHddsDatanodes().iterator().next()
.getDatanodeDetails().getUuid();
UUID dnUuid = cluster.getHddsDatanodes().iterator().next()
.getDatanodeDetails().getUuid();
CloseContainerCommand closeContainerCommand =
new CloseContainerCommand(selectedContainer.getContainerID(),
selectedContainer.getPipelineID(), false);
CloseContainerCommand closeContainerCommand =
new CloseContainerCommand(selectedContainer.getContainerID(),
selectedContainer.getPipelineID(), false);
CommandForDatanode commandForDatanode = new CommandForDatanode(
dnUuid, closeContainerCommand);
CommandForDatanode commandForDatanode = new CommandForDatanode(
dnUuid, closeContainerCommand);
verify(publisher).fireEvent(eq(SCMEvents.DATANODE_COMMAND), argThat(new
CloseContainerCommandMatcher(dnUuid, commandForDatanode)));
verify(publisher).fireEvent(eq(SCMEvents.DATANODE_COMMAND), argThat(new
CloseContainerCommandMatcher(dnUuid, commandForDatanode)));
} finally {
cluster.shutdown();
}
}
@SuppressWarnings("visibilitymodifier")

View File

@ -49,7 +49,10 @@
import static org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer
.writeChunkForContainer;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
@ -64,16 +67,28 @@ public class TestContainerReplication {
@Rule
public Timeout testTimeout = new Timeout(300000);
private OzoneConfiguration conf;
private MiniOzoneCluster cluster;
@Before
public void setup() throws Exception {
conf = newOzoneConfiguration();
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(2)
.setRandomContainerPort(true).build();
}
@After
public void teardown() {
if (cluster != null) {
cluster.shutdown();
}
}
@Test
public void testContainerReplication() throws Exception {
//GIVEN
OzoneConfiguration conf = newOzoneConfiguration();
long containerId = 1L;
MiniOzoneCluster cluster =
MiniOzoneCluster.newBuilder(conf).setNumDatanodes(2)
.setRandomContainerPort(true).build();
cluster.waitForClusterToBeReady();
HddsDatanodeService firstDatanode = cluster.getHddsDatanodes().get(0);

View File

@ -52,6 +52,7 @@
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.hadoop.ozone.container.common.utils.ReferenceCountedDB;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
@ -121,6 +122,13 @@ public static void init() throws Exception {
containerIdsWithDeletedBlocks = new HashSet<>();
}
@AfterClass
public static void cleanup() {
if (cluster != null) {
cluster.shutdown();
}
}
@Test
public void testBlockDeletion() throws Exception {
String volumeName = UUID.randomUUID().toString();

View File

@ -17,9 +17,7 @@
*/
package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
import java.io.IOException;
import java.util.HashMap;
import java.util.concurrent.TimeoutException;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
@ -41,23 +39,38 @@
import org.apache.hadoop.test.GenericTestUtils;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
/**
* Test to behaviour of the datanode when recieve close container command.
* Test to behaviour of the datanode when receive close container command.
*/
public class TestCloseContainerHandler {
@Test
public void test()
throws IOException, TimeoutException, InterruptedException {
private MiniOzoneCluster cluster;
private OzoneConfiguration conf;
@Before
public void setup() throws Exception {
//setup a cluster (1G free space is enough for a unit test)
OzoneConfiguration conf = new OzoneConfiguration();
conf = new OzoneConfiguration();
conf.set(OZONE_SCM_CONTAINER_SIZE, "1GB");
MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf)
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(1).build();
}
@After
public void teardown() {
if (cluster != null) {
cluster.shutdown();
}
}
@Test
public void test() throws Exception {
cluster.waitForClusterToBeReady();
//the easiest way to create an open container is creating a key
@ -109,7 +122,7 @@ public void test()
Assert.assertTrue(isContainerClosed(cluster, containerId.getId()));
}
private Boolean isContainerClosed(MiniOzoneCluster cluster,
private static Boolean isContainerClosed(MiniOzoneCluster cluster,
long containerID) {
ContainerData containerData;
containerData = cluster.getHddsDatanodes().get(0)
@ -118,4 +131,4 @@ private Boolean isContainerClosed(MiniOzoneCluster cluster,
return !containerData.isOpen();
}
}
}

View File

@ -332,7 +332,7 @@ public void testSCMSafeModeRestrictedOp() throws Exception {
@Test(timeout = 300_000)
public void testSCMSafeModeDisabled() throws Exception {
cluster.stop();
cluster.shutdown();
// If safe mode is disabled, cluster should not be in safe mode even if
// min number of datanodes are not started.

View File

@ -23,6 +23,7 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@ -66,6 +67,13 @@ public static void init() throws IOException, TimeoutException,
mbs = ManagementFactory.getPlatformMBeanServer();
}
@AfterClass
public static void cleanup() {
if (cluster != null) {
cluster.shutdown();
}
}
@Test
public void testDiskUsage() throws Exception {
ObjectName bean = new ObjectName(