From e543c3b31aa607e62b7bd718d9cb7f17e6b03072 Mon Sep 17 00:00:00 2001 From: Hanisha Koneru Date: Tue, 16 Apr 2019 13:51:39 -0700 Subject: [PATCH] HDDS-1376. Datanode exits while executing client command when scmId is null (#724) --- .../common/statemachine/DatanodeStateMachine.java | 1 - .../states/endpoint/VersionEndpointTask.java | 3 ++- .../ozone/container/ozoneimpl/OzoneContainer.java | 3 ++- .../common/TestDatanodeStateMachine.java | 15 +++++++++++++++ .../TestCloseContainerCommandHandler.java | 11 +++++------ .../container/ozoneimpl/TestOzoneContainer.java | 5 ++--- .../ozoneimpl/TestOzoneContainerWithTLS.java | 5 ++--- .../ozoneimpl/TestSecureOzoneContainer.java | 5 ++--- 8 files changed, 30 insertions(+), 18 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index ff391031a5..69782efbfd 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -166,7 +166,6 @@ public OzoneContainer getContainer() { private void start() throws IOException { long now = 0; - container.start(); reportManager.init(); initCommandHandlerThread(conf); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java index e4c0eb16aa..04eaa05f44 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java @@ -106,7 +106,8 @@ public EndpointStateMachine.EndPointStates call() throws Exception { volumeSet.writeUnlock(); } - ozoneContainer.getDispatcher().setScmId(scmId); + // Start the container services after getting the version information + ozoneContainer.start(scmId); EndpointStateMachine.EndPointStates nextState = rpcEndPoint.getState().getNextState(); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index 87266a99ce..ed7c88c520 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -180,12 +180,13 @@ private void stopContainerScrub() { * * @throws IOException */ - public void start() throws IOException { + public void start(String scmId) throws IOException { LOG.info("Attempting to start container services."); startContainerScrub(); writeChannel.start(); readChannel.start(); hddsDispatcher.init(); + hddsDispatcher.setScmId(scmId); } /** diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java index 29160eeca2..47438dc882 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java @@ -258,6 +258,21 @@ public void testDatanodeStateContext() throws IOException, task.execute(executorService); newState = task.await(10, TimeUnit.SECONDS); + + // Wait for GetVersion call (called by task.execute) to finish. After + // Earlier task.execute called into GetVersion. Wait for the execution + // to finish and the endPointState to move to REGISTER state. + GenericTestUtils.waitFor(() -> { + for (EndpointStateMachine endpoint : + stateMachine.getConnectionManager().getValues()) { + if (endpoint.getState() != + EndpointStateMachine.EndPointStates.REGISTER) { + return false; + } + } + return true; + }, 1000, 50000); + // If we are in running state, we should be in running. Assert.assertEquals(DatanodeStateMachine.DatanodeStates.RUNNING, newState); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java index 7962cb2f3b..6356d0be78 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerCommandHandler.java @@ -68,7 +68,7 @@ public void testCloseContainerViaRatis() final DatanodeDetails datanodeDetails = randomDatanodeDetails(); final OzoneContainer ozoneContainer = getOzoneContainer(conf, datanodeDetails); - ozoneContainer.start(); + ozoneContainer.start(UUID.randomUUID().toString()); try { final Container container = createContainer(conf, datanodeDetails, ozoneContainer); @@ -105,7 +105,7 @@ public void testCloseContainerViaStandalone() final DatanodeDetails datanodeDetails = randomDatanodeDetails(); final OzoneContainer ozoneContainer = getOzoneContainer(conf, datanodeDetails); - ozoneContainer.start(); + ozoneContainer.start(UUID.randomUUID().toString()); try { final Container container = createContainer(conf, datanodeDetails, ozoneContainer); @@ -140,7 +140,7 @@ public void testQuasiCloseToClose() throws Exception { final DatanodeDetails datanodeDetails = randomDatanodeDetails(); final OzoneContainer ozoneContainer = getOzoneContainer(conf, datanodeDetails); - ozoneContainer.start(); + ozoneContainer.start(UUID.randomUUID().toString()); try { final Container container = createContainer(conf, datanodeDetails, ozoneContainer); @@ -188,7 +188,7 @@ public void testForceCloseOpenContainer() throws Exception { final DatanodeDetails datanodeDetails = randomDatanodeDetails(); final OzoneContainer ozoneContainer = getOzoneContainer(conf, datanodeDetails); - ozoneContainer.start(); + ozoneContainer.start(UUID.randomUUID().toString()); try { final Container container = createContainer(conf, datanodeDetails, ozoneContainer); @@ -224,7 +224,7 @@ public void testQuasiCloseClosedContainer() final DatanodeDetails datanodeDetails = randomDatanodeDetails(); final OzoneContainer ozoneContainer = getOzoneContainer( conf, datanodeDetails); - ozoneContainer.start(); + ozoneContainer.start(UUID.randomUUID().toString()); try { final Container container = createContainer( conf, datanodeDetails, ozoneContainer); @@ -277,7 +277,6 @@ private OzoneContainer getOzoneContainer(final OzoneConfiguration conf, Mockito.when(context.getParent()).thenReturn(datanodeStateMachine); final OzoneContainer ozoneContainer = new OzoneContainer( datanodeDetails, conf, context, null); - ozoneContainer.getDispatcher().setScmId(UUID.randomUUID().toString()); return ozoneContainer; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index eeb6c5375c..fbdfbed75b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -82,9 +82,8 @@ public void testCreateOzoneContainer() throws Exception { Mockito.when(dsm.getDatanodeDetails()).thenReturn(datanodeDetails); Mockito.when(context.getParent()).thenReturn(dsm); container = new OzoneContainer(datanodeDetails, conf, context, null); - //Setting scmId, as we start manually ozone container. - container.getDispatcher().setScmId(UUID.randomUUID().toString()); - container.start(); + //Set scmId and manually start ozone container. + container.start(UUID.randomUUID().toString()); XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf); client.connect(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java index a6e8dd06e8..c2937a8726 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java @@ -155,9 +155,8 @@ public void testCreateOzoneContainer() throws Exception { OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false); container = new OzoneContainer(dn, conf, getContext(dn), null); - //Setting scmId, as we start manually ozone container. - container.getDispatcher().setScmId(UUID.randomUUID().toString()); - container.start(); + //Set scmId and manually start ozone container. + container.start(UUID.randomUUID().toString()); XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf); client.connect(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java index 07836ea281..c086f31925 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestSecureOzoneContainer.java @@ -143,9 +143,8 @@ public void testCreateOzoneContainer() throws Exception { DatanodeDetails dn = TestUtils.randomDatanodeDetails(); container = new OzoneContainer(dn, conf, getContext(dn), caClient); - //Setting scmId, as we start manually ozone container. - container.getDispatcher().setScmId(UUID.randomUUID().toString()); - container.start(); + //Set scmId and manually start ozone container. + container.start(UUID.randomUUID().toString()); UserGroupInformation ugi = UserGroupInformation.createUserForTesting( "user1", new String[] {"usergroup"});