From 3779fadc5c344923645d2dce1d9594714cf3fafd Mon Sep 17 00:00:00 2001 From: Nanda kumar Date: Wed, 11 Apr 2018 14:25:38 +0530 Subject: [PATCH] HDFS-13324. Ozone: Remove InfoPort and InfoSecurePort from DatanodeDetails. Contributed by Shashikant Banerjee. --- .../hadoop/hdds/protocol/DatanodeDetails.java | 88 +------------------ hadoop-hdds/common/src/main/proto/hdds.proto | 8 +- .../hadoop/ozone/HddsDatanodeService.java | 7 -- .../common/TestDatanodeStateMachine.java | 2 - .../org/apache/hadoop/hdds/scm/TestUtils.java | 2 - .../hdds/scm/block/TestDeletedBlockLog.java | 4 - .../ozone/container/ContainerTestHelper.java | 2 - .../hadoop/ozone/ksm/TestKSMSQLCli.java | 2 +- .../hadoop/ozone/scm/TestContainerSQLCli.java | 2 +- .../hadoop/ozone/scm/TestSCMMetrics.java | 2 - .../hadoop/ozone/ksm/KeySpaceManager.java | 6 -- .../hadoop/ozone/genesis/GenesisUtil.java | 2 - .../apache/hadoop/ozone/scm/cli/SQLCLI.java | 31 +++---- .../hadoop/ozone/scm/cli/package-info.java | 2 +- 14 files changed, 22 insertions(+), 138 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java index 1463591861..764b3cd1af 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java @@ -42,8 +42,6 @@ public final class DatanodeDetails implements Comparable { private String ipAddress; private String hostName; - private Integer infoPort; - private Integer infoSecurePort; private Integer containerPort; private Integer ratisPort; private Integer ozoneRestPort; @@ -55,21 +53,15 @@ public final class DatanodeDetails implements Comparable { * @param uuid DataNode's UUID * @param ipAddress IP Address of this DataNode * @param hostName DataNode's hostname - * @param infoPort HTTP Port - * @param infoSecurePort HTTPS Port * @param containerPort Container Port * @param ratisPort Ratis Port * @param ozoneRestPort Rest Port */ - private DatanodeDetails( - String uuid, String ipAddress, String hostName, Integer infoPort, - Integer infoSecurePort, Integer containerPort, Integer ratisPort, - Integer ozoneRestPort) { + private DatanodeDetails(String uuid, String ipAddress, String hostName, + Integer containerPort, Integer ratisPort, Integer ozoneRestPort) { this.uuid = UUID.fromString(uuid); this.ipAddress = ipAddress; this.hostName = hostName; - this.infoPort = infoPort; - this.infoSecurePort = infoSecurePort; this.containerPort = containerPort; this.ratisPort = ratisPort; this.ozoneRestPort = ozoneRestPort; @@ -129,41 +121,6 @@ public String getHostName() { return hostName; } - /** - * Sets the InfoPort. - * @param port InfoPort - */ - public void setInfoPort(int port) { - infoPort = port; - } - - /** - * Returns DataNodes Info Port. - * - * @return InfoPort - */ - public int getInfoPort() { - return infoPort; - } - - /** - * Sets the InfoSecurePort. - * - * @param port InfoSecurePort - */ - public void setInfoSecurePort(int port) { - infoSecurePort = port; - } - - /** - * Returns DataNodes Secure Info Port. - * - * @return InfoSecurePort - */ - public int getInfoSecurePort() { - return infoSecurePort; - } - /** * Sets the Container Port. * @param port ContainerPort @@ -231,12 +188,6 @@ public static DatanodeDetails getFromProtoBuf( if (datanodeDetailsProto.hasHostName()) { builder.setHostName(datanodeDetailsProto.getHostName()); } - if (datanodeDetailsProto.hasInfoPort()) { - builder.setInfoPort(datanodeDetailsProto.getInfoPort()); - } - if (datanodeDetailsProto.hasInfoSecurePort()) { - builder.setInfoSecurePort(datanodeDetailsProto.getInfoSecurePort()); - } if (datanodeDetailsProto.hasContainerPort()) { builder.setContainerPort(datanodeDetailsProto.getContainerPort()); } @@ -263,12 +214,6 @@ public HddsProtos.DatanodeDetailsProto getProtoBufMessage() { if (hostName != null) { builder.setHostName(hostName); } - if (infoPort != null) { - builder.setInfoPort(infoPort); - } - if (infoSecurePort != null) { - builder.setInfoSecurePort(infoSecurePort); - } if (containerPort != null) { builder.setContainerPort(containerPort); } @@ -312,8 +257,6 @@ public static class Builder { private String id; private String ipAddress; private String hostName; - private Integer infoPort; - private Integer infoSecurePort; private Integer containerPort; private Integer ratisPort; private Integer ozoneRestPort; @@ -350,29 +293,6 @@ public Builder setHostName(String host) { this.hostName = host; return this; } - - /** - * Sets the InfoPort. - * - * @param port InfoPort - * @return DatanodeDetails.Builder - */ - public Builder setInfoPort(Integer port) { - this.infoPort = port; - return this; - } - - /** - * Sets the Secure Info Port. - * - * @param port InfoSecurePort - * @return DatanodeDetails.Builder - */ - public Builder setInfoSecurePort(Integer port) { - this.infoSecurePort = port; - return this; - } - /** * Sets the ContainerPort. * @@ -413,8 +333,8 @@ public Builder setOzoneRestPort(Integer port) { */ public DatanodeDetails build() { Preconditions.checkNotNull(id); - return new DatanodeDetails(id, ipAddress, hostName, - infoPort, infoSecurePort, containerPort, ratisPort, ozoneRestPort); + return new DatanodeDetails(id, ipAddress, hostName, containerPort, + ratisPort, ozoneRestPort); } } diff --git a/hadoop-hdds/common/src/main/proto/hdds.proto b/hadoop-hdds/common/src/main/proto/hdds.proto index f7b2f72b48..0b650b45ea 100644 --- a/hadoop-hdds/common/src/main/proto/hdds.proto +++ b/hadoop-hdds/common/src/main/proto/hdds.proto @@ -33,11 +33,9 @@ message DatanodeDetailsProto { required string uuid = 1; // UUID assigned to the Datanode. required string ipAddress = 2; // IP address required string hostName = 3; // hostname - optional uint32 infoPort = 4; // datanode http port - optional uint32 infoSecurePort = 5 [default = 0]; // datanode https port - optional uint32 containerPort = 6 [default = 0]; // Ozone stand_alone protocol - optional uint32 ratisPort = 7 [default = 0]; //Ozone ratis port - optional uint32 ozoneRestPort = 8 [default = 0]; + optional uint32 containerPort = 4 [default = 0]; // Ozone stand_alone protocol + optional uint32 ratisPort = 5 [default = 0]; //Ozone ratis port + optional uint32 ozoneRestPort = 6 [default = 0]; } message PipelineChannel { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 1c30f2e46d..010445b293 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -72,13 +72,6 @@ public void start(Object service) { datanodeDetails = initializeDatanodeDetails(); datanodeDetails.setHostName(hostname); datanodeDetails.setIpAddress(ip); - - //Below block should be removed as part of HDFS-13324 - if (service != null) { - DataNode dataNode = (DataNode) service; - datanodeDetails.setInfoPort(dataNode.getInfoPort()); - datanodeDetails.setInfoSecurePort(dataNode.getInfoSecurePort()); - } datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf); startPlugins(); // Starting HDDS Daemons diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java index 9446ce275a..9a9aab1995 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java @@ -371,8 +371,6 @@ private DatanodeDetails getNewDatanodeDetails() { .setUuid(UUID.randomUUID().toString()) .setHostName("localhost") .setIpAddress("127.0.0.1") - .setInfoPort(0) - .setInfoSecurePort(0) .setContainerPort(0) .setRatisPort(0) .setOzoneRestPort(0) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java index e191dd54a1..13a78f65aa 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java @@ -87,8 +87,6 @@ private static DatanodeDetails getDatanodeDetails(String uuid) { builder.setUuid(uuid) .setHostName("localhost") .setIpAddress(ipAddress) - .setInfoPort(0) - .setInfoSecurePort(0) .setContainerPort(0) .setRatisPort(0) .setOzoneRestPort(0); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java index e820fa4427..7237ab4a55 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java @@ -265,8 +265,6 @@ public void testDeletedBlockTransactions() throws IOException { .setUuid("node1") .setIpAddress("127.0.0.1") .setHostName("localhost") - .setInfoPort(0) - .setInfoSecurePort(0) .setContainerPort(0) .setRatisPort(0) .setOzoneRestPort(0) @@ -275,8 +273,6 @@ public void testDeletedBlockTransactions() throws IOException { .setUuid("node2") .setIpAddress("127.0.0.1") .setHostName("localhost") - .setInfoPort(0) - .setInfoSecurePort(0) .setContainerPort(0) .setRatisPort(0) .setOzoneRestPort(0) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java index 7924862447..acab0b2e0e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java @@ -96,8 +96,6 @@ public static DatanodeDetails createDatanodeDetails() throws IOException { .setUuid(UUID.randomUUID().toString()) .setIpAddress(socket.getInetAddress().getHostAddress()) .setHostName(socket.getInetAddress().getHostName()) - .setInfoPort(port) - .setInfoSecurePort(port) .setContainerPort(port) .setRatisPort(port) .setOzoneRestPort(port) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java index 1db4a3e9e7..daf1fa7070 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java @@ -21,7 +21,7 @@ import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.hdds.scm.cli.SQLCLI; +import org.apache.hadoop.ozone.scm.cli.SQLCLI; import org.apache.hadoop.ozone.web.handlers.BucketArgs; import org.apache.hadoop.ozone.web.handlers.KeyArgs; import org.apache.hadoop.ozone.web.handlers.UserArgs; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java index 5fe4f0653b..7ec43e3a28 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java @@ -26,7 +26,6 @@ import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.block.BlockManagerImpl; -import org.apache.hadoop.hdds.scm.cli.SQLCLI; import org.apache.hadoop.hdds.scm.container.ContainerMapping; import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy; import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity; @@ -35,6 +34,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.ozone.scm.cli.SQLCLI; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java index d6a5017ee9..968a698323 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java @@ -246,8 +246,6 @@ private ContainerReportsRequestProto createContainerReport(int numReport, .setUuid(datanodeUuid) .setIpAddress("127.0.0.1") .setHostName("localhost") - .setInfoPort(0) - .setInfoSecurePort(0) .setContainerPort(0) .setRatisPort(0) .setOzoneRestPort(0) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java index e9acb35ca8..76312e7304 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java @@ -888,12 +888,6 @@ public List getServiceList() throws IOException { .setValue(datanode.getOzoneRestPort()) .build()); - if (datanode.hasInfoSecurePort() && datanode.getInfoSecurePort() > 0) { - dnServiceInfoBuilder.addServicePort(ServicePort.newBuilder() - .setType(ServicePort.Type.HTTPS) - .setValue(datanode.getOzoneRestPort()) - .build()); - } services.add(dnServiceInfoBuilder.build()); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java index 199ead58ea..f033b81224 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java @@ -65,8 +65,6 @@ public static DatanodeDetails createDatanodeDetails(String uuid) { builder.setUuid(uuid) .setHostName("localhost") .setIpAddress(ipAddress) - .setInfoPort(0) - .setInfoSecurePort(0) .setContainerPort(0) .setRatisPort(0) .setOzoneRestPort(0); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java index c9b44e318d..e15c72281b 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdds.scm.cli; +package org.apache.hadoop.ozone.scm.cli; import com.google.common.base.Preconditions; import org.apache.commons.cli.BasicParser; @@ -98,16 +98,14 @@ public class SQLCLI extends Configured implements Tool { "hostName TEXT NOT NULL, " + "datanodeUUId TEXT PRIMARY KEY NOT NULL," + "ipAddress TEXT, " + - "infoPort INTEGER," + - "infoSecurePort INTEGER," + "containerPort INTEGER NOT NULL);"; private static final String INSERT_CONTAINER_INFO = "INSERT INTO containerInfo (containerName, leaderUUID) " + "VALUES (\"%s\", \"%s\")"; private static final String INSERT_DATANODE_INFO = "INSERT INTO datanodeInfo (hostname, datanodeUUid, ipAddress, " + - "infoPort, infoSecurePort, containerPort) " + - "VALUES (\"%s\", \"%s\", \"%s\", %d, %d, %d, %d, %d)"; + "containerPort,) " + + "VALUES (\"%s\", \"%s\", \"%s\", %d"; private static final String INSERT_CONTAINER_MEMBERS = "INSERT INTO containerMembers (containerName, datanodeUUID) " + "VALUES (\"%s\", \"%s\")"; @@ -476,11 +474,11 @@ private enum KeyType { * * datanodeInfo: * --------------------------------------------------------- - * hostname | datanodeUUid* | xferPort | infoPort | ipcPort + * hostname | datanodeUUid* | xferPort | ipcPort * --------------------------------------------------------- * * -------------------------------- - * | infoSecurePort | containerPort + * | containerPort * -------------------------------- * * @param dbPath path to container db. @@ -541,13 +539,9 @@ private void insertContainerDB(Connection conn, String containerName, // but this seems a bit cleaner. String ipAddr = dd.getIpAddress(); String hostName = dd.getHostName(); - int infoPort = dd.hasInfoPort() ? dd.getInfoPort() : 0; - int securePort = - dd.hasInfoSecurePort() ? dd.getInfoSecurePort() : 0; int containerPort = dd.getContainerPort(); String insertMachineInfo = String.format( - INSERT_DATANODE_INFO, hostName, uuid, ipAddr, infoPort, - securePort, containerPort); + INSERT_DATANODE_INFO, hostName, uuid, ipAddr, containerPort); executeSQL(conn, insertMachineInfo); uuidChecked.add(uuid); } @@ -607,11 +601,11 @@ private void convertBlockDB(Path dbPath, Path outPath) throws Exception { * * datanodeInfo: * --------------------------------------------------------- - * hostname | datanodeUUid* | xferPort | infoPort | ipcPort + * hostname | datanodeUUid* | xferPort | ipcPort * --------------------------------------------------------- * * -------------------------------- - * | infoSecurePort | containerPort + * |containerPort * -------------------------------- * * @param dbPath path to container db. @@ -648,11 +642,10 @@ private void insertNodePoolDB(Connection conn, String blockPool, datanodeDetails.getUuidString(), blockPool); executeSQL(conn, insertNodePool); - String insertDatanodeDetails = String.format(INSERT_DATANODE_INFO, - datanodeDetails.getHostName(), datanodeDetails.getUuid(), - datanodeDetails.getIpAddress(), datanodeDetails.getInfoPort(), - datanodeDetails.getInfoSecurePort(), - datanodeDetails.getContainerPort()); + String insertDatanodeDetails = String + .format(INSERT_DATANODE_INFO, datanodeDetails.getHostName(), + datanodeDetails.getUuid(), datanodeDetails.getIpAddress(), + datanodeDetails.getContainerPort()); executeSQL(conn, insertDatanodeDetails); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java index fbef0d2ec6..4c38ae0bd2 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdds.scm.cli; +package org.apache.hadoop.ozone.scm.cli; /** * Command line helpers for scm management.