big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13324. Ozone: Remove InfoPort and InfoSecurePort from DatanodeDetails. Contributed by Shashikant Banerjee.

Browse Source
This commit is contained in:
Nanda kumar 2018-04-11 14:25:38 +05:30 committed by Owen O'Malley
parent 25f2398bbd
commit 3779fadc5c
14 changed files with 22 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
View File

@ -42,8 +42,6 @@ public final class DatanodeDetails implements Comparable<DatanodeDetails> {
private String ipAddress;
private String hostName;
private Integer infoPort;
private Integer infoSecurePort;
private Integer containerPort;
private Integer ratisPort;
private Integer ozoneRestPort;
@ -55,21 +53,15 @@ public final class DatanodeDetails implements Comparable<DatanodeDetails> {
* @param uuid DataNode's UUID
* @param ipAddress IP Address of this DataNode
* @param hostName DataNode's hostname
* @param infoPort HTTP Port
* @param infoSecurePort HTTPS Port
* @param containerPort Container Port
* @param ratisPort Ratis Port
* @param ozoneRestPort Rest Port
*/
private DatanodeDetails(
String uuid, String ipAddress, String hostName, Integer infoPort,
Integer infoSecurePort, Integer containerPort, Integer ratisPort,
Integer ozoneRestPort) {
private DatanodeDetails(String uuid, String ipAddress, String hostName,
Integer containerPort, Integer ratisPort, Integer ozoneRestPort) {
this.uuid = UUID.fromString(uuid);
this.ipAddress = ipAddress;
this.hostName = hostName;
this.infoPort = infoPort;
this.infoSecurePort = infoSecurePort;
this.containerPort = containerPort;
this.ratisPort = ratisPort;
this.ozoneRestPort = ozoneRestPort;
@ -129,41 +121,6 @@ public String getHostName() {
return hostName;
}
/**
* Sets the InfoPort.
* @param port InfoPort
*/
public void setInfoPort(int port) {
infoPort = port;
}
/**
* Returns DataNodes Info Port.
*
* @return InfoPort
*/
public int getInfoPort() {
return infoPort;
}
/**
* Sets the InfoSecurePort.
*
* @param port InfoSecurePort
*/
public void setInfoSecurePort(int port) {
infoSecurePort = port;
}
/**
* Returns DataNodes Secure Info Port.
*
* @return InfoSecurePort
*/
public int getInfoSecurePort() {
return infoSecurePort;
}
/**
* Sets the Container Port.
* @param port ContainerPort
@ -231,12 +188,6 @@ public static DatanodeDetails getFromProtoBuf(
if (datanodeDetailsProto.hasHostName()) {
builder.setHostName(datanodeDetailsProto.getHostName());
}
if (datanodeDetailsProto.hasInfoPort()) {
builder.setInfoPort(datanodeDetailsProto.getInfoPort());
}
if (datanodeDetailsProto.hasInfoSecurePort()) {
builder.setInfoSecurePort(datanodeDetailsProto.getInfoSecurePort());
}
if (datanodeDetailsProto.hasContainerPort()) {
builder.setContainerPort(datanodeDetailsProto.getContainerPort());
}
@ -263,12 +214,6 @@ public HddsProtos.DatanodeDetailsProto getProtoBufMessage() {
if (hostName != null) {
builder.setHostName(hostName);
}
if (infoPort != null) {
builder.setInfoPort(infoPort);
}
if (infoSecurePort != null) {
builder.setInfoSecurePort(infoSecurePort);
}
if (containerPort != null) {
builder.setContainerPort(containerPort);
}
@ -312,8 +257,6 @@ public static class Builder {
private String id;
private String ipAddress;
private String hostName;
private Integer infoPort;
private Integer infoSecurePort;
private Integer containerPort;
private Integer ratisPort;
private Integer ozoneRestPort;
@ -350,29 +293,6 @@ public Builder setHostName(String host) {
this.hostName = host;
return this;
}
/**
* Sets the InfoPort.
*
* @param port InfoPort
* @return DatanodeDetails.Builder
*/
public Builder setInfoPort(Integer port) {
this.infoPort = port;
return this;
}
/**
* Sets the Secure Info Port.
*
* @param port InfoSecurePort
* @return DatanodeDetails.Builder
*/
public Builder setInfoSecurePort(Integer port) {
this.infoSecurePort = port;
return this;
}
/**
* Sets the ContainerPort.
*
@ -413,8 +333,8 @@ public Builder setOzoneRestPort(Integer port) {
*/
public DatanodeDetails build() {
Preconditions.checkNotNull(id);
return new DatanodeDetails(id, ipAddress, hostName,
infoPort, infoSecurePort, containerPort, ratisPort, ozoneRestPort);
return new DatanodeDetails(id, ipAddress, hostName, containerPort,
ratisPort, ozoneRestPort);
}
}

8
hadoop-hdds/common/src/main/proto/hdds.proto
View File

@ -33,11 +33,9 @@ message DatanodeDetailsProto {
required string uuid = 1; // UUID assigned to the Datanode.
required string ipAddress = 2; // IP address
required string hostName = 3; // hostname
optional uint32 infoPort = 4; // datanode http port
optional uint32 infoSecurePort = 5 [default = 0]; // datanode https port
optional uint32 containerPort = 6 [default = 0]; // Ozone stand_alone protocol
optional uint32 ratisPort = 7 [default = 0]; //Ozone ratis port
optional uint32 ozoneRestPort = 8 [default = 0];
optional uint32 containerPort = 4 [default = 0]; // Ozone stand_alone protocol
optional uint32 ratisPort = 5 [default = 0]; //Ozone ratis port
optional uint32 ozoneRestPort = 6 [default = 0];
}
message PipelineChannel {

7
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
View File

@ -72,13 +72,6 @@ public void start(Object service) {
datanodeDetails = initializeDatanodeDetails();
datanodeDetails.setHostName(hostname);
datanodeDetails.setIpAddress(ip);
//Below block should be removed as part of HDFS-13324
if (service != null) {
DataNode dataNode = (DataNode) service;
datanodeDetails.setInfoPort(dataNode.getInfoPort());
datanodeDetails.setInfoSecurePort(dataNode.getInfoSecurePort());
}
datanodeStateMachine = new DatanodeStateMachine(datanodeDetails, conf);
startPlugins();
// Starting HDDS Daemons

2
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
View File

@ -371,8 +371,6 @@ private DatanodeDetails getNewDatanodeDetails() {
.setUuid(UUID.randomUUID().toString())
.setHostName("localhost")
.setIpAddress("127.0.0.1")
.setInfoPort(0)
.setInfoSecurePort(0)
.setContainerPort(0)
.setRatisPort(0)
.setOzoneRestPort(0)

2
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
View File

@ -87,8 +87,6 @@ private static DatanodeDetails getDatanodeDetails(String uuid) {
builder.setUuid(uuid)
.setHostName("localhost")
.setIpAddress(ipAddress)
.setInfoPort(0)
.setInfoSecurePort(0)
.setContainerPort(0)
.setRatisPort(0)
.setOzoneRestPort(0);

4
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
View File

@ -265,8 +265,6 @@ public void testDeletedBlockTransactions() throws IOException {
.setUuid("node1")
.setIpAddress("127.0.0.1")
.setHostName("localhost")
.setInfoPort(0)
.setInfoSecurePort(0)
.setContainerPort(0)
.setRatisPort(0)
.setOzoneRestPort(0)
@ -275,8 +273,6 @@ public void testDeletedBlockTransactions() throws IOException {
.setUuid("node2")
.setIpAddress("127.0.0.1")
.setHostName("localhost")
.setInfoPort(0)
.setInfoSecurePort(0)
.setContainerPort(0)
.setRatisPort(0)
.setOzoneRestPort(0)

2
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
View File

@ -96,8 +96,6 @@ public static DatanodeDetails createDatanodeDetails() throws IOException {
.setUuid(UUID.randomUUID().toString())
.setIpAddress(socket.getInetAddress().getHostAddress())
.setHostName(socket.getInetAddress().getHostName())
.setInfoPort(port)
.setInfoSecurePort(port)
.setContainerPort(port)
.setRatisPort(port)
.setOzoneRestPort(port)

2
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java
View File

@ -21,7 +21,7 @@
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.hdds.scm.cli.SQLCLI;
import org.apache.hadoop.ozone.scm.cli.SQLCLI;
import org.apache.hadoop.ozone.web.handlers.BucketArgs;
import org.apache.hadoop.ozone.web.handlers.KeyArgs;
import org.apache.hadoop.ozone.web.handlers.UserArgs;

2
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
View File

@ -26,7 +26,6 @@
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.block.BlockManagerImpl;
import org.apache.hadoop.hdds.scm.cli.SQLCLI;
import org.apache.hadoop.hdds.scm.container.ContainerMapping;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
@ -35,6 +34,7 @@
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.scm.cli.SQLCLI;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

2
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMetrics.java
View File

@ -246,8 +246,6 @@ private ContainerReportsRequestProto createContainerReport(int numReport,
.setUuid(datanodeUuid)
.setIpAddress("127.0.0.1")
.setHostName("localhost")
.setInfoPort(0)
.setInfoSecurePort(0)
.setContainerPort(0)
.setRatisPort(0)
.setOzoneRestPort(0)

6
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
View File

@ -888,12 +888,6 @@ public List<ServiceInfo> getServiceList() throws IOException {
.setValue(datanode.getOzoneRestPort())
.build());
if (datanode.hasInfoSecurePort() && datanode.getInfoSecurePort() > 0) {
dnServiceInfoBuilder.addServicePort(ServicePort.newBuilder()
.setType(ServicePort.Type.HTTPS)
.setValue(datanode.getOzoneRestPort())
.build());
}
services.add(dnServiceInfoBuilder.build());
}

2
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
View File

@ -65,8 +65,6 @@ public static DatanodeDetails createDatanodeDetails(String uuid) {
builder.setUuid(uuid)
.setHostName("localhost")
.setIpAddress(ipAddress)
.setInfoPort(0)
.setInfoSecurePort(0)
.setContainerPort(0)
.setRatisPort(0)
.setOzoneRestPort(0);

31
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.cli;
package org.apache.hadoop.ozone.scm.cli;
import com.google.common.base.Preconditions;
import org.apache.commons.cli.BasicParser;
@ -98,16 +98,14 @@ public class SQLCLI extends Configured implements Tool {
"hostName TEXT NOT NULL, " +
"datanodeUUId TEXT PRIMARY KEY NOT NULL," +
"ipAddress TEXT, " +
"infoPort INTEGER," +
"infoSecurePort INTEGER," +
"containerPort INTEGER NOT NULL);";
private static final String INSERT_CONTAINER_INFO =
"INSERT INTO containerInfo (containerName, leaderUUID) " +
"VALUES (\"%s\", \"%s\")";
private static final String INSERT_DATANODE_INFO =
"INSERT INTO datanodeInfo (hostname, datanodeUUid, ipAddress, " +
"infoPort, infoSecurePort, containerPort) " +
"VALUES (\"%s\", \"%s\", \"%s\", %d, %d, %d, %d, %d)";
"containerPort,) " +
"VALUES (\"%s\", \"%s\", \"%s\", %d";
private static final String INSERT_CONTAINER_MEMBERS =
"INSERT INTO containerMembers (containerName, datanodeUUID) " +
"VALUES (\"%s\", \"%s\")";
@ -476,11 +474,11 @@ private enum KeyType {
*
* datanodeInfo:
* ---------------------------------------------------------
* hostname | datanodeUUid* | xferPort | infoPort | ipcPort
* hostname | datanodeUUid* | xferPort | ipcPort
* ---------------------------------------------------------
*
* --------------------------------
* | infoSecurePort | containerPort
* | containerPort
* --------------------------------
*
* @param dbPath path to container db.
@ -541,13 +539,9 @@ private void insertContainerDB(Connection conn, String containerName,
// but this seems a bit cleaner.
String ipAddr = dd.getIpAddress();
String hostName = dd.getHostName();
int infoPort = dd.hasInfoPort() ? dd.getInfoPort() : 0;
int securePort =
dd.hasInfoSecurePort() ? dd.getInfoSecurePort() : 0;
int containerPort = dd.getContainerPort();
String insertMachineInfo = String.format(
INSERT_DATANODE_INFO, hostName, uuid, ipAddr, infoPort,
securePort, containerPort);
INSERT_DATANODE_INFO, hostName, uuid, ipAddr, containerPort);
executeSQL(conn, insertMachineInfo);
uuidChecked.add(uuid);
}
@ -607,11 +601,11 @@ private void convertBlockDB(Path dbPath, Path outPath) throws Exception {
*
* datanodeInfo:
* ---------------------------------------------------------
* hostname | datanodeUUid* | xferPort | infoPort | ipcPort
* hostname | datanodeUUid* | xferPort | ipcPort
* ---------------------------------------------------------
*
* --------------------------------
* | infoSecurePort | containerPort
* |containerPort
* --------------------------------
*
* @param dbPath path to container db.
@ -648,11 +642,10 @@ private void insertNodePoolDB(Connection conn, String blockPool,
datanodeDetails.getUuidString(), blockPool);
executeSQL(conn, insertNodePool);
String insertDatanodeDetails = String.format(INSERT_DATANODE_INFO,
datanodeDetails.getHostName(), datanodeDetails.getUuid(),
datanodeDetails.getIpAddress(), datanodeDetails.getInfoPort(),
datanodeDetails.getInfoSecurePort(),
datanodeDetails.getContainerPort());
String insertDatanodeDetails = String
.format(INSERT_DATANODE_INFO, datanodeDetails.getHostName(),
datanodeDetails.getUuid(), datanodeDetails.getIpAddress(),
datanodeDetails.getContainerPort());
executeSQL(conn, insertDatanodeDetails);
}

2
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java
View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdds.scm.cli;
package org.apache.hadoop.ozone.scm.cli;
/**
* Command line helpers for scm management.