big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-88. Create separate message structure to represent ports in DatanodeDetails.

Browse Source 
Contributed by Nanda Kumar.
This commit is contained in:
Anu Engineer 2018-05-30 08:52:07 -07:00
parent b24098bc8f
commit 3b34148c4f
30 changed files with 265 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClient.java
View File

@ -93,7 +93,7 @@ public void connect() throws Exception {
// read port from the data node, on failure use default configured
// port.
int port = leader.getContainerPort();
int port = leader.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
if (port == 0) {
port = config.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);

2
hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
View File

@ -80,7 +80,7 @@ public void connect() throws Exception {
// read port from the data node, on failure use default configured
// port.
int port = leader.getContainerPort();
int port = leader.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
if (port == 0) {
port = config.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);

229
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java
View File

@ -23,6 +23,8 @@
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
/**
@ -42,9 +44,7 @@ public final class DatanodeDetails implements Comparable<DatanodeDetails> {
private String ipAddress;
private String hostName;
private Integer containerPort;
private Integer ratisPort;
private Integer ozoneRestPort;
private List<Port> ports;
/**
@ -53,18 +53,14 @@ public final class DatanodeDetails implements Comparable<DatanodeDetails> {
* @param uuid DataNode's UUID
* @param ipAddress IP Address of this DataNode
* @param hostName DataNode's hostname
* @param containerPort Container Port
* @param ratisPort Ratis Port
* @param ozoneRestPort Rest Port
* @param ports Ports used by the DataNode
*/
private DatanodeDetails(String uuid, String ipAddress, String hostName,
Integer containerPort, Integer ratisPort, Integer ozoneRestPort) {
List<Port> ports) {
this.uuid = UUID.fromString(uuid);
this.ipAddress = ipAddress;
this.hostName = hostName;
this.containerPort = containerPort;
this.ratisPort = ratisPort;
this.ozoneRestPort = ozoneRestPort;
this.ports = ports;
}
/**
@ -122,54 +118,40 @@ public String getHostName() {
}
/**
* Sets the Container Port.
* @param port ContainerPort
*/
public void setContainerPort(int port) {
containerPort = port;
}
/**
* Returns standalone container Port.
* Sets a DataNode Port.
*
* @return Container Port
* @param port DataNode port
*/
public int getContainerPort() {
return containerPort;
public void setPort(Port port) {
// If the port is already in the list remove it first and add the
// new/updated port value.
ports.remove(port);
ports.add(port);
}
/**
* Sets Ratis Port.
* @param port RatisPort
* Returns all the Ports used by DataNode.
*
* @return DataNode Ports
*/
public void setRatisPort(int port) {
ratisPort = port;
}
/**
* Returns Ratis Port.
* @return Ratis Port
*/
public int getRatisPort() {
return ratisPort;
}
/**
* Sets OzoneRestPort.
* @param port OzoneRestPort
*/
public void setOzoneRestPort(int port) {
ozoneRestPort = port;
public List<Port> getPorts() {
return ports;
}
/**
* Returns Ozone Rest Port.
* @return OzoneRestPort
* Given the name returns port number, null if the asked port is not found.
*
* @param name Name of the port
*
* @return Port
*/
public int getOzoneRestPort() {
return ozoneRestPort;
public Port getPort(Port.Name name) {
for (Port port : ports) {
if (port.getName().equals(name)) {
return port;
}
}
return null;
}
/**
@ -188,14 +170,9 @@ public static DatanodeDetails getFromProtoBuf(
if (datanodeDetailsProto.hasHostName()) {
builder.setHostName(datanodeDetailsProto.getHostName());
}
if (datanodeDetailsProto.hasContainerPort()) {
builder.setContainerPort(datanodeDetailsProto.getContainerPort());
}
if (datanodeDetailsProto.hasRatisPort()) {
builder.setRatisPort(datanodeDetailsProto.getRatisPort());
}
if (datanodeDetailsProto.hasOzoneRestPort()) {
builder.setOzoneRestPort(datanodeDetailsProto.getOzoneRestPort());
for (HddsProtos.Port port : datanodeDetailsProto.getPortsList()) {
builder.addPort(newPort(
Port.Name.valueOf(port.getName().toUpperCase()), port.getValue()));
}
return builder.build();
}
@ -214,14 +191,11 @@ public HddsProtos.DatanodeDetailsProto getProtoBufMessage() {
if (hostName != null) {
builder.setHostName(hostName);
}
if (containerPort != null) {
builder.setContainerPort(containerPort);
}
if (ratisPort != null) {
builder.setRatisPort(ratisPort);
}
if (ozoneRestPort != null) {
builder.setOzoneRestPort(ozoneRestPort);
for (Port port : ports) {
builder.addPorts(HddsProtos.Port.newBuilder()
.setName(port.getName().toString())
.setValue(port.getValue())
.build());
}
return builder.build();
}
@ -268,9 +242,15 @@ public static class Builder {
private String id;
private String ipAddress;
private String hostName;
private Integer containerPort;
private Integer ratisPort;
private Integer ozoneRestPort;
private List<Port> ports;
/**
* Default private constructor. To create Builder instance use
* DatanodeDetails#newBuilder.
*/
private Builder() {
ports = new ArrayList<>();
}
/**
* Sets the DatanodeUuid.
@ -304,36 +284,16 @@ public Builder setHostName(String host) {
this.hostName = host;
return this;
}
/**
* Sets the ContainerPort.
*
* @param port ContainerPort
* @return DatanodeDetails.Builder
*/
public Builder setContainerPort(Integer port) {
this.containerPort = port;
return this;
}
/**
* Sets the RatisPort.
* Adds a DataNode Port.
*
* @param port DataNode port
*
* @param port RatisPort
* @return DatanodeDetails.Builder
*/
public Builder setRatisPort(Integer port) {
this.ratisPort = port;
return this;
}
/**
* Sets the OzoneRestPort.
*
* @param port OzoneRestPort
* @return DatanodeDetails.Builder
*/
public Builder setOzoneRestPort(Integer port) {
this.ozoneRestPort = port;
public Builder addPort(Port port) {
this.ports.add(port);
return this;
}
@ -344,10 +304,91 @@ public Builder setOzoneRestPort(Integer port) {
*/
public DatanodeDetails build() {
Preconditions.checkNotNull(id);
return new DatanodeDetails(id, ipAddress, hostName, containerPort,
ratisPort, ozoneRestPort);
return new DatanodeDetails(id, ipAddress, hostName, ports);
}
}
/**
* Constructs a new Port with name and value.
*
* @param name Name of the port
* @param value Port number
*
* @return {@code Port} instance
*/
public static Port newPort(Port.Name name, Integer value) {
return new Port(name, value);
}
/**
* Container to hold DataNode Port details.
*/
public static class Port {
/**
* Ports that are supported in DataNode.
*/
public enum Name {
STANDALONE, RATIS, REST
}
private Name name;
private Integer value;
/**
* Private constructor for constructing Port object. Use
* DatanodeDetails#newPort to create a new Port object.
*
* @param name
* @param value
*/
private Port(Name name, Integer value) {
this.name = name;
this.value = value;
}
/**
* Returns the name of the port.
*
* @return Port name
*/
public Name getName() {
return name;
}
/**
* Returns the port number.
*
* @return Port number
*/
public Integer getValue() {
return value;
}
@Override
public int hashCode() {
return name.hashCode();
}
/**
* Ports are considered equal if they have the same name.
*
* @param anObject
* The object to compare this {@code Port} against
* @return {@code true} if the given object represents a {@code Port}
and has the same name, {@code false} otherwise
*/
@Override
public boolean equals(Object anObject) {
if (this == anObject) {
return true;
}
if (anObject instanceof Port) {
return name.equals(((Port) anObject).name);
}
return false;
}
}
}

6
hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java
View File

@ -48,11 +48,13 @@ public interface RatisHelper {
Logger LOG = LoggerFactory.getLogger(RatisHelper.class);
static String toRaftPeerIdString(DatanodeDetails id) {
return id.getUuidString() + "_" + id.getRatisPort();
return id.getUuidString() + "_" +
id.getPort(DatanodeDetails.Port.Name.RATIS);
}
static String toRaftPeerAddressString(DatanodeDetails id) {
return id.getIpAddress() + ":" + id.getRatisPort();
return id.getIpAddress() + ":" +
id.getPort(DatanodeDetails.Port.Name.RATIS);
}
static RaftPeerId toRaftPeerId(DatanodeDetails id) {

10
hadoop-hdds/common/src/main/proto/hdds.proto
View File

@ -29,13 +29,15 @@ option java_generate_equals_and_hash = true;
package hadoop.hdds;
message DatanodeDetailsProto {
// TODO: make the port as a seperate proto message and use it here
required string uuid = 1; // UUID assigned to the Datanode.
required string ipAddress = 2; // IP address
required string hostName = 3; // hostname
optional uint32 containerPort = 4 [default = 0]; // Ozone stand_alone protocol
optional uint32 ratisPort = 5 [default = 0]; //Ozone ratis port
optional uint32 ozoneRestPort = 6 [default = 0];
repeated Port ports = 4;
}
message Port {
required string name = 1;
required uint32 value = 2;
}
message PipelineChannel {

3
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
View File

@ -80,7 +80,8 @@ public XceiverServer(DatanodeDetails datanodeDetails, Configuration conf,
+ "fallback to use default port {}", this.port, e);
}
}
datanodeDetails.setContainerPort(port);
datanodeDetails.setPort(
DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, port));
this.storageContainer = dispatcher;
}

3
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java
View File

@ -71,7 +71,8 @@ public XceiverServerGrpc(DatanodeDetails datanodeDetails, Configuration conf,
+ "fallback to use default port {}", this.port, e);
}
}
datanodeDetails.setContainerPort(port);
datanodeDetails.setPort(
DatanodeDetails.newPort(DatanodeDetails.Port.Name.STANDALONE, port));
server = ((NettyServerBuilder) ServerBuilder.forPort(port))
.maxMessageSize(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE)
.addService(new GrpcXceiverService(dispatcher))

3
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
View File

@ -203,7 +203,8 @@ public static XceiverServerRatis newXceiverServerRatis(
+ "fallback to use default port {}", localPort, e);
}
}
datanodeDetails.setRatisPort(localPort);
datanodeDetails.setPort(
DatanodeDetails.newPort(DatanodeDetails.Port.Name.RATIS, localPort));
return new XceiverServerRatis(datanodeDetails, localPort, storageDir,
dispatcher, ozoneConf);
}

16
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
View File

@ -209,8 +209,10 @@ public void testDatanodeStateContext() throws IOException,
conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID));
idPath.delete();
DatanodeDetails datanodeDetails = getNewDatanodeDetails();
datanodeDetails.setContainerPort(
DatanodeDetails.Port port = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.STANDALONE,
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
datanodeDetails.setPort(port);
ContainerUtils.writeDatanodeDetailsTo(datanodeDetails, idPath);
try (DatanodeStateMachine stateMachine =
@ -360,13 +362,19 @@ public void testDatanodeStateMachineWithInvalidConfiguration()
}
private DatanodeDetails getNewDatanodeDetails() {
DatanodeDetails.Port containerPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.STANDALONE, 0);
DatanodeDetails.Port ratisPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.RATIS, 0);
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, 0);
return DatanodeDetails.newBuilder()
.setUuid(UUID.randomUUID().toString())
.setHostName("localhost")
.setIpAddress("127.0.0.1")
.setContainerPort(0)
.setRatisPort(0)
.setOzoneRestPort(0)
.addPort(containerPort)
.addPort(ratisPort)
.addPort(restPort)
.build();
}
}

12
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
View File

@ -124,13 +124,19 @@ private static DatanodeDetails getDatanodeDetails(String uuid) {
.nextInt(256) + "." + random.nextInt(256);
String hostName = uuid;
DatanodeDetails.Port containerPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.STANDALONE, 0);
DatanodeDetails.Port ratisPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.RATIS, 0);
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, 0);
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
builder.setUuid(uuid)
.setHostName("localhost")
.setIpAddress(ipAddress)
.setContainerPort(0)
.setRatisPort(0)
.setOzoneRestPort(0);
.addPort(containerPort)
.addPort(ratisPort)
.addPort(restPort);
return builder.build();
}

18
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java
View File

@ -265,21 +265,27 @@ public void testDeletedBlockTransactions() throws IOException {
int count = 0;
long containerID = 0L;
DatanodeDetails.Port containerPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.STANDALONE, 0);
DatanodeDetails.Port ratisPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.RATIS, 0);
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, 0);
DatanodeDetails dnId1 = DatanodeDetails.newBuilder()
.setUuid(UUID.randomUUID().toString())
.setIpAddress("127.0.0.1")
.setHostName("localhost")
.setContainerPort(0)
.setRatisPort(0)
.setOzoneRestPort(0)
.addPort(containerPort)
.addPort(ratisPort)
.addPort(restPort)
.build();
DatanodeDetails dnId2 = DatanodeDetails.newBuilder()
.setUuid(UUID.randomUUID().toString())
.setIpAddress("127.0.0.1")
.setHostName("localhost")
.setContainerPort(0)
.setRatisPort(0)
.setOzoneRestPort(0)
.addPort(containerPort)
.addPort(ratisPort)
.addPort(restPort)
.build();
Mapping mappingService = mock(ContainerMapping.class);
// Creates {TXNum} TX in the log.

17
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
View File

@ -24,6 +24,7 @@
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ipc.Client;
@ -219,13 +220,15 @@ public void restartHddsDatanode(int i) {
datanodeService.stop();
datanodeService.join();
// ensure same ports are used across restarts.
Configuration config = datanodeService.getConf();
int currentPort = datanodeService.getDatanodeDetails().getContainerPort();
config.setInt(DFS_CONTAINER_IPC_PORT, currentPort);
config.setBoolean(DFS_CONTAINER_IPC_RANDOM_PORT, false);
int ratisPort = datanodeService.getDatanodeDetails().getRatisPort();
config.setInt(DFS_CONTAINER_RATIS_IPC_PORT, ratisPort);
config.setBoolean(DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, false);
Configuration conf = datanodeService.getConf();
int currentPort = datanodeService.getDatanodeDetails()
.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
conf.setInt(DFS_CONTAINER_IPC_PORT, currentPort);
conf.setBoolean(DFS_CONTAINER_IPC_RANDOM_PORT, false);
int ratisPort = datanodeService.getDatanodeDetails()
.getPort(DatanodeDetails.Port.Name.RATIS).getValue();
conf.setInt(DFS_CONTAINER_RATIS_IPC_PORT, ratisPort);
conf.setBoolean(DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, false);
datanodeService.start(null);
}

3
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java
View File

@ -22,6 +22,7 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.client.rpc.RpcClient;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.ratis.rpc.RpcType;
@ -78,7 +79,7 @@ public void close() {
public int getDatanodeOzoneRestPort() {
return cluster.getHddsDatanodes().get(0).getDatanodeDetails()
.getOzoneRestPort();
.getPort(DatanodeDetails.Port.Name.REST).getValue();
}
}

8
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
View File

@ -27,7 +27,6 @@
import org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.hdds.scm.XceiverClient;
import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineChannel;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
@ -44,6 +43,7 @@
import java.util.HashSet;
import java.util.List;
import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
@ -114,9 +114,9 @@ public void testDatanodeIDPersistent() throws Exception {
DatanodeDetails id1 = TestUtils.getDatanodeDetails();
DatanodeDetails id2 = TestUtils.getDatanodeDetails();
DatanodeDetails id3 = TestUtils.getDatanodeDetails();
id1.setContainerPort(1);
id2.setContainerPort(2);
id3.setContainerPort(3);
id1.setPort(DatanodeDetails.newPort(Port.Name.STANDALONE, 1));
id2.setPort(DatanodeDetails.newPort(Port.Name.STANDALONE, 2));
id3.setPort(DatanodeDetails.newPort(Port.Name.STANDALONE, 3));
// Write a single ID to the file and read it out
File validIdsFile = new File(WRITE_TMP, "valid-values.id");

12
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
View File

@ -94,13 +94,19 @@ public static String createLocalAddress() throws IOException {
public static DatanodeDetails createDatanodeDetails() throws IOException {
ServerSocket socket = new ServerSocket(0);
int port = socket.getLocalPort();
DatanodeDetails.Port containerPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.STANDALONE, port);
DatanodeDetails.Port ratisPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.RATIS, port);
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, port);
DatanodeDetails datanodeDetails = DatanodeDetails.newBuilder()
.setUuid(UUID.randomUUID().toString())
.setIpAddress(socket.getInetAddress().getHostAddress())
.setHostName(socket.getInetAddress().getHostName())
.setContainerPort(port)
.setRatisPort(port)
.setOzoneRestPort(port)
.addPort(containerPort)
.addPort(ratisPort)
.addPort(restPort)
.build();
socket.close();

3
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
View File

@ -65,7 +65,8 @@ public void testContainerMetrics() throws Exception {
.createSingleNodePipeline();
OzoneConfiguration conf = new OzoneConfiguration();
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
pipeline.getLeader().getContainerPort());
pipeline.getLeader()
.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
conf.setInt(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY,
interval);

11
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.container.ozoneimpl;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
@ -62,8 +63,8 @@ public void testCreateOzoneContainer() throws Exception {
// We don't start Ozone Container via data node, we will do it
// independently in our test path.
Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline();
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
pipeline.getLeader().getContainerPort());
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline.getLeader()
.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
container = new OzoneContainer(TestUtils.getDatanodeDetails(), conf);
container.start();
@ -101,7 +102,8 @@ public void testOzoneContainerViaDataNode() throws Exception {
Pipeline pipeline =
ContainerTestHelper.createSingleNodePipeline();
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
pipeline.getLeader().getContainerPort());
pipeline.getLeader()
.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
cluster = MiniOzoneCluster.newBuilder(conf)
.setRandomContainerPort(false)
@ -527,7 +529,8 @@ private static XceiverClient createClientForTesting(OzoneConfiguration conf)
Pipeline pipeline =
ContainerTestHelper.createSingleNodePipeline();
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
pipeline.getLeader().getContainerPort());
pipeline.getLeader()
.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
// This client talks to ozone container via datanode.
return new XceiverClient(pipeline, conf);

8
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
View File

@ -103,7 +103,8 @@ public void testClientServer() throws Exception {
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
runTestClientServer(1,
(pipeline, conf) -> conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
pipeline.getLeader().getContainerPort()),
pipeline.getLeader()
.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue()),
XceiverClient::new,
(dn, conf) -> new XceiverServer(datanodeDetails, conf,
new TestContainerDispatcher()),
@ -130,7 +131,7 @@ public void testClientServerRatisGrpc() throws Exception {
static XceiverServerRatis newXceiverServerRatis(
DatanodeDetails dn, OzoneConfiguration conf) throws IOException {
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT,
dn.getRatisPort());
dn.getPort(DatanodeDetails.Port.Name.RATIS).getValue());
final String dir = TEST_DIR + dn.getUuid();
conf.set(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR, dir);
@ -208,7 +209,8 @@ public void testClientServerWithContainerDispatcher() throws Exception {
Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline();
OzoneConfiguration conf = new OzoneConfiguration();
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
pipeline.getLeader().getContainerPort());
pipeline.getLeader()
.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
Dispatcher dispatcher =
new Dispatcher(mock(ContainerManager.class), conf);

5
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManagerRestInterface.java
View File

@ -118,8 +118,9 @@ public void testGetServiceList() throws Exception {
switch (type) {
case HTTP:
case HTTPS:
Assert.assertEquals(datanodeDetails.getOzoneRestPort(),
(int) ports.get(type));
Assert.assertEquals(
datanodeDetails.getPort(DatanodeDetails.Port.Name.REST).getValue(),
ports.get(type));
break;
default:
// KSM only sends Datanode's info port details

4
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestDistributedOzoneVolumes.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.web;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@ -67,7 +68,8 @@ public static void init() throws Exception {
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
port = cluster.getHddsDatanodes().get(0)
.getDatanodeDetails().getOzoneRestPort();
.getDatanodeDetails()
.getPort(DatanodeDetails.Port.Name.REST).getValue();
}
/**

4
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestLocalOzoneVolumes.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.ozone.web;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.TestOzoneHelper;
@ -70,7 +71,8 @@ public static void init() throws Exception {
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
port = cluster.getHddsDatanodes().get(0)
.getDatanodeDetails().getOzoneRestPort();
.getDatanodeDetails().getPort(
DatanodeDetails.Port.Name.REST).getValue();
}
/**

4
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestOzoneWebAccess.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.web;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@ -78,7 +79,8 @@ public static void init() throws Exception {
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
port = cluster.getHddsDatanodes().get(0)
.getDatanodeDetails().getOzoneRestPort();
.getDatanodeDetails().getPort(
DatanodeDetails.Port.Name.REST).getValue();
}
/**

4
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestOzoneClient.java
View File

@ -43,6 +43,7 @@
import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@ -96,7 +97,8 @@ public static void init() throws Exception {
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
int port = cluster.getHddsDatanodes().get(0)
.getDatanodeDetails().getOzoneRestPort();
.getDatanodeDetails()
.getPort(DatanodeDetails.Port.Name.REST).getValue();
endpoint = String.format("http://localhost:%d", port);
}

2
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
View File

@ -81,8 +81,6 @@ public static void init() throws Exception {
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
final int port = cluster.getHddsDatanodes().get(0)
.getDatanodeDetails().getOzoneRestPort();
client = new RpcClient(conf);
}

4
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.web.client;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@ -63,7 +64,8 @@ public static void init() throws Exception {
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3).build();
cluster.waitForClusterToBeReady();
final int port = cluster.getHddsDatanodes().get(0)
.getDatanodeDetails().getOzoneRestPort();
.getDatanodeDetails()
.getPort(DatanodeDetails.Port.Name.REST).getValue();
client = new RpcClient(conf);
}

5
hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/OzoneHddsDatanodeService.java
View File

@ -20,6 +20,7 @@
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.web.netty.ObjectStoreRestHttpServer;
@ -51,8 +52,10 @@ public void start(Object service) {
objectStoreRestHttpServer = new ObjectStoreRestHttpServer(
conf, null, handler);
objectStoreRestHttpServer.start();
hddsDatanodeService.getDatanodeDetails().setOzoneRestPort(
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST,
objectStoreRestHttpServer.getHttpAddress().getPort());
hddsDatanodeService.getDatanodeDetails().setPort(restPort);
} catch (IOException e) {
throw new RuntimeException("Can't start the Object Store Rest server",

4
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
View File

@ -21,6 +21,7 @@
import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@ -896,7 +897,8 @@ public List<ServiceInfo> getServiceList() throws IOException {
dnServiceInfoBuilder.addServicePort(ServicePort.newBuilder()
.setType(ServicePort.Type.HTTP)
.setValue(datanode.getOzoneRestPort())
.setValue(DatanodeDetails.getFromProtoBuf(datanode)
.getPort(DatanodeDetails.Port.Name.REST).getValue())
.build());
services.add(dnServiceInfoBuilder.build());

12
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/GenesisUtil.java
View File

@ -78,13 +78,19 @@ public static DatanodeDetails createDatanodeDetails(String uuid) {
random.nextInt(256) + "." + random.nextInt(256) + "." + random
.nextInt(256) + "." + random.nextInt(256);
DatanodeDetails.Port containerPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.STANDALONE, 0);
DatanodeDetails.Port ratisPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.RATIS, 0);
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, 0);
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
builder.setUuid(uuid)
.setHostName("localhost")
.setIpAddress(ipAddress)
.setContainerPort(0)
.setRatisPort(0)
.setOzoneRestPort(0);
.addPort(containerPort)
.addPort(ratisPort)
.addPort(restPort);
return builder.build();
}
}

6
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
View File

@ -530,7 +530,8 @@ private void insertContainerDB(Connection conn, long containerID,
// but this seems a bit cleaner.
String ipAddr = dd.getIpAddress();
String hostName = dd.getHostName();
int containerPort = dd.getContainerPort();
int containerPort = DatanodeDetails.getFromProtoBuf(dd)
.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
String insertMachineInfo = String.format(
INSERT_DATANODE_INFO, hostName, uuid, ipAddr, containerPort);
executeSQL(conn, insertMachineInfo);
@ -598,7 +599,8 @@ private void insertNodePoolDB(Connection conn, String blockPool,
String insertDatanodeDetails = String
.format(INSERT_DATANODE_INFO, datanodeDetails.getHostName(),
datanodeDetails.getUuidString(), datanodeDetails.getIpAddress(),
datanodeDetails.getContainerPort());
datanodeDetails.getPort(DatanodeDetails.Port.Name.STANDALONE)
.getValue());
executeSQL(conn, insertDatanodeDetails);
}

3
hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java
View File

@ -90,7 +90,8 @@ public static void init() throws Exception {
// Fetch the host and port for File System init
DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0)
.getDatanodeDetails();
int port = datanodeDetails.getOzoneRestPort();
int port = datanodeDetails
.getPort(DatanodeDetails.Port.Name.REST).getValue();
String host = datanodeDetails.getHostName();
// Set the fs.defaultFS and start the filesystem