big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-258. Helper methods to generate NodeReport and ContainerReport for testing. Contributed by Nanda Kumar.

Browse Source
This commit is contained in:
Xiaoyu Yao 2018-07-23 21:29:44 -07:00
parent 17a87977f2
commit 2ced3efe94
19 changed files with 418 additions and 250 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java
View File

@ -342,7 +342,8 @@ public void close() throws IOException {
public VersionResponse getVersion(SCMVersionRequestProto versionRequest) {
return VersionResponse.newBuilder()
.setVersion(this.version.getVersion())
.addValue(OzoneConsts.SCM_ID, this.scmManager.getScmStorage().getScmId())
.addValue(OzoneConsts.SCM_ID,
this.scmManager.getScmStorage().getScmId())
.addValue(OzoneConsts.CLUSTER_ID, this.scmManager.getScmStorage()
.getClusterID())
.build();
@ -364,15 +365,11 @@ public VersionResponse getVersion(SCMVersionRequestProto versionRequest) {
public RegisteredCommand register(
DatanodeDetails datanodeDetails, NodeReportProto nodeReport) {
String hostname = null;
String ip = null;
InetAddress dnAddress = Server.getRemoteIp();
if (dnAddress != null) {
// Mostly called inside an RPC, update ip and peer hostname
hostname = dnAddress.getHostName();
ip = dnAddress.getHostAddress();
datanodeDetails.setHostName(hostname);
datanodeDetails.setIpAddress(ip);
datanodeDetails.setHostName(dnAddress.getHostName());
datanodeDetails.setIpAddress(dnAddress.getHostAddress());
}
UUID dnId = datanodeDetails.getUuid();
try {
@ -390,14 +387,12 @@ public RegisteredCommand register(
LOG.trace("Datanode is already registered. Datanode: {}",
datanodeDetails.toString());
}
RegisteredCommand.Builder builder =
RegisteredCommand.newBuilder().setErrorCode(ErrorCode.success)
.setDatanodeUUID(datanodeDetails.getUuidString())
.setClusterID(this.clusterID);
if (hostname != null && ip != null) {
builder.setHostname(hostname).setIpAddress(ip);
}
return builder.build();
return RegisteredCommand.newBuilder().setErrorCode(ErrorCode.success)
.setDatanodeUUID(datanodeDetails.getUuidString())
.setClusterID(this.clusterID)
.setHostname(datanodeDetails.getHostName())
.setIpAddress(datanodeDetails.getIpAddress())
.build();
}
/**
@ -430,7 +425,7 @@ public List<SCMCommand> processHeartbeat(DatanodeDetails datanodeDetails) {
*/
@Override
public void processNodeReport(UUID dnUuid, NodeReportProto nodeReport) {
this.updateNodeStat(dnUuid, nodeReport);
this.updateNodeStat(dnUuid, nodeReport);
}
/**

396
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestUtils.java
View File

@ -18,7 +18,9 @@
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos;
.StorageContainerDatanodeProtocolProtos.ContainerInfo;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol
.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
import org.apache.hadoop.hdds.protocol
@ -31,40 +33,181 @@
.StorageContainerDatanodeProtocolProtos.StorageTypeProto;
import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
/**
* Stateless helper functions to handler scm/datanode connection.
*/
public final class TestUtils {
private static ThreadLocalRandom random = ThreadLocalRandom.current();
private TestUtils() {
}
public static DatanodeDetails getDatanodeDetails(SCMNodeManager nodeManager) {
return getDatanodeDetails(nodeManager, UUID.randomUUID().toString());
}
/**
* Create a new DatanodeDetails with NodeID set to the string.
* Creates DatanodeDetails with random UUID.
*
* @param uuid - node ID, it is generally UUID.
* @return DatanodeID.
* @return DatanodeDetails
*/
public static DatanodeDetails getDatanodeDetails(SCMNodeManager nodeManager,
String uuid) {
DatanodeDetails datanodeDetails = getDatanodeDetails(uuid);
nodeManager.register(datanodeDetails, null);
return datanodeDetails;
public static DatanodeDetails randomDatanodeDetails() {
return createDatanodeDetails(UUID.randomUUID());
}
/**
* Create Node Report object.
* Creates DatanodeDetails using the given UUID.
*
* @param uuid Datanode's UUID
*
* @return DatanodeDetails
*/
private static DatanodeDetails createDatanodeDetails(UUID uuid) {
String ipAddress = random.nextInt(256)
+ "." + random.nextInt(256)
+ "." + random.nextInt(256)
+ "." + random.nextInt(256);
return createDatanodeDetails(uuid.toString(), "localhost", ipAddress);
}
/**
* Generates DatanodeDetails from RegisteredCommand.
*
* @param registeredCommand registration response from SCM
*
* @return DatanodeDetails
*/
public static DatanodeDetails getDatanodeDetails(
RegisteredCommand registeredCommand) {
return createDatanodeDetails(registeredCommand.getDatanodeUUID(),
registeredCommand.getHostName(), registeredCommand.getIpAddress());
}
/**
* Creates DatanodeDetails with the given information.
*
* @param uuid Datanode's UUID
* @param hostname hostname of Datanode
* @param ipAddress ip address of Datanode
*
* @return DatanodeDetails
*/
private static DatanodeDetails createDatanodeDetails(String uuid,
String hostname, String ipAddress) {
DatanodeDetails.Port containerPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.STANDALONE, 0);
DatanodeDetails.Port ratisPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.RATIS, 0);
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, 0);
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
builder.setUuid(uuid)
.setHostName(hostname)
.setIpAddress(ipAddress)
.addPort(containerPort)
.addPort(ratisPort)
.addPort(restPort);
return builder.build();
}
/**
* Creates a random DatanodeDetails and register it with the given
* NodeManager.
*
* @param nodeManager NodeManager
*
* @return DatanodeDetails
*/
public static DatanodeDetails createRandomDatanodeAndRegister(
SCMNodeManager nodeManager) {
return getDatanodeDetails(
nodeManager.register(randomDatanodeDetails(), null));
}
/**
* Get specified number of DatanodeDetails and register them with node
* manager.
*
* @param nodeManager node manager to register the datanode ids.
* @param count number of DatanodeDetails needed.
*
* @return list of DatanodeDetails
*/
public static List<DatanodeDetails> getListOfRegisteredDatanodeDetails(
SCMNodeManager nodeManager, int count) {
ArrayList<DatanodeDetails> datanodes = new ArrayList<>();
for (int i = 0; i < count; i++) {
datanodes.add(createRandomDatanodeAndRegister(nodeManager));
}
return datanodes;
}
/**
* Generates a random NodeReport.
*
* @return NodeReportProto
*/
public static NodeReportProto getRandomNodeReport() {
return getRandomNodeReport(1);
}
/**
* Generates random NodeReport with the given number of storage report in it.
*
* @param numberOfStorageReport number of storage report this node report
* should have
* @return NodeReportProto
*/
public static NodeReportProto getRandomNodeReport(int numberOfStorageReport) {
UUID nodeId = UUID.randomUUID();
return getRandomNodeReport(nodeId, File.separator + nodeId,
numberOfStorageReport);
}
/**
* Generates random NodeReport for the given nodeId with the given
* base path and number of storage report in it.
*
* @param nodeId datanode id
* @param basePath base path of storage directory
* @param numberOfStorageReport number of storage report
*
* @return NodeReportProto
*/
public static NodeReportProto getRandomNodeReport(UUID nodeId,
String basePath, int numberOfStorageReport) {
List<StorageReportProto> storageReports = new ArrayList<>();
for (int i = 0; i < numberOfStorageReport; i++) {
storageReports.add(getRandomStorageReport(nodeId,
basePath + File.separator + i));
}
return createNodeReport(storageReports);
}
/**
* Creates NodeReport with the given storage reports.
*
* @param reports one or more storage report
*
* @return NodeReportProto
*/
public static NodeReportProto createNodeReport(
StorageReportProto... reports) {
return createNodeReport(Arrays.asList(reports));
}
/**
* Creates NodeReport with the given storage reports.
*
* @param reports storage reports to be included in the node report.
*
* @return NodeReportProto
*/
public static NodeReportProto createNodeReport(
@ -75,25 +218,150 @@ public static NodeReportProto createNodeReport(
}
/**
* Create SCM Storage Report object.
* @return list of SCMStorageReport
* Generates random storage report.
*
* @param nodeId datanode id for which the storage report belongs to
* @param path path of the storage
*
* @return StorageReportProto
*/
public static List<StorageReportProto> createStorageReport(long capacity,
long used, long remaining, String path, StorageTypeProto type, String id,
int count) {
List<StorageReportProto> reportList = new ArrayList<>();
for (int i = 0; i < count; i++) {
Preconditions.checkNotNull(path);
Preconditions.checkNotNull(id);
StorageReportProto.Builder srb = StorageReportProto.newBuilder();
srb.setStorageUuid(id).setStorageLocation(path).setCapacity(capacity)
.setScmUsed(used).setRemaining(remaining);
StorageTypeProto storageTypeProto =
type == null ? StorageTypeProto.DISK : type;
srb.setStorageType(storageTypeProto);
reportList.add(srb.build());
public static StorageReportProto getRandomStorageReport(UUID nodeId,
String path) {
return createStorageReport(nodeId, path,
random.nextInt(1000),
random.nextInt(500),
random.nextInt(500),
StorageTypeProto.DISK);
}
/**
* Creates storage report with the given information.
*
* @param nodeId datanode id
* @param path storage dir
* @param capacity storage size
* @param used space used
* @param remaining space remaining
* @param type type of storage
*
* @return StorageReportProto
*/
public static StorageReportProto createStorageReport(UUID nodeId, String path,
long capacity, long used, long remaining, StorageTypeProto type) {
Preconditions.checkNotNull(nodeId);
Preconditions.checkNotNull(path);
StorageReportProto.Builder srb = StorageReportProto.newBuilder();
srb.setStorageUuid(nodeId.toString())
.setStorageLocation(path)
.setCapacity(capacity)
.setScmUsed(used)
.setRemaining(remaining);
StorageTypeProto storageTypeProto =
type == null ? StorageTypeProto.DISK : type;
srb.setStorageType(storageTypeProto);
return srb.build();
}
/**
* Generates random container reports
*
* @return ContainerReportsProto
*/
public static ContainerReportsProto getRandomContainerReports() {
return getRandomContainerReports(1);
}
/**
* Generates random container report with the given number of containers.
*
* @param numberOfContainers number of containers to be in container report
*
* @return ContainerReportsProto
*/
public static ContainerReportsProto getRandomContainerReports(
int numberOfContainers) {
List<ContainerInfo> containerInfos = new ArrayList<>();
for (int i = 0; i < numberOfContainers; i++) {
containerInfos.add(getRandomContainerInfo(i));
}
return reportList;
return getContainerReports(containerInfos);
}
/**
* Creates container report with the given ContainerInfo(s).
*
* @param containerInfos one or more ContainerInfo
*
* @return ContainerReportsProto
*/
public static ContainerReportsProto getContainerReports(
ContainerInfo... containerInfos) {
return getContainerReports(Arrays.asList(containerInfos));
}
/**
* Creates container report with the given ContainerInfo(s).
*
* @param containerInfos list of ContainerInfo
*
* @return ContainerReportsProto
*/
public static ContainerReportsProto getContainerReports(
List<ContainerInfo> containerInfos) {
ContainerReportsProto.Builder
reportsBuilder = ContainerReportsProto.newBuilder();
for (ContainerInfo containerInfo : containerInfos) {
reportsBuilder.addReports(containerInfo);
}
return reportsBuilder.build();
}
/**
* Generates random ContainerInfo.
*
* @param containerId container id of the ContainerInfo
*
* @return ContainerInfo
*/
public static ContainerInfo getRandomContainerInfo(long containerId) {
return createContainerInfo(containerId,
OzoneConsts.GB * 5,
random.nextLong(1000),
OzoneConsts.GB * random.nextInt(5),
random.nextLong(1000),
OzoneConsts.GB * random.nextInt(2),
random.nextLong(1000),
OzoneConsts.GB * random.nextInt(5));
}
/**
* Creates ContainerInfo with the given details.
*
* @param containerId id of the container
* @param size size of container
* @param keyCount number of keys
* @param bytesUsed bytes used by the container
* @param readCount number of reads
* @param readBytes bytes read
* @param writeCount number of writes
* @param writeBytes bytes written
*
* @return ContainerInfo
*/
public static ContainerInfo createContainerInfo(
long containerId, long size, long keyCount, long bytesUsed,
long readCount, long readBytes, long writeCount, long writeBytes) {
return ContainerInfo.newBuilder()
.setContainerID(containerId)
.setSize(size)
.setKeyCount(keyCount)
.setUsed(bytesUsed)
.setReadCount(readCount)
.setReadBytes(readBytes)
.setWriteCount(writeCount)
.setWriteBytes(writeBytes)
.build();
}
/**
@ -109,66 +377,4 @@ public static CommandStatusReportsProto createCommandStatusReport(
}
/**
* Get specified number of DatanodeDetails and registered them with node
* manager.
*
* @param nodeManager - node manager to register the datanode ids.
* @param count - number of DatanodeDetails needed.
* @return
*/
public static List<DatanodeDetails> getListOfRegisteredDatanodeDetails(
SCMNodeManager nodeManager, int count) {
ArrayList<DatanodeDetails> datanodes = new ArrayList<>();
for (int i = 0; i < count; i++) {
datanodes.add(getDatanodeDetails(nodeManager));
}
return datanodes;
}
/**
* Get a datanode details.
*
* @return DatanodeDetails
*/
public static DatanodeDetails getDatanodeDetails() {
return getDatanodeDetails(UUID.randomUUID().toString());
}
private static DatanodeDetails getDatanodeDetails(String uuid) {
Random random = new Random();
String ipAddress =
random.nextInt(256) + "." + random.nextInt(256) + "." + random
.nextInt(256) + "." + random.nextInt(256);
String hostName = uuid;
DatanodeDetails.Port containerPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.STANDALONE, 0);
DatanodeDetails.Port ratisPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.RATIS, 0);
DatanodeDetails.Port restPort = DatanodeDetails.newPort(
DatanodeDetails.Port.Name.REST, 0);
DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
builder.setUuid(uuid)
.setHostName("localhost")
.setIpAddress(ipAddress)
.addPort(containerPort)
.addPort(ratisPort)
.addPort(restPort);
return builder.build();
}
/**
* Get specified number of list of DatanodeDetails.
*
* @param count - number of datanode IDs needed.
* @return
*/
public static List<DatanodeDetails> getListOfDatanodeDetails(int count) {
ArrayList<DatanodeDetails> datanodes = new ArrayList<>();
for (int i = 0; i < count; i++) {
datanodes.add(getDatanodeDetails());
}
return datanodes;
}
}

2
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/command/TestCommandStatusReportHandler.java
View File

@ -97,7 +97,7 @@ private CommandStatusReportFromDatanode getStatusReport(List<CommandStatus>
reports) {
CommandStatusReportsProto report = TestUtils.createCommandStatusReport
(reports);
DatanodeDetails dn = TestUtils.getDatanodeDetails();
DatanodeDetails dn = TestUtils.randomDatanodeDetails();
return new SCMDatanodeHeartbeatDispatcher.CommandStatusReportFromDatanode
(dn, report);
}

4
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.hadoop.hdds.scm.container;
import org.apache.hadoop.hdds.scm.TestUtils;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
import org.apache.hadoop.hdds.scm.node.NodeManager;
@ -41,7 +42,6 @@
import java.util.Map;
import java.util.UUID;
import static org.apache.hadoop.hdds.scm.TestUtils.getDatanodeDetails;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.DEAD;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState
.HEALTHY;
@ -80,7 +80,7 @@ public MockNodeManager(boolean initializeFakeNodes, int nodeCount) {
aggregateStat = new SCMNodeStat();
if (initializeFakeNodes) {
for (int x = 0; x < nodeCount; x++) {
DatanodeDetails dd = getDatanodeDetails();
DatanodeDetails dd = TestUtils.randomDatanodeDetails();
populateNodeMetric(dd, x);
}
}

4
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
View File

@ -192,7 +192,7 @@ public void testContainerCreationLeaseTimeout() throws IOException,
@Test
public void testFullContainerReport() throws IOException {
ContainerInfo info = createContainer();
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports =
new ArrayList<>();
StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder =
@ -226,7 +226,7 @@ public void testFullContainerReport() throws IOException {
@Test
public void testContainerCloseWithContainerReport() throws IOException {
ContainerInfo info = createContainer();
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports =
new ArrayList<>();

2
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java
View File

@ -219,7 +219,7 @@ private void sendContainerReport(ContainerInfo info, long used) throws
.setWriteBytes(2000000000L)
.setDeleteTransactionId(0);
reports.addReports(ciBuilder);
mapping.processContainerReports(TestUtils.getDatanodeDetails(),
mapping.processContainerReports(TestUtils.randomDatanodeDetails(),
reports.build());
}
}

2
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementCapacity.java
View File

@ -44,7 +44,7 @@ public void chooseDatanodes() throws SCMException {
List<DatanodeDetails> datanodes = new ArrayList<>();
for (int i = 0; i < 7; i++) {
datanodes.add(TestUtils.getDatanodeDetails());
datanodes.add(TestUtils.randomDatanodeDetails());
}
NodeManager mockNodeManager = Mockito.mock(NodeManager.class);

2
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRandom.java
View File

@ -43,7 +43,7 @@ public void chooseDatanodes() throws SCMException {
List<DatanodeDetails> datanodes = new ArrayList<>();
for (int i = 0; i < 5; i++) {
datanodes.add(TestUtils.getDatanodeDetails());
datanodes.add(TestUtils.randomDatanodeDetails());
}
NodeManager mockNodeManager = Mockito.mock(NodeManager.class);

7
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java
View File

@ -73,7 +73,12 @@ public class TestReplicationManager {
@Before
public void initReplicationManager() throws IOException {
listOfDatanodeDetails = TestUtils.getListOfDatanodeDetails(5);
listOfDatanodeDetails = new ArrayList<>();
listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails());
listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails());
listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails());
listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails());
listOfDatanodeDetails.add(TestUtils.randomDatanodeDetails());
containerPlacementPolicy =
(excludedNodes, nodesRequired, sizeRequired) -> listOfDatanodeDetails

4
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java
View File

@ -142,10 +142,6 @@ public void testContainerPlacementCapacity() throws IOException,
TestUtils.getListOfRegisteredDatanodeDetails(nodeManager, nodeCount);
try {
for (DatanodeDetails datanodeDetails : datanodes) {
String id = UUID.randomUUID().toString();
String path = testDir.getAbsolutePath() + "/" + id;
List<StorageReportProto> reports = TestUtils
.createStorageReport(capacity, used, remaining, path, null, id, 1);
nodeManager.processHeartbeat(datanodeDetails);
}

80
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
View File

@ -154,8 +154,8 @@ public void testScmHeartbeat() throws IOException,
try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
// Send some heartbeats from different nodes.
for (int x = 0; x < nodeManager.getMinimumChillModeNodes(); x++) {
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails(
nodeManager);
DatanodeDetails datanodeDetails = TestUtils
.createRandomDatanodeAndRegister(nodeManager);
nodeManager.processHeartbeat(datanodeDetails);
}
@ -200,7 +200,8 @@ public void testScmNotEnoughHeartbeats() throws IOException,
// Need 100 nodes to come out of chill mode, only one node is sending HB.
nodeManager.setMinimumChillModeNodes(100);
nodeManager.processHeartbeat(TestUtils.getDatanodeDetails(nodeManager));
nodeManager.processHeartbeat(TestUtils
.createRandomDatanodeAndRegister(nodeManager));
//TODO: wait for heartbeat to be processed
Thread.sleep(4 * 1000);
assertFalse("Not enough heartbeat, Node manager should have" +
@ -223,7 +224,7 @@ public void testScmSameNodeHeartbeats() throws IOException,
try (SCMNodeManager nodeManager = createNodeManager(getConf())) {
nodeManager.setMinimumChillModeNodes(3);
DatanodeDetails datanodeDetails = TestUtils
.getDatanodeDetails(nodeManager);
.createRandomDatanodeAndRegister(nodeManager);
// Send 10 heartbeat from same node, and assert we never leave chill mode.
for (int x = 0; x < 10; x++) {
@ -253,7 +254,8 @@ public void testScmShutdown() throws IOException, InterruptedException,
conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
100, TimeUnit.MILLISECONDS);
SCMNodeManager nodeManager = createNodeManager(conf);
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails(nodeManager);
DatanodeDetails datanodeDetails = TestUtils
.createRandomDatanodeAndRegister(nodeManager);
nodeManager.close();
// These should never be processed.
@ -276,14 +278,14 @@ public void testScmHeartbeatAfterRestart() throws Exception {
OzoneConfiguration conf = getConf();
conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
100, TimeUnit.MILLISECONDS);
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
String dnId = datanodeDetails.getUuidString();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
UUID dnId = datanodeDetails.getUuid();
String storagePath = testDir.getAbsolutePath() + "/" + dnId;
List<StorageReportProto> reports =
TestUtils.createStorageReport(100, 10, 90, storagePath, null, dnId, 1);
StorageReportProto report =
TestUtils.createStorageReport(dnId, storagePath, 100, 10, 90, null);
try (SCMNodeManager nodemanager = createNodeManager(conf)) {
nodemanager.register(datanodeDetails,
TestUtils.createNodeReport(reports));
TestUtils.createNodeReport(report));
List<SCMCommand> command = nodemanager.processHeartbeat(datanodeDetails);
Assert.assertTrue(nodemanager.getAllNodes().contains(datanodeDetails));
Assert.assertTrue("On regular HB calls, SCM responses a "
@ -331,8 +333,8 @@ public void testScmHealthyNodeCount() throws IOException,
try (SCMNodeManager nodeManager = createNodeManager(conf)) {
for (int x = 0; x < count; x++) {
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails(
nodeManager);
DatanodeDetails datanodeDetails = TestUtils
.createRandomDatanodeAndRegister(nodeManager);
nodeManager.processHeartbeat(datanodeDetails);
}
//TODO: wait for heartbeat to be processed
@ -421,7 +423,7 @@ public void testScmDetectStaleAndDeadNode() throws IOException,
List<DatanodeDetails> nodeList = createNodeSet(nodeManager, nodeCount);
DatanodeDetails staleNode = TestUtils.getDatanodeDetails(nodeManager);
DatanodeDetails staleNode = TestUtils.createRandomDatanodeAndRegister(nodeManager);
// Heartbeat once
nodeManager.processHeartbeat(staleNode);
@ -560,11 +562,11 @@ public void testScmClusterIsInExpectedState1() throws IOException,
*/
try (SCMNodeManager nodeManager = createNodeManager(conf)) {
DatanodeDetails healthyNode =
TestUtils.getDatanodeDetails(nodeManager);
TestUtils.createRandomDatanodeAndRegister(nodeManager);
DatanodeDetails staleNode =
TestUtils.getDatanodeDetails(nodeManager);
TestUtils.createRandomDatanodeAndRegister(nodeManager);
DatanodeDetails deadNode =
TestUtils.getDatanodeDetails(nodeManager);
TestUtils.createRandomDatanodeAndRegister(nodeManager);
nodeManager.processHeartbeat(healthyNode);
nodeManager.processHeartbeat(staleNode);
nodeManager.processHeartbeat(deadNode);
@ -693,8 +695,9 @@ private List<DatanodeDetails> createNodeSet(SCMNodeManager nodeManager, int
count) {
List<DatanodeDetails> list = new LinkedList<>();
for (int x = 0; x < count; x++) {
list.add(TestUtils.getDatanodeDetails(nodeManager, UUID.randomUUID()
.toString()));
DatanodeDetails datanodeDetails = TestUtils
.createRandomDatanodeAndRegister(nodeManager);
list.add(datanodeDetails);
}
return list;
}
@ -876,8 +879,8 @@ public void testScmEnterAndExitChillMode() throws IOException,
try (SCMNodeManager nodeManager = createNodeManager(conf)) {
nodeManager.setMinimumChillModeNodes(10);
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails(
nodeManager);
DatanodeDetails datanodeDetails = TestUtils
.createRandomDatanodeAndRegister(nodeManager);
nodeManager.processHeartbeat(datanodeDetails);
String status = nodeManager.getChillModeStatus();
Assert.assertThat(status, containsString("Still in chill " +
@ -904,7 +907,8 @@ public void testScmEnterAndExitChillMode() throws IOException,
// Assert that node manager force enter cannot be overridden by nodes HBs.
for (int x = 0; x < 20; x++) {
DatanodeDetails datanode = TestUtils.getDatanodeDetails(nodeManager);
DatanodeDetails datanode = TestUtils
.createRandomDatanodeAndRegister(nodeManager);
nodeManager.processHeartbeat(datanode);
}
@ -943,14 +947,13 @@ public void testScmStatsFromNodeReport() throws IOException,
try (SCMNodeManager nodeManager = createNodeManager(conf)) {
for (int x = 0; x < nodeCount; x++) {
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails(
nodeManager);
String dnId = datanodeDetails.getUuidString();
DatanodeDetails datanodeDetails = TestUtils
.createRandomDatanodeAndRegister(nodeManager);
UUID dnId = datanodeDetails.getUuid();
long free = capacity - used;
String storagePath = testDir.getAbsolutePath() + "/" + dnId;
List<StorageReportProto> reports = TestUtils
.createStorageReport(capacity, used, free, storagePath,
null, dnId, 1);
StorageReportProto report = TestUtils
.createStorageReport(dnId, storagePath, capacity, used, free, null);
nodeManager.processHeartbeat(datanodeDetails);
}
//TODO: wait for heartbeat to be processed
@ -990,17 +993,17 @@ public void testScmNodeReportUpdate() throws IOException,
try (SCMNodeManager nodeManager = createNodeManager(conf)) {
DatanodeDetails datanodeDetails =
TestUtils.getDatanodeDetails(nodeManager);
TestUtils.createRandomDatanodeAndRegister(nodeManager);
final long capacity = 2000;
final long usedPerHeartbeat = 100;
String dnId = datanodeDetails.getUuidString();
UUID dnId = datanodeDetails.getUuid();
for (int x = 0; x < heartbeatCount; x++) {
long scmUsed = x * usedPerHeartbeat;
long remaining = capacity - scmUsed;
String storagePath = testDir.getAbsolutePath() + "/" + dnId;
List<StorageReportProto> reports = TestUtils
.createStorageReport(capacity, scmUsed, remaining, storagePath,
null, dnId, 1);
StorageReportProto report = TestUtils
.createStorageReport(dnId, storagePath, capacity, scmUsed,
remaining, null);
nodeManager.processHeartbeat(datanodeDetails);
Thread.sleep(100);
@ -1106,21 +1109,20 @@ public void testHandlingSCMCommandEvent() {
conf.getTimeDuration(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
100, TimeUnit.MILLISECONDS);
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
String dnId = datanodeDetails.getUuidString();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
UUID dnId = datanodeDetails.getUuid();
String storagePath = testDir.getAbsolutePath() + "/" + dnId;
List<StorageReportProto> reports =
TestUtils.createStorageReport(100, 10, 90,
storagePath, null, dnId, 1);
StorageReportProto report =
TestUtils.createStorageReport(dnId, storagePath, 100, 10, 90, null);
EventQueue eq = new EventQueue();
try (SCMNodeManager nodemanager = createNodeManager(conf)) {
eq.addHandler(DATANODE_COMMAND, nodemanager);
nodemanager
.register(datanodeDetails, TestUtils.createNodeReport(reports));
.register(datanodeDetails, TestUtils.createNodeReport(report));
eq.fireEvent(DATANODE_COMMAND,
new CommandForDatanode(datanodeDetails.getUuid(),
new CommandForDatanode<>(datanodeDetails.getUuid(),
new CloseContainerCommand(1L, ReplicationType.STAND_ALONE)));
eq.processAll(1000L);

19
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeReportHandler.java
View File

@ -17,7 +17,6 @@
package org.apache.hadoop.hdds.scm.node;
import java.io.IOException;
import java.util.List;
import java.util.UUID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@ -55,24 +54,22 @@ public void resetEventCollector() throws IOException {
@Test
public void testNodeReport() throws IOException {
DatanodeDetails dn = TestUtils.getDatanodeDetails();
List<StorageReportProto> reports =
TestUtils.createStorageReport(100, 10, 90, storagePath, null,
dn.getUuid().toString(), 1);
DatanodeDetails dn = TestUtils.randomDatanodeDetails();
StorageReportProto storageOne = TestUtils
.createStorageReport(dn.getUuid(), storagePath, 100, 10, 90, null);
nodeReportHandler.onMessage(
getNodeReport(dn, reports), this);
getNodeReport(dn, storageOne), this);
SCMNodeMetric nodeMetric = nodeManager.getNodeStat(dn);
Assert.assertTrue(nodeMetric.get().getCapacity().get() == 100);
Assert.assertTrue(nodeMetric.get().getRemaining().get() == 90);
Assert.assertTrue(nodeMetric.get().getScmUsed().get() == 10);
reports =
TestUtils.createStorageReport(100, 10, 90, storagePath, null,
dn.getUuid().toString(), 2);
StorageReportProto storageTwo = TestUtils
.createStorageReport(dn.getUuid(), storagePath, 100, 10, 90, null);
nodeReportHandler.onMessage(
getNodeReport(dn, reports), this);
getNodeReport(dn, storageOne, storageTwo), this);
nodeMetric = nodeManager.getNodeStat(dn);
Assert.assertTrue(nodeMetric.get().getCapacity().get() == 200);
@ -82,7 +79,7 @@ public void testNodeReport() throws IOException {
}
private NodeReportFromDatanode getNodeReport(DatanodeDetails dn,
List<StorageReportProto> reports) {
StorageReportProto... reports) {
NodeReportProto nodeReportProto = TestUtils.createNodeReport(reports);
return new NodeReportFromDatanode(dn, nodeReportProto);
}

13
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeStorageStatMap.java
View File

@ -139,18 +139,17 @@ public void testProcessNodeReportCheckOneNode() throws IOException {
SCMNodeStorageStatMap map = new SCMNodeStorageStatMap(conf);
map.insertNewDatanode(key, reportSet);
Assert.assertTrue(map.isKnownDatanode(key));
String storageId = UUID.randomUUID().toString();
UUID storageId = UUID.randomUUID();
String path =
GenericTestUtils.getRandomizedTempPath().concat("/" + storageId);
StorageLocationReport report = reportSet.iterator().next();
long reportCapacity = report.getCapacity();
long reportScmUsed = report.getScmUsed();
long reportRemaining = report.getRemaining();
List<StorageReportProto> reports = TestUtils
.createStorageReport(reportCapacity, reportScmUsed, reportRemaining,
path, null, storageId, 1);
StorageReportProto storageReport = TestUtils.createStorageReport(storageId,
path, reportCapacity, reportScmUsed, reportRemaining, null);
StorageReportResult result =
map.processNodeReport(key, TestUtils.createNodeReport(reports));
map.processNodeReport(key, TestUtils.createNodeReport(storageReport));
Assert.assertEquals(result.getStatus(),
SCMNodeStorageStatMap.ReportStatus.ALL_IS_WELL);
StorageContainerDatanodeProtocolProtos.NodeReportProto.Builder nrb =
@ -162,8 +161,8 @@ public void testProcessNodeReportCheckOneNode() throws IOException {
SCMNodeStorageStatMap.ReportStatus.ALL_IS_WELL);
reportList.add(TestUtils
.createStorageReport(reportCapacity, reportCapacity, 0, path, null,
UUID.randomUUID().toString(), 1).get(0));
.createStorageReport(UUID.randomUUID(), path, reportCapacity,
reportCapacity, 0, null));
result = map.processNodeReport(key, TestUtils.createNodeReport(reportList));
Assert.assertEquals(result.getStatus(),
SCMNodeStorageStatMap.ReportStatus.STORAGE_OUT_OF_SPACE);

4
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMDatanodeHeartbeatDispatcher.java
View File

@ -75,7 +75,7 @@ public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
}
});
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
SCMHeartbeatRequestProto heartbeat =
SCMHeartbeatRequestProto.newBuilder()
@ -121,7 +121,7 @@ public <PAYLOAD, EVENT_TYPE extends Event<PAYLOAD>> void fireEvent(
}
});
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
SCMHeartbeatRequestProto heartbeat =
SCMHeartbeatRequestProto.newBuilder()

74
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java
View File

@ -16,15 +16,13 @@
*/
package org.apache.hadoop.ozone.container.common;
import java.util.List;
import java.util.Map;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.protocol.proto.
StorageContainerDatanodeProtocolProtos.CloseContainerCommandProto;
import org.apache.hadoop.hdds.protocol.proto.
@ -43,8 +41,6 @@
import org.apache.hadoop.hdds.scm.VersionInfo;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
import org.apache.hadoop.hdds.protocol.proto
@ -57,8 +53,6 @@
.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ContainerReport;
import org.apache.hadoop.ozone.container.common.statemachine
.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine
@ -84,10 +78,8 @@
import java.io.File;
import java.net.InetSocketAddress;
import java.util.List;
import java.util.UUID;
import static org.apache.hadoop.hdds.scm.TestUtils.getDatanodeDetails;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils
@ -159,7 +151,7 @@ public void testGetVersionTask() throws Exception {
OzoneConfiguration conf = SCMTestUtils.getConf();
try (EndpointStateMachine rpcEndPoint = createEndpoint(conf,
serverAddress, 1000)) {
OzoneContainer ozoneContainer = new OzoneContainer(getDatanodeDetails(),
OzoneContainer ozoneContainer = new OzoneContainer(TestUtils.randomDatanodeDetails(),
conf);
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint,
@ -183,8 +175,8 @@ public void testCheckVersionResponse() throws Exception {
serverAddress, 1000)) {
GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
.captureLogs(VersionEndpointTask.LOG);
OzoneContainer ozoneContainer = new OzoneContainer(getDatanodeDetails(),
conf);
OzoneContainer ozoneContainer = new OzoneContainer(TestUtils
.randomDatanodeDetails(), conf);
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint,
conf, ozoneContainer);
@ -235,7 +227,7 @@ public void testGetVersionToInvalidEndpoint() throws Exception {
try (EndpointStateMachine rpcEndPoint = createEndpoint(conf,
nonExistentServerAddress, 1000)) {
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
OzoneContainer ozoneContainer = new OzoneContainer(getDatanodeDetails(),
OzoneContainer ozoneContainer = new OzoneContainer(TestUtils.randomDatanodeDetails(),
conf);
VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint,
conf, ozoneContainer);
@ -262,7 +254,7 @@ public void testGetVersionAssertRpcTimeOut() throws Exception {
try (EndpointStateMachine rpcEndPoint = createEndpoint(conf,
serverAddress, (int) rpcTimeout)) {
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.GETVERSION);
OzoneContainer ozoneContainer = new OzoneContainer(getDatanodeDetails(),
OzoneContainer ozoneContainer = new OzoneContainer(TestUtils.randomDatanodeDetails(),
conf);
VersionEndpointTask versionTask = new VersionEndpointTask(rpcEndPoint,
conf, ozoneContainer);
@ -280,14 +272,14 @@ public void testGetVersionAssertRpcTimeOut() throws Exception {
@Test
public void testRegister() throws Exception {
DatanodeDetails nodeToRegister = getDatanodeDetails();
DatanodeDetails nodeToRegister = TestUtils.randomDatanodeDetails();
try (EndpointStateMachine rpcEndPoint = createEndpoint(
SCMTestUtils.getConf(), serverAddress, 1000)) {
SCMRegisteredResponseProto responseProto = rpcEndPoint.getEndPoint()
.register(nodeToRegister.getProtoBufMessage(), TestUtils
.createNodeReport(
getStorageReports(nodeToRegister.getUuidString())),
createContainerReport(10, nodeToRegister));
getStorageReports(nodeToRegister.getUuid())),
TestUtils.getRandomContainerReports(10));
Assert.assertNotNull(responseProto);
Assert.assertEquals(nodeToRegister.getUuidString(),
responseProto.getDatanodeUUID());
@ -298,9 +290,9 @@ public void testRegister() throws Exception {
}
}
private List<StorageReportProto> getStorageReports(String id) {
private StorageReportProto getStorageReports(UUID id) {
String storagePath = testDir.getAbsolutePath() + "/" + id;
return TestUtils.createStorageReport(100, 10, 90, storagePath, null, id, 1);
return TestUtils.createStorageReport(id, storagePath, 100, 10, 90, null);
}
private EndpointStateMachine registerTaskHelper(InetSocketAddress scmAddress,
@ -312,13 +304,13 @@ private EndpointStateMachine registerTaskHelper(InetSocketAddress scmAddress,
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.REGISTER);
OzoneContainer ozoneContainer = mock(OzoneContainer.class);
when(ozoneContainer.getNodeReport()).thenReturn(TestUtils
.createNodeReport(getStorageReports(UUID.randomUUID().toString())));
.createNodeReport(getStorageReports(UUID.randomUUID())));
when(ozoneContainer.getContainerReport()).thenReturn(
createContainerReport(10, null));
TestUtils.getRandomContainerReports(10));
RegisterEndpointTask endpointTask =
new RegisterEndpointTask(rpcEndPoint, conf, ozoneContainer);
if (!clearDatanodeDetails) {
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
endpointTask.setDatanodeDetails(datanodeDetails);
}
endpointTask.call();
@ -371,15 +363,14 @@ public void testRegisterRpcTimeout() throws Exception {
@Test
public void testHeartbeat() throws Exception {
DatanodeDetails dataNode = getDatanodeDetails();
DatanodeDetails dataNode = TestUtils.randomDatanodeDetails();
try (EndpointStateMachine rpcEndPoint =
createEndpoint(SCMTestUtils.getConf(),
serverAddress, 1000)) {
String storageId = UUID.randomUUID().toString();
SCMHeartbeatRequestProto request = SCMHeartbeatRequestProto.newBuilder()
.setDatanodeDetails(dataNode.getProtoBufMessage())
.setNodeReport(TestUtils.createNodeReport(
getStorageReports(storageId)))
getStorageReports(UUID.randomUUID())))
.build();
SCMHeartbeatResponseProto responseProto = rpcEndPoint.getEndPoint()
@ -391,11 +382,10 @@ public void testHeartbeat() throws Exception {
@Test
public void testHeartbeatWithCommandStatusReport() throws Exception {
DatanodeDetails dataNode = getDatanodeDetails();
DatanodeDetails dataNode = TestUtils.randomDatanodeDetails();
try (EndpointStateMachine rpcEndPoint =
createEndpoint(SCMTestUtils.getConf(),
serverAddress, 1000)) {
String storageId = UUID.randomUUID().toString();
// Add some scmCommands for heartbeat response
addScmCommands();
@ -403,7 +393,7 @@ public void testHeartbeatWithCommandStatusReport() throws Exception {
SCMHeartbeatRequestProto request = SCMHeartbeatRequestProto.newBuilder()
.setDatanodeDetails(dataNode.getProtoBufMessage())
.setNodeReport(TestUtils.createNodeReport(
getStorageReports(storageId)))
getStorageReports(UUID.randomUUID())))
.build();
SCMHeartbeatResponseProto responseProto = rpcEndPoint.getEndPoint()
@ -482,11 +472,11 @@ private StateContext heartbeatTaskHelper(InetSocketAddress scmAddress,
// Create a datanode state machine for stateConext used by endpoint task
try (DatanodeStateMachine stateMachine = new DatanodeStateMachine(
TestUtils.getDatanodeDetails(), conf);
EndpointStateMachine rpcEndPoint =
TestUtils.randomDatanodeDetails(), conf);
EndpointStateMachine rpcEndPoint =
createEndpoint(conf, scmAddress, rpcTimeout)) {
HddsProtos.DatanodeDetailsProto datanodeDetailsProto =
getDatanodeDetails().getProtoBufMessage();
TestUtils.randomDatanodeDetails().getProtoBufMessage();
rpcEndPoint.setState(EndpointStateMachine.EndPointStates.HEARTBEAT);
final StateContext stateContext =
@ -530,26 +520,4 @@ public void testHeartbeatTaskRpcTimeOut() throws Exception {
lessThanOrEqualTo(rpcTimeout + tolerance));
}
private ContainerReportsProto createContainerReport(
int count, DatanodeDetails datanodeDetails) {
StorageContainerDatanodeProtocolProtos.ContainerReportsProto.Builder
reportsBuilder = StorageContainerDatanodeProtocolProtos
.ContainerReportsProto.newBuilder();
for (int x = 0; x < count; x++) {
long containerID = RandomUtils.nextLong();
ContainerReport report = new ContainerReport(containerID,
DigestUtils.sha256Hex("Simulated"));
report.setKeyCount(1000);
report.setSize(OzoneConsts.GB * 5);
report.setBytesUsed(OzoneConsts.GB * 2);
report.setReadCount(100);
report.setReadBytes(OzoneConsts.GB * 1);
report.setWriteCount(50);
report.setWriteBytes(OzoneConsts.GB * 2);
reportsBuilder.addReports(report.getProtoBufMessage());
}
return reportsBuilder.build();
}
}

20
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
View File

@ -109,9 +109,9 @@ public void testStartMultipleDatanodes() throws Exception {
@Test
public void testDatanodeIDPersistent() throws Exception {
// Generate IDs for testing
DatanodeDetails id1 = TestUtils.getDatanodeDetails();
DatanodeDetails id2 = TestUtils.getDatanodeDetails();
DatanodeDetails id3 = TestUtils.getDatanodeDetails();
DatanodeDetails id1 = TestUtils.randomDatanodeDetails();
DatanodeDetails id2 = TestUtils.randomDatanodeDetails();
DatanodeDetails id3 = TestUtils.randomDatanodeDetails();
id1.setPort(DatanodeDetails.newPort(Port.Name.STANDALONE, 1));
id2.setPort(DatanodeDetails.newPort(Port.Name.STANDALONE, 2));
id3.setPort(DatanodeDetails.newPort(Port.Name.STANDALONE, 3));
@ -162,11 +162,11 @@ public void testContainerRandomPort() throws IOException {
true);
try (
DatanodeStateMachine sm1 = new DatanodeStateMachine(
TestUtils.getDatanodeDetails(), ozoneConf);
TestUtils.randomDatanodeDetails(), ozoneConf);
DatanodeStateMachine sm2 = new DatanodeStateMachine(
TestUtils.getDatanodeDetails(), ozoneConf);
TestUtils.randomDatanodeDetails(), ozoneConf);
DatanodeStateMachine sm3 = new DatanodeStateMachine(
TestUtils.getDatanodeDetails(), ozoneConf)
TestUtils.randomDatanodeDetails(), ozoneConf)
) {
HashSet<Integer> ports = new HashSet<Integer>();
assertTrue(ports.add(sm1.getContainer().getContainerServerPort()));
@ -185,11 +185,11 @@ public void testContainerRandomPort() throws IOException {
ozoneConf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
try (
DatanodeStateMachine sm1 = new DatanodeStateMachine(
TestUtils.getDatanodeDetails(), ozoneConf);
TestUtils.randomDatanodeDetails(), ozoneConf);
DatanodeStateMachine sm2 = new DatanodeStateMachine(
TestUtils.getDatanodeDetails(), ozoneConf);
TestUtils.randomDatanodeDetails(), ozoneConf);
DatanodeStateMachine sm3 = new DatanodeStateMachine(
TestUtils.getDatanodeDetails(), ozoneConf)
TestUtils.randomDatanodeDetails(), ozoneConf)
) {
HashSet<Integer> ports = new HashSet<Integer>();
assertTrue(ports.add(sm1.getContainer().getContainerServerPort()));
@ -204,7 +204,7 @@ public void testContainerRandomPort() throws IOException {
private void createMalformedIDFile(File malformedFile)
throws IOException{
malformedFile.delete();
DatanodeDetails id = TestUtils.getDatanodeDetails();
DatanodeDetails id = TestUtils.randomDatanodeDetails();
ContainerUtils.writeDatanodeDetailsTo(id, malformedFile);
FileOutputStream out = new FileOutputStream(malformedFile);

2
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
View File

@ -72,7 +72,7 @@ public void testContainerMetrics() throws Exception {
conf.setInt(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY,
interval);
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, path);
VolumeSet volumeSet = new VolumeSet(
datanodeDetails.getUuidString(), conf);

2
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
View File

@ -64,7 +64,7 @@ public void testCreateOzoneContainer() throws Exception {
.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
container = new OzoneContainer(TestUtils.getDatanodeDetails(),
container = new OzoneContainer(TestUtils.randomDatanodeDetails(),
conf);
//Setting scmId, as we start manually ozone container.
container.getDispatcher().setScmId(UUID.randomUUID().toString());

4
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
View File

@ -102,7 +102,7 @@ public void testPipeline() throws IOException {
@Test
public void testClientServer() throws Exception {
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
runTestClientServer(1,
(pipeline, conf) -> conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
pipeline.getLeader()
@ -216,7 +216,7 @@ public void testClientServerWithContainerDispatcher() throws Exception {
HddsDispatcher dispatcher = new HddsDispatcher(
conf, mock(ContainerSet.class), mock(VolumeSet.class));
dispatcher.init();
DatanodeDetails datanodeDetails = TestUtils.getDatanodeDetails();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
server = new XceiverServer(datanodeDetails, conf, dispatcher);
client = new XceiverClient(pipeline, conf);