big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "HDDS-1384. TestBlockOutputStreamWithFailures is failing"

Browse Source 
This reverts commit dead9b4049.
This commit is contained in:
Márton Elek 2019-05-06 20:17:00 +02:00
parent 12b7059ddc
commit fb7c1cad0e
No known key found for this signature in database
GPG Key ID: D51EA8F00EE79B28
1 changed files with 14 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.ozone; package org.apache.hadoop.ozone;
import java.io.File; import java.io.File;
import java.net.ServerSocket;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
@ -64,7 +63,6 @@
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState
@ -390,9 +388,6 @@ private void setCAClient(CertificateClient client) {
*/ */
public static class Builder extends MiniOzoneCluster.Builder { public static class Builder extends MiniOzoneCluster.Builder {
private static AtomicInteger lastUsedPort =
new AtomicInteger(1000);
/** /**
* Creates a new Builder. * Creates a new Builder.
* *
@ -534,16 +529,14 @@ OzoneManager createOM()
*/ */
List<HddsDatanodeService> createHddsDatanodes( List<HddsDatanodeService> createHddsDatanodes(
StorageContainerManager scm) throws IOException { StorageContainerManager scm) throws IOException {
configureHddsDatanodes();
String scmAddress = scm.getDatanodeRpcAddress().getHostString() + String scmAddress = scm.getDatanodeRpcAddress().getHostString() +
":" + scm.getDatanodeRpcAddress().getPort(); ":" + scm.getDatanodeRpcAddress().getPort();
String[] args = new String[] {}; String[] args = new String[] {};
conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, scmAddress); conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, scmAddress);
List<HddsDatanodeService> hddsDatanodes = new ArrayList<>(); List<HddsDatanodeService> hddsDatanodes = new ArrayList<>();
for (int i = 0; i < numOfDatanodes; i++) { for (int i = 0; i < numOfDatanodes; i++) {
OzoneConfiguration dnConf = new OzoneConfiguration(conf); OzoneConfiguration dnConf = new OzoneConfiguration(conf);
configureHddsDatanodes(dnConf);
String datanodeBaseDir = path + "/datanode-" + Integer.toString(i); String datanodeBaseDir = path + "/datanode-" + Integer.toString(i);
Path metaDir = Paths.get(datanodeBaseDir, "meta"); Path metaDir = Paths.get(datanodeBaseDir, "meta");
Path dataDir = Paths.get(datanodeBaseDir, "data", "containers"); Path dataDir = Paths.get(datanodeBaseDir, "data", "containers");
@ -570,14 +563,10 @@ List<HddsDatanodeService> createHddsDatanodes(
} }
private void configureSCM() { private void configureSCM() {
conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0");
"127.0.0.1:" + findPort()); conf.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, "127.0.0.1:0");
conf.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "127.0.0.1:0");
"127.0.0.1:" + findPort()); conf.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY, "127.0.0.1:0");
conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY,
"127.0.0.1:" + findPort());
conf.set(ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY,
"127.0.0.1:" + findPort());
conf.setInt(ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY, numOfScmHandlers); conf.setInt(ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY, numOfScmHandlers);
configureSCMheartbeat(); configureSCMheartbeat();
} }
@ -608,42 +597,19 @@ private void configureSCMheartbeat() {
private void configureOM() { private void configureOM() {
conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "127.0.0.1:" + findPort()); conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "127.0.0.1:0");
conf.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, conf.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, "127.0.0.1:0");
"127.0.0.1:" + findPort());
conf.setInt(OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY, numOfOmHandlers); conf.setInt(OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY, numOfOmHandlers);
} }
/** private void configureHddsDatanodes() {
* Return an available TCP port if available. conf.set(ScmConfigKeys.HDDS_REST_HTTP_ADDRESS_KEY, "0.0.0.0:0");
* <p> conf.set(HddsConfigKeys.HDDS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
* As we have a static counter the port should be unique inside the JVM.. conf.set(HDDS_DATANODE_PLUGINS_KEY,
*/
private int findPort() {
while (lastUsedPort.get() < 65536) {
try {
int nextPort = lastUsedPort.incrementAndGet();
ServerSocket socket = new ServerSocket(nextPort);
socket.close();
return nextPort;
} catch (IOException ex) {
//port is not available, let's try the next one.
continue;
}
}
throw new RuntimeException("No available port");
}
private void configureHddsDatanodes(OzoneConfiguration dnConf) {
dnConf.set(ScmConfigKeys.HDDS_REST_HTTP_ADDRESS_KEY,
"0.0.0.0:" + findPort());
dnConf.set(HddsConfigKeys.HDDS_DATANODE_HTTP_ADDRESS_KEY,
"0.0.0.0:" + findPort());
dnConf.set(HDDS_DATANODE_PLUGINS_KEY,
"org.apache.hadoop.ozone.web.OzoneHddsDatanodeService"); "org.apache.hadoop.ozone.web.OzoneHddsDatanodeService");
dnConf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT,
randomContainerPort); randomContainerPort);
dnConf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
randomContainerPort); randomContainerPort);
} }