big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-974. Add getServiceAddress method to ServiceInfo and use it in TestOzoneShell. Contributed by Doroszlai, Attila.

Browse Source
This commit is contained in:
Yiqun Lin 2019-01-28 17:03:12 +08:00
parent 3b49d7aeae
commit 8326450bca
4 changed files with 40 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
View File

@ -80,7 +80,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.HEAD;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.*;
@ -206,8 +205,8 @@ private InetSocketAddress getScmAddressForClient() throws IOException {
ServiceInfo scmInfo = services.stream().filter(
a -> a.getNodeType().equals(HddsProtos.NodeType.SCM))
.collect(Collectors.toList()).get(0);
return NetUtils.createSocketAddr(scmInfo.getHostname()+ ":" +
scmInfo.getPort(ServicePort.Type.RPC));
return NetUtils.createSocketAddr(
scmInfo.getServiceAddress(ServicePort.Type.RPC));
}
@Override

14
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java
View File

@ -109,17 +109,27 @@ public Map<ServicePort.Type, Integer> getPorts() {
}
/**
* Returns the port for given type, null if the service doesn't support
* the type.
* Returns the port for given type.
*
* @param type the type of port.
* ex: RPC, HTTP, HTTPS, etc..
* @throws NullPointerException if the service doesn't support the given type
*/
@JsonIgnore
public int getPort(ServicePort.Type type) {
return ports.get(type);
}
/**
* Returns the address of the service (hostname with port of the given type).
* @param portType the type of port, eg. RPC, HTTP, etc.
* @return service address (hostname with port of the given type)
*/
@JsonIgnore
public String getServiceAddress(ServicePort.Type portType) {
return hostname + ":" + getPort(portType);
}
/**
* Converts {@link ServiceInfo} to OzoneManagerProtocolProtos.ServiceInfo.
*

34
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java
View File

@ -21,25 +21,19 @@
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.cli.MissingSubcommandException;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.client.rest.RestClient;
import org.apache.hadoop.ozone.client.rpc.RpcClient;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.AfterClass;
@ -78,11 +72,9 @@ public class TestOzoneDatanodeShell {
@Rule
public Timeout testTimeout = new Timeout(300000);
private static String url;
private static File baseDir;
private static OzoneConfiguration conf = null;
private static MiniOzoneCluster cluster = null;
private static ClientProtocol client = null;
private static HddsDatanodeService datanode = null;
private final ByteArrayOutputStream out = new ByteArrayOutputStream();
@ -123,7 +115,6 @@ public static void init() throws Exception {
.build();
conf.setInt(OZONE_REPLICATION, ReplicationFactor.THREE.getValue());
conf.setQuietMode(false);
client = new RpcClient(conf);
cluster.waitForClusterToBeReady();
}
@ -145,26 +136,6 @@ public static void shutdown() {
public void setup() {
System.setOut(new PrintStream(out));
System.setErr(new PrintStream(err));
if(clientProtocol.equals(RestClient.class)) {
String hostName = cluster.getOzoneManager().getHttpServer()
.getHttpAddress().getHostName();
int port = cluster
.getOzoneManager().getHttpServer().getHttpAddress().getPort();
url = String.format("http://" + hostName + ":" + port);
} else {
List<ServiceInfo> services = null;
try {
services = cluster.getOzoneManager().getServiceList();
} catch (IOException e) {
LOG.error("Could not get service list from OM");
}
String hostName = services.stream().filter(
a -> a.getNodeType().equals(HddsProtos.NodeType.OM))
.collect(Collectors.toList()).get(0).getHostname();
String port = cluster.getOzoneManager().getRpcPort();
url = String.format("o3://" + hostName + ":" + port);
}
}
@After
@ -180,9 +151,8 @@ public void reset() {
private void executeDatanode(HddsDatanodeService hdds, String[] args) {
List<String> arguments = new ArrayList(Arrays.asList(args));
LOG.info("Executing ozone datanode command with args {}", arguments);
CommandLine cmd = datanode.getCmd();
LOG.info("Executing datanode command with args {}", Arrays.asList(args));
CommandLine cmd = hdds.getCmd();
IExceptionHandler2<List<Object>> exceptionHandler =
new IExceptionHandler2<List<Object>>() {

66
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
View File

@ -35,7 +35,6 @@
import java.util.Objects;
import java.util.Random;
import java.util.UUID;
import java.util.stream.Collectors;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.cli.MissingSubcommandException;
@ -58,6 +57,7 @@
import org.apache.hadoop.ozone.client.rest.RestClient;
import org.apache.hadoop.ozone.client.rpc.RpcClient;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
import org.apache.hadoop.ozone.web.ozShell.Shell;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo;
@ -177,25 +177,15 @@ public static void shutdown() {
public void setup() {
System.setOut(new PrintStream(out));
System.setErr(new PrintStream(err));
if(clientProtocol.equals(RestClient.class)) {
String hostName = cluster.getOzoneManager().getHttpServer()
.getHttpAddress().getHostName();
int port = cluster
.getOzoneManager().getHttpServer().getHttpAddress().getPort();
url = String.format("http://" + hostName + ":" + port);
url = String.format("http://%s:%d", hostName, port);
} else {
List<ServiceInfo> services = null;
try {
services = cluster.getOzoneManager().getServiceList();
} catch (IOException e) {
LOG.error("Could not get service list from OM");
}
String hostName = services.stream().filter(
a -> a.getNodeType().equals(HddsProtos.NodeType.OM))
.collect(Collectors.toList()).get(0).getHostname();
String port = cluster.getOzoneManager().getRpcPort();
url = String.format("o3://" + hostName + ":" + port);
url = "o3://" + getOmAddress();
}
}
@ -245,8 +235,7 @@ private void testCreateVolume(String volumeName, String errorMsg)
}
private void execute(Shell ozoneShell, String[] args) {
List<String> arguments = new ArrayList(Arrays.asList(args));
LOG.info("Executing shell command with args {}", arguments);
LOG.info("Executing shell command with args {}", Arrays.asList(args));
CommandLine cmd = ozoneShell.getCmd();
IExceptionHandler2<List<Object>> exceptionHandler =
@ -1154,17 +1143,8 @@ public void testListKey() throws Exception {
@Test
public void testS3BucketMapping() throws IOException {
List<ServiceInfo> services =
cluster.getOzoneManager().getServiceList();
String omHostName = services.stream().filter(
a -> a.getNodeType().equals(HddsProtos.NodeType.OM))
.collect(Collectors.toList()).get(0).getHostname();
String omPort = cluster.getOzoneManager().getRpcPort();
String setOmAddress =
"--set=" + OZONE_OM_ADDRESS_KEY + "=" + omHostName + ":" + omPort;
"--set=" + OZONE_OM_ADDRESS_KEY + "=" + getOmAddress();
String s3Bucket = "bucket1";
String commandOutput;
@ -1201,16 +1181,8 @@ public void testS3BucketMapping() throws IOException {
@Test
public void testS3Secret() throws Exception {
List<ServiceInfo> services =
cluster.getOzoneManager().getServiceList();
String omHostName = services.stream().filter(
a -> a.getNodeType().equals(HddsProtos.NodeType.OM))
.collect(Collectors.toList()).get(0).getHostname();
String omPort = cluster.getOzoneManager().getRpcPort();
String setOmAddress =
"--set=" + OZONE_OM_ADDRESS_KEY + "=" + omHostName + ":" + omPort;
"--set=" + OZONE_OM_ADDRESS_KEY + "=" + getOmAddress();
err.reset();
String outputFirstAttempt;
@ -1275,13 +1247,7 @@ private OzoneBucket creatBucket() throws OzoneException, IOException {
@Test
public void testTokenCommands() throws Exception {
String omHostName = cluster.getOzoneManager().getServiceList().stream()
.filter(a -> a.getNodeType().equals(HddsProtos.NodeType.OM))
.collect(Collectors.toList()).get(0).getHostname();
String omPort = cluster.getOzoneManager().getRpcPort();
String omAdd = "--set=" + OZONE_OM_ADDRESS_KEY + "=" + omHostName
+ ":" + omPort;
String omAdd = "--set=" + OZONE_OM_ADDRESS_KEY + "=" + getOmAddress();
List<String[]> shellCommands = new ArrayList<>(4);
// Case 1: Execution will fail when security is disabled.
shellCommands.add(new String[]{omAdd, "token", "get"});
@ -1359,4 +1325,20 @@ private String createTmpFile() throws Exception {
return tmpFile.getAbsolutePath();
}
private String getOmAddress() {
List<ServiceInfo> services;
try {
services = cluster.getOzoneManager().getServiceList();
} catch (IOException e) {
fail("Could not get service list from OM");
return null;
}
return services.stream()
.filter(a -> HddsProtos.NodeType.OM.equals(a.getNodeType()))
.findFirst()
.map(s -> s.getServiceAddress(ServicePort.Type.RPC))
.orElseThrow(IllegalStateException::new);
}
}