big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11071. Ozone: SCM: Move SCM config keys to ScmConfig. Contributed by Weiwei Yang.

Browse Source
This commit is contained in:
Xiaoyu Yao 2017-03-10 16:24:24 -08:00 committed by Owen O'Malley
parent 7b7d186b82
commit fc7d678d3d
10 changed files with 203 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java
View File

@ -37,4 +37,90 @@ public final class ScmConfigKeys {
// TODO : this is copied from OzoneConsts, may need to move to a better place // TODO : this is copied from OzoneConsts, may need to move to a better place
public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1 MB public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1 MB
public static final int OZONE_SCM_CLIENT_PORT_DEFAULT = 9860;
public static final int OZONE_SCM_DATANODE_PORT_DEFAULT = 9861;
public static final String OZONE_SCM_CLIENT_ADDRESS_KEY =
"ozone.scm.client.address";
public static final String OZONE_SCM_CLIENT_BIND_HOST_KEY =
"ozone.scm.client.bind.host";
public static final String OZONE_SCM_CLIENT_BIND_HOST_DEFAULT =
"0.0.0.0";
public static final String OZONE_SCM_DATANODE_ADDRESS_KEY =
"ozone.scm.datanode.address";
public static final String OZONE_SCM_DATANODE_BIND_HOST_KEY =
"ozone.scm.datanode.bind.host";
public static final String OZONE_SCM_DATANODE_BIND_HOST_DEFAULT =
"0.0.0.0";
public static final String OZONE_SCM_HANDLER_COUNT_KEY =
"ozone.scm.handler.count.key";
public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 10;
public static final String OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS =
"ozone.scm.heartbeat.interval.seconds";
public static final int OZONE_SCM_HEARBEAT_INTERVAL_SECONDS_DEFAULT =
30;
public static final String OZONE_SCM_DEADNODE_INTERVAL_MS =
"ozone.scm.dead.node.interval.ms";
public static final long OZONE_SCM_DEADNODE_INTERVAL_DEFAULT =
OZONE_SCM_HEARBEAT_INTERVAL_SECONDS_DEFAULT * 1000L * 20L;
public static final String OZONE_SCM_MAX_HB_COUNT_TO_PROCESS =
"ozone.scm.max.hb.count.to.process";
public static final int OZONE_SCM_MAX_HB_COUNT_TO_PROCESS_DEFAULT = 5000;
public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS =
"ozone.scm.heartbeat.thread.interval.ms";
public static final long OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS_DEFAULT =
3000;
public static final String OZONE_SCM_STALENODE_INTERVAL_MS =
"ozone.scm.stale.node.interval.ms";
public static final long OZONE_SCM_STALENODE_INTERVAL_DEFAULT =
OZONE_SCM_HEARBEAT_INTERVAL_SECONDS_DEFAULT * 1000L * 3L;
public static final String OZONE_SCM_HEARTBEAT_RPC_TIMEOUT =
"ozone.scm.heartbeat.rpc-timeout";
public static final long OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT =
100;
/**
* Defines how frequently we will log the missing of heartbeat to a specific
* SCM. In the default case we will write a warning message for each 10
* sequential heart beats that we miss to a specific SCM. This is to avoid
* overrunning the log with lots of HB missed Log statements.
*/
public static final String OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT =
"ozone.scm.heartbeat.log.warn.interval.count";
public static final int OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT =
10;
// ozone.scm.names key is a set of DNS | DNS:PORT | IP Address | IP:PORT.
// Written as a comma separated string. e.g. scm1, scm2:8020, 7.7.7.7:7777
//
// If this key is not specified datanodes will not be able to find
// SCM. The SCM membership can be dynamic, so this key should contain
// all possible SCM names. Once the SCM leader is discovered datanodes will
// get the right list of SCMs to heartbeat to from the leader.
// While it is good for the datanodes to know the names of all SCM nodes,
// it is sufficient to actually know the name of on working SCM. That SCM
// will be able to return the information about other SCMs that are part of
// the SCM replicated Log.
//
//In case of a membership change, any one of the SCM machines will be
// able to send back a new list to the datanodes.
public static final String OZONE_SCM_NAMES = "ozone.scm.names";
public static final int OZONE_SCM_DEFAULT_PORT = 9862;
// File Name and path where datanode ID is to written to.
// if this value is not set then container startup will fail.
public static final String OZONE_SCM_DATANODE_ID = "ozone.scm.datanode.id";
public static final String OZONE_SCM_DB_CACHE_SIZE_MB =
"ozone.scm.db.cache.size.mb";
public static final int OZONE_SCM_DB_CACHE_SIZE_DEFAULT = 128;
} }

74
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java
View File

@ -35,28 +35,27 @@
import java.util.Map; import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.ozone.OzoneConfigKeys.*; import static org.apache.hadoop.scm.ScmConfigKeys
import static org.apache.hadoop.ozone.OzoneConfigKeys
.OZONE_SCM_DEADNODE_INTERVAL_DEFAULT; .OZONE_SCM_DEADNODE_INTERVAL_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_DEADNODE_INTERVAL_MS; .OZONE_SCM_DEADNODE_INTERVAL_MS;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS; .OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT; .OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT; .OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS; .OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_HEARTBEAT_RPC_TIMEOUT; .OZONE_SCM_HEARTBEAT_RPC_TIMEOUT;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT; .OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_STALENODE_INTERVAL_DEFAULT; .OZONE_SCM_STALENODE_INTERVAL_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_STALENODE_INTERVAL_MS; .OZONE_SCM_STALENODE_INTERVAL_MS;
/** /**
@ -96,15 +95,15 @@ public static Collection<InetSocketAddress> getSCMAddresses(
Collection<InetSocketAddress> addresses = Collection<InetSocketAddress> addresses =
new HashSet<InetSocketAddress>(); new HashSet<InetSocketAddress>();
Collection<String> names = Collection<String> names =
conf.getTrimmedStringCollection(OzoneConfigKeys.OZONE_SCM_NAMES); conf.getTrimmedStringCollection(ScmConfigKeys.OZONE_SCM_NAMES);
if (names == null || names.isEmpty()) { if (names == null || names.isEmpty()) {
throw new IllegalArgumentException(OzoneConfigKeys.OZONE_SCM_NAMES throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_NAMES
+ " need to be a set of valid DNS names or IP addresses." + " need to be a set of valid DNS names or IP addresses."
+ " Null or empty address list found."); + " Null or empty address list found.");
} }
final com.google.common.base.Optional<Integer> final com.google.common.base.Optional<Integer>
defaultPort = com.google.common.base.Optional.of(OzoneConfigKeys defaultPort = com.google.common.base.Optional.of(ScmConfigKeys
.OZONE_SCM_DEFAULT_PORT); .OZONE_SCM_DEFAULT_PORT);
for (String address : names) { for (String address : names) {
com.google.common.base.Optional<String> hostname = com.google.common.base.Optional<String> hostname =
@ -131,20 +130,21 @@ public static Collection<InetSocketAddress> getSCMAddresses(
*/ */
public static InetSocketAddress getScmAddressForClients(Configuration conf) { public static InetSocketAddress getScmAddressForClients(Configuration conf) {
final Optional<String> host = getHostNameFromConfigKeys(conf, final Optional<String> host = getHostNameFromConfigKeys(conf,
OZONE_SCM_CLIENT_ADDRESS_KEY); ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
if (!host.isPresent()) { if (!host.isPresent()) {
throw new IllegalArgumentException(OZONE_SCM_CLIENT_ADDRESS_KEY + throw new IllegalArgumentException(
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY +
" must be defined. See" + " must be defined. See" +
" https://wiki.apache.org/hadoop/Ozone#Configuration for details" + " https://wiki.apache.org/hadoop/Ozone#Configuration for details" +
" on configuring Ozone."); " on configuring Ozone.");
} }
final Optional<Integer> port = getPortNumberFromConfigKeys(conf, final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
OZONE_SCM_CLIENT_ADDRESS_KEY); ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
return NetUtils.createSocketAddr(host.get() + ":" + return NetUtils.createSocketAddr(host.get() + ":" +
port.or(OZONE_SCM_CLIENT_PORT_DEFAULT)); port.or(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
} }
/** /**
@ -162,10 +162,12 @@ public static InetSocketAddress getScmAddressForDataNodes(
// - OZONE_SCM_CLIENT_ADDRESS_KEY // - OZONE_SCM_CLIENT_ADDRESS_KEY
// //
final Optional<String> host = getHostNameFromConfigKeys(conf, final Optional<String> host = getHostNameFromConfigKeys(conf,
OZONE_SCM_DATANODE_ADDRESS_KEY, OZONE_SCM_CLIENT_ADDRESS_KEY); ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY,
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
if (!host.isPresent()) { if (!host.isPresent()) {
throw new IllegalArgumentException(OZONE_SCM_CLIENT_ADDRESS_KEY + throw new IllegalArgumentException(
ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY +
" must be defined. See" + " must be defined. See" +
" https://wiki.apache.org/hadoop/Ozone#Configuration for details" + " https://wiki.apache.org/hadoop/Ozone#Configuration for details" +
" on configuring Ozone."); " on configuring Ozone.");
@ -173,10 +175,10 @@ public static InetSocketAddress getScmAddressForDataNodes(
// If no port number is specified then we'll just try the defaultBindPort. // If no port number is specified then we'll just try the defaultBindPort.
final Optional<Integer> port = getPortNumberFromConfigKeys(conf, final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
OZONE_SCM_DATANODE_ADDRESS_KEY); ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY);
InetSocketAddress addr = NetUtils.createSocketAddr(host.get() + ":" + InetSocketAddress addr = NetUtils.createSocketAddr(host.get() + ":" +
port.or(OZONE_SCM_DATANODE_PORT_DEFAULT)); port.or(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
return addr; return addr;
} }
@ -191,14 +193,14 @@ public static InetSocketAddress getScmAddressForDataNodes(
public static InetSocketAddress getScmClientBindAddress( public static InetSocketAddress getScmClientBindAddress(
Configuration conf) { Configuration conf) {
final Optional<String> host = getHostNameFromConfigKeys(conf, final Optional<String> host = getHostNameFromConfigKeys(conf,
OZONE_SCM_CLIENT_BIND_HOST_KEY); ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY);
final Optional<Integer> port = getPortNumberFromConfigKeys(conf, final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
OZONE_SCM_CLIENT_ADDRESS_KEY); ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
return NetUtils.createSocketAddr( return NetUtils.createSocketAddr(
host.or(OZONE_SCM_CLIENT_BIND_HOST_DEFAULT) + ":" + host.or(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT) + ":" +
port.or(OZONE_SCM_CLIENT_PORT_DEFAULT)); port.or(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
} }
/** /**
@ -211,15 +213,15 @@ public static InetSocketAddress getScmClientBindAddress(
public static InetSocketAddress getScmDataNodeBindAddress( public static InetSocketAddress getScmDataNodeBindAddress(
Configuration conf) { Configuration conf) {
final Optional<String> host = getHostNameFromConfigKeys(conf, final Optional<String> host = getHostNameFromConfigKeys(conf,
OZONE_SCM_DATANODE_BIND_HOST_KEY); ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY);
// If no port number is specified then we'll just try the defaultBindPort. // If no port number is specified then we'll just try the defaultBindPort.
final Optional<Integer> port = getPortNumberFromConfigKeys(conf, final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
OZONE_SCM_DATANODE_ADDRESS_KEY); ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY);
return NetUtils.createSocketAddr( return NetUtils.createSocketAddr(
host.or(OZONE_SCM_DATANODE_BIND_HOST_DEFAULT) + ":" + host.or(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_DEFAULT) + ":" +
port.or(OZONE_SCM_DATANODE_PORT_DEFAULT)); port.or(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
} }
/** /**
@ -351,7 +353,7 @@ private static long sanitizeUserArgs(long valueTocheck, long baseValue,
*/ */
public static long getScmheartbeatCheckerInterval(Configuration conf) { public static long getScmheartbeatCheckerInterval(Configuration conf) {
return conf.getLong(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, return conf.getLong(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS,
OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS_DEFAULT); ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS_DEFAULT);
} }
/** /**
@ -362,8 +364,10 @@ public static long getScmheartbeatCheckerInterval(Configuration conf) {
* @return - HB interval in seconds. * @return - HB interval in seconds.
*/ */
public static long getScmHeartbeatInterval(Configuration conf) { public static long getScmHeartbeatInterval(Configuration conf) {
return conf.getTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS, return conf.getTimeDuration(
OZONE_SCM_HEARBEAT_INTERVAL_SECONDS_DEFAULT, TimeUnit.SECONDS); OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS,
ScmConfigKeys.OZONE_SCM_HEARBEAT_INTERVAL_SECONDS_DEFAULT,
TimeUnit.SECONDS);
} }
/** /**
@ -444,8 +448,8 @@ public static long getDeadNodeInterval(Configuration conf) {
* @return - int -- Number of HBs to process * @return - int -- Number of HBs to process
*/ */
public static int getMaxHBToProcessPerLoop(Configuration conf) { public static int getMaxHBToProcessPerLoop(Configuration conf) {
return conf.getInt(OZONE_SCM_MAX_HB_COUNT_TO_PROCESS, return conf.getInt(ScmConfigKeys.OZONE_SCM_MAX_HB_COUNT_TO_PROCESS,
OZONE_SCM_MAX_HB_COUNT_TO_PROCESS_DEFAULT); ScmConfigKeys.OZONE_SCM_MAX_HB_COUNT_TO_PROCESS_DEFAULT);
} }
/** /**

88
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
View File

@ -49,98 +49,10 @@ public final class OzoneConfigKeys {
public static final String OZONE_KEY_CACHE = "ozone.key.cache.size"; public static final String OZONE_KEY_CACHE = "ozone.key.cache.size";
public static final int OZONE_KEY_CACHE_DEFAULT = 1024; public static final int OZONE_KEY_CACHE_DEFAULT = 1024;
public static final int OZONE_SCM_CLIENT_PORT_DEFAULT = 9860;
public static final int OZONE_SCM_DATANODE_PORT_DEFAULT = 9861;
public static final String OZONE_SCM_CLIENT_ADDRESS_KEY =
"ozone.scm.client.address";
public static final String OZONE_SCM_CLIENT_BIND_HOST_KEY =
"ozone.scm.client.bind.host";
public static final String OZONE_SCM_CLIENT_BIND_HOST_DEFAULT =
"0.0.0.0";
public static final String OZONE_SCM_DATANODE_ADDRESS_KEY =
"ozone.scm.datanode.address";
public static final String OZONE_SCM_DATANODE_BIND_HOST_KEY =
"ozone.scm.datanode.bind.host";
public static final String OZONE_SCM_DATANODE_BIND_HOST_DEFAULT =
"0.0.0.0";
public static final String OZONE_SCM_HANDLER_COUNT_KEY =
"ozone.scm.handler.count.key";
public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 10;
public static final String OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS =
"ozone.scm.heartbeat.interval.seconds";
public static final int OZONE_SCM_HEARBEAT_INTERVAL_SECONDS_DEFAULT =
30;
public static final String OZONE_SCM_DEADNODE_INTERVAL_MS =
"ozone.scm.dead.node.interval.ms";
public static final long OZONE_SCM_DEADNODE_INTERVAL_DEFAULT =
OZONE_SCM_HEARBEAT_INTERVAL_SECONDS_DEFAULT * 1000L * 20L;
public static final String OZONE_SCM_MAX_HB_COUNT_TO_PROCESS =
"ozone.scm.max.hb.count.to.process";
public static final int OZONE_SCM_MAX_HB_COUNT_TO_PROCESS_DEFAULT = 5000;
public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS =
"ozone.scm.heartbeat.thread.interval.ms";
public static final long OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS_DEFAULT =
3000;
public static final String OZONE_SCM_STALENODE_INTERVAL_MS =
"ozone.scm.stale.node.interval.ms";
public static final long OZONE_SCM_STALENODE_INTERVAL_DEFAULT =
OZONE_SCM_HEARBEAT_INTERVAL_SECONDS_DEFAULT * 1000L * 3L;
public static final String OZONE_SCM_HEARTBEAT_RPC_TIMEOUT =
"ozone.scm.heartbeat.rpc-timeout";
public static final long OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT =
100;
/**
* Defines how frequently we will log the missing of heartbeat to a specific
* SCM. In the default case we will write a warning message for each 10
* sequential heart beats that we miss to a specific SCM. This is to avoid
* overrunning the log with lots of HB missed Log statements.
*/
public static final String OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT =
"ozone.scm.heartbeat.log.warn.interval.count";
public static final int OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT =
10;
public static final String OZONE_CONTAINER_TASK_WAIT = public static final String OZONE_CONTAINER_TASK_WAIT =
"ozone.container.task.wait.seconds"; "ozone.container.task.wait.seconds";
public static final long OZONE_CONTAINER_TASK_WAIT_DEFAULT = 5; public static final long OZONE_CONTAINER_TASK_WAIT_DEFAULT = 5;
// ozone.scm.names key is a set of DNS | DNS:PORT | IP Address | IP:PORT.
// Written as a comma separated string. e.g. scm1, scm2:8020, 7.7.7.7:7777
//
// If this key is not specified datanodes will not be able to find
// SCM. The SCM membership can be dynamic, so this key should contain
// all possible SCM names. Once the SCM leader is discovered datanodes will
// get the right list of SCMs to heartbeat to from the leader.
// While it is good for the datanodes to know the names of all SCM nodes,
// it is sufficient to actually know the name of on working SCM. That SCM
// will be able to return the information about other SCMs that are part of
// the SCM replicated Log.
//
//In case of a membership change, any one of the SCM machines will be
// able to send back a new list to the datanodes.
public static final String OZONE_SCM_NAMES = "ozone.scm.names";
public static final int OZONE_SCM_DEFAULT_PORT = 9862;
// File Name and path where datanode ID is to written to.
// if this value is not set then container startup will fail.
public static final String OZONE_SCM_DATANODE_ID = "ozone.scm.datanode.id";
public static final String OZONE_SCM_DB_CACHE_SIZE_MB =
"ozone.scm.db.cache.size.mb";
public static final int OZONE_SCM_DB_CACHE_SIZE_DEFAULT = 128;
/** /**
* There is no need to instantiate this class. * There is no need to instantiate this class.
*/ */

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
View File

@ -21,7 +21,6 @@
import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.ozone.OzoneClientUtils; import org.apache.hadoop.ozone.OzoneClientUtils;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager; import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
@ -31,6 +30,7 @@
import org.apache.hadoop.ozone.container.common.states.endpoint.RegisterEndpointTask; import org.apache.hadoop.ozone.container.common.states.endpoint.RegisterEndpointTask;
import org.apache.hadoop.ozone.container.common.states.endpoint.VersionEndpointTask; import org.apache.hadoop.ozone.container.common.states.endpoint.VersionEndpointTask;
import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.scm.ScmConfigKeys;
import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Time;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -151,10 +151,10 @@ private DatanodeID createDatanodeID() throws UnknownHostException {
*/ */
private StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto private StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto
getContainerNodeID() { getContainerNodeID() {
String dataNodeIDPath = conf.get(OzoneConfigKeys.OZONE_SCM_DATANODE_ID); String dataNodeIDPath = conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID);
if (dataNodeIDPath == null || dataNodeIDPath.isEmpty()) { if (dataNodeIDPath == null || dataNodeIDPath.isEmpty()) {
LOG.error("A valid file path is needed for config setting {}", LOG.error("A valid file path is needed for config setting {}",
OzoneConfigKeys.OZONE_SCM_DATANODE_ID); ScmConfigKeys.OZONE_SCM_DATANODE_ID);
// This is an unrecoverable error. // This is an unrecoverable error.
this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN); this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN);

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
View File

@ -19,12 +19,12 @@
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.container.common.statemachine import org.apache.hadoop.ozone.container.common.statemachine
.EndpointStateMachine; .EndpointStateMachine;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto; .StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto;
import org.apache.hadoop.scm.ScmConfigKeys;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -99,7 +99,7 @@ public EndpointStateMachine.EndPointStates call() throws Exception {
// TODO : Add responses to the command Queue. // TODO : Add responses to the command Queue.
rpcEndPoint.getEndPoint().register(dnNodeID, rpcEndPoint.getEndPoint().register(dnNodeID,
conf.getStrings(OzoneConfigKeys.OZONE_SCM_NAMES)); conf.getStrings(ScmConfigKeys.OZONE_SCM_NAMES));
EndpointStateMachine.EndPointStates nextState = EndpointStateMachine.EndPointStates nextState =
rpcEndPoint.getState().getNextState(); rpcEndPoint.getState().getNextState();
rpcEndPoint.setState(nextState); rpcEndPoint.setState(nextState);

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
View File

@ -84,17 +84,17 @@
import java.util.Map; import java.util.Map;
import java.util.HashMap; import java.util.HashMap;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_CLIENT_ADDRESS_KEY; .OZONE_SCM_CLIENT_ADDRESS_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_DATANODE_ADDRESS_KEY; .OZONE_SCM_DATANODE_ADDRESS_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_DB_CACHE_SIZE_DEFAULT; .OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_DB_CACHE_SIZE_MB; .OZONE_SCM_DB_CACHE_SIZE_MB;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_HANDLER_COUNT_DEFAULT; .OZONE_SCM_HANDLER_COUNT_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_HANDLER_COUNT_KEY; .OZONE_SCM_HANDLER_COUNT_KEY;
import static org.apache.hadoop.util.ExitUtil.terminate; import static org.apache.hadoop.util.ExitUtil.terminate;

17
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
View File

@ -26,6 +26,7 @@
import org.apache.hadoop.ipc.Client; import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.scm.ScmConfigKeys;
import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB; import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
import org.apache.hadoop.ozone.scm.StorageContainerManager; import org.apache.hadoop.ozone.scm.StorageContainerManager;
@ -298,15 +299,15 @@ public MiniOzoneCluster build() throws IOException {
configureSCMheartbeat(); configureSCMheartbeat();
configScmMetadata(); configScmMetadata();
conf.set(OzoneConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0");
conf.set(OzoneConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "127.0.0.1:0"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "127.0.0.1:0");
StorageContainerManager scm = new StorageContainerManager(conf); StorageContainerManager scm = new StorageContainerManager(conf);
scm.start(); scm.start();
String addressString = scm.getDatanodeRpcAddress().getHostString() + String addressString = scm.getDatanodeRpcAddress().getHostString() +
":" + scm.getDatanodeRpcAddress().getPort(); ":" + scm.getDatanodeRpcAddress().getPort();
conf.setStrings(OzoneConfigKeys.OZONE_SCM_NAMES, addressString); conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, addressString);
MiniOzoneCluster cluster = new MiniOzoneCluster(this, scm); MiniOzoneCluster cluster = new MiniOzoneCluster(this, scm);
try { try {
@ -344,7 +345,7 @@ private void configScmMetadata() throws IOException {
// TODO : Fix this, we need a more generic mechanism to map // TODO : Fix this, we need a more generic mechanism to map
// different datanode ID for different datanodes when we have lots of // different datanode ID for different datanodes when we have lots of
// datanodes in the cluster. // datanodes in the cluster.
conf.setStrings(OzoneConfigKeys.OZONE_SCM_DATANODE_ID, conf.setStrings(ScmConfigKeys.OZONE_SCM_DATANODE_ID,
scmPath.toString() + "/datanode.id"); scmPath.toString() + "/datanode.id");
} }
@ -368,19 +369,19 @@ private void configureTrace() {
private void configureSCMheartbeat() { private void configureSCMheartbeat() {
if (hbSeconds.isPresent()) { if (hbSeconds.isPresent()) {
conf.setInt(OzoneConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS, conf.setInt(ScmConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS,
hbSeconds.get()); hbSeconds.get());
} else { } else {
conf.setInt(OzoneConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS, conf.setInt(ScmConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS,
defaultHBSeconds); defaultHBSeconds);
} }
if (hbProcessorInterval.isPresent()) { if (hbProcessorInterval.isPresent()) {
conf.setInt(OzoneConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, conf.setInt(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS,
hbProcessorInterval.get()); hbProcessorInterval.get());
} else { } else {
conf.setInt(OzoneConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, conf.setInt(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS,
defaultProcessorMs); defaultProcessorMs);
} }

92
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.ozone; package org.apache.hadoop.ozone;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.scm.ScmConfigKeys;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.ExpectedException; import org.junit.rules.ExpectedException;
@ -30,7 +31,6 @@
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import static org.apache.hadoop.ozone.OzoneConfigKeys.*;
import static org.hamcrest.core.Is.is; import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat; import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@ -67,14 +67,14 @@ public void testGetScmClientAddress() {
// First try a client address with just a host name. Verify it falls // First try a client address with just a host name. Verify it falls
// back to the default port. // back to the default port.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4");
InetSocketAddress addr = OzoneClientUtils.getScmAddressForClients(conf); InetSocketAddress addr = OzoneClientUtils.getScmAddressForClients(conf);
assertThat(addr.getHostString(), is("1.2.3.4")); assertThat(addr.getHostString(), is("1.2.3.4"));
assertThat(addr.getPort(), is(OZONE_SCM_CLIENT_PORT_DEFAULT)); assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
// Next try a client address with a host name and port. Verify both // Next try a client address with a host name and port. Verify both
// are used correctly. // are used correctly.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100");
addr = OzoneClientUtils.getScmAddressForClients(conf); addr = OzoneClientUtils.getScmAddressForClients(conf);
assertThat(addr.getHostString(), is("1.2.3.4")); assertThat(addr.getHostString(), is("1.2.3.4"));
assertThat(addr.getPort(), is(100)); assertThat(addr.getPort(), is(100));
@ -102,31 +102,31 @@ public void testGetScmDataNodeAddress() {
// First try a client address with just a host name. Verify it falls // First try a client address with just a host name. Verify it falls
// back to the default port. // back to the default port.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4");
InetSocketAddress addr = OzoneClientUtils.getScmAddressForDataNodes(conf); InetSocketAddress addr = OzoneClientUtils.getScmAddressForDataNodes(conf);
assertThat(addr.getHostString(), is("1.2.3.4")); assertThat(addr.getHostString(), is("1.2.3.4"));
assertThat(addr.getPort(), is(OZONE_SCM_DATANODE_PORT_DEFAULT)); assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
// Next try a client address with just a host name and port. Verify the port // Next try a client address with just a host name and port. Verify the port
// is ignored and the default DataNode port is used. // is ignored and the default DataNode port is used.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100");
addr = OzoneClientUtils.getScmAddressForDataNodes(conf); addr = OzoneClientUtils.getScmAddressForDataNodes(conf);
assertThat(addr.getHostString(), is("1.2.3.4")); assertThat(addr.getHostString(), is("1.2.3.4"));
assertThat(addr.getPort(), is(OZONE_SCM_DATANODE_PORT_DEFAULT)); assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
// Set both OZONE_SCM_CLIENT_ADDRESS_KEY and OZONE_SCM_DATANODE_ADDRESS_KEY. // Set both OZONE_SCM_CLIENT_ADDRESS_KEY and OZONE_SCM_DATANODE_ADDRESS_KEY.
// Verify that the latter overrides and the port number is still the default. // Verify that the latter overrides and the port number is still the default.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100");
conf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, "5.6.7.8"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "5.6.7.8");
addr = OzoneClientUtils.getScmAddressForDataNodes(conf); addr = OzoneClientUtils.getScmAddressForDataNodes(conf);
assertThat(addr.getHostString(), is("5.6.7.8")); assertThat(addr.getHostString(), is("5.6.7.8"));
assertThat(addr.getPort(), is(OZONE_SCM_DATANODE_PORT_DEFAULT)); assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
// Set both OZONE_SCM_CLIENT_ADDRESS_KEY and OZONE_SCM_DATANODE_ADDRESS_KEY. // Set both OZONE_SCM_CLIENT_ADDRESS_KEY and OZONE_SCM_DATANODE_ADDRESS_KEY.
// Verify that the latter overrides and the port number from the latter is // Verify that the latter overrides and the port number from the latter is
// used. // used.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100");
conf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, "5.6.7.8:200"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "5.6.7.8:200");
addr = OzoneClientUtils.getScmAddressForDataNodes(conf); addr = OzoneClientUtils.getScmAddressForDataNodes(conf);
assertThat(addr.getHostString(), is("5.6.7.8")); assertThat(addr.getHostString(), is("5.6.7.8"));
assertThat(addr.getPort(), is(200)); assertThat(addr.getPort(), is(200));
@ -142,16 +142,16 @@ public void testScmClientBindHostDefault() {
// The bind host should be 0.0.0.0 unless OZONE_SCM_CLIENT_BIND_HOST_KEY // The bind host should be 0.0.0.0 unless OZONE_SCM_CLIENT_BIND_HOST_KEY
// is set differently. // is set differently.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4");
InetSocketAddress addr = OzoneClientUtils.getScmClientBindAddress(conf); InetSocketAddress addr = OzoneClientUtils.getScmClientBindAddress(conf);
assertThat(addr.getHostString(), is("0.0.0.0")); assertThat(addr.getHostString(), is("0.0.0.0"));
assertThat(addr.getPort(), is(OZONE_SCM_CLIENT_PORT_DEFAULT)); assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
// The bind host should be 0.0.0.0 unless OZONE_SCM_CLIENT_BIND_HOST_KEY // The bind host should be 0.0.0.0 unless OZONE_SCM_CLIENT_BIND_HOST_KEY
// is set differently. The port number from OZONE_SCM_CLIENT_ADDRESS_KEY // is set differently. The port number from OZONE_SCM_CLIENT_ADDRESS_KEY
// should be respected. // should be respected.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100");
conf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200");
addr = OzoneClientUtils.getScmClientBindAddress(conf); addr = OzoneClientUtils.getScmClientBindAddress(conf);
assertThat(addr.getHostString(), is("0.0.0.0")); assertThat(addr.getHostString(), is("0.0.0.0"));
assertThat(addr.getPort(), is(100)); assertThat(addr.getPort(), is(100));
@ -159,19 +159,19 @@ public void testScmClientBindHostDefault() {
// OZONE_SCM_CLIENT_BIND_HOST_KEY should be respected. // OZONE_SCM_CLIENT_BIND_HOST_KEY should be respected.
// Port number should be default if none is specified via // Port number should be default if none is specified via
// OZONE_SCM_DATANODE_ADDRESS_KEY. // OZONE_SCM_DATANODE_ADDRESS_KEY.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4");
conf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4");
conf.set(OZONE_SCM_CLIENT_BIND_HOST_KEY, "5.6.7.8"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY, "5.6.7.8");
addr = OzoneClientUtils.getScmClientBindAddress(conf); addr = OzoneClientUtils.getScmClientBindAddress(conf);
assertThat(addr.getHostString(), is("5.6.7.8")); assertThat(addr.getHostString(), is("5.6.7.8"));
assertThat(addr.getPort(), is(OZONE_SCM_CLIENT_PORT_DEFAULT)); assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
// OZONE_SCM_CLIENT_BIND_HOST_KEY should be respected. // OZONE_SCM_CLIENT_BIND_HOST_KEY should be respected.
// Port number from OZONE_SCM_CLIENT_ADDRESS_KEY should be // Port number from OZONE_SCM_CLIENT_ADDRESS_KEY should be
// respected. // respected.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100");
conf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200");
conf.set(OZONE_SCM_CLIENT_BIND_HOST_KEY, "5.6.7.8"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY, "5.6.7.8");
addr = OzoneClientUtils.getScmClientBindAddress(conf); addr = OzoneClientUtils.getScmClientBindAddress(conf);
assertThat(addr.getHostString(), is("5.6.7.8")); assertThat(addr.getHostString(), is("5.6.7.8"));
assertThat(addr.getPort(), is(100)); assertThat(addr.getPort(), is(100));
@ -187,16 +187,16 @@ public void testScmDataNodeBindHostDefault() {
// The bind host should be 0.0.0.0 unless OZONE_SCM_DATANODE_BIND_HOST_KEY // The bind host should be 0.0.0.0 unless OZONE_SCM_DATANODE_BIND_HOST_KEY
// is set differently. // is set differently.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4");
InetSocketAddress addr = OzoneClientUtils.getScmDataNodeBindAddress(conf); InetSocketAddress addr = OzoneClientUtils.getScmDataNodeBindAddress(conf);
assertThat(addr.getHostString(), is("0.0.0.0")); assertThat(addr.getHostString(), is("0.0.0.0"));
assertThat(addr.getPort(), is(OZONE_SCM_DATANODE_PORT_DEFAULT)); assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
// The bind host should be 0.0.0.0 unless OZONE_SCM_DATANODE_BIND_HOST_KEY // The bind host should be 0.0.0.0 unless OZONE_SCM_DATANODE_BIND_HOST_KEY
// is set differently. The port number from OZONE_SCM_DATANODE_ADDRESS_KEY // is set differently. The port number from OZONE_SCM_DATANODE_ADDRESS_KEY
// should be respected. // should be respected.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100");
conf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200");
addr = OzoneClientUtils.getScmDataNodeBindAddress(conf); addr = OzoneClientUtils.getScmDataNodeBindAddress(conf);
assertThat(addr.getHostString(), is("0.0.0.0")); assertThat(addr.getHostString(), is("0.0.0.0"));
assertThat(addr.getPort(), is(200)); assertThat(addr.getPort(), is(200));
@ -204,19 +204,19 @@ public void testScmDataNodeBindHostDefault() {
// OZONE_SCM_DATANODE_BIND_HOST_KEY should be respected. // OZONE_SCM_DATANODE_BIND_HOST_KEY should be respected.
// Port number should be default if none is specified via // Port number should be default if none is specified via
// OZONE_SCM_DATANODE_ADDRESS_KEY. // OZONE_SCM_DATANODE_ADDRESS_KEY.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100");
conf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4");
conf.set(OZONE_SCM_DATANODE_BIND_HOST_KEY, "5.6.7.8"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY, "5.6.7.8");
addr = OzoneClientUtils.getScmDataNodeBindAddress(conf); addr = OzoneClientUtils.getScmDataNodeBindAddress(conf);
assertThat(addr.getHostString(), is("5.6.7.8")); assertThat(addr.getHostString(), is("5.6.7.8"));
assertThat(addr.getPort(), is(OZONE_SCM_DATANODE_PORT_DEFAULT)); assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
// OZONE_SCM_DATANODE_BIND_HOST_KEY should be respected. // OZONE_SCM_DATANODE_BIND_HOST_KEY should be respected.
// Port number from OZONE_SCM_DATANODE_ADDRESS_KEY should be // Port number from OZONE_SCM_DATANODE_ADDRESS_KEY should be
// respected. // respected.
conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100");
conf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200");
conf.set(OZONE_SCM_DATANODE_BIND_HOST_KEY, "5.6.7.8"); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY, "5.6.7.8");
addr = OzoneClientUtils.getScmDataNodeBindAddress(conf); addr = OzoneClientUtils.getScmDataNodeBindAddress(conf);
assertThat(addr.getHostString(), is("5.6.7.8")); assertThat(addr.getHostString(), is("5.6.7.8"));
assertThat(addr.getPort(), is(200)); assertThat(addr.getPort(), is(200));
@ -230,23 +230,23 @@ public void testGetSCMAddresses() {
Iterator<InetSocketAddress> it = null; Iterator<InetSocketAddress> it = null;
// Verify valid IP address setup // Verify valid IP address setup
conf.setStrings(OZONE_SCM_NAMES, "1.2.3.4"); conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "1.2.3.4");
addresses = OzoneClientUtils.getSCMAddresses(conf); addresses = OzoneClientUtils.getSCMAddresses(conf);
assertThat(addresses.size(), is(1)); assertThat(addresses.size(), is(1));
addr = addresses.iterator().next(); addr = addresses.iterator().next();
assertThat(addr.getHostName(), is("1.2.3.4")); assertThat(addr.getHostName(), is("1.2.3.4"));
assertThat(addr.getPort(), is(OZONE_SCM_DEFAULT_PORT)); assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DEFAULT_PORT));
// Verify valid hostname setup // Verify valid hostname setup
conf.setStrings(OZONE_SCM_NAMES, "scm1"); conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1");
addresses = OzoneClientUtils.getSCMAddresses(conf); addresses = OzoneClientUtils.getSCMAddresses(conf);
assertThat(addresses.size(), is(1)); assertThat(addresses.size(), is(1));
addr = addresses.iterator().next(); addr = addresses.iterator().next();
assertThat(addr.getHostName(), is("scm1")); assertThat(addr.getHostName(), is("scm1"));
assertThat(addr.getPort(), is(OZONE_SCM_DEFAULT_PORT)); assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DEFAULT_PORT));
// Verify valid hostname and port // Verify valid hostname and port
conf.setStrings(OZONE_SCM_NAMES, "scm1:1234"); conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234");
addresses = OzoneClientUtils.getSCMAddresses(conf); addresses = OzoneClientUtils.getSCMAddresses(conf);
assertThat(addresses.size(), is(1)); assertThat(addresses.size(), is(1));
addr = addresses.iterator().next(); addr = addresses.iterator().next();
@ -260,7 +260,7 @@ public void testGetSCMAddresses() {
hostsAndPorts.put("scm3", 3456); hostsAndPorts.put("scm3", 3456);
// Verify multiple hosts and port // Verify multiple hosts and port
conf.setStrings(OZONE_SCM_NAMES, "scm1:1234,scm2:2345,scm3:3456"); conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234,scm2:2345,scm3:3456");
addresses = OzoneClientUtils.getSCMAddresses(conf); addresses = OzoneClientUtils.getSCMAddresses(conf);
assertThat(addresses.size(), is(3)); assertThat(addresses.size(), is(3));
it = addresses.iterator(); it = addresses.iterator();
@ -273,7 +273,7 @@ public void testGetSCMAddresses() {
assertTrue(expected1.isEmpty()); assertTrue(expected1.isEmpty());
// Verify names with spaces // Verify names with spaces
conf.setStrings(OZONE_SCM_NAMES, " scm1:1234, scm2:2345 , scm3:3456 "); conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, " scm1:1234, scm2:2345 , scm3:3456 ");
addresses = OzoneClientUtils.getSCMAddresses(conf); addresses = OzoneClientUtils.getSCMAddresses(conf);
assertThat(addresses.size(), is(3)); assertThat(addresses.size(), is(3));
it = addresses.iterator(); it = addresses.iterator();
@ -286,7 +286,7 @@ public void testGetSCMAddresses() {
assertTrue(expected2.isEmpty()); assertTrue(expected2.isEmpty());
// Verify empty value // Verify empty value
conf.setStrings(OZONE_SCM_NAMES, ""); conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "");
try { try {
addresses = OzoneClientUtils.getSCMAddresses(conf); addresses = OzoneClientUtils.getSCMAddresses(conf);
fail("Empty value should cause an IllegalArgumentException"); fail("Empty value should cause an IllegalArgumentException");
@ -295,7 +295,7 @@ public void testGetSCMAddresses() {
} }
// Verify invalid hostname // Verify invalid hostname
conf.setStrings(OZONE_SCM_NAMES, "s..x..:1234"); conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "s..x..:1234");
try { try {
addresses = OzoneClientUtils.getSCMAddresses(conf); addresses = OzoneClientUtils.getSCMAddresses(conf);
fail("An invalid hostname should cause an IllegalArgumentException"); fail("An invalid hostname should cause an IllegalArgumentException");
@ -304,7 +304,7 @@ public void testGetSCMAddresses() {
} }
// Verify invalid port // Verify invalid port
conf.setStrings(OZONE_SCM_NAMES, "scm:xyz"); conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm:xyz");
try { try {
addresses = OzoneClientUtils.getSCMAddresses(conf); addresses = OzoneClientUtils.getSCMAddresses(conf);
fail("An invalid port should cause an IllegalArgumentException"); fail("An invalid port should cause an IllegalArgumentException");
@ -313,7 +313,7 @@ public void testGetSCMAddresses() {
} }
// Verify a mixed case (valid and invalid value both appears) // Verify a mixed case (valid and invalid value both appears)
conf.setStrings(OZONE_SCM_NAMES, "scm1:1234, scm:xyz"); conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234, scm:xyz");
try { try {
addresses = OzoneClientUtils.getSCMAddresses(conf); addresses = OzoneClientUtils.getSCMAddresses(conf);
fail("An invalid value should cause an IllegalArgumentException"); fail("An invalid value should cause an IllegalArgumentException");

10
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
View File

@ -19,7 +19,7 @@
import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine; import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager; import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
@ -49,7 +49,7 @@
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_HEARTBEAT_RPC_TIMEOUT; .OZONE_SCM_HEARTBEAT_RPC_TIMEOUT;
/** /**
@ -83,7 +83,7 @@ public void setUp() throws Exception {
mockServers.add(mock); mockServers.add(mock);
} }
conf.setStrings(OzoneConfigKeys.OZONE_SCM_NAMES, conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES,
serverAddresses.toArray(new String[0])); serverAddresses.toArray(new String[0]));
URL p = this.getClass().getResource(""); URL p = this.getClass().getResource("");
@ -96,7 +96,7 @@ public void setUp() throws Exception {
conf.set(DFS_DATANODE_DATA_DIR_KEY, path); conf.set(DFS_DATANODE_DATA_DIR_KEY, path);
path = Paths.get(path.toString(), path = Paths.get(path.toString(),
TestDatanodeStateMachine.class.getSimpleName() + ".id").toString(); TestDatanodeStateMachine.class.getSimpleName() + ".id").toString();
conf.set(OzoneConfigKeys.OZONE_SCM_DATANODE_ID, path); conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ID, path);
executorService = HadoopExecutors.newCachedThreadPool( executorService = HadoopExecutors.newCachedThreadPool(
new ThreadFactoryBuilder().setDaemon(true) new ThreadFactoryBuilder().setDaemon(true)
.setNameFormat("Test Data Node State Machine Thread - %d").build()); .setNameFormat("Test Data Node State Machine Thread - %d").build());
@ -275,7 +275,7 @@ public void testDatanodeStateMachineWithInvalidSCMNames()
"scm:xyz", // Invalid port "scm:xyz", // Invalid port
"scm:123456" // Port out of range "scm:123456" // Port out of range
}) { }) {
conf.setStrings(OzoneConfigKeys.OZONE_SCM_NAMES, name); conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, name);
final DatanodeStateMachine stateMachine = final DatanodeStateMachine stateMachine =
new DatanodeStateMachine(conf); new DatanodeStateMachine(conf);
DatanodeStateMachine.DatanodeStates currentState = DatanodeStateMachine.DatanodeStates currentState =

14
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java
View File

@ -19,13 +19,13 @@
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.container.common.SCMTestUtils; import org.apache.hadoop.ozone.container.common.SCMTestUtils;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMNodeReport; .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMStorageReport; .StorageContainerDatanodeProtocolProtos.SCMStorageReport;
import org.apache.hadoop.scm.ScmConfigKeys;
import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils;
import org.hamcrest.CoreMatchers; import org.hamcrest.CoreMatchers;
import org.junit.Assert; import org.junit.Assert;
@ -40,11 +40,11 @@
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_DEADNODE_INTERVAL_MS; import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL_MS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS; import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_INTERVAL_SECONDS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS; import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_MAX_HB_COUNT_TO_PROCESS; import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_MAX_HB_COUNT_TO_PROCESS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_STALENODE_INTERVAL_MS; import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL_MS;
import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.HEALTHY; import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.HEALTHY;
import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.STALE; import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.STALE;
import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.DEAD; import static org.apache.hadoop.ozone.scm.node.NodeManager.NODESTATE.DEAD;
@ -202,7 +202,7 @@ public void testScmSameNodeHeartbeats() throws IOException,
public void testScmShutdown() throws IOException, InterruptedException, public void testScmShutdown() throws IOException, InterruptedException,
TimeoutException { TimeoutException {
Configuration conf = getConf(); Configuration conf = getConf();
conf.setInt(OzoneConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, 100); conf.setInt(ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, 100);
SCMNodeManager nodeManager = createNodeManager(conf); SCMNodeManager nodeManager = createNodeManager(conf);
DatanodeID datanodeID = SCMTestUtils.getDatanodeID(nodeManager); DatanodeID datanodeID = SCMTestUtils.getDatanodeID(nodeManager);
nodeManager.close(); nodeManager.close();