big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "HDDS-1474. ozone.scm.datanode.id config should take path for a dir (#792)" (#808)

Browse Source 
This reverts commit 18cca583ac.
This commit is contained in:
Hanisha Koneru 2019-05-09 11:18:46 -07:00 committed by GitHub
parent ec36126346
commit 2d31ccc1b3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 45 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
View File

@ -296,10 +296,11 @@ public final class ScmConfigKeys {
public static final int OZONE_SCM_DEFAULT_PORT =
OZONE_SCM_DATANODE_PORT_DEFAULT;
// The path where datanode ID is to be written to.
// File Name and path where datanode ID is to written to.
// if this value is not set then container startup will fail.
public static final String OZONE_SCM_DATANODE_ID_DIR =
"ozone.scm.datanode.id.dir";
public static final String OZONE_SCM_DATANODE_ID = "ozone.scm.datanode.id";
public static final String OZONE_SCM_DATANODE_ID_PATH_DEFAULT = "datanode.id";
public static final String OZONE_SCM_DB_CACHE_SIZE_MB =
"ozone.scm.db.cache.size.mb";

5
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
View File

@ -195,11 +195,6 @@ public static Versioning getVersioning(boolean versioning) {
public static final int INVALID_PORT = -1;
/**
* Default SCM Datanode ID file name.
*/
public static final String OZONE_SCM_DATANODE_ID_FILE_DEFAULT = "datanode.id";
// The ServiceListJSONServlet context attribute where OzoneManager
// instance gets stored.
public static final String OM_CONTEXT_ATTRIBUTE = "ozone.om";

2
hadoop-hdds/common/src/main/resources/ozone-default.xml
View File

@ -883,7 +883,7 @@
</description>
</property>
<property>
<name>ozone.scm.datanode.id.dir</name>
<name>ozone.scm.datanode.id</name>
<value/>
<tag>OZONE, MANAGEMENT</tag>
<description>The path that datanodes will use to store the datanode ID.

13
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/hdds/scm/HddsServerUtil.java
View File

@ -23,7 +23,6 @@
import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -381,9 +380,8 @@ public static String getDefaultRatisDirectory(Configuration conf) {
* @return the path of datanode id as string
*/
public static String getDatanodeIdFilePath(Configuration conf) {
String dataNodeIDDirPath =
conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR);
if (dataNodeIDDirPath == null) {
String dataNodeIDPath = conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID);
if (dataNodeIDPath == null) {
File metaDirPath = ServerUtils.getOzoneMetaDirPath(conf);
if (metaDirPath == null) {
// this means meta data is not found, in theory should not happen at
@ -391,10 +389,9 @@ public static String getDatanodeIdFilePath(Configuration conf) {
throw new IllegalArgumentException("Unable to locate meta data" +
"directory when getting datanode id path");
}
dataNodeIDDirPath = metaDirPath.toString();
dataNodeIDPath = new File(metaDirPath,
ScmConfigKeys.OZONE_SCM_DATANODE_ID_PATH_DEFAULT).toString();
}
// Use default datanode id file name for file path
return new File(dataNodeIDDirPath,
OzoneConsts.OZONE_SCM_DATANODE_ID_FILE_DEFAULT).toString();
return dataNodeIDPath;
}
}

14
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
View File

@ -320,10 +320,9 @@ private DatanodeDetails initializeDatanodeDetails()
throws IOException {
String idFilePath = HddsServerUtil.getDatanodeIdFilePath(conf);
if (idFilePath == null || idFilePath.isEmpty()) {
LOG.error("A valid path is needed for config setting {}",
ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR);
throw new IllegalArgumentException(
ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR +
LOG.error("A valid file path is needed for config setting {}",
ScmConfigKeys.OZONE_SCM_DATANODE_ID);
throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_DATANODE_ID +
" must be defined. See" +
" https://wiki.apache.org/hadoop/Ozone#Configuration" +
" for details on configuring Ozone.");
@ -351,10 +350,9 @@ private void persistDatanodeDetails(DatanodeDetails dnDetails)
throws IOException {
String idFilePath = HddsServerUtil.getDatanodeIdFilePath(conf);
if (idFilePath == null || idFilePath.isEmpty()) {
LOG.error("A valid path is needed for config setting {}",
ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR);
throw new IllegalArgumentException(
ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR +
LOG.error("A valid file path is needed for config setting {}",
ScmConfigKeys.OZONE_SCM_DATANODE_ID);
throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_DATANODE_ID +
" must be defined. See" +
" https://wiki.apache.org/hadoop/Ozone#Configuration" +
" for details on configuring Ozone.");

4
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
View File

@ -119,8 +119,8 @@ public DatanodeStateMachine.DatanodeStates call() throws Exception {
private void persistContainerDatanodeDetails() {
String dataNodeIDPath = HddsServerUtil.getDatanodeIdFilePath(conf);
if (Strings.isNullOrEmpty(dataNodeIDPath)) {
LOG.error("A valid path is needed for config setting {}",
ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR);
LOG.error("A valid file path is needed for config setting {}",
ScmConfigKeys.OZONE_SCM_DATANODE_ID);
this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN);
return;
}

17
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java
View File

@ -25,7 +25,6 @@
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.statemachine
.DatanodeStateMachine;
@ -50,6 +49,7 @@
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -117,8 +117,9 @@ public void setUp() throws Exception {
}
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS,
new File(testRoot, "scm").getAbsolutePath());
path = new File(testRoot, "datanodeID").getAbsolutePath();
conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR, path);
path = Paths.get(path.toString(),
TestDatanodeStateMachine.class.getSimpleName() + ".id").toString();
conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ID, path);
executorService = HadoopExecutors.newCachedThreadPool(
new ThreadFactoryBuilder().setDaemon(true)
.setNameFormat("Test Data Node State Machine Thread - %d").build());
@ -211,8 +212,7 @@ public void testDatanodeStateContext() throws IOException,
// There is no mini cluster started in this test,
// create a ID file so that state machine could load a fake datanode ID.
File idPath = new File(
conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR),
OzoneConsts.OZONE_SCM_DATANODE_ID_FILE_DEFAULT);
conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID));
idPath.delete();
DatanodeDetails datanodeDetails = getNewDatanodeDetails();
DatanodeDetails.Port port = DatanodeDetails.newPort(
@ -334,8 +334,7 @@ public void testDatanodeStateContext() throws IOException,
public void testDatanodeStateMachineWithIdWriteFail() throws Exception {
File idPath = new File(
conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR),
OzoneConsts.OZONE_SCM_DATANODE_ID_FILE_DEFAULT);
conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID));
idPath.delete();
DatanodeDetails datanodeDetails = getNewDatanodeDetails();
DatanodeDetails.Port port = DatanodeDetails.newPort(
@ -397,10 +396,10 @@ public void testDatanodeStateMachineWithInvalidConfiguration()
/** Port out of range **/
confList.add(Maps.immutableEntry(
ScmConfigKeys.OZONE_SCM_NAMES, "scm:123456"));
// Invalid ozone.scm.datanode.id.dir
// Invalid ozone.scm.datanode.id
/** Empty **/
confList.add(Maps.immutableEntry(
ScmConfigKeys.OZONE_SCM_DATANODE_ID_DIR, ""));
ScmConfigKeys.OZONE_SCM_DATANODE_ID, ""));
confList.forEach((entry) -> {
Configuration perTestConf = new Configuration(conf);

6
hadoop-hdds/docs/content/Settings.md
View File

@ -105,14 +105,14 @@ Here is an example,
</property>
{{< /highlight >}}
* **ozone.scm.datanode.id.dir** Data nodes generate a Unique ID called Datanode
ID. This identity is written to the file datanode.id in a directory specified by this path. *Data nodes
* **ozone.scm.datanode.id** Data nodes generate a Unique ID called Datanode
ID. This identity is written to the file specified by this path. *Data nodes
will create this path if it doesn't exist already.*
Here is an example,
{{< highlight xml >}}
<property>
<name>ozone.scm.datanode.id.dir</name>
<name>ozone.scm.datanode.id</name>
<value>/data/disk1/meta/node/datanode.id</value>
</property>
{{< /highlight >}}

2
hadoop-ozone/dist/src/main/compose/ozone-hdfs/docker-config vendored
View File

@ -18,7 +18,7 @@ OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=true
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.scm.client.address=scm

2
hadoop-ozone/dist/src/main/compose/ozone-om-ha/docker-config vendored
View File

@ -21,7 +21,7 @@ OZONE-SITE.XML_ozone.om.address.om3=om3
OZONE-SITE.XML_ozone.om.ratis.enable=true
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed

2
hadoop-ozone/dist/src/main/compose/ozone-recon/docker-config vendored
View File

@ -18,7 +18,7 @@ OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon

2
hadoop-ozone/dist/src/main/compose/ozone/docker-config vendored
View File

@ -18,7 +18,7 @@ OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed

2
hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config vendored
View File

@ -18,7 +18,7 @@ OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed

2
hadoop-ozone/dist/src/main/compose/ozonefs/docker-config vendored
View File

@ -19,7 +19,7 @@ OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed

2
hadoop-ozone/dist/src/main/compose/ozoneperf/docker-config vendored
View File

@ -18,7 +18,7 @@ OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=true
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.scm.client.address=scm

2
hadoop-ozone/dist/src/main/compose/ozones3/docker-config vendored
View File

@ -17,7 +17,7 @@
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=true
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.scm.client.address=scm

2
hadoop-ozone/dist/src/main/compose/ozonescripts/docker-config vendored
View File

@ -19,7 +19,7 @@ CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000
OZONE-SITE.XML_ozone.ksm.address=ksm
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=true
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.block.client.address=scm

2
hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config vendored
View File

@ -18,7 +18,7 @@ OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed

2
hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config vendored
View File

@ -20,7 +20,7 @@ OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.om.http-address=om:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed

2
hadoop-ozone/dist/src/main/compose/ozonetrace/docker-config vendored
View File

@ -17,7 +17,7 @@
OZONE-SITE.XML_ozone.om.address=om
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=true
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id
OZONE-SITE.XML_ozone.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.scm.client.address=scm

3
hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml vendored
View File

@ -18,9 +18,8 @@ kind: ConfigMap
metadata:
name: config
data:
OZONE-SITE.XML_hdds.datanode.dir: "/data/storage"
OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data"
OZONE-SITE.XML_ozone.scm.datanode.id: "/data/datanode.id"
OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata"
OZONE-SITE.XML_ozone.scm.block.client.address: "scm-0.scm"
OZONE-SITE.XML_ozone.om.address: "om-0.om"

2
hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml vendored
View File

@ -20,7 +20,7 @@ metadata:
name: config
data:
OZONE-SITE.XML_hdds.datanode.dir: /data/storage
OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/datanode.id
OZONE-SITE.XML_ozone.scm.datanode.id: /data/datanode.id
OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm
OZONE-SITE.XML_ozone.om.address: om-0.om

2
hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml vendored
View File

@ -20,7 +20,7 @@ metadata:
name: config
data:
OZONE-SITE.XML_hdds.datanode.dir: /data/storage
OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/datanode.id
OZONE-SITE.XML_ozone.scm.datanode.id: /data/datanode.id
OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm
OZONE-SITE.XML_ozone.om.address: om-0.om

2
hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml vendored
View File

@ -20,7 +20,7 @@ metadata:
name: config
data:
OZONE-SITE.XML_hdds.datanode.dir: /data/storage
OZONE-SITE.XML_ozone.scm.datanode.id.dir: /data/datanode.id
OZONE-SITE.XML_ozone.scm.datanode.id: /data/datanode.id
OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata
OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm
OZONE-SITE.XML_ozone.om.address: om-0.om