+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.conf; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.concurrent.TimeUnit; + +/** + * Mark field to be configurable from ozone-site.xml. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +public @interface Config { + + /** + * Configuration fragment relative to the prefix defined with @ConfigGroup. + */ + String key(); + + /** + * Type of configuration. Use AUTO to decide it based on the java type. + */ + ConfigType type() default ConfigType.AUTO; + + /** + * If type == TIME the unit should be defined with this attribute. + */ + TimeUnit timeUnit() default TimeUnit.MILLISECONDS; +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigGroup.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigGroup.java new file mode 100644 index 0000000000..dd24ccbf00 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigGroup.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.conf; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Mark pojo which holds configuration variables. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface ConfigGroup { + String prefix(); +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigType.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigType.java new file mode 100644 index 0000000000..23a81042b2 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigType.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.conf; + +/** + * Possible type of injected configuration. + *
+ * AUTO means that the exact type will be identified based on the java type of + * the configuration field. + */ +public enum ConfigType { + AUTO, + STRING, + BOOLEAN, + INT, + LONG, + TIME, + SIZE +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationException.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationException.java new file mode 100644 index 0000000000..9c6b213fe9 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationException.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.conf;
+
+/**
+ * Exeception to throw in case of a configuration problem.
+ */
+public class ConfigurationException extends RuntimeException {
+ public ConfigurationException() {
+ }
+
+ public ConfigurationException(String message) {
+ super(message);
+ }
+
+ public ConfigurationException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
index b7166c7f47..b4dc94a2e3 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
@@ -28,6 +28,8 @@
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.net.URL;
import java.util.ArrayList;
import java.util.Enumeration;
@@ -61,6 +63,108 @@ public List
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.conf;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Example configuration to test the configuration injection.
+ */
+@ConfigGroup(prefix = "ozone.scm.client")
+public class SimpleConfiguration {
+
+ private String clientAddress;
+
+ private String bindHost;
+
+ private boolean enabled;
+
+ private int port = 1234;
+
+ private long waitTime = 1;
+
+ @Config(key = "address")
+ public void setClientAddress(String clientAddress) {
+ this.clientAddress = clientAddress;
+ }
+
+ @Config(key = "bind.host")
+ public void setBindHost(String bindHost) {
+ this.bindHost = bindHost;
+ }
+
+ @Config(key = "enabled")
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
+
+ @Config(key = "port")
+ public void setPort(int port) {
+ this.port = port;
+ }
+
+ @Config(key = "wait", type = ConfigType.TIME, timeUnit =
+ TimeUnit.SECONDS)
+ public void setWaitTime(long waitTime) {
+ this.waitTime = waitTime;
+ }
+
+ public String getClientAddress() {
+ return clientAddress;
+ }
+
+ public String getBindHost() {
+ return bindHost;
+ }
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public long getWaitTime() {
+ return waitTime;
+ }
+}
diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
index ef6e5a84ee..bf8ac04145 100644
--- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
+++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
@@ -29,6 +29,7 @@
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.util.concurrent.TimeUnit;
/**
* Test class for OzoneConfiguration.
@@ -97,6 +98,37 @@ public void testGetAllPropertiesByTags() throws Exception {
.getProperty("dfs.cblock.trace.io"));
}
+ @Test
+ public void getConfigurationObject() {
+ OzoneConfiguration ozoneConfig = new OzoneConfiguration();
+ ozoneConfig.set("ozone.scm.client.address", "address");
+ ozoneConfig.set("ozone.scm.client.bind.host", "host");
+ ozoneConfig.setBoolean("ozone.scm.client.enabled", true);
+ ozoneConfig.setInt("ozone.scm.client.port", 5555);
+ ozoneConfig.setTimeDuration("ozone.scm.client.wait", 10, TimeUnit.MINUTES);
+
+ SimpleConfiguration configuration =
+ ozoneConfig.getObject(SimpleConfiguration.class);
+
+ Assert.assertEquals("host", configuration.getBindHost());
+ Assert.assertEquals("address", configuration.getClientAddress());
+ Assert.assertEquals(true, configuration.isEnabled());
+ Assert.assertEquals(5555, configuration.getPort());
+ Assert.assertEquals(600, configuration.getWaitTime());
+ }
+
+ @Test
+ public void getConfigurationObjectWithDefault() {
+ OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
+
+ SimpleConfiguration configuration =
+ ozoneConfiguration.getObject(SimpleConfiguration.class);
+
+ Assert.assertEquals(false, configuration.isEnabled());
+ Assert.assertEquals(9860, configuration.getPort());
+ }
+
+
private void appendProperty(BufferedWriter out, String name, String val)
throws IOException {
this.appendProperty(out, name, val, false);
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
index 1dce81b9c6..8f62243e7d 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ReplicationManager.java
@@ -21,12 +21,13 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.GeneratedMessage;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.conf.ConfigType;
+import org.apache.hadoop.hdds.conf.ConfigGroup;
+import org.apache.hadoop.hdds.conf.Config;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.placement.algorithms
.ContainerPlacementPolicy;
import org.apache.hadoop.hdds.scm.events.SCMEvents;
@@ -108,15 +109,9 @@ public class ReplicationManager {
private final Thread replicationMonitor;
/**
- * The frequency in which ReplicationMonitor thread should run.
+ * ReplicationManager specific configuration.
*/
- private final long interval;
-
- /**
- * Timeout for container replication & deletion command issued by
- * ReplicationManager.
- */
- private final long eventTimeout;
+ private final ReplicationManagerConfiguration conf;
/**
* Flag used for checking if the ReplicationMonitor thread is running or
@@ -132,27 +127,21 @@ public class ReplicationManager {
* @param containerPlacement ContainerPlacementPolicy
* @param eventPublisher EventPublisher
*/
- public ReplicationManager(final Configuration conf,
+ public ReplicationManager(final ReplicationManagerConfiguration conf,
final ContainerManager containerManager,
final ContainerPlacementPolicy containerPlacement,
- final EventPublisher eventPublisher) {
+ final EventPublisher eventPublisher,
+ final LockManager lockManager) {
this.containerManager = containerManager;
this.containerPlacement = containerPlacement;
this.eventPublisher = eventPublisher;
- this.lockManager = new LockManager<>(conf);
+ this.lockManager = lockManager;
this.inflightReplication = new HashMap<>();
this.inflightDeletion = new HashMap<>();
this.replicationMonitor = new Thread(this::run);
this.replicationMonitor.setName("ReplicationMonitor");
this.replicationMonitor.setDaemon(true);
- this.interval = conf.getTimeDuration(
- ScmConfigKeys.HDDS_SCM_REPLICATION_THREAD_INTERVAL,
- ScmConfigKeys.HDDS_SCM_REPLICATION_THREAD_INTERVAL_DEFAULT,
- TimeUnit.MILLISECONDS);
- this.eventTimeout = conf.getTimeDuration(
- ScmConfigKeys.HDDS_SCM_REPLICATION_EVENT_TIMEOUT,
- ScmConfigKeys.HDDS_SCM_REPLICATION_EVENT_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
+ this.conf = conf;
this.running = false;
}
@@ -217,7 +206,7 @@ private synchronized void run() {
" processing {} containers.", Time.monotonicNow() - start,
containerIds.size());
- wait(interval);
+ wait(conf.getInterval());
}
} catch (Throwable t) {
// When we get runtime exception, we should terminate SCM.
@@ -337,7 +326,7 @@ private void updateInflightAction(final ContainerInfo container,
final Map