big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12018. Ozone: Misc: Add CBlocks defaults to ozone-defaults.xml. Contributed by Chen Liang.

Browse Source
This commit is contained in:
Weiwei Yang 2017-06-27 12:55:08 +08:00 committed by Owen O'Malley
parent fea1ab6192
commit 7e834c839d
4 changed files with 173 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
View File

@ -25,28 +25,19 @@
public final class CBlockConfigKeys {
public static final String DFS_CBLOCK_SERVICERPC_ADDRESS_KEY =
"dfs.cblock.servicerpc-address";
public static final String DFS_CBLOCK_SERVICERPC_PORT_KEY =
"dfs.cblock.servicerpc.port";
public static final int DFS_CBLOCK_SERVICERPC_PORT_DEFAULT =
9810;
public static final String DFS_CBLOCK_SERVICERPC_HOSTNAME_KEY =
"dfs.cblock.servicerpc.hostname";
public static final String DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT =
"0.0.0.0";
public static final String DFS_CBLOCK_SERVICERPC_ADDRESS_DEFAULT =
DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT
+ ":" + DFS_CBLOCK_SERVICERPC_PORT_DEFAULT;
public static final String DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY =
"dfs.cblock.jscsi-address";
//The port on CBlockManager node for jSCSI to ask
public static final String DFS_CBLOCK_JSCSI_PORT_KEY =
"dfs.cblock.jscsi.port";
public static final int DFS_CBLOCK_JSCSI_PORT_DEFAULT =
9811;
public static final String DFS_CBLOCK_JSCSIRPC_ADDRESS_DEFAULT =
DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT
+ ":" + DFS_CBLOCK_JSCSI_PORT_DEFAULT;
public static final String DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY =
"dfs.cblock.service.rpc-bind-host";
@ -97,9 +88,9 @@ public final class CBlockConfigKeys {
/**
* Minimum Number of threads that cache pool will use for background I/O.
*/
public static final String DFS_CBLOCK_CACHE_CORE_POOL_SIZE =
"dfs.cblock.cache.core.pool.size";
public static final int DFS_CBLOCK_CACHE_CORE_POOL_SIZE_DEFAULT = 16;
public static final String DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE =
"dfs.cblock.cache.core.min.pool.size";
public static final int DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT = 16;
/**
* Maximum Number of threads that cache pool will use for background I/O.

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
View File

@ -61,9 +61,9 @@
import static org.apache.hadoop.cblock.CBlockConfigKeys
.DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys
.DFS_CBLOCK_CACHE_CORE_POOL_SIZE;
.DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE;
import static org.apache.hadoop.cblock.CBlockConfigKeys
.DFS_CBLOCK_CACHE_CORE_POOL_SIZE_DEFAULT;
.DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys
.DFS_CBLOCK_CACHE_KEEP_ALIVE_SECONDS;
import static org.apache.hadoop.cblock.CBlockConfigKeys
@ -114,8 +114,8 @@ public ContainerCacheFlusher(Configuration config,
CBlockTargetMetrics metrics) {
int queueSize = config.getInt(DFS_CBLOCK_CACHE_QUEUE_SIZE_KB,
DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT) * 1024;
int corePoolSize = config.getInt(DFS_CBLOCK_CACHE_CORE_POOL_SIZE,
DFS_CBLOCK_CACHE_CORE_POOL_SIZE_DEFAULT);
int corePoolSize = config.getInt(DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE,
DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT);
int maxPoolSize = config.getInt(DFS_CBLOCK_CACHE_MAX_POOL_SIZE,
DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT);
long keepAlive = config.getLong(DFS_CBLOCK_CACHE_KEEP_ALIVE_SECONDS,

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
View File

@ -36,6 +36,7 @@
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY;
import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
@ -67,6 +68,10 @@ public static void main(String[] args) throws Exception {
DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY,
DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT
);
int cbmPort = ozoneConf.getInt(
DFS_CBLOCK_JSCSI_PORT_KEY,
DFS_CBLOCK_JSCSI_PORT_DEFAULT
);
String scmAddress = ozoneConf.get(OZONE_SCM_CLIENT_BIND_HOST_KEY,
OZONE_SCM_CLIENT_BIND_HOST_DEFAULT);
@ -82,7 +87,7 @@ public static void main(String[] args) throws Exception {
ozoneConf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, scmDataodeAddress);
InetSocketAddress cbmAddress = new InetSocketAddress(
cbmIPAddress, DFS_CBLOCK_JSCSI_PORT_DEFAULT);
cbmIPAddress, cbmPort);
long version = RPC.getProtocolVersion(
CBlockServiceProtocolPB.class);
CBlockClientProtocolClientSideTranslatorPB cbmClient =

159
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml
View File

@ -579,5 +579,162 @@
The keytab file for Kerberos authentication in KSM.
</description>
</property>
<property>
<name>dfs.cblock.servicerpc-address</name>
<value></value>
<description>
The address that cblock will be bind to, should be a host:port format,
this setting is required for cblock server to start.
</description>
</property>
<property>
<name>dfs.cblock.jscsi.port</name>
<value>9811</value>
<description>
The port on CBlockManager node for jSCSI to talk to.
</description>
</property>
<property>
<name>dfs.storage.service.handler.count</name>
<value>10</value>
<description>
Default number of handlers for CBlock service rpc.
</description>
</property>
<property>
<name>dfs.cblock.service.leveldb.path</name>
<value>/tmp/cblock_levelDB.dat</value>
<description>
Default path for the cblock meta data disk store.
</description>
</property>
<property>
<name>dfs.cblock.disk.cache.path</name>
<value>/tmp/cblockCacheDB</value>
<description>
Default path for the cblock local cache.
</description>
</property>
<property>
<name>dfs.cblock.trace.io</name>
<value>false</value>
<description>
Default flag for enabling trace io.
</description>
</property>
<property>
<name>dfs.cblock.short.circuit.io</name>
<value>false</value>
<description>
Default flag to enable cblock local cache.
</description>
</property>
<property>
<name>dfs.cblock.cache.cache.size.in.kb</name>
<value>256</value>
<description>
Default cblock cache queue size, in number of kb.
</description>
</property>
<property>
<name>dfs.cblock.cache.core.min.pool.size</name>
<value>16</value>
<description>
Minimum number of thread pool thread that cBlock cache will use for background I/O.
</description>
</property>
<property>
<name>dfs.cblock.cache.max.pool.size</name>
<value>256</value>
<description>
Maximum number of thread pool thread that cBlcok cache will use for background I/O.
</description>
</property>
<property>
<name>dfs.cblock.cache.keep.alive.seconds</name>
<value>60</value>
<description>
Number of seconds to keep the Thread alive when it is idle.
</description>
</property>
<property>
<name>dfs.cblock.cache.thread.priority</name>
<value>5</value>
<description>
Priority of cache flusher thread, affecting the relative
performance of write and read.upported values are 1, 5, 10.
Use 10 for high priority and 1 for low priority.
</description>
</property>
<property>
<name>dfs.cblock.cache.block.buffer.size</name>
<value>512</value>
<description>
Block Buffer size in terms of blockID entries, in number of blockIDs.
</description>
</property>
<property>
<name>dfs.cblock.block.buffer.flush.interval.seconds</name>
<value>60</value>
<description>
The interval to flush cblock buffer in seconds.
</description>
</property>
<property>
<name>dfs.cblock.jscsi.server.address</name>
<value>127.0.0.1</value>
<description>
The address that jscsi server will be running, should have one
local jscsi server for each client that tries to mount cblock.
</description>
</property>
<property>
<name>dfs.cblock.jscsi.cblock.server.address</name>
<value>127.0.0.1</value>
<description>
The address local jscsi server will use to talk to cblock manager.
</description>
</property>
<property>
<name>dfs.cblock.container.size</name>
<value>5</value>
<description>
The size of ozone container in number of GBs. Note that this is
not setting container size for ozone, but rather, this is setting
that assumption that cblock manager will make about ozone.
</description>
</property>
<property>
<name>dfs.cblock.cache.leveldb.cache.size.mb</name>
<value>256</value>
<description>
The size of LevelDB cache file which uses an off-heap cache in LevelDB.
</description>
</property>
<property>
<name>dfs.cblock.cache.max.retry</name>
<value>65536</value>
<description>
The maximum number of retries when writing a block to container.
</description>
</property>
</configuration>