big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-6878. Change MiniDFSCluster to support StorageType configuration for individual directories. (Arpit Agarwal)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1619271 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arpit Agarwal 2014-08-21 00:42:27 +00:00
parent 8304001861
commit 8f00897a4b
4 changed files with 61 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -499,6 +499,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6858. Allow dfs.data.transfer.saslproperties.resolver.class default to HDFS-6858. Allow dfs.data.transfer.saslproperties.resolver.class default to
hadoop.security.saslproperties.resolver.class. (Benoy Antony via cnauroth) hadoop.security.saslproperties.resolver.class. (Benoy Antony via cnauroth)
HDFS-6878. Change MiniDFSCluster to support StorageType configuration
for individual directories (Arpit Agarwal)
OPTIMIZATIONS OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang) HDFS-6690. Deduplicate xattr names in memory. (wang)

82
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
View File

@ -144,7 +144,7 @@ public static class Builder {
private int nameNodeHttpPort = 0; private int nameNodeHttpPort = 0;
private final Configuration conf; private final Configuration conf;
private int numDataNodes = 1; private int numDataNodes = 1;
private StorageType storageType = StorageType.DEFAULT; private StorageType[][] storageTypes = null;
private boolean format = true; private boolean format = true;
private boolean manageNameDfsDirs = true; private boolean manageNameDfsDirs = true;
private boolean manageNameDfsSharedDirs = true; private boolean manageNameDfsSharedDirs = true;
@ -193,10 +193,26 @@ public Builder numDataNodes(int val) {
} }
/** /**
* Default: StorageType.DEFAULT * Set the same storage type configuration for each datanode.
* If storageTypes is uninitialized or passed null then
* StorageType.DEFAULT is used.
*/ */
public Builder storageType(StorageType type) { public Builder storageTypes(StorageType[] types) {
this.storageType = type; assert types.length == DIRS_PER_DATANODE;
this.storageTypes = new StorageType[numDataNodes][types.length];
for (int i = 0; i < numDataNodes; ++i) {
this.storageTypes[i] = types;
}
return this;
}
/**
* Set custom storage type configuration for each datanode.
* If storageTypes is uninitialized or passed null then
* StorageType.DEFAULT is used.
*/
public Builder storageTypes(StorageType[][] types) {
this.storageTypes = types;
return this; return this;
} }
@ -369,7 +385,8 @@ protected MiniDFSCluster(Builder builder) throws IOException {
builder.nnTopology = MiniDFSNNTopology.simpleSingleNN( builder.nnTopology = MiniDFSNNTopology.simpleSingleNN(
builder.nameNodePort, builder.nameNodeHttpPort); builder.nameNodePort, builder.nameNodeHttpPort);
} }
assert builder.storageTypes == null ||
builder.storageTypes.length == builder.numDataNodes;
final int numNameNodes = builder.nnTopology.countNameNodes(); final int numNameNodes = builder.nnTopology.countNameNodes();
LOG.info("starting cluster: numNameNodes=" + numNameNodes LOG.info("starting cluster: numNameNodes=" + numNameNodes
+ ", numDataNodes=" + builder.numDataNodes); + ", numDataNodes=" + builder.numDataNodes);
@ -377,7 +394,7 @@ protected MiniDFSCluster(Builder builder) throws IOException {
initMiniDFSCluster(builder.conf, initMiniDFSCluster(builder.conf,
builder.numDataNodes, builder.numDataNodes,
builder.storageType, builder.storageTypes,
builder.format, builder.format,
builder.manageNameDfsDirs, builder.manageNameDfsDirs,
builder.manageNameDfsSharedDirs, builder.manageNameDfsSharedDirs,
@ -477,8 +494,8 @@ public MiniDFSCluster() {
* Servers will be started on free ports. * Servers will be started on free ports.
* <p> * <p>
* The caller must manage the creation of NameNode and DataNode directories * The caller must manage the creation of NameNode and DataNode directories
* and have already set {@link #DFS_NAMENODE_NAME_DIR_KEY} and * and have already set {@link DFSConfigKeys#DFS_NAMENODE_NAME_DIR_KEY} and
* {@link #DFS_DATANODE_DATA_DIR_KEY} in the given conf. * {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} in the given conf.
* *
* @param conf the base configuration to use in starting the servers. This * @param conf the base configuration to use in starting the servers. This
* will be modified as necessary. * will be modified as necessary.
@ -554,8 +571,8 @@ public MiniDFSCluster(Configuration conf,
* @param format if true, format the NameNode and DataNodes before starting * @param format if true, format the NameNode and DataNodes before starting
* up * up
* @param manageDfsDirs if true, the data directories for servers will be * @param manageDfsDirs if true, the data directories for servers will be
* created and {@link #DFS_NAMENODE_NAME_DIR_KEY} and * created and {@link DFSConfigKeys#DFS_NAMENODE_NAME_DIR_KEY} and
* {@link #DFS_DATANODE_DATA_DIR_KEY} will be set in * {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be set in
* the conf * the conf
* @param operation the operation with which to start the servers. If null * @param operation the operation with which to start the servers. If null
* or StartupOption.FORMAT, then StartupOption.REGULAR will be used. * or StartupOption.FORMAT, then StartupOption.REGULAR will be used.
@ -586,8 +603,8 @@ public MiniDFSCluster(int nameNodePort,
* @param numDataNodes Number of DataNodes to start; may be zero * @param numDataNodes Number of DataNodes to start; may be zero
* @param format if true, format the NameNode and DataNodes before starting up * @param format if true, format the NameNode and DataNodes before starting up
* @param manageDfsDirs if true, the data directories for servers will be * @param manageDfsDirs if true, the data directories for servers will be
* created and {@link #DFS_NAMENODE_NAME_DIR_KEY} and * created and {@link DFSConfigKeys#DFS_NAMENODE_NAME_DIR_KEY} and
* {@link #DFS_DATANODE_DATA_DIR_KEY} will be set in * {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be set in
* the conf * the conf
* @param operation the operation with which to start the servers. If null * @param operation the operation with which to start the servers. If null
* or StartupOption.FORMAT, then StartupOption.REGULAR will be used. * or StartupOption.FORMAT, then StartupOption.REGULAR will be used.
@ -620,11 +637,11 @@ public MiniDFSCluster(int nameNodePort,
* @param numDataNodes Number of DataNodes to start; may be zero * @param numDataNodes Number of DataNodes to start; may be zero
* @param format if true, format the NameNode and DataNodes before starting up * @param format if true, format the NameNode and DataNodes before starting up
* @param manageNameDfsDirs if true, the data directories for servers will be * @param manageNameDfsDirs if true, the data directories for servers will be
* created and {@link #DFS_NAMENODE_NAME_DIR_KEY} and * created and {@link DFSConfigKeys#DFS_NAMENODE_NAME_DIR_KEY} and
* {@link #DFS_DATANODE_DATA_DIR_KEY} will be set in * {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be set in
* the conf * the conf
* @param manageDataDfsDirs if true, the data directories for datanodes will * @param manageDataDfsDirs if true, the data directories for datanodes will
* be created and {@link #DFS_DATANODE_DATA_DIR_KEY} * be created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY}
* set to same in the conf * set to same in the conf
* @param operation the operation with which to start the servers. If null * @param operation the operation with which to start the servers. If null
* or StartupOption.FORMAT, then StartupOption.REGULAR will be used. * or StartupOption.FORMAT, then StartupOption.REGULAR will be used.
@ -643,7 +660,7 @@ public MiniDFSCluster(int nameNodePort,
String[] racks, String hosts[], String[] racks, String hosts[],
long[] simulatedCapacities) throws IOException { long[] simulatedCapacities) throws IOException {
this.nameNodes = new NameNodeInfo[1]; // Single namenode in the cluster this.nameNodes = new NameNodeInfo[1]; // Single namenode in the cluster
initMiniDFSCluster(conf, numDataNodes, StorageType.DEFAULT, format, initMiniDFSCluster(conf, numDataNodes, null, format,
manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs, manageNameDfsDirs, true, manageDataDfsDirs, manageDataDfsDirs,
operation, null, racks, hosts, operation, null, racks, hosts,
simulatedCapacities, null, true, false, simulatedCapacities, null, true, false,
@ -652,7 +669,7 @@ public MiniDFSCluster(int nameNodePort,
private void initMiniDFSCluster( private void initMiniDFSCluster(
Configuration conf, Configuration conf,
int numDataNodes, StorageType storageType, boolean format, boolean manageNameDfsDirs, int numDataNodes, StorageType[][] storageTypes, boolean format, boolean manageNameDfsDirs,
boolean manageNameDfsSharedDirs, boolean enableManagedDfsDirsRedundancy, boolean manageNameDfsSharedDirs, boolean enableManagedDfsDirsRedundancy,
boolean manageDataDfsDirs, StartupOption startOpt, boolean manageDataDfsDirs, StartupOption startOpt,
StartupOption dnStartOpt, String[] racks, StartupOption dnStartOpt, String[] racks,
@ -725,7 +742,7 @@ private void initMiniDFSCluster(
} }
// Start the DataNodes // Start the DataNodes
startDataNodes(conf, numDataNodes, storageType, manageDataDfsDirs, startDataNodes(conf, numDataNodes, storageTypes, manageDataDfsDirs,
dnStartOpt != null ? dnStartOpt : startOpt, dnStartOpt != null ? dnStartOpt : startOpt,
racks, hosts, simulatedCapacities, setupHostsFile, racks, hosts, simulatedCapacities, setupHostsFile,
checkDataNodeAddrConfig, checkDataNodeHostConfig, dnConfOverlays); checkDataNodeAddrConfig, checkDataNodeHostConfig, dnConfOverlays);
@ -1100,15 +1117,18 @@ public void waitClusterUp() throws IOException {
} }
} }
String makeDataNodeDirs(int dnIndex, StorageType storageType) throws IOException { String makeDataNodeDirs(int dnIndex, StorageType[] storageTypes) throws IOException {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
assert storageTypes == null || storageTypes.length == DIRS_PER_DATANODE;
for (int j = 0; j < DIRS_PER_DATANODE; ++j) { for (int j = 0; j < DIRS_PER_DATANODE; ++j) {
File dir = getInstanceStorageDir(dnIndex, j); File dir = getInstanceStorageDir(dnIndex, j);
dir.mkdirs(); dir.mkdirs();
if (!dir.isDirectory()) { if (!dir.isDirectory()) {
throw new IOException("Mkdirs failed to create directory for DataNode " + dir); throw new IOException("Mkdirs failed to create directory for DataNode " + dir);
} }
sb.append((j > 0 ? "," : "") + "[" + storageType + "]" + fileAsURI(dir)); sb.append((j > 0 ? "," : "") + "[" +
(storageTypes == null ? StorageType.DEFAULT : storageTypes[j]) +
"]" + fileAsURI(dir));
} }
return sb.toString(); return sb.toString();
} }
@ -1127,7 +1147,7 @@ String makeDataNodeDirs(int dnIndex, StorageType storageType) throws IOException
* will be modified as necessary. * will be modified as necessary.
* @param numDataNodes Number of DataNodes to start; may be zero * @param numDataNodes Number of DataNodes to start; may be zero
* @param manageDfsDirs if true, the data directories for DataNodes will be * @param manageDfsDirs if true, the data directories for DataNodes will be
* created and {@link #DFS_DATANODE_DATA_DIR_KEY} will be set * created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be set
* in the conf * in the conf
* @param operation the operation with which to start the DataNodes. If null * @param operation the operation with which to start the DataNodes. If null
* or StartupOption.FORMAT, then StartupOption.REGULAR will be used. * or StartupOption.FORMAT, then StartupOption.REGULAR will be used.
@ -1159,7 +1179,7 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
* will be modified as necessary. * will be modified as necessary.
* @param numDataNodes Number of DataNodes to start; may be zero * @param numDataNodes Number of DataNodes to start; may be zero
* @param manageDfsDirs if true, the data directories for DataNodes will be * @param manageDfsDirs if true, the data directories for DataNodes will be
* created and {@link #DFS_DATANODE_DATA_DIR_KEY} will be * created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be
* set in the conf * set in the conf
* @param operation the operation with which to start the DataNodes. If null * @param operation the operation with which to start the DataNodes. If null
* or StartupOption.FORMAT, then StartupOption.REGULAR will be used. * or StartupOption.FORMAT, then StartupOption.REGULAR will be used.
@ -1175,21 +1195,17 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
String[] racks, String[] hosts, String[] racks, String[] hosts,
long[] simulatedCapacities, long[] simulatedCapacities,
boolean setupHostsFile) throws IOException { boolean setupHostsFile) throws IOException {
startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs, operation, racks, hosts, startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, hosts,
simulatedCapacities, setupHostsFile, false, false, null); simulatedCapacities, setupHostsFile, false, false, null);
} }
/**
* @see MiniDFSCluster#startDataNodes(Configuration, int, boolean, StartupOption,
* String[], String[], long[], boolean, boolean, boolean)
*/
public synchronized void startDataNodes(Configuration conf, int numDataNodes, public synchronized void startDataNodes(Configuration conf, int numDataNodes,
boolean manageDfsDirs, StartupOption operation, boolean manageDfsDirs, StartupOption operation,
String[] racks, String[] hosts, String[] racks, String[] hosts,
long[] simulatedCapacities, long[] simulatedCapacities,
boolean setupHostsFile, boolean setupHostsFile,
boolean checkDataNodeAddrConfig) throws IOException { boolean checkDataNodeAddrConfig) throws IOException {
startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs, operation, racks, hosts, startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, hosts,
simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false, null); simulatedCapacities, setupHostsFile, checkDataNodeAddrConfig, false, null);
} }
@ -1207,7 +1223,7 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
* will be modified as necessary. * will be modified as necessary.
* @param numDataNodes Number of DataNodes to start; may be zero * @param numDataNodes Number of DataNodes to start; may be zero
* @param manageDfsDirs if true, the data directories for DataNodes will be * @param manageDfsDirs if true, the data directories for DataNodes will be
* created and {@link #DFS_DATANODE_DATA_DIR_KEY} will be * created and {@link DFSConfigKeys#DFS_DATANODE_DATA_DIR_KEY} will be
* set in the conf * set in the conf
* @param operation the operation with which to start the DataNodes. If null * @param operation the operation with which to start the DataNodes. If null
* or StartupOption.FORMAT, then StartupOption.REGULAR will be used. * or StartupOption.FORMAT, then StartupOption.REGULAR will be used.
@ -1222,13 +1238,15 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
* @throws IllegalStateException if NameNode has been shutdown * @throws IllegalStateException if NameNode has been shutdown
*/ */
public synchronized void startDataNodes(Configuration conf, int numDataNodes, public synchronized void startDataNodes(Configuration conf, int numDataNodes,
StorageType storageType, boolean manageDfsDirs, StartupOption operation, StorageType[][] storageTypes, boolean manageDfsDirs, StartupOption operation,
String[] racks, String[] hosts, String[] racks, String[] hosts,
long[] simulatedCapacities, long[] simulatedCapacities,
boolean setupHostsFile, boolean setupHostsFile,
boolean checkDataNodeAddrConfig, boolean checkDataNodeAddrConfig,
boolean checkDataNodeHostConfig, boolean checkDataNodeHostConfig,
Configuration[] dnConfOverlays) throws IOException { Configuration[] dnConfOverlays) throws IOException {
assert storageTypes == null || storageTypes.length == numDataNodes;
if (operation == StartupOption.RECOVER) { if (operation == StartupOption.RECOVER) {
return; return;
} }
@ -1289,7 +1307,7 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
// Set up datanode address // Set up datanode address
setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig); setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig);
if (manageDfsDirs) { if (manageDfsDirs) {
String dirs = makeDataNodeDirs(i, storageType); String dirs = makeDataNodeDirs(i, storageTypes == null ? null : storageTypes[i]);
dnConf.set(DFS_DATANODE_DATA_DIR_KEY, dirs); dnConf.set(DFS_DATANODE_DATA_DIR_KEY, dirs);
conf.set(DFS_DATANODE_DATA_DIR_KEY, dirs); conf.set(DFS_DATANODE_DATA_DIR_KEY, dirs);
} }
@ -2173,7 +2191,7 @@ public void injectBlocks(int dataNodeIndex,
} }
/** /**
* Multiple-NameNode version of {@link #injectBlocks(Iterable[])}. * Multiple-NameNode version of injectBlocks.
*/ */
public void injectBlocks(int nameNodeIndex, int dataNodeIndex, public void injectBlocks(int nameNodeIndex, int dataNodeIndex,
Iterable<Block> blocksToInject) throws IOException { Iterable<Block> blocksToInject) throws IOException {

12
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
View File

@ -50,12 +50,14 @@ public static void setNodeGroups (String[] nodeGroups) {
} }
public synchronized void startDataNodes(Configuration conf, int numDataNodes, public synchronized void startDataNodes(Configuration conf, int numDataNodes,
StorageType storageType, boolean manageDfsDirs, StartupOption operation, StorageType[][] storageTypes, boolean manageDfsDirs, StartupOption operation,
String[] racks, String[] nodeGroups, String[] hosts, String[] racks, String[] nodeGroups, String[] hosts,
long[] simulatedCapacities, long[] simulatedCapacities,
boolean setupHostsFile, boolean setupHostsFile,
boolean checkDataNodeAddrConfig, boolean checkDataNodeAddrConfig,
boolean checkDataNodeHostConfig) throws IOException { boolean checkDataNodeHostConfig) throws IOException {
assert storageTypes == null || storageTypes.length == numDataNodes;
if (operation == StartupOption.RECOVER) { if (operation == StartupOption.RECOVER) {
return; return;
} }
@ -112,7 +114,7 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
// Set up datanode address // Set up datanode address
setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig); setupDatanodeAddress(dnConf, setupHostsFile, checkDataNodeAddrConfig);
if (manageDfsDirs) { if (manageDfsDirs) {
String dirs = makeDataNodeDirs(i, storageType); String dirs = makeDataNodeDirs(i, storageTypes == null ? null : storageTypes[i]);
dnConf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dirs); dnConf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dirs);
conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dirs); conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dirs);
} }
@ -190,7 +192,7 @@ public synchronized void startDataNodes(Configuration conf, int numDataNodes,
String[] racks, String[] nodeGroups, String[] hosts, String[] racks, String[] nodeGroups, String[] hosts,
long[] simulatedCapacities, long[] simulatedCapacities,
boolean setupHostsFile) throws IOException { boolean setupHostsFile) throws IOException {
startDataNodes(conf, numDataNodes, StorageType.DEFAULT, manageDfsDirs, operation, racks, nodeGroups, startDataNodes(conf, numDataNodes, null, manageDfsDirs, operation, racks, nodeGroups,
hosts, simulatedCapacities, setupHostsFile, false, false); hosts, simulatedCapacities, setupHostsFile, false, false);
} }
@ -205,14 +207,14 @@ public void startDataNodes(Configuration conf, int numDataNodes,
// This is for initialize from parent class. // This is for initialize from parent class.
@Override @Override
public synchronized void startDataNodes(Configuration conf, int numDataNodes, public synchronized void startDataNodes(Configuration conf, int numDataNodes,
StorageType storageType, boolean manageDfsDirs, StartupOption operation, StorageType[][] storageTypes, boolean manageDfsDirs, StartupOption operation,
String[] racks, String[] hosts, String[] racks, String[] hosts,
long[] simulatedCapacities, long[] simulatedCapacities,
boolean setupHostsFile, boolean setupHostsFile,
boolean checkDataNodeAddrConfig, boolean checkDataNodeAddrConfig,
boolean checkDataNodeHostConfig, boolean checkDataNodeHostConfig,
Configuration[] dnConfOverlays) throws IOException { Configuration[] dnConfOverlays) throws IOException {
startDataNodes(conf, numDataNodes, storageType, manageDfsDirs, operation, racks, startDataNodes(conf, numDataNodes, storageTypes, manageDfsDirs, operation, racks,
NODE_GROUPS, hosts, simulatedCapacities, setupHostsFile, NODE_GROUPS, hosts, simulatedCapacities, setupHostsFile,
checkDataNodeAddrConfig, checkDataNodeHostConfig); checkDataNodeAddrConfig, checkDataNodeHostConfig);
} }

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java
View File

@ -58,7 +58,7 @@ public void startUpCluster() throws IOException {
conf = new HdfsConfiguration(); conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf) cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(REPL_FACTOR) .numDataNodes(REPL_FACTOR)
.storageType(storageType) .storageTypes(new StorageType[] { storageType, storageType } )
.build(); .build();
fs = cluster.getFileSystem(); fs = cluster.getFileSystem();
bpid = cluster.getNamesystem().getBlockPoolId(); bpid = cluster.getNamesystem().getBlockPoolId();