big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13408. MiniDFSCluster to support being built on randomized base directory. Contributed by Xiao Liang

Browse Source
This commit is contained in:
Chris Douglas 2018-04-23 11:13:18 -07:00
parent c533c77047
commit f411de6a79
2 changed files with 28 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
View File

@ -202,6 +202,26 @@ public Builder(Configuration conf) {
this.conf = conf; this.conf = conf;
this.storagesPerDatanode = this.storagesPerDatanode =
FsDatasetTestUtils.Factory.getFactory(conf).getDefaultNumOfDataDirs(); FsDatasetTestUtils.Factory.getFactory(conf).getDefaultNumOfDataDirs();
if (null == conf.get(HDFS_MINIDFS_BASEDIR)) {
conf.set(HDFS_MINIDFS_BASEDIR,
new File(getBaseDirectory()).getAbsolutePath());
}
}
public Builder(Configuration conf, File basedir) {
this.conf = conf;
this.storagesPerDatanode =
FsDatasetTestUtils.Factory.getFactory(conf).getDefaultNumOfDataDirs();
if (null == basedir) {
throw new IllegalArgumentException(
"MiniDFSCluster base directory cannot be null");
}
String cdir = conf.get(HDFS_MINIDFS_BASEDIR);
if (cdir != null) {
throw new IllegalArgumentException(
"MiniDFSCluster base directory already defined (" + cdir + ")");
}
conf.set(HDFS_MINIDFS_BASEDIR, basedir.getAbsolutePath());
} }
/** /**

16
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
View File

@ -201,9 +201,8 @@ private MiniDFSCluster newCluster(
public void testIsClusterUpAfterShutdown() throws Throwable { public void testIsClusterUpAfterShutdown() throws Throwable {
Configuration conf = new HdfsConfiguration(); Configuration conf = new HdfsConfiguration();
File testDataCluster4 = new File(testDataPath, CLUSTER_4); File testDataCluster4 = new File(testDataPath, CLUSTER_4);
String c4Path = testDataCluster4.getAbsolutePath(); MiniDFSCluster cluster4 =
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c4Path); new MiniDFSCluster.Builder(conf, testDataCluster4).build();
MiniDFSCluster cluster4 = new MiniDFSCluster.Builder(conf).build();
try { try {
DistributedFileSystem dfs = cluster4.getFileSystem(); DistributedFileSystem dfs = cluster4.getFileSystem();
dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
@ -222,12 +221,11 @@ public void testClusterSetDatanodeHostname() throws Throwable {
Configuration conf = new HdfsConfiguration(); Configuration conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "MYHOST"); conf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY, "MYHOST");
File testDataCluster5 = new File(testDataPath, CLUSTER_5); File testDataCluster5 = new File(testDataPath, CLUSTER_5);
String c5Path = testDataCluster5.getAbsolutePath(); try (MiniDFSCluster cluster5 =
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, c5Path); new MiniDFSCluster.Builder(conf, testDataCluster5)
try (MiniDFSCluster cluster5 = new MiniDFSCluster.Builder(conf) .numDataNodes(1)
.numDataNodes(1) .checkDataNodeHostConfig(true)
.checkDataNodeHostConfig(true) .build()) {
.build()) {
assertEquals("DataNode hostname config not respected", "MYHOST", assertEquals("DataNode hostname config not respected", "MYHOST",
cluster5.getDataNodes().get(0).getDatanodeId().getHostName()); cluster5.getDataNodes().get(0).getDatanodeId().getHostName());
} }