HDFS-13503. Fix TestFsck test failures on Windows. Contributed by Xiao Liang.
This commit is contained in:
Inigo Goiri
parent
4e1382aca4
commit
9e2cfb2d3f
@ -2924,7 +2924,8 @@ public File getProvidedStorageDir(int dnIndex, int dirIndex) {
|
||||
* @return Storage directory
|
||||
*/
|
||||
public File getStorageDir(int dnIndex, int dirIndex) {
|
||||
return new File(getBaseDirectory(), getStorageDirPath(dnIndex, dirIndex));
|
||||
return new File(determineDfsBaseDir(),
|
||||
getStorageDirPath(dnIndex, dirIndex));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -18,6 +18,7 @@
|
||||
|
||||
package org.apache.hadoop.hdfs.server.namenode;
|
||||
|
||||
import static org.apache.hadoop.hdfs.MiniDFSCluster.HDFS_MINIDFS_BASEDIR;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
@ -209,7 +210,9 @@ public void testFsck() throws Exception {
|
||||
conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
|
||||
precision);
|
||||
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(4).build();
|
||||
fs = cluster.getFileSystem();
|
||||
final String fileName = "/srcdat";
|
||||
util.createFiles(fs, fileName);
|
||||
@ -297,7 +300,9 @@ public void testFsckNonExistent() throws Exception {
|
||||
setNumFiles(20).build();
|
||||
FileSystem fs = null;
|
||||
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(4).build();
|
||||
fs = cluster.getFileSystem();
|
||||
util.createFiles(fs, "/srcdat");
|
||||
util.waitReplication(fs, "/srcdat", (short)3);
|
||||
@ -315,7 +320,9 @@ public void testFsckPermission() throws Exception {
|
||||
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
|
||||
|
||||
// Create a cluster with the current user, write some files
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(4).build();
|
||||
final MiniDFSCluster c2 = cluster;
|
||||
final String dir = "/dfsck";
|
||||
final Path dirpath = new Path(dir);
|
||||
@ -361,8 +368,9 @@ public void testFsckMove() throws Exception {
|
||||
DFSTestUtil util = new DFSTestUtil("TestFsck", 5, 3,
|
||||
(5 * dfsBlockSize) + (dfsBlockSize - 1), 5 * dfsBlockSize);
|
||||
FileSystem fs = null;
|
||||
cluster = new MiniDFSCluster.Builder(conf).
|
||||
numDataNodes(numDatanodes).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(numDatanodes).build();
|
||||
String topDir = "/srcdat";
|
||||
fs = cluster.getFileSystem();
|
||||
cluster.waitActive();
|
||||
@ -568,7 +576,9 @@ public void testFsckMoveAndDelete() throws Exception {
|
||||
FileSystem fs = null;
|
||||
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
|
||||
conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(4).build();
|
||||
String topDir = "/srcdat";
|
||||
fs = cluster.getFileSystem();
|
||||
cluster.waitActive();
|
||||
@ -632,7 +642,9 @@ public void testFsckOpenFiles() throws Exception {
|
||||
setNumFiles(4).build();
|
||||
FileSystem fs = null;
|
||||
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(4).build();
|
||||
String topDir = "/srcdat";
|
||||
String randomString = "HADOOP ";
|
||||
fs = cluster.getFileSystem();
|
||||
@ -685,7 +697,8 @@ public void testFsckOpenECFiles() throws Exception {
|
||||
final int numAllUnits = dataBlocks + ecPolicy.getNumParityUnits();
|
||||
int blockSize = 2 * cellSize;
|
||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).numDataNodes(
|
||||
numAllUnits + 1).build();
|
||||
String topDir = "/myDir";
|
||||
cluster.waitActive();
|
||||
@ -776,7 +789,9 @@ public void testCorruptBlock() throws Exception {
|
||||
String outStr = null;
|
||||
short factor = 1;
|
||||
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(1).build();
|
||||
cluster.waitActive();
|
||||
fs = cluster.getFileSystem();
|
||||
Path file1 = new Path("/testCorruptBlock");
|
||||
@ -847,7 +862,9 @@ public void testUnderMinReplicatedBlock() throws Exception {
|
||||
Random random = new Random();
|
||||
String outStr = null;
|
||||
short factor = 1;
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(2).build();
|
||||
cluster.waitActive();
|
||||
fs = cluster.getFileSystem();
|
||||
Path file1 = new Path("/testUnderMinReplicatedBlock");
|
||||
@ -919,9 +936,9 @@ public void testFsckReplicaDetails() throws Exception {
|
||||
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
|
||||
|
||||
DistributedFileSystem dfs;
|
||||
cluster =
|
||||
new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts)
|
||||
.racks(racks).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(numDn).hosts(hosts).racks(racks).build();
|
||||
cluster.waitClusterUp();
|
||||
dfs = cluster.getFileSystem();
|
||||
|
||||
@ -1068,7 +1085,8 @@ public Boolean get() {
|
||||
@Test
|
||||
public void testFsckError() throws Exception {
|
||||
// bring up a one-node cluster
|
||||
cluster = new MiniDFSCluster.Builder(conf).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build();
|
||||
String fileName = "/test.txt";
|
||||
Path filePath = new Path(fileName);
|
||||
FileSystem fs = cluster.getFileSystem();
|
||||
@ -1100,7 +1118,8 @@ public void testFsckListCorruptFilesBlocks() throws Exception {
|
||||
conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
|
||||
FileSystem fs = null;
|
||||
|
||||
cluster = new MiniDFSCluster.Builder(conf).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build();
|
||||
cluster.waitActive();
|
||||
fs = cluster.getFileSystem();
|
||||
DFSTestUtil util = new DFSTestUtil.Builder().
|
||||
@ -1163,7 +1182,8 @@ public void testFsckListCorruptFilesBlocks() throws Exception {
|
||||
@Test
|
||||
public void testToCheckTheFsckCommandOnIllegalArguments() throws Exception {
|
||||
// bring up a one-node cluster
|
||||
cluster = new MiniDFSCluster.Builder(conf).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build();
|
||||
String fileName = "/test.txt";
|
||||
Path filePath = new Path(fileName);
|
||||
FileSystem fs = cluster.getFileSystem();
|
||||
@ -1207,8 +1227,9 @@ public void testFsckMissingReplicas() throws IOException {
|
||||
DistributedFileSystem dfs = null;
|
||||
|
||||
// Startup a minicluster
|
||||
cluster =
|
||||
new MiniDFSCluster.Builder(conf).numDataNodes(numReplicas).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(numReplicas).build();
|
||||
assertNotNull("Failed Cluster Creation", cluster);
|
||||
cluster.waitClusterUp();
|
||||
dfs = cluster.getFileSystem();
|
||||
@ -1268,9 +1289,9 @@ public void testFsckMisPlacedReplicas() throws IOException {
|
||||
DistributedFileSystem dfs = null;
|
||||
|
||||
// Startup a minicluster
|
||||
cluster =
|
||||
new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts)
|
||||
.racks(racks).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(numDn).hosts(hosts).racks(racks).build();
|
||||
assertNotNull("Failed Cluster Creation", cluster);
|
||||
cluster.waitClusterUp();
|
||||
dfs = cluster.getFileSystem();
|
||||
@ -1377,7 +1398,9 @@ public void testFsckSymlink() throws Exception {
|
||||
conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
|
||||
precision);
|
||||
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(4).build();
|
||||
fs = cluster.getFileSystem();
|
||||
final String fileName = "/srcdat";
|
||||
util.createFiles(fs, fileName);
|
||||
@ -1404,7 +1427,8 @@ public void testFsckSymlink() throws Exception {
|
||||
*/
|
||||
@Test
|
||||
public void testFsckForSnapshotFiles() throws Exception {
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).numDataNodes(1)
|
||||
.build();
|
||||
String runFsck = runFsck(conf, 0, true, "/", "-includeSnapshots",
|
||||
"-files");
|
||||
@ -1439,9 +1463,9 @@ public void testBlockIdCK() throws Exception {
|
||||
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2);
|
||||
|
||||
DistributedFileSystem dfs = null;
|
||||
cluster =
|
||||
new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts)
|
||||
.racks(racks).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(numDn).hosts(hosts).racks(racks).build();
|
||||
|
||||
assertNotNull("Failed Cluster Creation", cluster);
|
||||
cluster.waitClusterUp();
|
||||
@ -1494,9 +1518,9 @@ public void testBlockIdCKDecommission() throws Exception {
|
||||
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2);
|
||||
|
||||
DistributedFileSystem dfs;
|
||||
cluster =
|
||||
new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts)
|
||||
.racks(racks).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(numDn).hosts(hosts).racks(racks).build();
|
||||
|
||||
assertNotNull("Failed Cluster Creation", cluster);
|
||||
cluster.waitClusterUp();
|
||||
@ -1579,7 +1603,8 @@ public void testBlockIdCKMaintenance() throws Exception {
|
||||
replFactor);
|
||||
|
||||
DistributedFileSystem dfs;
|
||||
cluster = new MiniDFSCluster.Builder(conf)
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(numDn)
|
||||
.hosts(hosts)
|
||||
.racks(racks)
|
||||
@ -1700,9 +1725,9 @@ public void testBlockIdCKCorruption() throws Exception {
|
||||
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
|
||||
|
||||
DistributedFileSystem dfs = null;
|
||||
cluster =
|
||||
new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts)
|
||||
.racks(racks).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(numDn).hosts(hosts).racks(racks).build();
|
||||
|
||||
assertNotNull("Failed Cluster Creation", cluster);
|
||||
cluster.waitClusterUp();
|
||||
@ -1769,7 +1794,8 @@ private void writeFile(final DistributedFileSystem dfs,
|
||||
*/
|
||||
@Test
|
||||
public void testStoragePoliciesCK() throws Exception {
|
||||
cluster = new MiniDFSCluster.Builder(conf)
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(3)
|
||||
.storageTypes(
|
||||
new StorageType[] {StorageType.DISK, StorageType.ARCHIVE})
|
||||
@ -1812,9 +1838,9 @@ public void testFsckWithDecommissionedReplicas() throws Exception {
|
||||
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
|
||||
|
||||
DistributedFileSystem dfs;
|
||||
cluster =
|
||||
new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts)
|
||||
.racks(racks).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(numDn).hosts(hosts).racks(racks).build();
|
||||
|
||||
assertNotNull("Failed Cluster Creation", cluster);
|
||||
cluster.waitClusterUp();
|
||||
@ -1894,7 +1920,8 @@ public void testFsckWithMaintenanceReplicas() throws Exception {
|
||||
replFactor);
|
||||
|
||||
DistributedFileSystem dfs;
|
||||
cluster = new MiniDFSCluster.Builder(conf)
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(numDn)
|
||||
.hosts(hosts)
|
||||
.racks(racks)
|
||||
@ -2002,7 +2029,9 @@ public void testECFsck() throws Exception {
|
||||
int parityBlocks =
|
||||
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
||||
int totalSize = dataBlocks + parityBlocks;
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(totalSize).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(totalSize).build();
|
||||
fs = cluster.getFileSystem();
|
||||
fs.enableErasureCodingPolicy(
|
||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||
@ -2069,7 +2098,8 @@ public void testFsckListCorruptSnapshotFiles() throws Exception {
|
||||
|
||||
int numFiles = 3;
|
||||
int numSnapshots = 0;
|
||||
cluster = new MiniDFSCluster.Builder(conf).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build();
|
||||
cluster.waitActive();
|
||||
hdfs = cluster.getFileSystem();
|
||||
DFSTestUtil util = new DFSTestUtil.Builder().
|
||||
@ -2149,7 +2179,8 @@ public void testFsckMoveAfterCorruption() throws Exception {
|
||||
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L);
|
||||
conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
|
||||
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replication);
|
||||
cluster = new MiniDFSCluster.Builder(conf).build();
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build();
|
||||
DistributedFileSystem dfs = cluster.getFileSystem();
|
||||
cluster.waitActive();
|
||||
|
||||
@ -2244,6 +2275,7 @@ private void testUpgradeDomain(boolean defineUpgradeDomain,
|
||||
HostsFileWriter hostsFileWriter = new HostsFileWriter();
|
||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replFactor);
|
||||
conf.set(HDFS_MINIDFS_BASEDIR, GenericTestUtils.getRandomizedTempPath());
|
||||
if (defineUpgradeDomain) {
|
||||
conf.setClass(DFSConfigKeys.DFS_NAMENODE_HOSTS_PROVIDER_CLASSNAME_KEY,
|
||||
CombinedHostFileManager.class, HostConfigManager.class);
|
||||
@ -2295,7 +2327,8 @@ public void testFsckCorruptECFile() throws Exception {
|
||||
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
||||
int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
|
||||
int totalSize = dataBlocks + parityBlocks;
|
||||
cluster = new MiniDFSCluster.Builder(conf)
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(totalSize).build();
|
||||
fs = cluster.getFileSystem();
|
||||
fs.enableErasureCodingPolicy(
|
||||
@ -2366,7 +2399,8 @@ public void testFsckMissingECFile() throws Exception {
|
||||
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
||||
int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
|
||||
int totalSize = dataBlocks + parityBlocks;
|
||||
cluster = new MiniDFSCluster.Builder(conf)
|
||||
File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
|
||||
cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
|
||||
.numDataNodes(totalSize).build();
|
||||
fs = cluster.getFileSystem();
|
||||
fs.enableErasureCodingPolicy(
|
||||
@ -2427,7 +2461,8 @@ public Boolean get() {
|
||||
@Test(timeout = 300000)
|
||||
public void testFsckCorruptWhenOneReplicaIsCorrupt()
|
||||
throws Exception {
|
||||
try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
|
||||
try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf,
|
||||
new File(GenericTestUtils.getRandomizedTempPath()))
|
||||
.nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(2)
|
||||
.build()) {
|
||||
cluster.waitActive();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user