From 8511d80804de052618168a456a475ee0f7aa6d8c Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sun, 19 Apr 2015 16:09:06 -0700 Subject: [PATCH] HDFS-8152. Refactoring of lazy persist storage cases. (Arpit Agarwal) --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../fsdataset/impl/LazyPersistTestCase.java | 170 ++++++++++-------- .../fsdataset/impl/TestLazyPersistFiles.java | 20 +-- .../fsdataset/impl/TestLazyPersistPolicy.java | 8 +- .../impl/TestLazyPersistReplicaPlacement.java | 12 +- .../impl/TestLazyPersistReplicaRecovery.java | 4 +- .../fsdataset/impl/TestLazyWriter.java | 14 +- .../impl/TestScrLazyPersistFiles.java | 48 +++-- 8 files changed, 161 insertions(+), 117 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 30a460fb1e..39ee33e583 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -448,6 +448,8 @@ Release 2.8.0 - UNRELEASED HDFS-8165. Move GRANDFATHER_GENERATION_STAMP and GRANDFATER_INODE_ID to hdfs-client. (wheat9) + HDFS-8152. Refactoring of lazy persist storage cases. (Arpit Agarwal) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java index 6adec2091b..d46964bbac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; +import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; + import static org.apache.hadoop.fs.CreateFlag.CREATE; import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST; import static org.apache.hadoop.fs.StorageType.DEFAULT; @@ -45,6 +47,7 @@ import java.util.Set; import java.util.UUID; +import com.google.common.base.Preconditions; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -228,11 +231,15 @@ protected final void makeTestFile(Path path, long length, * If ramDiskStorageLimit is >=0, then RAM_DISK capacity is artificially * capped. If ramDiskStorageLimit < 0 then it is ignored. */ - protected final void startUpCluster(boolean hasTransientStorage, - final int ramDiskReplicaCapacity, - final boolean useSCR, - final boolean useLegacyBlockReaderLocal) - throws IOException { + protected final void startUpCluster( + int numDatanodes, + boolean hasTransientStorage, + StorageType[] storageTypes, + int ramDiskReplicaCapacity, + long ramDiskStorageLimit, + long evictionLowWatermarkReplicas, + boolean useSCR, + boolean useLegacyBlockReaderLocal) throws IOException { Configuration conf = new Configuration(); conf.setLong(DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); @@ -243,17 +250,17 @@ protected final void startUpCluster(boolean hasTransientStorage, HEARTBEAT_RECHECK_INTERVAL_MSEC); conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC, LAZY_WRITER_INTERVAL_SEC); - conf.setInt(DFS_DATANODE_RAM_DISK_LOW_WATERMARK_BYTES, - EVICTION_LOW_WATERMARK * BLOCK_SIZE); + conf.setLong(DFS_DATANODE_RAM_DISK_LOW_WATERMARK_BYTES, + evictionLowWatermarkReplicas * BLOCK_SIZE); if (useSCR) { conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, true); // Do not share a client context across tests. conf.set(DFS_CLIENT_CONTEXT, UUID.randomUUID().toString()); + conf.set(DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY, + UserGroupInformation.getCurrentUser().getShortUserName()); if (useLegacyBlockReaderLocal) { conf.setBoolean(DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL, true); - conf.set(DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY, - UserGroupInformation.getCurrentUser().getShortUserName()); } else { sockDir = new TemporarySocketDirectory(); conf.set(DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), @@ -261,22 +268,29 @@ protected final void startUpCluster(boolean hasTransientStorage, } } - long[] capacities = null; + Preconditions.checkState( + ramDiskReplicaCapacity < 0 || ramDiskStorageLimit < 0, + "Cannot specify non-default values for both ramDiskReplicaCapacity " + + "and ramDiskStorageLimit"); + + long[] capacities; if (hasTransientStorage && ramDiskReplicaCapacity >= 0) { // Convert replica count to byte count, add some delta for .meta and // VERSION files. - long ramDiskStorageLimit = ((long) ramDiskReplicaCapacity * BLOCK_SIZE) + + ramDiskStorageLimit = ((long) ramDiskReplicaCapacity * BLOCK_SIZE) + (BLOCK_SIZE - 1); - capacities = new long[] { ramDiskStorageLimit, -1 }; } + capacities = new long[] { ramDiskStorageLimit, -1 }; cluster = new MiniDFSCluster .Builder(conf) - .numDataNodes(REPL_FACTOR) + .numDataNodes(numDatanodes) .storageCapacities(capacities) - .storageTypes(hasTransientStorage ? - new StorageType[]{ RAM_DISK, DEFAULT } : null) + .storageTypes(storageTypes != null ? storageTypes : + (hasTransientStorage ? new StorageType[]{RAM_DISK, DEFAULT} : null)) .build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); client = fs.getClient(); try { @@ -287,65 +301,77 @@ protected final void startUpCluster(boolean hasTransientStorage, LOG.info("Cluster startup complete"); } - /** - * If ramDiskStorageLimit is >=0, then RAM_DISK capacity is artificially - * capped. If ramDiskStorageLimit < 0 then it is ignored. - */ - protected final void startUpCluster(final int numDataNodes, - final StorageType[] storageTypes, - final long ramDiskStorageLimit, - final boolean useSCR) - throws IOException { - - Configuration conf = new Configuration(); - conf.setLong(DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); - conf.setInt(DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC, - LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC); - conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL_SEC); - conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, - HEARTBEAT_RECHECK_INTERVAL_MSEC); - conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC, - LAZY_WRITER_INTERVAL_SEC); - - if (useSCR) - { - conf.setBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY,useSCR); - conf.set(DFS_CLIENT_CONTEXT, UUID.randomUUID().toString()); - sockDir = new TemporarySocketDirectory(); - conf.set(DFS_DOMAIN_SOCKET_PATH_KEY, new File(sockDir.getDir(), - this.getClass().getSimpleName() + "._PORT.sock").getAbsolutePath()); - conf.set(DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY, - UserGroupInformation.getCurrentUser().getShortUserName()); - } - - cluster = new MiniDFSCluster - .Builder(conf) - .numDataNodes(numDataNodes) - .storageTypes(storageTypes != null ? - storageTypes : new StorageType[] { DEFAULT, DEFAULT }) - .build(); - fs = cluster.getFileSystem(); - client = fs.getClient(); - - // Artificially cap the storage capacity of the RAM_DISK volume. - if (ramDiskStorageLimit >= 0) { - List volumes = - cluster.getDataNodes().get(0).getFSDataset().getVolumes(); - - for (FsVolumeSpi volume : volumes) { - if (volume.getStorageType() == RAM_DISK) { - ((FsVolumeImpl) volume).setCapacityForTesting(ramDiskStorageLimit); - } - } - } - - LOG.info("Cluster startup complete"); + ClusterWithRamDiskBuilder getClusterBuilder() { + return new ClusterWithRamDiskBuilder(); } - protected final void startUpCluster(boolean hasTransientStorage, - final int ramDiskReplicaCapacity) - throws IOException { - startUpCluster(hasTransientStorage, ramDiskReplicaCapacity, false, false); + /** + * Builder class that allows controlling RAM disk-specific properties for a + * MiniDFSCluster. + */ + class ClusterWithRamDiskBuilder { + public ClusterWithRamDiskBuilder setNumDatanodes( + int numDatanodes) { + this.numDatanodes = numDatanodes; + return this; + } + + public ClusterWithRamDiskBuilder setStorageTypes( + StorageType[] storageTypes) { + this.storageTypes = storageTypes; + return this; + } + + public ClusterWithRamDiskBuilder setRamDiskReplicaCapacity( + int ramDiskReplicaCapacity) { + this.ramDiskReplicaCapacity = ramDiskReplicaCapacity; + return this; + } + + public ClusterWithRamDiskBuilder setRamDiskStorageLimit( + long ramDiskStorageLimit) { + this.ramDiskStorageLimit = ramDiskStorageLimit; + return this; + } + + public ClusterWithRamDiskBuilder setUseScr(boolean useScr) { + this.useScr = useScr; + return this; + } + + public ClusterWithRamDiskBuilder setHasTransientStorage( + boolean hasTransientStorage) { + this.hasTransientStorage = hasTransientStorage; + return this; + } + + public ClusterWithRamDiskBuilder setUseLegacyBlockReaderLocal( + boolean useLegacyBlockReaderLocal) { + this.useLegacyBlockReaderLocal = useLegacyBlockReaderLocal; + return this; + } + + public ClusterWithRamDiskBuilder setEvictionLowWatermarkReplicas( + long evictionLowWatermarkReplicas) { + this.evictionLowWatermarkReplicas = evictionLowWatermarkReplicas; + return this; + } + + public void build() throws IOException { + LazyPersistTestCase.this.startUpCluster( + numDatanodes, hasTransientStorage, storageTypes, ramDiskReplicaCapacity, + ramDiskStorageLimit, evictionLowWatermarkReplicas, + useScr, useLegacyBlockReaderLocal); + } + + private int numDatanodes = REPL_FACTOR; + private StorageType[] storageTypes = null; + private int ramDiskReplicaCapacity = -1; + private long ramDiskStorageLimit = -1; + private boolean hasTransientStorage = true; + private boolean useScr = false; + private boolean useLegacyBlockReaderLocal = false; + private long evictionLowWatermarkReplicas = EVICTION_LOW_WATERMARK; } protected final void triggerBlockReport() diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java index 30e5d260c6..41398c84d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java @@ -19,11 +19,6 @@ import com.google.common.util.concurrent.Uninterruptibles; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Test; @@ -34,17 +29,12 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.ArrayList; -import java.util.Collections; -import static org.apache.hadoop.fs.StorageType.DEFAULT; import static org.apache.hadoop.fs.StorageType.RAM_DISK; import static org.apache.hadoop.hdfs.DFSConfigKeys.*; import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsNot.not; import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class TestLazyPersistFiles extends LazyPersistTestCase { @@ -56,7 +46,7 @@ public class TestLazyPersistFiles extends LazyPersistTestCase { */ @Test public void testAppendIsDenied() throws IOException { - startUpCluster(true, -1); + getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); @@ -77,7 +67,7 @@ public void testAppendIsDenied() throws IOException { */ @Test public void testTruncateIsDenied() throws IOException { - startUpCluster(true, -1); + getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); @@ -99,7 +89,7 @@ public void testTruncateIsDenied() throws IOException { @Test public void testCorruptFilesAreDiscarded() throws IOException, InterruptedException { - startUpCluster(true, 2); + getClusterBuilder().setRamDiskReplicaCapacity(2).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path1 = new Path("/" + METHOD_NAME + ".01.dat"); @@ -136,7 +126,7 @@ public void testCorruptFilesAreDiscarded() @Test public void testConcurrentRead() throws Exception { - startUpCluster(true, 2); + getClusterBuilder().setRamDiskReplicaCapacity(2).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); final Path path1 = new Path("/" + METHOD_NAME + ".dat"); @@ -187,7 +177,7 @@ public void run() { @Test public void testConcurrentWrites() throws IOException, InterruptedException { - startUpCluster(true, 9); + getClusterBuilder().setRamDiskReplicaCapacity(9).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); final int SEED = 0xFADED; final int NUM_WRITERS = 4; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistPolicy.java index 873e2b0f59..6b198f1bae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistPolicy.java @@ -34,7 +34,7 @@ public class TestLazyPersistPolicy extends LazyPersistTestCase { @Test public void testPolicyNotSetByDefault() throws IOException { - startUpCluster(false, -1); + getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); @@ -47,7 +47,7 @@ public void testPolicyNotSetByDefault() throws IOException { @Test public void testPolicyPropagation() throws IOException { - startUpCluster(false, -1); + getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); @@ -59,7 +59,7 @@ public void testPolicyPropagation() throws IOException { @Test public void testPolicyPersistenceInEditLog() throws IOException { - startUpCluster(false, -1); + getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); @@ -73,7 +73,7 @@ public void testPolicyPersistenceInEditLog() throws IOException { @Test public void testPolicyPersistenceInFsImage() throws IOException { - startUpCluster(false, -1); + getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java index 4e1a893688..018eabafe6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaPlacement.java @@ -33,7 +33,7 @@ public class TestLazyPersistReplicaPlacement extends LazyPersistTestCase { @Test public void testPlacementOnRamDisk() throws IOException { - startUpCluster(true, -1); + getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); @@ -43,7 +43,7 @@ public void testPlacementOnRamDisk() throws IOException { @Test public void testPlacementOnSizeLimitedRamDisk() throws IOException { - startUpCluster(true, 3); + getClusterBuilder().setRamDiskReplicaCapacity(3).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path1 = new Path("/" + METHOD_NAME + ".01.dat"); Path path2 = new Path("/" + METHOD_NAME + ".02.dat"); @@ -62,7 +62,7 @@ public void testPlacementOnSizeLimitedRamDisk() throws IOException { */ @Test public void testFallbackToDisk() throws IOException { - startUpCluster(false, -1); + getClusterBuilder().setHasTransientStorage(false).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); @@ -76,7 +76,7 @@ public void testFallbackToDisk() throws IOException { */ @Test public void testFallbackToDiskFull() throws Exception { - startUpCluster(false, 0); + getClusterBuilder().setRamDiskReplicaCapacity(0).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); @@ -95,7 +95,7 @@ public void testFallbackToDiskFull() throws Exception { @Test public void testFallbackToDiskPartial() throws IOException, InterruptedException { - startUpCluster(true, 2); + getClusterBuilder().setRamDiskReplicaCapacity(2).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); @@ -134,7 +134,7 @@ public void testFallbackToDiskPartial() */ @Test public void testRamDiskNotChosenByDefault() throws IOException { - startUpCluster(true, -1); + getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java index b8b2653400..231353ad94 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistReplicaRecovery.java @@ -32,7 +32,7 @@ public class TestLazyPersistReplicaRecovery extends LazyPersistTestCase { public void testDnRestartWithSavedReplicas() throws IOException, InterruptedException { - startUpCluster(true, -1); + getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path1 = new Path("/" + METHOD_NAME + ".01.dat"); @@ -57,7 +57,7 @@ public void testDnRestartWithSavedReplicas() public void testDnRestartWithUnsavedReplicas() throws IOException, InterruptedException { - startUpCluster(true, 1); + getClusterBuilder().build(); FsDatasetTestUtil.stopLazyWriter(cluster.getDataNodes().get(0)); final String METHOD_NAME = GenericTestUtils.getMethodName(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyWriter.java index 23203c72db..ee8aaf08c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyWriter.java @@ -39,7 +39,7 @@ public class TestLazyWriter extends LazyPersistTestCase { @Test public void testLazyPersistBlocksAreSaved() throws IOException, InterruptedException { - startUpCluster(true, -1); + getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); @@ -63,7 +63,7 @@ public void testLazyPersistBlocksAreSaved() */ @Test public void testRamDiskEviction() throws Exception { - startUpCluster(true, 1 + EVICTION_LOW_WATERMARK); + getClusterBuilder().setRamDiskReplicaCapacity(1 + EVICTION_LOW_WATERMARK).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path1 = new Path("/" + METHOD_NAME + ".01.dat"); Path path2 = new Path("/" + METHOD_NAME + ".02.dat"); @@ -99,7 +99,7 @@ public void testRamDiskEviction() throws Exception { @Test public void testRamDiskEvictionBeforePersist() throws IOException, InterruptedException { - startUpCluster(true, 1); + getClusterBuilder().setRamDiskReplicaCapacity(1).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path1 = new Path("/" + METHOD_NAME + ".01.dat"); Path path2 = new Path("/" + METHOD_NAME + ".02.dat"); @@ -133,7 +133,7 @@ public void testRamDiskEvictionBeforePersist() public void testRamDiskEvictionIsLru() throws Exception { final int NUM_PATHS = 5; - startUpCluster(true, NUM_PATHS + EVICTION_LOW_WATERMARK); + getClusterBuilder().setRamDiskReplicaCapacity(NUM_PATHS + EVICTION_LOW_WATERMARK).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path paths[] = new Path[NUM_PATHS * 2]; @@ -194,7 +194,7 @@ public void testRamDiskEvictionIsLru() @Test public void testDeleteBeforePersist() throws Exception { - startUpCluster(true, -1); + getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); FsDatasetTestUtil.stopLazyWriter(cluster.getDataNodes().get(0)); @@ -221,7 +221,7 @@ public void testDeleteBeforePersist() @Test public void testDeleteAfterPersist() throws Exception { - startUpCluster(true, -1); + getClusterBuilder().build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); @@ -249,7 +249,7 @@ public void testDeleteAfterPersist() @Test public void testDfsUsageCreateDelete() throws IOException, InterruptedException { - startUpCluster(true, 4); + getClusterBuilder().setRamDiskReplicaCapacity(4).build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java index d54e5a4a99..7c7ba64369 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestScrLazyPersistFiles.java @@ -70,9 +70,11 @@ public void before() { @Test public void testRamDiskShortCircuitRead() throws IOException, InterruptedException { - startUpCluster(REPL_FACTOR, - new StorageType[]{RAM_DISK, DEFAULT}, - 2 * BLOCK_SIZE - 1, true); // 1 replica + delta, SCR read + getClusterBuilder().setNumDatanodes(REPL_FACTOR) + .setStorageTypes(new StorageType[]{RAM_DISK, DEFAULT}) + .setRamDiskStorageLimit(2 * BLOCK_SIZE - 1) + .setUseScr(true) + .build(); final String METHOD_NAME = GenericTestUtils.getMethodName(); final int SEED = 0xFADED; Path path = new Path("/" + METHOD_NAME + ".dat"); @@ -111,8 +113,14 @@ public void testRamDiskShortCircuitRead() @Test public void testRamDiskEvictionWithShortCircuitReadHandle() throws IOException, InterruptedException { - startUpCluster(REPL_FACTOR, new StorageType[] { RAM_DISK, DEFAULT }, - (6 * BLOCK_SIZE -1), true); // 5 replica + delta, SCR. + // 5 replica + delta, SCR. + getClusterBuilder().setNumDatanodes(REPL_FACTOR) + .setStorageTypes(new StorageType[]{RAM_DISK, DEFAULT}) + .setRamDiskStorageLimit(6 * BLOCK_SIZE - 1) + .setEvictionLowWatermarkReplicas(3) + .setUseScr(true) + .build(); + final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path1 = new Path("/" + METHOD_NAME + ".01.dat"); Path path2 = new Path("/" + METHOD_NAME + ".02.dat"); @@ -156,14 +164,20 @@ public void testRamDiskEvictionWithShortCircuitReadHandle() public void testShortCircuitReadAfterEviction() throws IOException, InterruptedException { Assume.assumeThat(DomainSocket.getLoadingFailureReason(), equalTo(null)); - startUpCluster(true, 1 + EVICTION_LOW_WATERMARK, true, false); + getClusterBuilder().setRamDiskReplicaCapacity(1 + EVICTION_LOW_WATERMARK) + .setUseScr(true) + .setUseLegacyBlockReaderLocal(false) + .build(); doShortCircuitReadAfterEvictionTest(); } @Test public void testLegacyShortCircuitReadAfterEviction() throws IOException, InterruptedException { - startUpCluster(true, 1 + EVICTION_LOW_WATERMARK, true, true); + getClusterBuilder().setRamDiskReplicaCapacity(1 + EVICTION_LOW_WATERMARK) + .setUseScr(true) + .setUseLegacyBlockReaderLocal(true) + .build(); doShortCircuitReadAfterEvictionTest(); } @@ -220,14 +234,20 @@ private void doShortCircuitReadAfterEvictionTest() throws IOException, public void testShortCircuitReadBlockFileCorruption() throws IOException, InterruptedException { Assume.assumeThat(DomainSocket.getLoadingFailureReason(), equalTo(null)); - startUpCluster(true, 1 + EVICTION_LOW_WATERMARK, true, false); + getClusterBuilder().setRamDiskReplicaCapacity(1 + EVICTION_LOW_WATERMARK) + .setUseScr(true) + .setUseLegacyBlockReaderLocal(false) + .build(); doShortCircuitReadBlockFileCorruptionTest(); } @Test public void testLegacyShortCircuitReadBlockFileCorruption() throws IOException, InterruptedException { - startUpCluster(true, 1 + EVICTION_LOW_WATERMARK, true, true); + getClusterBuilder().setRamDiskReplicaCapacity(1 + EVICTION_LOW_WATERMARK) + .setUseScr(true) + .setUseLegacyBlockReaderLocal(true) + .build(); doShortCircuitReadBlockFileCorruptionTest(); } @@ -260,14 +280,20 @@ public void doShortCircuitReadBlockFileCorruptionTest() throws IOException, public void testShortCircuitReadMetaFileCorruption() throws IOException, InterruptedException { Assume.assumeThat(DomainSocket.getLoadingFailureReason(), equalTo(null)); - startUpCluster(true, 1 + EVICTION_LOW_WATERMARK, true, false); + getClusterBuilder().setRamDiskReplicaCapacity(1 + EVICTION_LOW_WATERMARK) + .setUseScr(true) + .setUseLegacyBlockReaderLocal(false) + .build(); doShortCircuitReadMetaFileCorruptionTest(); } @Test public void testLegacyShortCircuitReadMetaFileCorruption() throws IOException, InterruptedException { - startUpCluster(true, 1 + EVICTION_LOW_WATERMARK, true, true); + getClusterBuilder().setRamDiskReplicaCapacity(1 + EVICTION_LOW_WATERMARK) + .setUseScr(true) + .setUseLegacyBlockReaderLocal(true) + .build(); doShortCircuitReadMetaFileCorruptionTest(); }