big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-1380. Add functonality to write from multiple clients in MiniOzoneChaosCluster. Contributed by Shashikant Banerjee.

Browse Source
This commit is contained in:
Shashikant Banerjee 2019-04-16 18:52:07 +05:30
parent 008766c119
commit bfcb6534cd
2 changed files with 19 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneLoadGenerator.java
View File

@ -56,10 +56,11 @@ public class MiniOzoneLoadGenerator {
private AtomicBoolean isWriteThreadRunning;
private final OzoneBucket ozoneBucket;
private final List<OzoneBucket> ozoneBuckets;
MiniOzoneLoadGenerator(OzoneBucket bucket, int numThreads, int numBuffers) {
this.ozoneBucket = bucket;
MiniOzoneLoadGenerator(List<OzoneBucket> bucket, int numThreads,
int numBuffers) {
this.ozoneBuckets = bucket;
this.numWriteThreads = numThreads;
this.numBuffers = numBuffers;
this.writeExecutor = new ThreadPoolExecutor(numThreads, numThreads, 100,
@ -94,7 +95,9 @@ private void load(long runTimeMillis) {
int bufferCapacity = buffer.capacity();
String keyName = threadName + "-" + index;
try (OzoneOutputStream stream = ozoneBucket.createKey(keyName,
OzoneBucket bucket =
ozoneBuckets.get((int) (Math.random() * ozoneBuckets.size()));
try (OzoneOutputStream stream = bucket.createKey(keyName,
bufferCapacity, ReplicationType.RATIS, ReplicationFactor.THREE,
new HashMap<>())) {
stream.write(buffer.array());
@ -106,7 +109,7 @@ private void load(long runTimeMillis) {
// to closed container. add a break here once that is fixed.
}
try (OzoneInputStream stream = ozoneBucket.readKey(keyName)) {
try (OzoneInputStream stream = bucket.readKey(keyName)) {
byte[] readBuffer = new byte[bufferCapacity];
int readLen = stream.read(readBuffer);

14
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
View File

@ -29,7 +29,8 @@
import picocli.CommandLine.Option;
import picocli.CommandLine;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
/**
@ -55,6 +56,10 @@ public class TestMiniChaosOzoneCluster implements Runnable {
description = "total run time")
private static int numMinutes = 1440; // 1 day by default
@Option(names = {"-n", "--numClients"},
description = "no of clients writing to OM")
private static int numClients = 3;
@Option(names = {"-i", "--failureInterval"},
description = "time between failure events in seconds")
private static int failureInterval = 5; // 5 second period between failures.
@ -74,9 +79,12 @@ public static void init() throws Exception {
store.createVolume(volumeName);
OzoneVolume volume = store.getVolume(volumeName);
volume.createBucket(bucketName);
OzoneBucket ozoneBucket = volume.getBucket(bucketName);
List<OzoneBucket> ozoneBuckets = new ArrayList<>(numClients);
for (int i = 0; i < numClients; i++) {
ozoneBuckets.add(volume.getBucket(bucketName));
}
loadGenerator =
new MiniOzoneLoadGenerator(ozoneBucket, numThreads, numBuffers);
new MiniOzoneLoadGenerator(ozoneBuckets, numThreads, numBuffers);
}
/**