HDDS-1785. OOM error in Freon due to the concurrency handling

Closes #1085

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java

@@ -25,9 +25,11 @@
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -95,8 +97,6 @@ enum FreonOps {
     KEY_WRITE
   }
 
-  private static final String RATIS = "ratis";
-
   private static final String DURATION_FORMAT = "HH:mm:ss,SSS";
 
   private static final int QUANTILES = 10;
@@ -112,8 +112,8 @@ enum FreonOps {
   private static final Logger LOG =
       LoggerFactory.getLogger(RandomKeyGenerator.class);
 
-  private boolean completed = false;
+  private volatile boolean completed = false;
-  private Exception exception = null;
+  private volatile Exception exception = null;
 
   @Option(names = "--numOfThreads",
       description = "number of threads to be launched for the run",
@@ -193,6 +193,14 @@ enum FreonOps {
 
   private AtomicLong totalBytesWritten;
 
+  private int totalBucketCount;
+  private long totalKeyCount;
+  private AtomicInteger volumeCounter;
+  private AtomicInteger bucketCounter;
+  private AtomicLong keyCounter;
+  private Map<Integer, OzoneVolume> volumes;
+  private Map<Integer, OzoneBucket> buckets;
+
   private AtomicInteger numberOfVolumesCreated;
   private AtomicInteger numberOfBucketsCreated;
   private AtomicLong numberOfKeysAdded;
@@ -226,6 +234,11 @@ public void init(OzoneConfiguration configuration) throws IOException {
     numberOfVolumesCreated = new AtomicInteger();
     numberOfBucketsCreated = new AtomicInteger();
     numberOfKeysAdded = new AtomicLong();
+    volumeCounter = new AtomicInteger();
+    bucketCounter = new AtomicInteger();
+    keyCounter = new AtomicLong();
+    volumes = new ConcurrentHashMap<>();
+    buckets = new ConcurrentHashMap<>();
     ozoneClient = OzoneClientFactory.getClient(configuration);
     objectStore = ozoneClient.getObjectStore();
     for (FreonOps ops : FreonOps.values()) {
@@ -259,6 +272,9 @@ public Void call() throws Exception {
       }
     }
 
+    totalBucketCount = numOfVolumes * numOfBuckets;
+    totalKeyCount = totalBucketCount * numOfKeys;
+
     LOG.info("Number of Threads: " + numOfThreads);
     threadPoolSize = numOfThreads;
     executor = Executors.newFixedThreadPool(threadPoolSize);
@@ -269,9 +285,8 @@ public Void call() throws Exception {
     LOG.info("Number of Keys per Bucket: {}.", numOfKeys);
     LOG.info("Key size: {} bytes", keySize);
     LOG.info("Buffer size: {} bytes", bufferSize);
-    for (int i = 0; i < numOfVolumes; i++) {
+    for (int i = 0; i < numOfThreads; i++) {
-      String volumeName = "vol-" + i + "-" + RandomStringUtils.randomNumeric(5);
+      executor.submit(new ObjectCreator());
-      executor.submit(new VolumeProcessor(volumeName));
     }
 
     Thread validator = null;
@@ -286,22 +301,15 @@ public Void call() throws Exception {
       LOG.info("Data validation is enabled.");
     }
 
-    Supplier<Long> currentValue;
+    Supplier<Long> currentValue = numberOfKeysAdded::get;
-    long maxValue;
+    progressbar = new ProgressBar(System.out, totalKeyCount, currentValue);
-
-    currentValue = () -> numberOfKeysAdded.get();
-    maxValue = numOfVolumes *
-            numOfBuckets *
-            numOfKeys;
-
-    progressbar = new ProgressBar(System.out, maxValue, currentValue);
 
     LOG.info("Starting progress bar Thread.");
 
     progressbar.start();
 
     // wait until all keys are added or exception occurred.
-    while ((numberOfKeysAdded.get() != numOfVolumes * numOfBuckets * numOfKeys)
+    while ((numberOfKeysAdded.get() != totalKeyCount)
            && exception == null) {
       try {
         Thread.sleep(CHECK_INTERVAL_MILLIS);
@@ -570,7 +578,7 @@ private static class KeyValidate {
      *
      * @param bucket    bucket part
      * @param keyName   key part
-     * @param keyName   digest of this key's full value
+     * @param digest    digest of this key's full value
      */
     KeyValidate(OzoneBucket bucket, String keyName, byte[] digest) {
       this.bucket = bucket;
@@ -579,61 +587,70 @@ private static class KeyValidate {
     }
   }
 
-  private class VolumeProcessor implements Runnable {
+  private class ObjectCreator implements Runnable {
-    private String volumeName;
+    @Override
-
+    public void run() {
-    VolumeProcessor(String volumeName) {
+      int v;
-      this.volumeName = volumeName;
+      while ((v = volumeCounter.getAndIncrement()) < numOfVolumes) {
+        if (!createVolume(v)) {
+          return;
+        }
       }
 
-    @Override
+      int b;
-    @SuppressFBWarnings("REC_CATCH_EXCEPTION")
+      while ((b = bucketCounter.getAndIncrement()) < totalBucketCount) {
-    public void run() {
+        if (!createBucket(b)) {
+          return;
+        }
+      }
+
+      long k;
+      while ((k = keyCounter.getAndIncrement()) < totalKeyCount) {
+        if (!createKey(k)) {
+          return;
+        }
+      }
+    }
+  }
+
+  private boolean createVolume(int volumeNumber) {
+    String volumeName = "vol-" + volumeNumber + "-"
+        + RandomStringUtils.randomNumeric(5);
     LOG.trace("Creating volume: {}", volumeName);
-      long start = System.nanoTime();
+    try (Scope ignored = GlobalTracer.get().buildSpan("createVolume")
-      OzoneVolume volume;
-      try (Scope scope = GlobalTracer.get().buildSpan("createVolume")
         .startActive(true)) {
+      long start = System.nanoTime();
       objectStore.createVolume(volumeName);
       long volumeCreationDuration = System.nanoTime() - start;
       volumeCreationTime.getAndAdd(volumeCreationDuration);
       histograms.get(FreonOps.VOLUME_CREATE.ordinal())
           .update(volumeCreationDuration);
       numberOfVolumesCreated.getAndIncrement();
-        volume = objectStore.getVolume(volumeName);
+
+      OzoneVolume volume = objectStore.getVolume(volumeName);
+      volumes.put(volumeNumber, volume);
+      return true;
     } catch (IOException e) {
       exception = e;
       LOG.error("Could not create volume", e);
-        return;
+      return false;
+    }
   }
 
-      for (int i = 0; i < numOfBuckets; i++) {
+  private boolean createBucket(int globalBucketNumber) {
-        String bucketName = "bucket-" + i + "-" +
+    int volumeNumber = globalBucketNumber % numOfVolumes;
+    int bucketNumber = globalBucketNumber / numOfVolumes;
+    OzoneVolume volume = getVolume(volumeNumber);
+    if (volume == null) {
+      return false;
+    }
+    String bucketName = "bucket-" + bucketNumber + "-" +
         RandomStringUtils.randomNumeric(5);
-        BucketProcessor bp = new BucketProcessor(volume, bucketName);
-        executor.submit(bp);
-      }
-    }
-  }
-
-  private class BucketProcessor implements Runnable {
-    private OzoneVolume volume;
-    private String bucketName;
-
-    BucketProcessor(OzoneVolume volume, String bucketName) {
-      this.volume = volume;
-      this.bucketName = bucketName;
-    }
-
-    @Override
-    @SuppressFBWarnings("REC_CATCH_EXCEPTION")
-    public void run() {
     LOG.trace("Creating bucket: {} in volume: {}",
         bucketName, volume.getName());
-      long start = System.nanoTime();
+    try (Scope ignored = GlobalTracer.get().buildSpan("createBucket")
-      OzoneBucket bucket;
-      try (Scope scope = GlobalTracer.get().buildSpan("createBucket")
         .startActive(true)) {
+      long start = System.nanoTime();
       volume.createBucket(bucketName);
       long bucketCreationDuration = System.nanoTime() - start;
       histograms.get(FreonOps.BUCKET_CREATE.ordinal())
@@ -641,42 +658,35 @@ public void run() {
       bucketCreationTime.getAndAdd(bucketCreationDuration);
       numberOfBucketsCreated.getAndIncrement();
 
-        bucket = volume.getBucket(bucketName);
+      OzoneBucket bucket = volume.getBucket(bucketName);
+      buckets.put(globalBucketNumber, bucket);
+      return true;
     } catch (IOException e) {
       exception = e;
       LOG.error("Could not create bucket ", e);
-        return;
+      return false;
-      }
-
-      for (int i = 0; i < numOfKeys; i++) {
-        String keyName = "key-" + i + "-" + RandomStringUtils.randomNumeric(5);
-        KeyProcessor kp = new KeyProcessor(bucket, keyName);
-        executor.submit(kp);
-      }
     }
   }
 
-  private class KeyProcessor implements Runnable {
-    private OzoneBucket bucket;
-    private String keyName;
-
-    KeyProcessor(OzoneBucket bucket, String keyName) {
-      this.bucket = bucket;
-      this.keyName = keyName;
-    }
-
-    @Override
   @SuppressFBWarnings("REC_CATCH_EXCEPTION")
-    public void run() {
+  private boolean createKey(long globalKeyNumber) {
+    int globalBucketNumber = (int) (globalKeyNumber % totalBucketCount);
+    long keyNumber = globalKeyNumber / totalBucketCount;
+    OzoneBucket bucket = getBucket(globalBucketNumber);
+    if (bucket == null) {
+      return false;
+    }
     String bucketName = bucket.getName();
     String volumeName = bucket.getVolumeName();
+    String keyName = "key-" + keyNumber + "-"
+        + RandomStringUtils.randomNumeric(5);
     LOG.trace("Adding key: {} in bucket: {} of volume: {}",
         keyName, bucketName, volumeName);
     byte[] randomValue = DFSUtil.string2Bytes(UUID.randomUUID().toString());
     try {
-        long keyCreateStart = System.nanoTime();
       try (Scope scope = GlobalTracer.get().buildSpan("createKey")
           .startActive(true)) {
+        long keyCreateStart = System.nanoTime();
         OzoneOutputStream os = bucket.createKey(keyName, keySize, type,
             factor, new HashMap<>());
         long keyCreationDuration = System.nanoTime() - keyCreateStart;
@@ -684,12 +694,12 @@ public void run() {
             .update(keyCreationDuration);
         keyCreationTime.getAndAdd(keyCreationDuration);
 
-          long keyWriteStart = System.nanoTime();
         try (Scope writeScope = GlobalTracer.get().buildSpan("writeKeyData")
             .startActive(true)) {
+          long keyWriteStart = System.nanoTime();
           for (long nrRemaining = keySize - randomValue.length;
                nrRemaining > 0; nrRemaining -= bufferSize) {
-              int curSize = (int)Math.min(bufferSize, nrRemaining);
+            int curSize = (int) Math.min(bufferSize, nrRemaining);
             os.write(keyValueBuffer, 0, curSize);
           }
           os.write(randomValue);
@@ -705,20 +715,50 @@ public void run() {
       }
 
       if (validateWrites) {
-          MessageDigest tmpMD = (MessageDigest)commonInitialMD.clone();
+        MessageDigest tmpMD = (MessageDigest) commonInitialMD.clone();
         tmpMD.update(randomValue);
         boolean validate = validationQueue.offer(
             new KeyValidate(bucket, keyName, tmpMD.digest()));
         if (validate) {
-            LOG.trace("Key {}, is queued for validation.", keyName);
+          LOG.trace("Key {} is queued for validation.", keyName);
         }
       }
+
+      return true;
     } catch (Exception e) {
       exception = e;
       LOG.error("Exception while adding key: {} in bucket: {}" +
           " of volume: {}.", keyName, bucketName, volumeName, e);
+      return false;
     }
   }
+
+  private OzoneVolume getVolume(Integer volumeNumber) {
+    return waitUntilAddedToMap(volumes, volumeNumber);
+  }
+
+  private OzoneBucket getBucket(Integer bucketNumber) {
+    return waitUntilAddedToMap(buckets, bucketNumber);
+  }
+
+  /**
+   * Looks up volume or bucket from the cache.  Waits for it to be created if
+   * needed (can happen for the last few items depending on the number of
+   * threads).
+   *
+   * @return may return null if this thread is interrupted, or if any other
+   *   thread encounters an exception (and stores it to {@code exception})
+   */
+  private <T> T waitUntilAddedToMap(Map<Integer, T> map, Integer i) {
+    while (exception == null && !map.containsKey(i)) {
+      try {
+        Thread.sleep(10);
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        return null;
+      }
+    }
+    return map.get(i);
   }
 
   private final class FreonJobInfo {