big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-1863. Freon RandomKeyGenerator even if keySize is set to 0, it returns some random data to key. (#1167)

Browse Source
This commit is contained in:
Bharat Viswanadham 2019-08-08 15:40:19 -07:00 committed by GitHub
parent 6ad9a11494
commit aa5f445fb9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 21 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
View File

@ -26,7 +26,6 @@
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
@ -263,9 +262,7 @@ public Void call() throws Exception {
// Compute the common initial digest for all keys without their UUID
if (validateWrites) {
commonInitialMD = DigestUtils.getDigest(DIGEST_ALGORITHM);
int uuidLength = UUID.randomUUID().toString().length();
keySize = Math.max(uuidLength, keySize);
for (long nrRemaining = keySize - uuidLength; nrRemaining > 0;
for (long nrRemaining = keySize; nrRemaining > 0;
nrRemaining -= bufferSize) {
int curSize = (int)Math.min(bufferSize, nrRemaining);
commonInitialMD.update(keyValueBuffer, 0, curSize);
@ -682,7 +679,6 @@ private boolean createKey(long globalKeyNumber) {
+ RandomStringUtils.randomNumeric(5);
LOG.trace("Adding key: {} in bucket: {} of volume: {}",
keyName, bucketName, volumeName);
byte[] randomValue = DFSUtil.string2Bytes(UUID.randomUUID().toString());
try {
try (Scope scope = GlobalTracer.get().buildSpan("createKey")
.startActive(true)) {
@ -697,12 +693,11 @@ private boolean createKey(long globalKeyNumber) {
try (Scope writeScope = GlobalTracer.get().buildSpan("writeKeyData")
.startActive(true)) {
long keyWriteStart = System.nanoTime();
for (long nrRemaining = keySize - randomValue.length;
for (long nrRemaining = keySize;
nrRemaining > 0; nrRemaining -= bufferSize) {
int curSize = (int) Math.min(bufferSize, nrRemaining);
os.write(keyValueBuffer, 0, curSize);
}
os.write(randomValue);
os.close();
long keyWriteDuration = System.nanoTime() - keyWriteStart;
@ -716,7 +711,6 @@ private boolean createKey(long globalKeyNumber) {
if (validateWrites) {
MessageDigest tmpMD = (MessageDigest) commonInitialMD.clone();
tmpMD.update(randomValue);
boolean validate = validationQueue.offer(
new KeyValidate(bucket, keyName, tmpMD.digest()));
if (validate) {

19
hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
View File

@ -128,6 +128,25 @@ public void bigFileThan2GB() throws Exception {
Assert.assertEquals(1, randomKeyGenerator.getSuccessfulValidationCount());
}
@Test
public void fileWithSizeZero() throws Exception {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
randomKeyGenerator.setNumOfVolumes(1);
randomKeyGenerator.setNumOfBuckets(1);
randomKeyGenerator.setNumOfKeys(1);
randomKeyGenerator.setNumOfThreads(1);
randomKeyGenerator.setKeySize(0);
randomKeyGenerator.setFactor(ReplicationFactor.THREE);
randomKeyGenerator.setType(ReplicationType.RATIS);
randomKeyGenerator.setValidateWrites(true);
randomKeyGenerator.call();
Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
Assert.assertEquals(1, randomKeyGenerator.getSuccessfulValidationCount());
}
@Test
public void testThreadPoolSize() throws Exception {
RandomKeyGenerator randomKeyGenerator =