big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-398. Support multiple tests in freon.

Browse Source 
Contributed by Elek, Marton.
This commit is contained in:
Anu Engineer 2018-09-07 13:27:15 -07:00
parent 73fcbdd296
commit ca3adf588c
6 changed files with 1205 additions and 1325 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-ozone/acceptance-test/src/test/acceptance/basic/basic.robot
View File

@ -45,6 +45,6 @@ Check webui static resources
Should contain ${result} 200
Start freon testing
${result} = Execute on ozoneManager ozone freon -numOfVolumes 5 -numOfBuckets 5 -numOfKeys 5 -numOfThreads 10
${result} = Execute on ozoneManager ozone freon randomkeys --numOfVolumes 5 --numOfBuckets 5 --numOfKeys 5 --numOfThreads 10
Wait Until Keyword Succeeds 3min 10sec Should contain ${result} Number of Keys added: 125
Should Not Contain ${result} ERROR

117
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidate.java
View File

@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -18,20 +18,15 @@
package org.apache.hadoop.ozone.freon;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.util.ToolRunner;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;
/**
* Tests Freon, with MiniOzoneCluster and validate data.
*/
@ -45,7 +40,6 @@ public class TestDataValidate {
* <p>
* Ozone is made active by setting OZONE_ENABLED = true
*
* @throws IOException
*/
@BeforeClass
public static void init() throws Exception {
@ -67,74 +61,55 @@ public static void shutdown() {
@Test
public void ratisTestLargeKey() throws Exception {
List<String> args = new ArrayList<>();
args.add("-validateWrites");
args.add("-numOfVolumes");
args.add("1");
args.add("-numOfBuckets");
args.add("1");
args.add("-numOfKeys");
args.add("1");
args.add("-ratis");
args.add("3");
args.add("-keySize");
args.add("104857600");
Freon freon = new Freon(conf);
int res = ToolRunner.run(conf, freon,
args.toArray(new String[0]));
Assert.assertEquals(1, freon.getNumberOfVolumesCreated());
Assert.assertEquals(1, freon.getNumberOfBucketsCreated());
Assert.assertEquals(1, freon.getNumberOfKeysAdded());
Assert.assertEquals(0, freon.getUnsuccessfulValidationCount());
Assert.assertEquals(0, res);
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
randomKeyGenerator.setNumOfVolumes(1);
randomKeyGenerator.setNumOfBuckets(1);
randomKeyGenerator.setNumOfKeys(1);
randomKeyGenerator.setType(ReplicationType.RATIS);
randomKeyGenerator.setFactor(ReplicationFactor.THREE);
randomKeyGenerator.setKeySize(104857600);
randomKeyGenerator.setValidateWrites(true);
randomKeyGenerator.call();
Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
Assert.assertEquals(0, randomKeyGenerator.getUnsuccessfulValidationCount());
}
@Test
public void standaloneTestLargeKey() throws Exception {
List<String> args = new ArrayList<>();
args.add("-validateWrites");
args.add("-numOfVolumes");
args.add("1");
args.add("-numOfBuckets");
args.add("1");
args.add("-numOfKeys");
args.add("1");
args.add("-keySize");
args.add("104857600");
Freon freon = new Freon(conf);
int res = ToolRunner.run(conf, freon,
args.toArray(new String[0]));
Assert.assertEquals(1, freon.getNumberOfVolumesCreated());
Assert.assertEquals(1, freon.getNumberOfBucketsCreated());
Assert.assertEquals(1, freon.getNumberOfKeysAdded());
Assert.assertEquals(0, freon.getUnsuccessfulValidationCount());
Assert.assertEquals(0, res);
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
randomKeyGenerator.setNumOfVolumes(1);
randomKeyGenerator.setNumOfBuckets(1);
randomKeyGenerator.setNumOfKeys(1);
randomKeyGenerator.setKeySize(104857600);
randomKeyGenerator.setValidateWrites(true);
randomKeyGenerator.call();
Assert.assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
Assert.assertEquals(0, randomKeyGenerator.getUnsuccessfulValidationCount());
}
@Test
public void validateWriteTest() throws Exception {
PrintStream originalStream = System.out;
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
System.setOut(new PrintStream(outStream));
List<String> args = new ArrayList<>();
args.add("-validateWrites");
args.add("-numOfVolumes");
args.add("2");
args.add("-numOfBuckets");
args.add("5");
args.add("-numOfKeys");
args.add("10");
Freon freon = new Freon(conf);
int res = ToolRunner.run(conf, freon,
args.toArray(new String[0]));
Assert.assertEquals(0, res);
Assert.assertEquals(2, freon.getNumberOfVolumesCreated());
Assert.assertEquals(10, freon.getNumberOfBucketsCreated());
Assert.assertEquals(100, freon.getNumberOfKeysAdded());
Assert.assertTrue(freon.getValidateWrites());
Assert.assertNotEquals(0, freon.getTotalKeysValidated());
Assert.assertNotEquals(0, freon.getSuccessfulValidationCount());
Assert.assertEquals(0, freon.getUnsuccessfulValidationCount());
System.setOut(originalStream);
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
randomKeyGenerator.setNumOfVolumes(2);
randomKeyGenerator.setNumOfBuckets(5);
randomKeyGenerator.setNumOfKeys(10);
randomKeyGenerator.setValidateWrites(true);
randomKeyGenerator.call();
Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
Assert.assertTrue(randomKeyGenerator.getValidateWrites());
Assert.assertNotEquals(0, randomKeyGenerator.getTotalKeysValidated());
Assert.assertNotEquals(0, randomKeyGenerator
.getSuccessfulValidationCount());
Assert.assertEquals(0, randomKeyGenerator
.getUnsuccessfulValidationCount());
}
}

129
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreon.java
View File

@ -1,129 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.freon;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.util.ToolRunner;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* Tests Freon, with MiniOzoneCluster.
*/
public class TestFreon {
private static MiniOzoneCluster cluster;
private static OzoneConfiguration conf;
/**
* Create a MiniDFSCluster for testing.
* <p>
* Ozone is made active by setting OZONE_ENABLED = true
*
* @throws IOException
*/
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
cluster.waitForClusterToBeReady();
}
/**
* Shutdown MiniDFSCluster.
*/
@AfterClass
public static void shutdown() {
if (cluster != null) {
cluster.shutdown();
}
}
@Test
public void defaultTest() throws Exception {
List<String> args = new ArrayList<>();
args.add("-numOfVolumes");
args.add("2");
args.add("-numOfBuckets");
args.add("5");
args.add("-numOfKeys");
args.add("10");
Freon freon = new Freon(conf);
int res = ToolRunner.run(conf, freon,
args.toArray(new String[0]));
Assert.assertEquals(2, freon.getNumberOfVolumesCreated());
Assert.assertEquals(10, freon.getNumberOfBucketsCreated());
Assert.assertEquals(100, freon.getNumberOfKeysAdded());
Assert.assertEquals(10240 - 36, freon.getKeyValueLength());
Assert.assertEquals(0, res);
}
@Test
public void multiThread() throws Exception {
List<String> args = new ArrayList<>();
args.add("-numOfVolumes");
args.add("10");
args.add("-numOfBuckets");
args.add("1");
args.add("-numOfKeys");
args.add("10");
args.add("-numOfThread");
args.add("10");
args.add("-keySize");
args.add("10240");
Freon freon = new Freon(conf);
int res = ToolRunner.run(conf, freon,
args.toArray(new String[0]));
Assert.assertEquals(10, freon.getNumberOfVolumesCreated());
Assert.assertEquals(10, freon.getNumberOfBucketsCreated());
Assert.assertEquals(100, freon.getNumberOfKeysAdded());
Assert.assertEquals(0, res);
}
@Test
public void ratisTest3() throws Exception {
List<String> args = new ArrayList<>();
args.add("-numOfVolumes");
args.add("10");
args.add("-numOfBuckets");
args.add("1");
args.add("-numOfKeys");
args.add("10");
args.add("-ratis");
args.add("3");
args.add("-numOfThread");
args.add("10");
args.add("-keySize");
args.add("10240");
Freon freon = new Freon(conf);
int res = ToolRunner.run(conf, freon,
args.toArray(new String[0]));
Assert.assertEquals(10, freon.getNumberOfVolumesCreated());
Assert.assertEquals(10, freon.getNumberOfBucketsCreated());
Assert.assertEquals(100, freon.getNumberOfKeysAdded());
Assert.assertEquals(0, res);
}
}

106
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java Normal file
View File

@ -0,0 +1,106 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.freon;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
/**
* Tests Freon, with MiniOzoneCluster.
*/
public class TestRandomKeyGenerator {
private static MiniOzoneCluster cluster;
private static OzoneConfiguration conf;
/**
* Create a MiniDFSCluster for testing.
* <p>
* Ozone is made active by setting OZONE_ENABLED = true
*
*/
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
cluster.waitForClusterToBeReady();
}
/**
* Shutdown MiniDFSCluster.
*/
@AfterClass
public static void shutdown() {
if (cluster != null) {
cluster.shutdown();
}
}
@Test
public void defaultTest() throws Exception {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
randomKeyGenerator.setNumOfVolumes(2);
randomKeyGenerator.setNumOfBuckets(5);
randomKeyGenerator.setNumOfKeys(10);
randomKeyGenerator.call();
Assert.assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
Assert.assertEquals(10240 - 36, randomKeyGenerator.getKeyValueLength());
}
@Test
public void multiThread() throws Exception {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
randomKeyGenerator.setNumOfVolumes(10);
randomKeyGenerator.setNumOfBuckets(1);
randomKeyGenerator.setNumOfKeys(10);
randomKeyGenerator.setNumOfThreads(10);
randomKeyGenerator.setKeySize(10240);
randomKeyGenerator.call();
Assert.assertEquals(10, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
}
@Test
public void ratisTest3() throws Exception {
RandomKeyGenerator randomKeyGenerator =
new RandomKeyGenerator((OzoneConfiguration) cluster.getConf());
randomKeyGenerator.setNumOfVolumes(10);
randomKeyGenerator.setNumOfBuckets(1);
randomKeyGenerator.setNumOfKeys(10);
randomKeyGenerator.setNumOfThreads(10);
randomKeyGenerator.setKeySize(10240);
randomKeyGenerator.setFactor(ReplicationFactor.THREE);
randomKeyGenerator.setType(ReplicationType.RATIS);
randomKeyGenerator.call();
Assert.assertEquals(10, randomKeyGenerator.getNumberOfVolumesCreated());
Assert.assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
Assert.assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
}
}

1136
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java
View File

File diff suppressed because it is too large Load Diff

1038
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java Normal file
View File

File diff suppressed because it is too large Load Diff