big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-10420. Fix Ozone unit tests to use MiniOzoneCluster. Contributed by Arpit Agarwal.

Browse Source
This commit is contained in:
Arpit Agarwal 2016-05-20 15:19:09 -07:00 committed by Owen O'Malley
parent 3beee0b3d1
commit e3aa2b687c
16 changed files with 201 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java
View File

@ -110,8 +110,10 @@ public static InetSocketAddress getScmAddressForDataNodes(
final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
OZONE_SCM_DATANODE_ADDRESS_KEY);
return NetUtils.createSocketAddr(host.get() + ":" +
InetSocketAddress addr = NetUtils.createSocketAddr(host.get() + ":" +
port.or(OZONE_SCM_DATANODE_PORT_DEFAULT));
return addr;
}
/**

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
View File

@ -49,7 +49,7 @@ public class ContainerLocationManagerImpl implements ContainerLocationManager {
private final Configuration conf;
private final FsDatasetSpi dataset;
private final FsDatasetSpi<? extends FsVolumeSpi> dataset;
private final Path[] volumePaths;
private int currentIndex;
private final List<Path> locations;
@ -59,8 +59,9 @@ public class ContainerLocationManagerImpl implements ContainerLocationManager {
* Constructs a Location Manager.
* @param conf - Configuration.
*/
public ContainerLocationManagerImpl(Configuration conf, List<Path> locations,
FsDatasetSpi dataset) throws IOException {
public ContainerLocationManagerImpl(
Configuration conf, List<Path> locations,
FsDatasetSpi<? extends FsVolumeSpi> dataset) throws IOException {
this.conf = conf;
this.dataset = dataset;
List<Path> pathList = new LinkedList<>();

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
View File

@ -25,6 +25,7 @@
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
@ -83,9 +84,9 @@ public class ContainerManagerImpl implements ContainerManager {
* @throws IOException
*/
@Override
public void init(Configuration config, List<Path> containerDirs,
FsDatasetSpi dataset)
throws IOException {
public void init(
Configuration config, List<Path> containerDirs,
FsDatasetSpi<? extends FsVolumeSpi> dataset) throws IOException {
Preconditions.checkNotNull(config);
Preconditions.checkNotNull(containerDirs);

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
View File

@ -22,6 +22,7 @@
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.util.RwLock;
import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
import org.apache.hadoop.ozone.container.common.helpers.Pipeline;
@ -42,11 +43,10 @@ public interface ContainerManager extends RwLock {
*
* @param config - Configuration.
* @param containerDirs - List of Metadata Container locations.
* @param dataset - FSDataset.
* @throws IOException
*/
void init(Configuration config, List<Path> containerDirs,
FsDatasetSpi dataset)
FsDatasetSpi<? extends FsVolumeSpi> dataset)
throws IOException;
/**

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
View File

@ -48,7 +48,7 @@ public class OzoneContainer {
LoggerFactory.getLogger(OzoneContainer.class);
private final Configuration ozoneConfig;
private final FsDatasetSpi dataSet;
private final FsDatasetSpi<? extends FsVolumeSpi> dataSet;
private final ContainerDispatcher dispatcher;
private final ContainerManager manager;
private final XceiverServer server;
@ -62,8 +62,9 @@ public class OzoneContainer {
* @param dataSet - FsDataset.
* @throws IOException
*/
public OzoneContainer(Configuration ozoneConfig, FsDatasetSpi dataSet) throws
Exception {
public OzoneContainer(
Configuration ozoneConfig,
FsDatasetSpi<? extends FsVolumeSpi> dataSet) throws Exception {
List<Path> locations = new LinkedList<>();
String[] paths = ozoneConfig.getStrings(OzoneConfigKeys
.OZONE_METADATA_DIRS);
@ -150,8 +151,9 @@ public void stop() {
* @param pathList - List of paths.
* @throws IOException
*/
private void getDataDir(FsDatasetSpi dataset, List<Path> pathList) throws
IOException {
private void getDataDir(
FsDatasetSpi<? extends FsVolumeSpi> dataset,
List<Path> pathList) throws IOException {
FsDatasetSpi.FsVolumeReferences references;
try {
synchronized (dataset) {

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/storage/StorageContainerManager.java
View File

@ -155,7 +155,7 @@ public StorageContainerManager(OzoneConfiguration conf)
DatanodeProtocolPB.class, dnProtoPbService, handlerCount);
datanodeRpcAddress = updateListenAddress(conf,
OZONE_SCM_DATANODE_ADDRESS_KEY, datanodeRpcAddr, datanodeRpcServer);
LOG.info(buildRpcServerStartMessage("Service RPC server",
LOG.info(buildRpcServerStartMessage("RPC server for DataNodes",
datanodeRpcAddress));
BlockingService storageProtoPbService =
@ -522,7 +522,7 @@ private static InetSocketAddress updateListenAddress(OzoneConfiguration conf,
InetSocketAddress updatedAddr = new InetSocketAddress(
addr.getHostString(), listenAddr.getPort());
conf.set(rpcAddressKey,
addr.getHostString() + ":" + listenAddr.getPort());
addr.getHostString() + ":" + updatedAddr.getPort());
return updatedAddr;
}

60
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
View File

@ -22,9 +22,14 @@
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.util.Random;
import java.util.concurrent.TimeoutException;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.test.GenericTestUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -41,7 +46,6 @@
import org.apache.hadoop.ozone.web.client.OzoneClient;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
/**
* MiniOzoneCluster creates a complete in-process Ozone cluster suitable for
@ -82,6 +86,7 @@ public static class Builder
extends org.apache.hadoop.hdfs.MiniDFSCluster.Builder {
private final OzoneConfiguration conf;
private Optional<String> ozoneHandlerType = Optional.absent();
/**
* Creates a new Builder.
@ -100,13 +105,35 @@ public Builder numDataNodes(int val) {
return this;
}
public Builder setHandlerType(String handler) {
ozoneHandlerType = Optional.of(handler);
return this;
}
@Override
public MiniOzoneCluster build() throws IOException {
if (!ozoneHandlerType.isPresent()) {
throw new IllegalArgumentException(
"The Ozone handler type must be specified.");
}
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
conf.setBoolean(OzoneConfigKeys.OZONE_TRACE_ENABLED_KEY, true);
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, ozoneHandlerType.get());
conf.set(OzoneConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0");
conf.set(OzoneConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "127.0.0.1:0");
StorageContainerManager scm = new StorageContainerManager(conf);
scm.start();
return new MiniOzoneCluster(this, scm);
MiniOzoneCluster cluster = new MiniOzoneCluster(this, scm);
try {
cluster.waitOzoneReady();
} catch(Exception e) {
// A workaround to propagate MiniOzoneCluster failures without
// changing the method signature (which would require cascading
// changes to hundreds of unrelated HDFS tests).
throw new IOException("Failed to start MiniOzoneCluster", e);
}
return cluster;
}
}
@ -176,22 +203,19 @@ public OzoneClient createOzoneClient() throws OzoneException {
/**
* Waits for the Ozone cluster to be ready for processing requests.
*/
public void waitOzoneReady() {
long begin = Time.monotonicNow();
while (scm.getDatanodeReport(DatanodeReportType.LIVE).length <
numDataNodes) {
if (Time.monotonicNow() - begin > 20000) {
throw new IllegalStateException(
"Timed out waiting for Ozone cluster to become ready.");
public void waitOzoneReady() throws TimeoutException, InterruptedException {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
final DatanodeInfo[] reports =
scm.getDatanodeReport(DatanodeReportType.LIVE);
if (reports.length >= numDataNodes) {
return true;
}
LOG.info("Waiting for cluster to be ready. Got {} of {} DN reports.",
reports.length, numDataNodes);
return false;
}
LOG.info("Waiting for Ozone cluster to become ready");
try {
Thread.sleep(100);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IllegalStateException(
"Interrupted while waiting for Ozone cluster to become ready.");
}
}
}, 100, 45000);
}
}

36
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/storage/TestStorageContainerManager.java → hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
View File

@ -33,8 +33,17 @@
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.protocol.LocatedContainer;
import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.junit.rules.Timeout;
/**
* Test class that exercises the StorageContainerManager.
*/
public class TestStorageContainerManager {
/**
* Set the timeout for every test.
*/
@Rule
public Timeout testTimeout = new Timeout(300000);
private static MiniOzoneCluster cluster;
private static OzoneConfiguration conf;
@ -47,9 +56,6 @@ public class TestStorageContainerManager {
@BeforeClass
public static void init() throws IOException {
conf = new OzoneConfiguration();
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, "distributed");
conf.setBoolean(OzoneConfigKeys.OZONE_TRACE_ENABLED_KEY, true);
}
@After
@ -58,9 +64,9 @@ public void shutdown() throws InterruptedException {
}
@Test
public void testLocationsForSingleKey() throws IOException {
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(1).build();
cluster.waitOzoneReady();
public void testLocationsForSingleKey() throws Exception {
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(1)
.setHandlerType("distributed").build();
storageContainerLocationClient =
cluster.createStorageContainerLocationClient();
Set<LocatedContainer> containers =
@ -72,9 +78,9 @@ public void testLocationsForSingleKey() throws IOException {
}
@Test
public void testLocationsForMultipleKeys() throws IOException {
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(1).build();
cluster.waitOzoneReady();
public void testLocationsForMultipleKeys() throws Exception {
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(1)
.setHandlerType("distributed").build();
storageContainerLocationClient =
cluster.createStorageContainerLocationClient();
Set<LocatedContainer> containers =
@ -88,9 +94,9 @@ public void testLocationsForMultipleKeys() throws IOException {
}
@Test
public void testNoDataNodes() throws IOException {
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(0).build();
cluster.waitOzoneReady();
public void testNoDataNodes() throws Exception {
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(0)
.setHandlerType("distributed").build();
storageContainerLocationClient =
cluster.createStorageContainerLocationClient();
exception.expect(IOException.class);
@ -100,9 +106,9 @@ public void testNoDataNodes() throws IOException {
}
@Test
public void testMultipleDataNodes() throws IOException {
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(3).build();
cluster.waitOzoneReady();
public void testMultipleDataNodes() throws Exception {
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(3)
.setHandlerType("distributed").build();
storageContainerLocationClient =
cluster.createStorageContainerLocationClient();
Set<LocatedContainer> containers =

39
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
View File

@ -20,9 +20,10 @@
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
@ -41,6 +42,7 @@
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout;
import java.io.File;
import java.io.IOException;
@ -69,21 +71,26 @@
* Simple tests to verify that container persistence works as expected.
*/
public class TestContainerPersistence {
static String path;
static ContainerManagerImpl containerManager;
static ChunkManagerImpl chunkManager;
static KeyManagerImpl keyManager;
static OzoneConfiguration conf;
static FsDatasetSpi fsDataSet;
static MiniDFSCluster cluster;
static List<Path> pathLists = new LinkedList<>();
@Rule
public ExpectedException exception = ExpectedException.none();
/**
* Set the timeout for every test.
*/
@Rule
public Timeout testTimeout = new Timeout(300000);
private static String path;
private static ContainerManagerImpl containerManager;
private static ChunkManagerImpl chunkManager;
private static KeyManagerImpl keyManager;
private static OzoneConfiguration conf;
private static FsDatasetSpi<? extends FsVolumeSpi> fsDataSet;
private static MiniOzoneCluster cluster;
private static List<Path> pathLists = new LinkedList<>();
@BeforeClass
public static void init() throws IOException {
public static void init() throws Throwable {
conf = new OzoneConfiguration();
URL p = conf.getClass().getResource("");
path = p.getPath().concat(
@ -91,8 +98,6 @@ public static void init() throws IOException {
path += conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, "local");
File containerDir = new File(path);
if (containerDir.exists()) {
@ -101,8 +106,8 @@ public static void init() throws IOException {
Assert.assertTrue(containerDir.mkdirs());
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local").build();
fsDataSet = cluster.getDataNodes().get(0).getFSDataset();
containerManager = new ContainerManagerImpl();
chunkManager = new ChunkManagerImpl(containerManager);
@ -601,4 +606,4 @@ public void testDeleteKeyTwice() throws IOException,
}
}
}

32
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
View File

@ -17,10 +17,8 @@
*/
package org.apache.hadoop.ozone.container.ozoneimpl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
@ -28,15 +26,23 @@
import org.apache.hadoop.ozone.container.common.transport.client.XceiverClient;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import java.net.URL;
public class TestOzoneContainer {
/**
* Set the timeout for every test.
*/
@Rule
public Timeout testTimeout = new Timeout(300000);
@Test
public void testCreateOzoneContainer() throws Exception {
String containerName = OzoneUtils.getRequestID();
Configuration conf = new OzoneConfiguration();
OzoneConfiguration conf = new OzoneConfiguration();
URL p = conf.getClass().getResource("");
String path = p.getPath().concat(
TestOzoneContainer.class.getSimpleName());
@ -44,15 +50,11 @@ public void testCreateOzoneContainer() throws Exception {
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
MiniOzoneCluster cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local").build();
// We don't start Ozone Container via data node, we will do it
// independently in our test path.
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, false);
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, "local");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline
(containerName);
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
@ -78,7 +80,7 @@ public void testCreateOzoneContainer() throws Exception {
public void testOzoneContainerViaDataNode() throws Exception {
String keyName = OzoneUtils.getRequestID();
String containerName = OzoneUtils.getRequestID();
Configuration conf = new OzoneConfiguration();
OzoneConfiguration conf = new OzoneConfiguration();
URL p = conf.getClass().getResource("");
String path = p.getPath().concat(
TestOzoneContainer.class.getSimpleName());
@ -87,16 +89,14 @@ public void testOzoneContainerViaDataNode() throws Exception {
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
// Start ozone container Via Datanode create.
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, "local");
Pipeline pipeline =
ContainerTestHelper.createSingleNodePipeline(containerName);
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
pipeline.getLeader().getContainerPort());
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
MiniOzoneCluster cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local").build();
// This client talks to ozone container via datanode.
XceiverClient client = new XceiverClient(pipeline, conf);

14
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
View File

@ -31,17 +31,22 @@
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.web.client.OzoneBucket;
import org.apache.hadoop.ozone.web.client.OzoneClient;
import org.apache.hadoop.ozone.web.client.OzoneVolume;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.junit.rules.Timeout;
/**
* End-to-end testing of Ozone REST operations.
*/
public class TestOzoneRestWithMiniCluster {
/**
* Set the timeout for every test.
*/
@Rule
public Timeout testTimeout = new Timeout(300000);
private static MiniOzoneCluster cluster;
private static OzoneConfiguration conf;
@ -54,11 +59,8 @@ public class TestOzoneRestWithMiniCluster {
@BeforeClass
public static void init() throws Exception {
conf = new OzoneConfiguration();
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, "distributed");
conf.setBoolean(OzoneConfigKeys.OZONE_TRACE_ENABLED_KEY, true);
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(3).build();
cluster.waitOzoneReady();
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(3)
.setHandlerType("distributed").build();
ozoneClient = cluster.createOzoneClient();
}

23
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneVolumes.java
View File

@ -17,8 +17,8 @@
*/
package org.apache.hadoop.ozone.web;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
@ -35,7 +35,9 @@
import org.apache.log4j.Logger;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import javax.ws.rs.core.HttpHeaders;
import java.io.IOException;
@ -49,8 +51,14 @@
import static org.junit.Assert.assertEquals;
public class TestOzoneVolumes {
static MiniDFSCluster cluster = null;
static int port = 0;
/**
* Set the timeout for every test.
*/
@Rule
public Timeout testTimeout = new Timeout(300000);
private static MiniOzoneCluster cluster = null;
private static int port = 0;
/**
* Create a MiniDFSCluster for testing.
@ -62,7 +70,7 @@ public class TestOzoneVolumes {
* @throws IOException
*/
@BeforeClass
public static void init() throws IOException {
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
URL p = conf.getClass().getResource("");
@ -71,13 +79,10 @@ public static void init() throws IOException {
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, "local");
conf.setBoolean(OzoneConfigKeys.OZONE_TRACE_ENABLED_KEY, true);
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local").build();
DataNode dataNode = cluster.getDataNodes().get(0);
port = dataNode.getInfoPort();
}

50
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneWebAccess.java
View File

@ -18,43 +18,43 @@
package org.apache.hadoop.ozone.web;
import static java.net.HttpURLConnection.HTTP_CREATED;
import static org.apache.hadoop.ozone.web.utils.OzoneUtils.*;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import javax.ws.rs.core.HttpHeaders;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.web.headers.Header;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.headers.Header;
import org.apache.hadoop.util.Time;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import javax.ws.rs.core.HttpHeaders;
import java.io.IOException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import static java.net.HttpURLConnection.HTTP_CREATED;
import static org.apache.hadoop.ozone.web.utils.OzoneUtils.getRequestID;
import static org.junit.Assert.assertEquals;
public class TestOzoneWebAccess {
private static MiniDFSCluster cluster;
private static int port;
/**
* Set the timeout for every test
*/
@Rule
public Timeout timeout = new Timeout(30000);
public Timeout testTimeout = new Timeout(300000);
private static MiniOzoneCluster cluster;
private static int port;
/**
* Create a MiniDFSCluster for testing.
@ -66,23 +66,21 @@ public class TestOzoneWebAccess {
* @throws IOException
*/
@BeforeClass
public static void init() throws IOException {
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
URL p = conf.getClass().getResource("");
String path = p.getPath().concat(TestOzoneWebAccess.class.getSimpleName());
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, "local");
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local").build();
DataNode dataNode = cluster.getDataNodes().get(0);
port = dataNode.getInfoPort();
}
/**
* shutdown MiniDFSCluster
* shutdown MiniOzoneCluster.
*/
@AfterClass
public static void shutdown() {

21
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
View File

@ -18,8 +18,8 @@
package org.apache.hadoop.ozone.web.client;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
@ -27,7 +27,9 @@
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import java.io.IOException;
import java.net.URISyntaxException;
@ -39,9 +41,13 @@
import static org.junit.Assert.fail;
public class TestBuckets {
/**
* Set the timeout for every test.
*/
@Rule
public Timeout testTimeout = new Timeout(300000);
static MiniDFSCluster cluster = null;
static int port = 0;
private static MiniOzoneCluster cluster = null;
private static OzoneClient client = null;
/**
@ -64,13 +70,10 @@ public static void init() throws IOException,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, "local");
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local").build();
DataNode dataNode = cluster.getDataNodes().get(0);
port = dataNode.getInfoPort();
final int port = dataNode.getInfoPort();
client = new OzoneClient(String.format("http://localhost:%d", port));
}

30
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
View File

@ -19,8 +19,8 @@
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
@ -31,13 +31,14 @@
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
@ -49,12 +50,16 @@
import static org.junit.Assert.fail;
public class TestKeys {
static MiniDFSCluster cluster = null;
static int port = 0;
/**
* Set the timeout for every test.
*/
@Rule
public Timeout testTimeout = new Timeout(300000);
private static MiniOzoneCluster cluster = null;
static private String path;
private static OzoneClient client = null;
/**
* Create a MiniDFSCluster for testing.
*
@ -65,27 +70,20 @@ public class TestKeys {
* @throws IOException
*/
@BeforeClass
public static void init()
throws IOException, OzoneException, URISyntaxException {
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
URL p = conf.getClass().getResource("");
path = p.getPath().concat(TestKeys.class.getSimpleName());
path += conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, "local");
conf.setBoolean(OzoneConfigKeys.OZONE_TRACE_ENABLED_KEY, true);
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local").build();
DataNode dataNode = cluster.getDataNodes().get(0);
port = dataNode.getInfoPort();
final int port = dataNode.getInfoPort();
client = new OzoneClient(String.format("http://localhost:%d", port));
}

18
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
View File

@ -19,8 +19,8 @@
package org.apache.hadoop.ozone.web.client;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
@ -36,7 +36,6 @@
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.List;
@ -46,8 +45,7 @@
import static org.junit.Assert.assertTrue;
public class TestVolume {
private static MiniDFSCluster cluster = null;
private static int port = 0;
private static MiniOzoneCluster cluster = null;
private static OzoneClient client = null;
/**
@ -60,8 +58,7 @@ public class TestVolume {
* @throws IOException
*/
@BeforeClass
public static void init() throws IOException, OzoneException,
URISyntaxException {
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
URL p = conf.getClass().getResource("");
@ -71,15 +68,12 @@ public static void init() throws IOException, OzoneException,
FileUtils.deleteDirectory(new File(path));
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, "local");
conf.setBoolean(OzoneConfigKeys.OZONE_TRACE_ENABLED_KEY, true);
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local").build();
DataNode dataNode = cluster.getDataNodes().get(0);
port = dataNode.getInfoPort();
final int port = dataNode.getInfoPort();
client = new OzoneClient(String.format("http://localhost:%d", port));
}