HDDS-332. Remove the ability to configure ozone.handler.type

Contributed by Nandakumar and Anu Engineer.
This commit is contained in:
Anu Engineer 2018-08-28 09:56:02 -07:00
parent 2172399c55
commit df21e1b1dd
37 changed files with 185 additions and 2096 deletions

View File

@ -66,16 +66,9 @@ public final class OzoneConfigKeys {
"dfs.container.ratis.ipc.random.port"; "dfs.container.ratis.ipc.random.port";
public static final boolean DFS_CONTAINER_RATIS_IPC_RANDOM_PORT_DEFAULT = public static final boolean DFS_CONTAINER_RATIS_IPC_RANDOM_PORT_DEFAULT =
false; false;
public static final String OZONE_LOCALSTORAGE_ROOT =
"ozone.localstorage.root";
public static final String OZONE_LOCALSTORAGE_ROOT_DEFAULT = "/tmp/ozone";
public static final String OZONE_ENABLED = public static final String OZONE_ENABLED =
"ozone.enabled"; "ozone.enabled";
public static final boolean OZONE_ENABLED_DEFAULT = false; public static final boolean OZONE_ENABLED_DEFAULT = false;
public static final String OZONE_HANDLER_TYPE_KEY =
"ozone.handler.type";
public static final String OZONE_HANDLER_TYPE_DEFAULT = "distributed";
public static final String OZONE_TRACE_ENABLED_KEY = public static final String OZONE_TRACE_ENABLED_KEY =
"ozone.trace.enabled"; "ozone.trace.enabled";
public static final boolean OZONE_TRACE_ENABLED_DEFAULT = false; public static final boolean OZONE_TRACE_ENABLED_DEFAULT = false;

View File

@ -114,7 +114,6 @@ public final class OzoneConsts {
* Ozone handler types. * Ozone handler types.
*/ */
public static final String OZONE_HANDLER_DISTRIBUTED = "distributed"; public static final String OZONE_HANDLER_DISTRIBUTED = "distributed";
public static final String OZONE_HANDLER_LOCAL = "local";
public static final String DELETING_KEY_PREFIX = "#deleting#"; public static final String DELETING_KEY_PREFIX = "#deleting#";
public static final String DELETED_KEY_PREFIX = "#deleted#"; public static final String DELETED_KEY_PREFIX = "#deleted#";

View File

@ -277,17 +277,6 @@
Please note: By default ozone is disabled on a hadoop cluster. Please note: By default ozone is disabled on a hadoop cluster.
</description> </description>
</property> </property>
<property>
<name>ozone.handler.type</name>
<value>distributed</value>
<tag>OZONE, REST</tag>
<description>
Tells ozone which storage handler to use. The possible values are:
distributed - The Ozone distributed storage handler, which speaks to
OM/SCM on the backend and provides REST services to clients.
local - Local Storage handler strictly for testing - To be removed.
</description>
</property>
<property> <property>
<name>ozone.key.deleting.limit.per.task</name> <name>ozone.key.deleting.limit.per.task</name>
<value>1000</value> <value>1000</value>
@ -416,16 +405,6 @@
Default user permissions used in OM. Default user permissions used in OM.
</description> </description>
</property> </property>
<property>
<name>ozone.localstorage.root</name>
<value>${hadoop.tmp.dir}/ozone</value>
<tag>OZONE, DEBUG</tag>
<description>
This is used only for testing purposes. This value is used by the local
storage handler to simulate a REST backend. This is useful only when
debugging the REST front end independent of OM and SCM. To be removed.
</description>
</property>
<property> <property>
<name>ozone.metadata.dirs</name> <name>ozone.metadata.dirs</name>
<value/> <value/>

View File

@ -50,10 +50,9 @@ public interface RatisTestHelper {
private final MiniOzoneCluster cluster; private final MiniOzoneCluster cluster;
/** /**
* Create a {@link MiniOzoneCluster} for testing by setting * Create a {@link MiniOzoneCluster} for testing by setting.
* OZONE_ENABLED = true, * OZONE_ENABLED = true
* RATIS_ENABLED = true, and * RATIS_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "distributed".
*/ */
public RatisTestSuite(final Class<?> clazz) public RatisTestSuite(final Class<?> clazz)
throws IOException, TimeoutException, InterruptedException { throws IOException, TimeoutException, InterruptedException {
@ -88,7 +87,6 @@ public interface RatisTestHelper {
static OzoneConfiguration newOzoneConfiguration( static OzoneConfiguration newOzoneConfiguration(
Class<?> clazz, RpcType rpc) { Class<?> clazz, RpcType rpc) {
final OzoneConfiguration conf = new OzoneConfiguration(); final OzoneConfiguration conf = new OzoneConfiguration();
ContainerTestHelper.setOzoneLocalStorageRoot(clazz, conf);
initRatisConf(rpc, conf); initRatisConf(rpc, conf);
return conf; return conf;
} }

View File

@ -22,8 +22,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneBucket;
@ -65,16 +63,13 @@ public class TestOzoneRestClient {
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "distributed"
* *
* @throws IOException * @throws IOException
*/ */
@BeforeClass @BeforeClass
public static void init() throws Exception { public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration(); OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build(); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build();
cluster.waitForClusterToBeReady(); cluster.waitForClusterToBeReady();
InetSocketAddress omHttpAddress = cluster.getOzoneManager() InetSocketAddress omHttpAddress = cluster.getOzoneManager()

View File

@ -81,8 +81,6 @@ public class TestCloseContainerHandlingByClient {
@BeforeClass @BeforeClass
public static void init() throws Exception { public static void init() throws Exception {
conf = new OzoneConfiguration(); conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
chunkSize = (int)OzoneConsts.MB; chunkSize = (int)OzoneConsts.MB;
blockSize = 4 * chunkSize; blockSize = 4 * chunkSize;
conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize); conf.setInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize);

View File

@ -24,9 +24,7 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneBucket;
@ -81,16 +79,13 @@ public class TestOzoneRpcClient {
/** /**
* Create a MiniOzoneCluster for testing. * Create a MiniOzoneCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "distributed"
* *
* @throws IOException * @throws IOException
*/ */
@BeforeClass @BeforeClass
public static void init() throws Exception { public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration(); OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 1); conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 1);
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10).build(); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10).build();
cluster.waitForClusterToBeReady(); cluster.waitForClusterToBeReady();
@ -439,7 +434,7 @@ public class TestOzoneRpcClient {
String volumeName = UUID.randomUUID().toString(); String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString(); String bucketName = UUID.randomUUID().toString();
String value = RandomStringUtils.random(RandomUtils.nextInt(0,1024)); String value = RandomStringUtils.random(RandomUtils.nextInt(0, 1024));
store.createVolume(volumeName); store.createVolume(volumeName);
OzoneVolume volume = store.getVolume(volumeName); OzoneVolume volume = store.getVolume(volumeName);
volume.createBucket(bucketName); volume.createBucket(bucketName);

View File

@ -38,7 +38,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
@ -73,15 +72,6 @@ public final class ContainerTestHelper {
private ContainerTestHelper() { private ContainerTestHelper() {
} }
public static void setOzoneLocalStorageRoot(
Class<?> clazz, OzoneConfiguration conf) {
String path = GenericTestUtils.getTempPath(clazz.getSimpleName());
path += conf.getTrimmed(
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
}
// TODO: mock multi-node pipeline // TODO: mock multi-node pipeline
/** /**
* Create a pipeline with single node replica. * Create a pipeline with single node replica.

View File

@ -28,7 +28,6 @@ import java.util.Random;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.apache.hadoop.ozone.container.ContainerTestHelper;
@ -45,17 +44,14 @@ import org.junit.Test;
*/ */
public class TestContainerDeletionChoosingPolicy { public class TestContainerDeletionChoosingPolicy {
private static String path; private static String path;
private static ContainerSet containerSet; private ContainerSet containerSet;
private static OzoneConfiguration conf; private OzoneConfiguration conf;
@Before @Before
public void init() throws Throwable { public void init() throws Throwable {
conf = new OzoneConfiguration(); conf = new OzoneConfiguration();
path = GenericTestUtils path = GenericTestUtils
.getTempPath(TestContainerDeletionChoosingPolicy.class.getSimpleName()); .getTempPath(TestContainerDeletionChoosingPolicy.class.getSimpleName());
path += conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
} }
@Test @Test

View File

@ -21,21 +21,20 @@ import com.google.common.collect.Maps;
import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.binary.Hex;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.helpers.KeyData;
import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy; import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy;
.RoundRobinVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
@ -45,8 +44,6 @@ import org.apache.hadoop.ozone.container.keyvalue.impl.KeyManagerImpl;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager; import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.KeyManager; import org.apache.hadoop.ozone.container.keyvalue.interfaces.KeyManager;
import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.helpers.KeyData;
import org.apache.hadoop.utils.MetadataStore; import org.apache.hadoop.utils.MetadataStore;
import org.junit.After; import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
@ -68,113 +65,80 @@ import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.ArrayList;
import java.util.UUID; import java.util.UUID;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Stage.COMBINED;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_ROOT_PREFIX; import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_ROOT_PREFIX;
import static org.apache.hadoop.ozone.container.ContainerTestHelper.getChunk; import static org.apache.hadoop.ozone.container.ContainerTestHelper.getChunk;
import static org.apache.hadoop.ozone.container.ContainerTestHelper.getData; import static org.apache.hadoop.ozone.container.ContainerTestHelper.getData;
import static org.apache.hadoop.ozone.container.ContainerTestHelper import static org.apache.hadoop.ozone.container.ContainerTestHelper.setDataChecksum;
.setDataChecksum;
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.Stage.COMBINED;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
/** /**
* Simple tests to verify that container persistence works as expected. * Simple tests to verify that container persistence works as expected. Some of
* Some of these tests are specific to {@link KeyValueContainer}. If a new * these tests are specific to {@link KeyValueContainer}. If a new {@link
* {@link ContainerProtos.ContainerType} is added, the tests need to be * ContainerProtos.ContainerType} is added, the tests need to be modified.
* modified.
*/ */
public class TestContainerPersistence { public class TestContainerPersistence {
@Rule private static final String DATANODE_UUID = UUID.randomUUID().toString();
public ExpectedException exception = ExpectedException.none(); private static final String SCM_ID = UUID.randomUUID().toString();
/**
* Set the timeout for every test.
*/
@Rule
public Timeout testTimeout = new Timeout(300000);
private static Logger log = private static Logger log =
LoggerFactory.getLogger(TestContainerPersistence.class); LoggerFactory.getLogger(TestContainerPersistence.class);
private static String hddsPath; private static String hddsPath;
private static String path;
private static OzoneConfiguration conf; private static OzoneConfiguration conf;
private static List<StorageLocation> pathLists = new LinkedList<>();
private Long containerID = 8888L;;
private static final String datanodeUuid = UUID.randomUUID().toString();
private static final String scmId = UUID.randomUUID().toString();
private static ContainerSet containerSet; private static ContainerSet containerSet;
private static VolumeSet volumeSet; private static VolumeSet volumeSet;
private static VolumeChoosingPolicy volumeChoosingPolicy; private static VolumeChoosingPolicy volumeChoosingPolicy;
private static KeyManager keyManager; private static KeyManager keyManager;
private static ChunkManager chunkManager; private static ChunkManager chunkManager;
@Rule
public ExpectedException exception = ExpectedException.none();
/**
* Set the timeout for every test.
*/
@Rule
public Timeout testTimeout = new Timeout(300000);
private Long containerID = 8888L;
@BeforeClass @BeforeClass
public static void init() throws Throwable { public static void init() throws Throwable {
conf = new OzoneConfiguration(); conf = new OzoneConfiguration();
hddsPath = GenericTestUtils hddsPath = GenericTestUtils
.getTempPath(TestContainerPersistence.class.getSimpleName()); .getTempPath(TestContainerPersistence.class.getSimpleName());
path = hddsPath + conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, hddsPath); conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, hddsPath);
File containerDir = new File(path);
if (containerDir.exists()) {
FileUtils.deleteDirectory(new File(path));
}
Assert.assertTrue(containerDir.mkdirs());
volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy(); volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy();
} }
@AfterClass @AfterClass
public static void shutdown() throws IOException { public static void shutdown() throws IOException {
FileUtils.deleteDirectory(new File(path));
FileUtils.deleteDirectory(new File(hddsPath)); FileUtils.deleteDirectory(new File(hddsPath));
} }
@Before @Before
public void setupPaths() throws IOException { public void setupPaths() throws IOException {
if (!new File(path).exists() && !new File(path).mkdirs()) {
throw new IOException("Unable to create paths. " + path);
}
StorageLocation loc = StorageLocation.parse(
Paths.get(path).resolve(CONTAINER_ROOT_PREFIX).toString());
pathLists.clear();
containerSet = new ContainerSet(); containerSet = new ContainerSet();
volumeSet = new VolumeSet(datanodeUuid, conf); volumeSet = new VolumeSet(DATANODE_UUID, conf);
keyManager = new KeyManagerImpl(conf); keyManager = new KeyManagerImpl(conf);
chunkManager = new ChunkManagerImpl(); chunkManager = new ChunkManagerImpl();
if (!new File(loc.getNormalizedUri()).mkdirs()) {
throw new IOException("unable to create paths. " +
loc.getNormalizedUri());
}
pathLists.add(loc);
for (String dir : conf.getStrings(DFS_DATANODE_DATA_DIR_KEY)) { for (String dir : conf.getStrings(DFS_DATANODE_DATA_DIR_KEY)) {
StorageLocation location = StorageLocation.parse(dir); StorageLocation location = StorageLocation.parse(dir);
FileUtils.forceMkdir(new File(location.getNormalizedUri())); FileUtils.forceMkdir(new File(location.getNormalizedUri()));
} }
} }
@After @After
public void cleanupDir() throws IOException { public void cleanupDir() throws IOException {
// Clean up SCM metadata // Clean up SCM metadata
log.info("Deleting {}", path);
FileUtils.deleteDirectory(new File(path));
log.info("Deleting {}", hddsPath); log.info("Deleting {}", hddsPath);
FileUtils.deleteDirectory(new File(hddsPath)); FileUtils.deleteDirectory(new File(hddsPath));
@ -196,9 +160,9 @@ public class TestContainerPersistence {
data.addMetadata("VOLUME", "shire"); data.addMetadata("VOLUME", "shire");
data.addMetadata("owner)", "bilbo"); data.addMetadata("owner)", "bilbo");
KeyValueContainer container = new KeyValueContainer(data, conf); KeyValueContainer container = new KeyValueContainer(data, conf);
container.create(volumeSet, volumeChoosingPolicy, scmId); container.create(volumeSet, volumeChoosingPolicy, SCM_ID);
containerSet.addContainer(container); containerSet.addContainer(container);
return container; return container;
} }
@Test @Test
@ -209,7 +173,7 @@ public class TestContainerPersistence {
.containsKey(testContainerID)); .containsKey(testContainerID));
KeyValueContainerData kvData = KeyValueContainerData kvData =
(KeyValueContainerData) containerSet.getContainer(testContainerID) (KeyValueContainerData) containerSet.getContainer(testContainerID)
.getContainerData(); .getContainerData();
Assert.assertNotNull(kvData); Assert.assertNotNull(kvData);
Assert.assertTrue(new File(kvData.getMetadataPath()).exists()); Assert.assertTrue(new File(kvData.getMetadataPath()).exists());
@ -287,7 +251,7 @@ public class TestContainerPersistence {
} }
@Test @Test
public void testGetContainerReports() throws Exception{ public void testGetContainerReports() throws Exception {
final int count = 10; final int count = 10;
List<Long> containerIDs = new ArrayList<>(); List<Long> containerIDs = new ArrayList<>();
@ -296,7 +260,7 @@ public class TestContainerPersistence {
Container container = addContainer(containerSet, testContainerID); Container container = addContainer(containerSet, testContainerID);
// Close a bunch of containers. // Close a bunch of containers.
if (i%3 == 0) { if (i % 3 == 0) {
container.close(); container.close();
} }
containerIDs.add(testContainerID); containerIDs.add(testContainerID);
@ -307,7 +271,8 @@ public class TestContainerPersistence {
List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports = List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports =
containerSet.getContainerReport().getReportsList(); containerSet.getContainerReport().getReportsList();
Assert.assertEquals(10, reports.size()); Assert.assertEquals(10, reports.size());
for(StorageContainerDatanodeProtocolProtos.ContainerInfo report : reports) { for (StorageContainerDatanodeProtocolProtos.ContainerInfo report :
reports) {
long actualContainerID = report.getContainerID(); long actualContainerID = report.getContainerID();
Assert.assertTrue(containerIDs.remove(actualContainerID)); Assert.assertTrue(containerIDs.remove(actualContainerID));
} }
@ -315,8 +280,8 @@ public class TestContainerPersistence {
} }
/** /**
* This test creates 50 containers and reads them back 5 containers at a * This test creates 50 containers and reads them back 5 containers at a time
* time and verifies that we did get back all containers. * and verifies that we did get back all containers.
* *
* @throws IOException * @throws IOException
*/ */
@ -426,7 +391,7 @@ public class TestContainerPersistence {
sha.update(FileUtils.readFileToByteArray(fname.toFile())); sha.update(FileUtils.readFileToByteArray(fname.toFile()));
String val = Hex.encodeHexString(sha.digest()); String val = Hex.encodeHexString(sha.digest());
Assert.assertEquals(fileHashMap.get(fname.getFileName().toString()) Assert.assertEquals(fileHashMap.get(fname.getFileName().toString())
.getChecksum(), val); .getChecksum(), val);
count++; count++;
sha.reset(); sha.reset();
} }
@ -454,8 +419,8 @@ public class TestContainerPersistence {
@Test @Test
public void testPartialRead() throws Exception { public void testPartialRead() throws Exception {
final int datalen = 1024; final int datalen = 1024;
final int start = datalen/4; final int start = datalen / 4;
final int length = datalen/2; final int length = datalen / 2;
long testContainerID = getTestContainerID(); long testContainerID = getTestContainerID();
Container container = addContainer(containerSet, testContainerID); Container container = addContainer(containerSet, testContainerID);
@ -544,7 +509,8 @@ public class TestContainerPersistence {
} }
// Request to read the whole data in a single go. // Request to read the whole data in a single go.
ChunkInfo largeChunk = getChunk(blockID.getLocalID(), 0, 0, datalen * chunkCount); ChunkInfo largeChunk = getChunk(blockID.getLocalID(), 0, 0,
datalen * chunkCount);
byte[] newdata = chunkManager.readChunk(container, blockID, largeChunk); byte[] newdata = chunkManager.readChunk(container, blockID, largeChunk);
MessageDigest newSha = MessageDigest.getInstance(OzoneConsts.FILE_HASH); MessageDigest newSha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
newSha.update(newdata); newSha.update(newdata);
@ -701,8 +667,8 @@ public class TestContainerPersistence {
} }
/** /**
* Tries to update an existing and non-existing container. * Tries to update an existing and non-existing container. Verifies container
* Verifies container map and persistent data both updated. * map and persistent data both updated.
* *
* @throws IOException * @throws IOException
*/ */
@ -743,7 +709,7 @@ public class TestContainerPersistence {
orgContainerFile.getAbsolutePath(), orgContainerFile.getAbsolutePath(),
newContainerFile.getAbsolutePath()); newContainerFile.getAbsolutePath());
ContainerData actualContainerData = ContainerDataYaml.readContainerFile( ContainerData actualContainerData = ContainerDataYaml.readContainerFile(
newContainerFile); newContainerFile);
Assert.assertEquals("shire_new", Assert.assertEquals("shire_new",
actualContainerData.getMetadata().get("VOLUME")); actualContainerData.getMetadata().get("VOLUME"));

View File

@ -31,7 +31,6 @@ import org.apache.hadoop.hdds.scm.block.SCMBlockDeletingService;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.OzoneTestUtils; import org.apache.hadoop.ozone.OzoneTestUtils;
import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.ObjectStore;
@ -88,16 +87,11 @@ public class TestBlockDeletion {
File baseDir = new File(path); File baseDir = new File(path);
baseDir.mkdirs(); baseDir.mkdirs();
path += conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
conf.setQuietMode(false);
conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100,
TimeUnit.MILLISECONDS); TimeUnit.MILLISECONDS);
conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200, conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200,
TimeUnit.MILLISECONDS); TimeUnit.MILLISECONDS);
conf.setQuietMode(false);
cluster = MiniOzoneCluster.newBuilder(conf) cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(1) .setNumDatanodes(1)
.setHbInterval(200) .setHbInterval(200)

View File

@ -1,19 +1,18 @@
/** /**
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one or more
* or more contributor license agreements. See the NOTICE file * contributor license agreements. See the NOTICE file distributed with this
* distributed with this work for additional information * work for additional information regarding copyright ownership. The ASF
* regarding copyright ownership. The ASF licenses this file * licenses this file to you under the Apache License, Version 2.0 (the
* to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License.
* "License"); you may not use this file except in compliance * You may obtain a copy of the License at
* with the License. You may obtain a copy of the License at
* <p> * <p>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* <p> * <p>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* See the License for the specific language governing permissions and * License for the specific language governing permissions and limitations under
* limitations under the License. * the License.
*/ */
package org.apache.hadoop.ozone.container.common.statemachine.commandhandler; package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
@ -26,8 +25,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneClientFactory;
@ -54,20 +51,16 @@ public class TestCloseContainerByPipeline {
private static OzoneClient client; private static OzoneClient client;
private static ObjectStore objectStore; private static ObjectStore objectStore;
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "distributed"
* *
* @throws IOException * @throws IOException
*/ */
@BeforeClass @BeforeClass
public static void init() throws Exception { public static void init() throws Exception {
conf = new OzoneConfiguration(); conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
cluster = MiniOzoneCluster.newBuilder(conf) cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(3).build(); .setNumDatanodes(3).build();
cluster.waitForClusterToBeReady(); cluster.waitForClusterToBeReady();
@ -243,7 +236,8 @@ public class TestCloseContainerByPipeline {
() -> isContainerClosed(cluster, containerID, datanodeDetails), 500, () -> isContainerClosed(cluster, containerID, datanodeDetails), 500,
15 * 1000); 15 * 1000);
//double check if it's really closed (waitFor also throws an exception) //double check if it's really closed (waitFor also throws an exception)
Assert.assertTrue(isContainerClosed(cluster, containerID, datanodeDetails)); Assert.assertTrue(isContainerClosed(cluster,
containerID, datanodeDetails));
} }
Assert.assertFalse(logCapturer.getOutput().contains( Assert.assertFalse(logCapturer.getOutput().contains(
"submitting CloseContainer request over STAND_ALONE " "submitting CloseContainer request over STAND_ALONE "
@ -257,13 +251,14 @@ public class TestCloseContainerByPipeline {
private Boolean isContainerClosed(MiniOzoneCluster cluster, long containerID, private Boolean isContainerClosed(MiniOzoneCluster cluster, long containerID,
DatanodeDetails datanode) { DatanodeDetails datanode) {
ContainerData containerData; ContainerData containerData;
for (HddsDatanodeService datanodeService : cluster.getHddsDatanodes()) for (HddsDatanodeService datanodeService : cluster.getHddsDatanodes()) {
if (datanode.equals(datanodeService.getDatanodeDetails())) { if (datanode.equals(datanodeService.getDatanodeDetails())) {
containerData = containerData =
datanodeService.getDatanodeStateMachine().getContainer() datanodeService.getDatanodeStateMachine().getContainer()
.getContainerSet().getContainer(containerID).getContainerData(); .getContainerSet().getContainer(containerID).getContainerData();
return containerData.isClosed(); return containerData.isClosed();
} }
}
return false; return false;
} }
} }

View File

@ -91,8 +91,6 @@ public class TestOzoneContainer {
static OzoneConfiguration newOzoneConfiguration() { static OzoneConfiguration newOzoneConfiguration() {
final OzoneConfiguration conf = new OzoneConfiguration(); final OzoneConfiguration conf = new OzoneConfiguration();
ContainerTestHelper.setOzoneLocalStorageRoot(
TestOzoneContainer.class, conf);
return conf; return conf;
} }

View File

@ -54,8 +54,6 @@ public class TestOzoneContainerRatis {
static OzoneConfiguration newOzoneConfiguration() { static OzoneConfiguration newOzoneConfiguration() {
final OzoneConfiguration conf = new OzoneConfiguration(); final OzoneConfiguration conf = new OzoneConfiguration();
ContainerTestHelper.setOzoneLocalStorageRoot(
TestOzoneContainerRatis.class, conf);
return conf; return conf;
} }

View File

@ -47,8 +47,6 @@ public class TestRatisManager {
static OzoneConfiguration newOzoneConfiguration() { static OzoneConfiguration newOzoneConfiguration() {
final OzoneConfiguration conf = new OzoneConfiguration(); final OzoneConfiguration conf = new OzoneConfiguration();
ContainerTestHelper.setOzoneLocalStorageRoot(
TestRatisManager.class, conf);
return conf; return conf;
} }

View File

@ -20,8 +20,6 @@ package org.apache.hadoop.ozone.freon;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.util.ToolRunner;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.Assert; import org.junit.Assert;
@ -45,16 +43,13 @@ public class TestDataValidate {
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "distributed"
* *
* @throws IOException * @throws IOException
*/ */
@BeforeClass @BeforeClass
public static void init() throws Exception { public static void init() throws Exception {
conf = new OzoneConfiguration(); conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
cluster = MiniOzoneCluster.newBuilder(conf) cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(5).build(); .setNumDatanodes(5).build();
cluster.waitForClusterToBeReady(); cluster.waitForClusterToBeReady();

View File

@ -41,8 +41,7 @@ public class TestFreon {
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "distributed"
* *
* @throws IOException * @throws IOException
*/ */

View File

@ -24,8 +24,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.*; import org.apache.hadoop.ozone.client.*;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@ -33,7 +31,6 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Rule; import org.junit.Rule;
@ -60,16 +57,13 @@ public class TestContainerReportWithKeys {
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "distributed"
* *
* @throws IOException * @throws IOException
*/ */
@BeforeClass @BeforeClass
public static void init() throws Exception { public static void init() throws Exception {
conf = new OzoneConfiguration(); conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
cluster = MiniOzoneCluster.newBuilder(conf).build(); cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady(); cluster.waitForClusterToBeReady();
scm = cluster.getStorageContainerManager(); scm = cluster.getStorageContainerManager();
@ -117,10 +111,6 @@ public class TestContainerReportWithKeys {
cluster.getOzoneManager().lookupKey(keyArgs).getKeyLocationVersions() cluster.getOzoneManager().lookupKey(keyArgs).getKeyLocationVersions()
.get(0).getBlocksLatestVersionOnly().get(0); .get(0).getBlocksLatestVersionOnly().get(0);
ContainerData cd = getContainerData(keyInfo.getContainerID());
/* LOG.info("DN Container Data: keyCount: {} used: {} ",
cd.getKeyCount(), cd.getBytesUsed());*/
ContainerInfo cinfo = scm.getContainerInfo(keyInfo.getContainerID()); ContainerInfo cinfo = scm.getContainerInfo(keyInfo.getContainerID());

View File

@ -62,8 +62,7 @@ public class TestMultipleContainerReadWrite {
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "distributed"
* *
* @throws IOException * @throws IOException
*/ */
@ -73,8 +72,6 @@ public class TestMultipleContainerReadWrite {
// set to as small as 100 bytes per block. // set to as small as 100 bytes per block.
conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 1); conf.setLong(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB, 1);
conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 5); conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 5);
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
cluster = MiniOzoneCluster.newBuilder(conf).build(); cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady(); cluster.waitForClusterToBeReady();
storageHandler = new ObjectStoreHandler(conf).getStorageHandler(); storageHandler = new ObjectStoreHandler(conf).getStorageHandler();

View File

@ -22,8 +22,6 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler; import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
@ -67,16 +65,13 @@ public class TestOmBlockVersioning {
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "distributed"
* *
* @throws IOException * @throws IOException
*/ */
@BeforeClass @BeforeClass
public static void init() throws Exception { public static void init() throws Exception {
conf = new OzoneConfiguration(); conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
cluster = MiniOzoneCluster.newBuilder(conf).build(); cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady(); cluster.waitForClusterToBeReady();
storageHandler = new ObjectStoreHandler(conf).getStorageHandler(); storageHandler = new ObjectStoreHandler(conf).getStorageHandler();

View File

@ -23,9 +23,7 @@ import java.io.IOException;
import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@ -52,8 +50,6 @@ public class TestOmMetrics {
@Before @Before
public void setup() throws Exception { public void setup() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration(); OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
cluster = MiniOzoneCluster.newBuilder(conf).build(); cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady(); cluster.waitForClusterToBeReady();
ozoneManager = cluster.getOzoneManager(); ozoneManager = cluster.getOzoneManager();
@ -104,7 +100,8 @@ public class TestOmMetrics {
Mockito.doThrow(exception).when(mockVm).setOwner(null, null); Mockito.doThrow(exception).when(mockVm).setOwner(null, null);
Mockito.doThrow(exception).when(mockVm).listVolumes(null, null, null, 0); Mockito.doThrow(exception).when(mockVm).listVolumes(null, null, null, 0);
org.apache.hadoop.test.Whitebox.setInternalState(ozoneManager, "volumeManager", mockVm); org.apache.hadoop.test.Whitebox.setInternalState(ozoneManager,
"volumeManager", mockVm);
doVolumeOps(); doVolumeOps();
omMetrics = getMetrics("OMMetrics"); omMetrics = getMetrics("OMMetrics");

View File

@ -20,7 +20,6 @@ import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.scm.cli.SQLCLI; import org.apache.hadoop.ozone.scm.cli.SQLCLI;
import org.apache.hadoop.ozone.web.handlers.BucketArgs; import org.apache.hadoop.ozone.web.handlers.BucketArgs;
import org.apache.hadoop.ozone.web.handlers.KeyArgs; import org.apache.hadoop.ozone.web.handlers.KeyArgs;
@ -97,16 +96,13 @@ public class TestOmSQLCli {
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "distributed"
* *
* @throws IOException * @throws IOException
*/ */
@Before @Before
public void setup() throws Exception { public void setup() throws Exception {
conf = new OzoneConfiguration(); conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
cluster = MiniOzoneCluster.newBuilder(conf).build(); cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady(); cluster.waitForClusterToBeReady();
storageHandler = new ObjectStoreHandler(conf).getStorageHandler(); storageHandler = new ObjectStoreHandler(conf).getStorageHandler();

View File

@ -108,8 +108,7 @@ public class TestOzoneManager {
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "distributed"
* *
* @throws IOException * @throws IOException
*/ */
@ -119,8 +118,6 @@ public class TestOzoneManager {
clusterId = UUID.randomUUID().toString(); clusterId = UUID.randomUUID().toString();
scmId = UUID.randomUUID().toString(); scmId = UUID.randomUUID().toString();
omId = UUID.randomUUID().toString(); omId = UUID.randomUUID().toString();
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2); conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2);
cluster = MiniOzoneCluster.newBuilder(conf) cluster = MiniOzoneCluster.newBuilder(conf)
.setClusterId(clusterId) .setClusterId(clusterId)

View File

@ -48,7 +48,6 @@ import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl.OzoneACLRights; import org.apache.hadoop.ozone.OzoneAcl.OzoneACLRights;
import org.apache.hadoop.ozone.OzoneAcl.OzoneACLType; import org.apache.hadoop.ozone.OzoneAcl.OzoneACLType;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneBucket;
@ -134,11 +133,6 @@ public class TestOzoneShell {
baseDir = new File(path); baseDir = new File(path);
baseDir.mkdirs(); baseDir.mkdirs();
path += conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
conf.setQuietMode(false);
shell = new Shell(); shell = new Shell();
shell.setConf(conf); shell.setConf(conf);
@ -146,6 +140,7 @@ public class TestOzoneShell {
.setNumDatanodes(3) .setNumDatanodes(3)
.build(); .build();
conf.setInt(OZONE_REPLICATION, ReplicationFactor.THREE.getValue()); conf.setInt(OZONE_REPLICATION, ReplicationFactor.THREE.getValue());
conf.setQuietMode(false);
client = new RpcClient(conf); client = new RpcClient(conf);
cluster.waitForClusterToBeReady(); cluster.waitForClusterToBeReady();
} }
@ -208,12 +203,15 @@ public class TestOzoneShell {
testCreateVolume(volumeName, ""); testCreateVolume(volumeName, "");
volumeName = "volume" + RandomStringUtils.randomNumeric(5); volumeName = "volume" + RandomStringUtils.randomNumeric(5);
testCreateVolume("/////" + volumeName, ""); testCreateVolume("/////" + volumeName, "");
testCreateVolume("/////", "Volume name is required to create a volume"); testCreateVolume("/////", "Volume name is required " +
"to create a volume");
testCreateVolume("/////vol/123", testCreateVolume("/////vol/123",
"Illegal argument: Bucket or Volume name has an unsupported character : /"); "Illegal argument: Bucket or Volume name has " +
"an unsupported character : /");
} }
private void testCreateVolume(String volumeName, String errorMsg) throws Exception { private void testCreateVolume(String volumeName, String errorMsg)
throws Exception {
err.reset(); err.reset();
String userName = "bilbo"; String userName = "bilbo";
String[] args = new String[] {"-createVolume", url + "/" + volumeName, String[] args = new String[] {"-createVolume", url + "/" + volumeName,
@ -397,7 +395,7 @@ public class TestOzoneShell {
// test -prefix option // test -prefix option
out.reset(); out.reset();
args = new String[] { "-listVolume", url + "/", "-user", user1, "-length", args = new String[] {"-listVolume", url + "/", "-user", user1, "-length",
"100", "-prefix", "test-vol-" + protocol + "1" }; "100", "-prefix", "test-vol-" + protocol + "1" };
assertEquals(0, ToolRunner.run(shell, args)); assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString(); commandOutput = out.toString();
@ -414,7 +412,7 @@ public class TestOzoneShell {
// test -start option // test -start option
out.reset(); out.reset();
args = new String[] { "-listVolume", url + "/", "-user", user2, "-length", args = new String[] {"-listVolume", url + "/", "-user", user2, "-length",
"100", "-start", "test-vol-" + protocol + "15" }; "100", "-start", "test-vol-" + protocol + "15" };
assertEquals(0, ToolRunner.run(shell, args)); assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString(); commandOutput = out.toString();

View File

@ -1,187 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.web;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.TestOzoneHelper;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import java.io.IOException;
/**
* Test ozone volume in the local storage handler scenario.
*/
public class TestLocalOzoneVolumes extends TestOzoneHelper {
/**
* Set the timeout for every test.
*/
@Rule
public Timeout testTimeout = new Timeout(300000);
private static MiniOzoneCluster cluster = null;
private static int port = 0;
/**
* Create a MiniDFSCluster for testing.
* <p>
* Ozone is made active by setting OZONE_ENABLED = true and
* OZONE_HANDLER_TYPE_KEY = "local" , which uses a local directory to
* emulate Ozone backend.
*
* @throws IOException
*/
@BeforeClass
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
String path = GenericTestUtils
.getTempPath(TestLocalOzoneVolumes.class.getSimpleName());
path += conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
port = cluster.getHddsDatanodes().get(0)
.getDatanodeDetails().getPort(
DatanodeDetails.Port.Name.REST).getValue();
}
/**
* Shutdown MiniDFSCluster.
*/
@AfterClass
public static void shutdown() {
if (cluster != null) {
cluster.shutdown();
}
}
/**
* Creates Volumes on Ozone Store.
*
* @throws IOException
*/
@Test
public void testCreateVolumes() throws IOException {
super.testCreateVolumes(port);
}
/**
* Create Volumes with Quota.
*
* @throws IOException
*/
@Test
public void testCreateVolumesWithQuota() throws IOException {
super.testCreateVolumesWithQuota(port);
}
/**
* Create Volumes with Invalid Quota.
*
* @throws IOException
*/
@Test
public void testCreateVolumesWithInvalidQuota() throws IOException {
super.testCreateVolumesWithInvalidQuota(port);
}
/**
* To create a volume a user name must be specified using OZONE_USER header.
* This test verifies that we get an error in case we call without a OZONE
* user name.
*
* @throws IOException
*/
@Test
public void testCreateVolumesWithInvalidUser() throws IOException {
super.testCreateVolumesWithInvalidUser(port);
}
/**
* Only Admins can create volumes in Ozone. This test uses simple userauth as
* backend and hdfs and root are admin users in the simple backend.
* <p>
* This test tries to create a volume as user bilbo.
*
* @throws IOException
*/
@Test
public void testCreateVolumesWithOutAdminRights() throws IOException {
super.testCreateVolumesWithOutAdminRights(port);
}
/**
* Create a bunch of volumes in a loop.
*
* @throws IOException
*/
//@Test
public void testCreateVolumesInLoop() throws IOException {
super.testCreateVolumesInLoop(port);
}
/**
* Get volumes owned by the user.
*
* @throws IOException
*/
@Test
public void testGetVolumesByUser() throws IOException {
super.testGetVolumesByUser(port);
}
/**
* Admins can read volumes belonging to other users.
*
* @throws IOException
*/
@Test
public void testGetVolumesOfAnotherUser() throws IOException {
super.testGetVolumesOfAnotherUser(port);
}
/**
* if you try to read volumes belonging to another user,
* then server always ignores it.
*
* @throws IOException
*/
@Test @Ignore
public void testGetVolumesOfAnotherUserShouldFail() throws IOException {
super.testGetVolumesOfAnotherUserShouldFail(port);
}
@Test
public void testListKeyOnEmptyBucket() throws IOException {
super.testListKeyOnEmptyBucket(port);
}
}

View File

@ -19,9 +19,7 @@ package org.apache.hadoop.ozone.web;
import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.TestOzoneHelper; import org.apache.hadoop.ozone.TestOzoneHelper;
import org.apache.log4j.Level; import org.apache.log4j.Level;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
@ -39,9 +37,9 @@ import java.io.IOException;
/** /**
* Test ozone volume in the distributed storage handler scenario. * Test ozone volume in the distributed storage handler scenario.
*/ */
public class TestDistributedOzoneVolumes extends TestOzoneHelper { public class TestOzoneVolumes extends TestOzoneHelper {
private static final org.slf4j.Logger LOG = private static final org.slf4j.Logger LOG =
LoggerFactory.getLogger(TestDistributedOzoneVolumes.class); LoggerFactory.getLogger(TestOzoneVolumes.class);
/** /**
* Set the timeout for every test. * Set the timeout for every test.
*/ */
@ -54,8 +52,7 @@ public class TestDistributedOzoneVolumes extends TestOzoneHelper {
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "distributed"
* *
* @throws IOException * @throws IOException
*/ */
@ -63,8 +60,6 @@ public class TestDistributedOzoneVolumes extends TestOzoneHelper {
public static void init() throws Exception { public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration(); OzoneConfiguration conf = new OzoneConfiguration();
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG); Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
cluster = MiniOzoneCluster.newBuilder(conf).build(); cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady(); cluster.waitForClusterToBeReady();
port = cluster.getHddsDatanodes().get(0) port = cluster.getHddsDatanodes().get(0)

View File

@ -20,7 +20,6 @@ package org.apache.hadoop.ozone.web;
import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.rest.headers.Header; import org.apache.hadoop.ozone.client.rest.headers.Header;
@ -62,20 +61,13 @@ public class TestOzoneWebAccess {
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* *
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "local" , which uses a local directory to
* emulate Ozone backend.
*
* @throws IOException * @throws IOException
*/ */
@BeforeClass @BeforeClass
public static void init() throws Exception { public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration(); OzoneConfiguration conf = new OzoneConfiguration();
String path = GenericTestUtils
.getTempPath(TestOzoneWebAccess.class.getSimpleName());
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
cluster = MiniOzoneCluster.newBuilder(conf).build(); cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady(); cluster.waitForClusterToBeReady();
port = cluster.getHddsDatanodes().get(0) port = cluster.getHddsDatanodes().get(0)

View File

@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.web.client;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.VolumeArgs; import org.apache.hadoop.ozone.client.VolumeArgs;
@ -86,9 +85,7 @@ public class TestBuckets {
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "local" , which uses a local directory to
* emulate Ozone backend.
* *
* @throws IOException * @throws IOException
*/ */
@ -100,10 +97,6 @@ public class TestBuckets {
String path = GenericTestUtils String path = GenericTestUtils
.getTempPath(TestBuckets.class.getSimpleName()); .getTempPath(TestBuckets.class.getSimpleName());
path += conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
cluster = MiniOzoneCluster.newBuilder(conf) cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(3) .setNumDatanodes(3)
.build(); .build();

View File

@ -19,9 +19,9 @@ package org.apache.hadoop.ozone.web.client;
import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.RatisTestHelper; import org.apache.hadoop.ozone.RatisTestHelper;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Rule; import org.junit.Rule;
@ -58,7 +58,7 @@ public class TestKeysRatis {
@BeforeClass @BeforeClass
public static void init() throws Exception { public static void init() throws Exception {
suite = new RatisTestHelper.RatisTestSuite(TestBucketsRatis.class); suite = new RatisTestHelper.RatisTestSuite(TestBucketsRatis.class);
path = suite.getConf().get(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT); path = GenericTestUtils.getTempPath(TestKeysRatis.class.getSimpleName());
ozoneCluster = suite.getCluster(); ozoneCluster = suite.getCluster();
ozoneCluster.waitForClusterToBeReady(); ozoneCluster.waitForClusterToBeReady();
client = suite.newOzoneClient(); client = suite.newOzoneClient();

View File

@ -45,7 +45,6 @@ import io.netty.handler.logging.LoggingHandler;
import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.rest.headers.Header; import org.apache.hadoop.ozone.client.rest.headers.Header;
@ -92,8 +91,6 @@ public class TestOzoneClient {
public static void init() throws Exception { public static void init() throws Exception {
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.ALL); Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.ALL);
OzoneConfiguration conf = new OzoneConfiguration(); OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
cluster = MiniOzoneCluster.newBuilder(conf).build(); cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady(); cluster.waitForClusterToBeReady();
int port = cluster.getHddsDatanodes().get(0) int port = cluster.getHddsDatanodes().get(0)

View File

@ -23,7 +23,6 @@ import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.client.OzoneQuota; import org.apache.hadoop.hdds.client.OzoneQuota;
import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.client.VolumeArgs; import org.apache.hadoop.ozone.client.VolumeArgs;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.client.rest.RestClient; import org.apache.hadoop.ozone.client.rest.RestClient;
@ -79,9 +78,7 @@ public class TestVolume {
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "local" , which uses a local directory to
* emulate Ozone backend.
* *
* @throws IOException * @throws IOException
*/ */
@ -91,11 +88,8 @@ public class TestVolume {
String path = GenericTestUtils String path = GenericTestUtils
.getTempPath(TestVolume.class.getSimpleName()); .getTempPath(TestVolume.class.getSimpleName());
path += conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
FileUtils.deleteDirectory(new File(path)); FileUtils.deleteDirectory(new File(path));
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG); Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
cluster = MiniOzoneCluster.newBuilder(conf).build(); cluster = MiniOzoneCluster.newBuilder(conf).build();
@ -221,7 +215,8 @@ public class TestVolume {
client.createVolume(volumeName); client.createVolume(volumeName);
client.setVolumeQuota(volumeName, OzoneQuota.parseQuota("1000MB")); client.setVolumeQuota(volumeName, OzoneQuota.parseQuota("1000MB"));
OzoneVolume newVol = client.getVolumeDetails(volumeName); OzoneVolume newVol = client.getVolumeDetails(volumeName);
assertEquals(newVol.getQuota(), OzoneQuota.parseQuota("1000MB").sizeInBytes()); assertEquals(newVol.getQuota(),
OzoneQuota.parseQuota("1000MB").sizeInBytes());
// verify if the creation time is missing after setting quota operation // verify if the creation time is missing after setting quota operation
assertTrue(newVol.getCreationTime() > 0); assertTrue(newVol.getCreationTime() > 0);
} }

View File

@ -72,11 +72,8 @@ public class TestVolumeRatis {
String path = GenericTestUtils String path = GenericTestUtils
.getTempPath(TestVolume.class.getSimpleName()); .getTempPath(TestVolume.class.getSimpleName());
path += conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
FileUtils.deleteDirectory(new File(path)); FileUtils.deleteDirectory(new File(path));
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG); Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3).build(); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(3).build();

View File

@ -1,28 +1,44 @@
/** /**
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one or more
* or more contributor license agreements. See the NOTICE file * contributor license agreements. See the NOTICE file distributed with this
* distributed with this work for additional information * work for additional information regarding copyright ownership. The ASF
* regarding copyright ownership. The ASF licenses this file * licenses this file to you under the Apache License, Version 2.0 (the
* to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License.
* "License"); you may not use this file except in compliance * You may obtain a copy of the License at
* with the License. You may obtain a copy of the License at * <p>
* * http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0 * <p>
*
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* See the License for the specific language governing permissions and * License for the specific language governing permissions and limitations under
* limitations under the License. * the License.
*/ */
package org.apache.hadoop.hdfs.server.datanode; package org.apache.hadoop.hdfs.server.datanode;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients; import com.sun.jersey.api.container.ContainerFactory;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients; import com.sun.jersey.api.core.ApplicationAdapter;
import static org.apache.hadoop.ozone.OmUtils.getOmAddress; import org.apache.hadoop.conf.Configuration;
import static org.apache.hadoop.ozone.OzoneConfigKeys.*; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import static com.sun.jersey.api.core.ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS; import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolClientSideTranslatorPB;
import static com.sun.jersey.api.core.ResourceConfig.FEATURE_TRACE; import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
import org.apache.hadoop.ozone.web.ObjectStoreApplication;
import org.apache.hadoop.ozone.web.handlers.ServiceFilter;
import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
import org.apache.hadoop.ozone.web.netty.ObjectStoreJerseyContainer;
import org.apache.hadoop.ozone.web.storage.DistributedStorageHandler;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable; import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
@ -30,35 +46,13 @@ import java.net.InetSocketAddress;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import com.sun.jersey.api.container.ContainerFactory; import static com.sun.jersey.api.core.ResourceConfig.FEATURE_TRACE;
import com.sun.jersey.api.core.ApplicationAdapter; import static com.sun.jersey.api.core.ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients;
import org.apache.hadoop.io.IOUtils; import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; import static org.apache.hadoop.ozone.OmUtils.getOmAddress;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TRACE_ENABLED_DEFAULT;
import org.apache.hadoop.ozone.OzoneConsts; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TRACE_ENABLED_KEY;
import org.apache.hadoop.ozone.web.ObjectStoreApplication;
import org.apache.hadoop.ozone.web.handlers.ServiceFilter;
import org.apache.hadoop.ozone.web.netty.ObjectStoreJerseyContainer;
import org.apache.hadoop.hdds.scm.protocolPB
.ScmBlockLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB;
import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
import org.apache.hadoop.ozone.web.storage.DistributedStorageHandler;
import org.apache.hadoop.ozone.web.localstorage.LocalStorageHandler;
import org.apache.hadoop.security.UserGroupInformation;
/** /**
* Implements object store handling within the DataNode process. This class is * Implements object store handling within the DataNode process. This class is
@ -86,69 +80,49 @@ public final class ObjectStoreHandler implements Closeable {
* @throws IOException if there is an I/O error * @throws IOException if there is an I/O error
*/ */
public ObjectStoreHandler(Configuration conf) throws IOException { public ObjectStoreHandler(Configuration conf) throws IOException {
String shType = conf.getTrimmed(OZONE_HANDLER_TYPE_KEY,
OZONE_HANDLER_TYPE_DEFAULT);
LOG.info("ObjectStoreHandler initializing with {}: {}",
OZONE_HANDLER_TYPE_KEY, shType);
boolean ozoneTrace = conf.getBoolean(OZONE_TRACE_ENABLED_KEY, boolean ozoneTrace = conf.getBoolean(OZONE_TRACE_ENABLED_KEY,
OZONE_TRACE_ENABLED_DEFAULT); OZONE_TRACE_ENABLED_DEFAULT);
// Initialize Jersey container for object store web application. // Initialize Jersey container for object store web application.
if (OzoneConsts.OZONE_HANDLER_DISTRIBUTED.equalsIgnoreCase(shType)) { RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class, ProtobufRpcEngine.class);
ProtobufRpcEngine.class); long scmVersion =
long scmVersion = RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class);
RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class);
InetSocketAddress scmAddress = InetSocketAddress scmAddress =
getScmAddressForClients(conf); getScmAddressForClients(conf);
this.storageContainerLocationClient = this.storageContainerLocationClient =
new StorageContainerLocationProtocolClientSideTranslatorPB( new StorageContainerLocationProtocolClientSideTranslatorPB(
RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion, RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion,
scmAddress, UserGroupInformation.getCurrentUser(), conf, scmAddress, UserGroupInformation.getCurrentUser(), conf,
NetUtils.getDefaultSocketFactory(conf), NetUtils.getDefaultSocketFactory(conf),
Client.getRpcTimeout(conf))); Client.getRpcTimeout(conf)));
InetSocketAddress scmBlockAddress = InetSocketAddress scmBlockAddress =
getScmAddressForBlockClients(conf); getScmAddressForBlockClients(conf);
this.scmBlockLocationClient = this.scmBlockLocationClient =
new ScmBlockLocationProtocolClientSideTranslatorPB( new ScmBlockLocationProtocolClientSideTranslatorPB(
RPC.getProxy(ScmBlockLocationProtocolPB.class, scmVersion, RPC.getProxy(ScmBlockLocationProtocolPB.class, scmVersion,
scmBlockAddress, UserGroupInformation.getCurrentUser(), conf, scmBlockAddress, UserGroupInformation.getCurrentUser(), conf,
NetUtils.getDefaultSocketFactory(conf), NetUtils.getDefaultSocketFactory(conf),
Client.getRpcTimeout(conf))); Client.getRpcTimeout(conf)));
RPC.setProtocolEngine(conf, OzoneManagerProtocolPB.class, RPC.setProtocolEngine(conf, OzoneManagerProtocolPB.class,
ProtobufRpcEngine.class); ProtobufRpcEngine.class);
long omVersion = long omVersion =
RPC.getProtocolVersion(OzoneManagerProtocolPB.class); RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
InetSocketAddress omAddress = getOmAddress(conf); InetSocketAddress omAddress = getOmAddress(conf);
this.ozoneManagerClient = this.ozoneManagerClient =
new OzoneManagerProtocolClientSideTranslatorPB( new OzoneManagerProtocolClientSideTranslatorPB(
RPC.getProxy(OzoneManagerProtocolPB.class, omVersion, RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
omAddress, UserGroupInformation.getCurrentUser(), conf, omAddress, UserGroupInformation.getCurrentUser(), conf,
NetUtils.getDefaultSocketFactory(conf), NetUtils.getDefaultSocketFactory(conf),
Client.getRpcTimeout(conf))); Client.getRpcTimeout(conf)));
storageHandler = new DistributedStorageHandler( storageHandler = new DistributedStorageHandler(
new OzoneConfiguration(conf), new OzoneConfiguration(conf),
this.storageContainerLocationClient, this.storageContainerLocationClient,
this.ozoneManagerClient); this.ozoneManagerClient);
} else {
if (OzoneConsts.OZONE_HANDLER_LOCAL.equalsIgnoreCase(shType)) {
storageHandler = new LocalStorageHandler(conf);
this.storageContainerLocationClient = null;
this.scmBlockLocationClient = null;
this.ozoneManagerClient = null;
} else {
throw new IllegalArgumentException(
String.format("Unrecognized value for %s: %s,"
+ " Allowed values are %s,%s",
OZONE_HANDLER_TYPE_KEY, shType,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED,
OzoneConsts.OZONE_HANDLER_LOCAL));
}
}
ApplicationAdapter aa = ApplicationAdapter aa =
new ApplicationAdapter(new ObjectStoreApplication()); new ApplicationAdapter(new ObjectStoreApplication());
Map<String, Object> settingsMap = new HashMap<>(); Map<String, Object> settingsMap = new HashMap<>();

View File

@ -19,10 +19,11 @@
package org.apache.hadoop.ozone.web.handlers; package org.apache.hadoop.ozone.web.handlers;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.web.interfaces.StorageHandler; import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
import org.apache.hadoop.ozone.web.localstorage.LocalStorageHandler; import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
/** /**
* This class is responsible for providing a {@link StorageHandler} * This class is responsible for providing a {@link StorageHandler}
@ -31,6 +32,9 @@ import org.apache.hadoop.ozone.web.localstorage.LocalStorageHandler;
@InterfaceAudience.Private @InterfaceAudience.Private
public final class StorageHandlerBuilder { public final class StorageHandlerBuilder {
private static final Logger LOG =
LoggerFactory.getLogger(StorageHandlerBuilder.class);
private static final ThreadLocal<StorageHandler> private static final ThreadLocal<StorageHandler>
STORAGE_HANDLER_THREAD_LOCAL = new ThreadLocal<>(); STORAGE_HANDLER_THREAD_LOCAL = new ThreadLocal<>();
@ -40,15 +44,15 @@ public final class StorageHandlerBuilder {
* *
* @return StorageHandler from thread-local storage * @return StorageHandler from thread-local storage
*/ */
public static StorageHandler getStorageHandler() { public static StorageHandler getStorageHandler() throws IOException {
StorageHandler storageHandler = STORAGE_HANDLER_THREAD_LOCAL.get(); StorageHandler storageHandler = STORAGE_HANDLER_THREAD_LOCAL.get();
if (storageHandler != null) { if (storageHandler != null) {
return storageHandler; return storageHandler;
} else { } else {
// This only happens while using mvn jetty:run for testing. LOG.error("No Storage Handler Configured.");
Configuration conf = new OzoneConfiguration(); throw new IOException("Invalid Handler Configuration");
return new LocalStorageHandler(conf);
} }
} }
/** /**

View File

@ -1,385 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.web.localstorage;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ozone.client.io.LengthInputStream;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.web.handlers.BucketArgs;
import org.apache.hadoop.ozone.web.handlers.KeyArgs;
import org.apache.hadoop.ozone.web.handlers.ListArgs;
import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.KeyInfo;
import org.apache.hadoop.ozone.web.response.ListBuckets;
import org.apache.hadoop.ozone.web.response.ListKeys;
import org.apache.hadoop.ozone.web.response.ListVolumes;
import org.apache.hadoop.ozone.web.response.VolumeInfo;
import java.io.IOException;
import java.io.OutputStream;
/**
* PLEASE NOTE : This file is a dummy backend for test purposes and prototyping
* effort only. It does not handle any Object semantics correctly, neither does
* it take care of security.
*/
@InterfaceAudience.Private
public class LocalStorageHandler implements StorageHandler {
private final Configuration conf;
/**
* Constructs LocalStorageHandler.
*
* @param conf ozone conf.
*/
public LocalStorageHandler(Configuration conf) {
this.conf = conf;
}
/**
* Creates Storage Volume.
*
* @param args - volumeArgs
* @throws IOException
*/
@Override
public void createVolume(VolumeArgs args) throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
oz.createVolume(args);
}
/**
* setVolumeOwner - sets the owner of the volume.
*
* @param args volumeArgs
* @throws IOException
*/
@Override
public void setVolumeOwner(VolumeArgs args)
throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
oz.setVolumeProperty(args, OzoneMetadataManager.VolumeProperty.OWNER);
}
/**
* Set Volume Quota Info.
*
* @param args - volumeArgs
* @param remove - true if the request is to remove the quota
* @throws IOException
*/
@Override
public void setVolumeQuota(VolumeArgs args, boolean remove)
throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
if (remove) {
OzoneQuota quota = new OzoneQuota();
args.setQuota(quota);
}
oz.setVolumeProperty(args, OzoneMetadataManager.VolumeProperty.QUOTA);
}
/**
* Checks if a Volume exists and the user specified has access to the volume.
*
* @param volume - Volume Name
* @param acl - Ozone acl which needs to be compared for access
* @return - Boolean - True if the user can modify the volume. This is
* possible for owners of the volume and admin users
* @throws IOException
*/
@Override
public boolean checkVolumeAccess(String volume, OzoneAcl acl)
throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
return oz.checkVolumeAccess(volume, acl);
}
/**
* Returns Info about the specified Volume.
*
* @param args - volumeArgs
* @return VolumeInfo
* @throws IOException
*/
@Override
public VolumeInfo getVolumeInfo(VolumeArgs args)
throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
return oz.getVolumeInfo(args);
}
/**
* Deletes an Empty Volume.
*
* @param args - Volume Args
* @throws IOException
*/
@Override
public void deleteVolume(VolumeArgs args) throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
oz.deleteVolume(args);
}
/**
* Returns the List of Volumes owned by the specific user.
*
* @param args - ListArgs
* @return - List of Volumes
* @throws IOException
*/
@Override
public ListVolumes listVolumes(ListArgs args)
throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
return oz.listVolumes(args);
}
/**
* true if the bucket exists and user has read access to the bucket else
* throws Exception.
*
* @param args Bucket args structure
* @throws IOException
*/
@Override
public void checkBucketAccess(BucketArgs args)
throws IOException, OzoneException {
}
/**
* Creates a Bucket in specified Volume.
*
* @param args BucketArgs- BucketName, UserName and Acls
* @throws IOException
*/
@Override
public void createBucket(BucketArgs args) throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
oz.createBucket(args);
}
/**
* Adds or Removes ACLs from a Bucket.
*
* @param args - BucketArgs
* @throws IOException
*/
@Override
public void setBucketAcls(BucketArgs args)
throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
oz.setBucketProperty(args, OzoneMetadataManager.BucketProperty.ACLS);
}
/**
* Enables or disables Bucket Versioning.
*
* @param args - BucketArgs
* @throws IOException
*/
@Override
public void setBucketVersioning(BucketArgs args)
throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
oz.setBucketProperty(args, OzoneMetadataManager.BucketProperty.VERSIONING);
}
/**
* Sets the Storage Class of a Bucket.
*
* @param args - BucketArgs
* @throws IOException
*/
@Override
public void setBucketStorageClass(BucketArgs args)
throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
oz.setBucketProperty(args, OzoneMetadataManager.BucketProperty.STORAGETYPE);
}
/**
* Deletes a bucket if it is empty.
*
* @param args Bucket args structure
* @throws IOException
*/
@Override
public void deleteBucket(BucketArgs args) throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
oz.deleteBucket(args);
}
/**
* Returns all Buckets of a specified Volume.
*
* @param args --User Args
* @return ListAllBuckets
* @throws OzoneException
*/
@Override
public ListBuckets listBuckets(ListArgs args)
throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
return oz.listBuckets(args);
}
/**
* Returns Bucket's Metadata as a String.
*
* @param args Bucket args structure
* @return Info about the bucket
* @throws IOException
*/
@Override
public BucketInfo getBucketInfo(BucketArgs args)
throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
return oz.getBucketInfo(args);
}
/**
* Writes a key in an existing bucket.
*
* @param args KeyArgs
* @return InputStream
* @throws OzoneException
*/
@Override
public OutputStream newKeyWriter(KeyArgs args) throws IOException,
OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
return oz.createKey(args);
}
/**
* Tells the file system that the object has been written out completely and
* it can do any house keeping operation that needs to be done.
*
* @param args Key Args
* @param stream
* @throws IOException
*/
@Override
public void commitKey(KeyArgs args, OutputStream stream) throws
IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
oz.commitKey(args, stream);
}
/**
* Reads a key from an existing bucket.
*
* @param args KeyArgs
* @return LengthInputStream
* @throws IOException
*/
@Override
public LengthInputStream newKeyReader(KeyArgs args) throws IOException,
OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
return oz.newKeyReader(args);
}
/**
* Deletes an existing key.
*
* @param args KeyArgs
* @throws OzoneException
*/
@Override
public void deleteKey(KeyArgs args) throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
oz.deleteKey(args);
}
@Override
public void renameKey(KeyArgs args, String toKeyName)
throws IOException, OzoneException {
throw new UnsupportedOperationException("Not yet implemented");
}
/**
* Returns a list of Key.
*
* @param args KeyArgs
* @return BucketList
* @throws IOException
*/
@Override
public ListKeys listKeys(ListArgs args) throws IOException, OzoneException {
OzoneMetadataManager oz =
OzoneMetadataManager.getOzoneMetadataManager(conf);
return oz.listKeys(args);
}
/**
* Get information of the specified Key.
*
* @param args Key Args
*
* @return KeyInfo
*
* @throws IOException
* @throws OzoneException
*/
@Override
public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException {
OzoneMetadataManager oz = OzoneMetadataManager
.getOzoneMetadataManager(conf);
return oz.getKeyInfo(args);
}
@Override
public void close() {
//No resource to close, do nothing.
}
}

View File

@ -57,8 +57,7 @@ public class TestOzoneFSInputStream {
/** /**
* Create a MiniDFSCluster for testing. * Create a MiniDFSCluster for testing.
* <p> * <p>
* Ozone is made active by setting OZONE_ENABLED = true and * Ozone is made active by setting OZONE_ENABLED = true
* OZONE_HANDLER_TYPE_KEY = "distributed"
* *
* @throws IOException * @throws IOException
*/ */
@ -90,9 +89,6 @@ public class TestOzoneFSInputStream {
// Fetch the host and port for File System init // Fetch the host and port for File System init
DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0) DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0)
.getDatanodeDetails(); .getDatanodeDetails();
int port = datanodeDetails
.getPort(DatanodeDetails.Port.Name.REST).getValue();
String host = datanodeDetails.getHostName();
// Set the fs.defaultFS and start the filesystem // Set the fs.defaultFS and start the filesystem
String uri = String.format("%s://%s.%s/", String uri = String.format("%s://%s.%s/",