big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-1670. Add limit support to /api/containers and /api/containers/{id} endpoints (#954)

Browse Source
This commit is contained in:
Vivek Ratnavel Subramanian 2019-06-18 08:51:16 -07:00 committed by Bharat Viswanadham
parent 335c1c9938
commit fb1ce0d50a
4 changed files with 80 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java
View File

@ -21,17 +21,19 @@
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
@ -72,10 +74,11 @@ public class ContainerKeyService {
* @return {@link Response}
*/
@GET
public Response getContainers() {
public Response getContainers(
@DefaultValue("-1") @QueryParam("limit") int limit) {
Map<Long, ContainerMetadata> containersMap;
try {
containersMap = containerDBServiceProvider.getContainers();
containersMap = containerDBServiceProvider.getContainers(limit);
} catch (IOException ioEx) {
throw new WebApplicationException(ioEx,
Response.Status.INTERNAL_SERVER_ERROR);
@ -92,8 +95,10 @@ public Response getContainers() {
*/
@GET
@Path("/{id}")
public Response getKeysForContainer(@PathParam("id") Long containerId) {
Map<String, KeyMetadata> keyMetadataMap = new HashMap<>();
public Response getKeysForContainer(
@PathParam("id") Long containerId,
@DefaultValue("-1") @QueryParam("limit") int limit) {
Map<String, KeyMetadata> keyMetadataMap = new LinkedHashMap<>();
try {
Map<ContainerKeyPrefix, Integer> containerKeyPrefixMap =
containerDBServiceProvider.getKeyPrefixesForContainer(containerId);
@ -143,6 +148,10 @@ public Response getKeysForContainer(@PathParam("id") Long containerId) {
Collections.singletonMap(containerKeyPrefix.getKeyVersion(),
blockIds));
} else {
// break the for loop if limit has been reached
if (keyMetadataMap.size() == limit) {
break;
}
KeyMetadata keyMetadata = new KeyMetadata();
keyMetadata.setBucket(omKeyInfo.getBucketName());
keyMetadata.setVolume(omKeyInfo.getVolumeName());

12
hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
View File

@ -70,13 +70,23 @@ Map<ContainerKeyPrefix, Integer> getKeyPrefixesForContainer(long containerId)
throws IOException;
/**
* Get a Map of containerID, containerMetadata of all Containers.
* Get a Map of containerID, containerMetadata of all the Containers.
*
* @return Map of containerID -> containerMetadata.
* @throws IOException
*/
Map<Long, ContainerMetadata> getContainers() throws IOException;
/**
* Get a Map of containerID, containerMetadata of Containers only for the
* given limit. If the limit is -1 or any integer <0, then return all
* the containers without any limit.
*
* @return Map of containerID -> containerMetadata.
* @throws IOException
*/
Map<Long, ContainerMetadata> getContainers(int limit) throws IOException;
/**
* Delete an entry in the container DB.
* @param containerKeyPrefix container key prefix to be deleted.

27
hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
View File

@ -22,7 +22,6 @@
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
@ -139,7 +138,7 @@ public Integer getCountForForContainerKeyPrefix(
public Map<ContainerKeyPrefix, Integer> getKeyPrefixesForContainer(
long containerId) throws IOException {
Map<ContainerKeyPrefix, Integer> prefixes = new HashMap<>();
Map<ContainerKeyPrefix, Integer> prefixes = new LinkedHashMap<>();
TableIterator<ContainerKeyPrefix, ? extends KeyValue<ContainerKeyPrefix,
Integer>> containerIterator = containerKeyTable.iterator();
containerIterator.seek(new ContainerKeyPrefix(containerId));
@ -166,13 +165,29 @@ public Map<ContainerKeyPrefix, Integer> getKeyPrefixesForContainer(
}
/**
* Iterate the DB to construct a Map of containerID -> containerMetadata.
* Get all the containers.
*
* @return Map of containerID -> containerMetadata.
* @throws IOException
*/
@Override
public Map<Long, ContainerMetadata> getContainers() throws IOException {
// Set a negative limit to get all the containers.
return getContainers(-1);
}
/**
* Iterate the DB to construct a Map of containerID -> containerMetadata
* only for the given limit.
*
* Return all the containers if limit < 0.
*
* @return Map of containerID -> containerMetadata.
* @throws IOException
*/
@Override
public Map<Long, ContainerMetadata> getContainers(int limit)
throws IOException {
Map<Long, ContainerMetadata> containers = new LinkedHashMap<>();
TableIterator<ContainerKeyPrefix, ? extends KeyValue<ContainerKeyPrefix,
Integer>> containerIterator = containerKeyTable.iterator();
@ -181,6 +196,12 @@ public Map<Long, ContainerMetadata> getContainers() throws IOException {
Long containerID = keyValue.getKey().getContainerId();
Integer numberOfKeys = keyValue.getValue();
// break the loop if limit has been reached
// and one more new entity needs to be added to the containers map
if (containers.size() == limit && !containers.containsKey(containerID)) {
break;
}
// initialize containerMetadata with 0 as number of keys.
containers.computeIfAbsent(containerID, ContainerMetadata::new);
// increment number of keys for the containerID

51
hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
View File

@ -20,6 +20,7 @@
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
@ -200,56 +201,67 @@ protected void configure() {
@Test
public void testGetKeysForContainer() {
Response response = containerKeyService.getKeysForContainer(1L);
Response response = containerKeyService.getKeysForContainer(1L, -1);
Collection<KeyMetadata> keyMetadataList =
(Collection<KeyMetadata>) response.getEntity();
assertTrue(keyMetadataList.size() == 2);
assertEquals(keyMetadataList.size(), 2);
Iterator<KeyMetadata> iterator = keyMetadataList.iterator();
KeyMetadata keyMetadata = iterator.next();
assertTrue(keyMetadata.getKey().equals("key_one"));
assertTrue(keyMetadata.getVersions().size() == 1);
assertTrue(keyMetadata.getBlockIds().size() == 1);
assertEquals(keyMetadata.getKey(), "key_one");
assertEquals(keyMetadata.getVersions().size(), 1);
assertEquals(keyMetadata.getBlockIds().size(), 1);
Map<Long, List<KeyMetadata.ContainerBlockMetadata>> blockIds =
keyMetadata.getBlockIds();
assertTrue(blockIds.get(0L).iterator().next().getLocalID() == 101);
assertEquals(blockIds.get(0L).iterator().next().getLocalID(), 101);
keyMetadata = iterator.next();
assertTrue(keyMetadata.getKey().equals("key_two"));
assertTrue(keyMetadata.getVersions().size() == 2);
assertEquals(keyMetadata.getKey(), "key_two");
assertEquals(keyMetadata.getVersions().size(), 2);
assertTrue(keyMetadata.getVersions().contains(0L) && keyMetadata
.getVersions().contains(1L));
assertTrue(keyMetadata.getBlockIds().size() == 2);
assertEquals(keyMetadata.getBlockIds().size(), 2);
blockIds = keyMetadata.getBlockIds();
assertTrue(blockIds.get(0L).iterator().next().getLocalID() == 103);
assertTrue(blockIds.get(1L).iterator().next().getLocalID() == 104);
assertEquals(blockIds.get(0L).iterator().next().getLocalID(), 103);
assertEquals(blockIds.get(1L).iterator().next().getLocalID(), 104);
response = containerKeyService.getKeysForContainer(3L);
response = containerKeyService.getKeysForContainer(3L, -1);
keyMetadataList = (Collection<KeyMetadata>) response.getEntity();
assertTrue(keyMetadataList.isEmpty());
// test if limit works as expected
response = containerKeyService.getKeysForContainer(1L, 1);
keyMetadataList = (Collection<KeyMetadata>) response.getEntity();
assertEquals(keyMetadataList.size(), 1);
}
@Test
public void testGetContainers() {
Response response = containerKeyService.getContainers();
Response response = containerKeyService.getContainers(-1);
List<ContainerMetadata> containers = new ArrayList<>(
(Collection<ContainerMetadata>) response.getEntity());
assertTrue(containers.size() == 2);
Iterator<ContainerMetadata> iterator = containers.iterator();
ContainerMetadata containerMetadata = iterator.next();
assertTrue(containerMetadata.getContainerID() == 1L);
assertTrue(containerMetadata.getNumberOfKeys() == 3L);
assertEquals(containerMetadata.getContainerID(), 1L);
// Number of keys for CID:1 should be 3 because of two different versions
// of key_two stored in CID:1
assertEquals(containerMetadata.getNumberOfKeys(), 3L);
containerMetadata = iterator.next();
assertTrue(containerMetadata.getContainerID() == 2L);
assertTrue(containerMetadata.getNumberOfKeys() == 2L);
assertEquals(containerMetadata.getContainerID(), 2L);
assertEquals(containerMetadata.getNumberOfKeys(), 2L);
// test if limit works as expected
response = containerKeyService.getContainers(1);
containers = new ArrayList<>(
(Collection<ContainerMetadata>) response.getEntity());
assertEquals(containers.size(), 1);
}
/**
@ -266,5 +278,4 @@ private OzoneConfiguration getTestOzoneConfiguration()
.getAbsolutePath());
return configuration;
}
}