HDDS-1358 : Recon Server REST API not working as expected. (#668)

This commit is contained in:
avijayanhwx 2019-04-03 14:52:06 -07:00 committed by Arpit Agarwal
parent e62cbcbc83
commit 8ff41d6243
12 changed files with 356 additions and 41 deletions

View File

@ -27,6 +27,20 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
<exclusions>
<exclusion>
<artifactId>jersey-server</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-core</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-servlet</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@ -40,20 +54,63 @@
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
<version>4.1.0</version>
<scope>compile</scope>
<version>${guice.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId>
<version>2.27</version>
<exclusions>
<exclusion>
<groupId>org.glassfish.hk2</groupId>
<artifactId>hk2-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet-core</artifactId>
<version>2.27</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.glassfish.hk2</groupId>
<artifactId>guice-bridge</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-server</artifactId>
<version>2.27</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>2.27</version>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-assistedinject</artifactId>
<version>4.1.0</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.inject</groupId>
<artifactId>jersey-hk2</artifactId>
<version>2.27</version>
<exclusions>
<exclusion>
<artifactId>hk2-api</artifactId>
<groupId>org.glassfish.hk2</groupId>
</exclusion>
<exclusion>
<groupId>org.glassfish.hk2.external</groupId>
<artifactId>aopalliance-repackaged</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.hk2</groupId>
<artifactId>hk2-utils</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
@ -70,6 +127,12 @@
<artifactId>powermock-module-junit4</artifactId>
<version>1.7.4</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.javassist</groupId>
<artifactId>javassist</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.powermock</groupId>

View File

@ -15,15 +15,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.recon;
import org.glassfish.jersey.server.ResourceConfig;
import com.google.inject.Injector;
import com.google.inject.servlet.GuiceServletContextListener;
/**
* JaxRS resource definition.
* Servlet Context Listener that provides the Guice injector.
*/
public class ReconApplication extends ResourceConfig {
public ReconApplication() {
packages("org.apache.hadoop.ozone.recon.api");
public class ReconGuiceServletContextListener
extends GuiceServletContextListener {
private static Injector injector;
@Override
public Injector getInjector() {
return injector;
}
static void setInjector(Injector inj) {
injector = inj;
}
}

View File

@ -0,0 +1,134 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.recon;
import java.net.URL;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.glassfish.hk2.api.ServiceLocator;
import org.glassfish.jersey.internal.inject.InjectionManager;
import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.server.spi.Container;
import org.glassfish.jersey.server.spi.ContainerLifecycleListener;
import org.glassfish.jersey.servlet.ServletContainer;
import org.jvnet.hk2.guice.bridge.api.GuiceBridge;
import org.jvnet.hk2.guice.bridge.api.GuiceIntoHK2Bridge;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.inject.Injector;
import com.google.inject.Scopes;
import com.google.inject.servlet.ServletModule;
/**
* Class to scan API Service classes and bind them to the injector.
*/
public abstract class ReconRestServletModule extends ServletModule {
private static final Logger LOG =
LoggerFactory.getLogger(ReconRestServletModule.class);
@Override
abstract protected void configureServlets();
/**
* Interface to provide packages for scanning.
*/
public interface RestKeyBindingBuilder {
void packages(String... packages);
}
protected RestKeyBindingBuilder rest(String... urlPatterns) {
return new RestKeyBindingBuilderImpl(Arrays.asList(urlPatterns));
}
private class RestKeyBindingBuilderImpl implements RestKeyBindingBuilder {
private List<String> paths;
RestKeyBindingBuilderImpl(List<String> paths) {
this.paths = paths;
}
private void checkIfPackageExistsAndLog(String pkg) {
String resourcePath = pkg.replace(".", "/");
URL resource = getClass().getClassLoader().getResource(resourcePath);
if (resource != null) {
LOG.info("rest(" + paths + ").packages(" + pkg + ")");
} else {
LOG.info("No Beans in '" + pkg + "' found. Requests " + paths
+ " will fail.");
}
}
@Override
public void packages(String... packages) {
StringBuilder sb = new StringBuilder();
for (String pkg : packages) {
if (sb.length() > 0) {
sb.append(',');
}
checkIfPackageExistsAndLog(pkg);
sb.append(pkg);
}
Map<String, String> params = new HashMap<>();
params.put("javax.ws.rs.Application",
GuiceResourceConfig.class.getCanonicalName());
if (sb.length() > 0) {
params.put("jersey.config.server.provider.packages", sb.toString());
}
bind(ServletContainer.class).in(Scopes.SINGLETON);
for (String path : paths) {
serve(path).with(ServletContainer.class, params);
}
}
}
}
/**
* Class to bridge Guice bindings to Jersey hk2 bindings.
*/
class GuiceResourceConfig extends ResourceConfig {
GuiceResourceConfig() {
register(new ContainerLifecycleListener() {
public void onStartup(Container container) {
ServletContainer servletContainer = (ServletContainer) container;
InjectionManager injectionManager = container.getApplicationHandler()
.getInjectionManager();
ServiceLocator serviceLocator = injectionManager
.getInstance(ServiceLocator.class);
GuiceBridge.getGuiceBridge().initializeGuiceBridge(serviceLocator);
GuiceIntoHK2Bridge guiceBridge = serviceLocator
.getService(GuiceIntoHK2Bridge.class);
Injector injector = (Injector) servletContainer.getServletContext()
.getAttribute(Injector.class.getName());
guiceBridge.bridgeGuiceInjector(injector);
}
public void onReload(Container container) {
}
public void onShutdown(Container container) {
}
});
}
}

View File

@ -62,7 +62,17 @@ public Void call() throws Exception {
OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
OzoneConfigurationProvider.setConfiguration(ozoneConfiguration);
injector = Guice.createInjector(new ReconControllerModule());
injector = Guice.createInjector(new
ReconControllerModule(), new ReconRestServletModule() {
@Override
protected void configureServlets() {
rest("/api/*")
.packages("org.apache.hadoop.ozone.recon.api");
}
});
//Pass on injector to listener that does the Guice - Jersey HK2 bridging.
ReconGuiceServletContextListener.setInjector(injector);
httpServer = injector.getInstance(ReconHttpServer.class);
LOG.info("Starting Recon server");

View File

@ -19,32 +19,45 @@
import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.recon.ReconServer;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.inject.Inject;
/**
* Endpoint for querying keys that belong to a container.
*/
@Path("/containers")
@Produces(MediaType.APPLICATION_JSON)
public class ContainerKeyService {
private static final Logger LOG = LoggerFactory.getLogger(ReconServer.class);
@Inject
private ContainerDBServiceProvider containerDBServiceProvider;
@ -59,7 +72,7 @@ public class ContainerKeyService {
* @return {@link Response}
*/
@GET
@Path("{id}")
@Path("/{id}")
public Response getKeysForContainer(@PathParam("id") Long containerId) {
Map<String, KeyMetadata> keyMetadataMap = new HashMap<>();
try {
@ -80,19 +93,36 @@ public Response getKeysForContainer(@PathParam("id") Long containerId) {
}
// Filter keys by version.
List<Long> matchedVersions = omKeyInfo.getKeyLocationVersions()
List<OmKeyLocationInfoGroup> matchedKeys = omKeyInfo
.getKeyLocationVersions()
.stream()
.filter(k -> (k.getVersion() == containerKeyPrefix.getKeyVersion()))
.mapToLong(OmKeyLocationInfoGroup::getVersion)
.boxed()
.collect(Collectors.toList());
List<ContainerBlockMetadata> blockIds = new ArrayList<>();
for (OmKeyLocationInfoGroup omKeyLocationInfoGroup : matchedKeys) {
List<OmKeyLocationInfo> omKeyLocationInfos = omKeyLocationInfoGroup
.getLocationList()
.stream()
.filter(c -> c.getContainerID() == containerId)
.collect(Collectors.toList());
for (OmKeyLocationInfo omKeyLocationInfo : omKeyLocationInfos) {
blockIds.add(new ContainerBlockMetadata(omKeyLocationInfo
.getContainerID(), omKeyLocationInfo.getLocalID()));
}
}
String ozoneKey = omMetadataManager.getOzoneKey(
omKeyInfo.getVolumeName(),
omKeyInfo.getBucketName(),
omKeyInfo.getKeyName());
if (keyMetadataMap.containsKey(ozoneKey)) {
keyMetadataMap.get(ozoneKey).getVersions().addAll(matchedVersions);
keyMetadataMap.get(ozoneKey).getVersions()
.add(containerKeyPrefix.getKeyVersion());
keyMetadataMap.get(ozoneKey).getBlockIds().putAll(
Collections.singletonMap(containerKeyPrefix.getKeyVersion(),
blockIds));
} else {
KeyMetadata keyMetadata = new KeyMetadata();
keyMetadata.setBucket(omKeyInfo.getBucketName());
@ -103,8 +133,14 @@ public Response getKeysForContainer(@PathParam("id") Long containerId) {
keyMetadata.setModificationTime(
Instant.ofEpochMilli(omKeyInfo.getModificationTime()));
keyMetadata.setDataSize(omKeyInfo.getDataSize());
keyMetadata.setVersions(matchedVersions);
keyMetadata.setVersions(new ArrayList<Long>() {{
add(containerKeyPrefix.getKeyVersion());
}});
keyMetadataMap.put(ozoneKey, keyMetadata);
keyMetadata.setBlockIds(new TreeMap<Long,
List<ContainerBlockMetadata>>() {{
put(containerKeyPrefix.getKeyVersion(), blockIds);
}});
}
}
} catch (IOException ioEx) {

View File

@ -19,15 +19,18 @@
import java.time.Instant;
import java.util.List;
import java.util.Map;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
/**
* Metadata object represents one key in the object store.
*/
@XmlRootElement (name = "KeyMetadata")
@XmlAccessorType(XmlAccessType.FIELD)
public class KeyMetadata {
@ -46,6 +49,9 @@ public class KeyMetadata {
@XmlElement(name = "Versions")
private List<Long> versions;
@XmlElement(name = "Blocks")
private Map<Long, List<ContainerBlockMetadata>> blockIds;
@XmlJavaTypeAdapter(IsoDateAdapter.class)
@XmlElement(name = "CreationTime")
private Instant creationTime;
@ -109,4 +115,33 @@ public List<Long> getVersions() {
public void setVersions(List<Long> versions) {
this.versions = versions;
}
public Map<Long, List<ContainerBlockMetadata>> getBlockIds() {
return blockIds;
}
public void setBlockIds(Map<Long, List<ContainerBlockMetadata>> blockIds) {
this.blockIds = blockIds;
}
/**
* Class to hold ContainerID and BlockID.
*/
public static class ContainerBlockMetadata {
private long containerID;
private long localID;
public ContainerBlockMetadata(long containerID, long localID) {
this.containerID = containerID;
this.localID = localID;
}
public long getContainerID() {
return containerID;
}
public long getLocalID() {
return localID;
}
}
}

View File

@ -88,7 +88,7 @@ public void updateOmDB(File newDbLocation) throws IOException {
if (oldDBLocation.exists()) {
LOG.info("Cleaning up old OM snapshot db at {}.",
oldDBLocation.getAbsolutePath());
FileUtils.deleteQuietly(oldDBLocation);
FileUtils.deleteDirectory(oldDBLocation);
}
}
initializeNewRdbStore(newDbLocation);

View File

@ -88,7 +88,7 @@ public void initNewContainerDB(Map<ContainerKeyPrefix, Integer>
if (oldDBLocation.exists()) {
LOG.info("Cleaning up old Recon Container DB at {}.",
oldDBLocation.getAbsolutePath());
FileUtils.deleteQuietly(oldDBLocation);
FileUtils.deleteDirectory(oldDBLocation);
}
for (Map.Entry<ContainerKeyPrefix, Integer> entry :
containerKeyPrefixCounts.entrySet()) {

View File

@ -85,8 +85,13 @@ public void run() {
long containerId = omKeyLocationInfo.getContainerID();
ContainerKeyPrefix containerKeyPrefix = new ContainerKeyPrefix(
containerId, key.toString(), keyVersion);
containerDBServiceProvider.storeContainerKeyMapping(
containerKeyPrefix, 1);
if (containerDBServiceProvider.getCountForForContainerKeyPrefix(
containerKeyPrefix) == 0) {
// Save on writes. No need to save same container-key prefix
// mapping again.
containerDBServiceProvider.storeContainerKeyMapping(
containerKeyPrefix, 1);
}
containerCount++;
}
}

View File

@ -14,20 +14,15 @@
<web-app version="3.0" xmlns="http://java.sun.com/xml/ns/javaee"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd">
<servlet>
<servlet-name>jaxrs</servlet-name>
<servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
<init-param>
<param-name>javax.ws.rs.Application</param-name>
<param-value>org.apache.hadoop.ozone.recon.ReconApplication</param-value>
</init-param>
<load-on-startup>1</load-on-startup>
</servlet>
<servlet-mapping>
<servlet-name>jaxrs</servlet-name>
<listener>
<listener-class>org.apache.hadoop.ozone.recon.ReconGuiceServletContextListener</listener-class>
</listener>
<filter>
<filter-name>guiceFilter</filter-name>
<filter-class>com.google.inject.servlet.GuiceFilter</filter-class>
</filter>
<filter-mapping>
<filter-name>guiceFilter</filter-name>
<url-pattern>/*</url-pattern>
</servlet-mapping>
</filter-mapping>
</web-app>

View File

@ -31,6 +31,7 @@
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.ws.rs.core.Response;
@ -81,7 +82,7 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
private ReconOMMetadataManager reconOMMetadataManager;
private Injector injector;
private OzoneManagerServiceProviderImpl ozoneManagerServiceProvider;
private ContainerKeyService containerKeyService = new ContainerKeyService();
private ContainerKeyService containerKeyService;
@Before
public void setUp() throws Exception {
@ -98,11 +99,12 @@ protected void configure() {
in(Singleton.class);
bind(ContainerDBServiceProvider.class).to(
ContainerDBServiceProviderImpl.class).in(Singleton.class);
bind(ContainerKeyService.class).toInstance(containerKeyService);
ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(
getTestOzoneConfiguration());
bind(OzoneManagerServiceProvider.class)
.toInstance(ozoneManagerServiceProvider);
containerKeyService = new ContainerKeyService();
bind(ContainerKeyService.class).toInstance(containerKeyService);
} catch (IOException e) {
Assert.fail();
}
@ -119,12 +121,12 @@ public void testGetKeysForContainer() throws Exception {
Pipeline pipeline = getRandomPipeline();
List<OmKeyLocationInfo> omKeyLocationInfoList = new ArrayList<>();
BlockID blockID1 = new BlockID(1, 1);
BlockID blockID1 = new BlockID(1, 101);
OmKeyLocationInfo omKeyLocationInfo1 = getOmKeyLocationInfo(blockID1,
pipeline);
omKeyLocationInfoList.add(omKeyLocationInfo1);
BlockID blockID2 = new BlockID(2, 1);
BlockID blockID2 = new BlockID(2, 102);
OmKeyLocationInfo omKeyLocationInfo2 = getOmKeyLocationInfo(blockID2,
pipeline);
omKeyLocationInfoList.add(omKeyLocationInfo2);
@ -138,7 +140,7 @@ public void testGetKeysForContainer() throws Exception {
Collections.singletonList(omKeyLocationInfoGroup));
List<OmKeyLocationInfoGroup> infoGroups = new ArrayList<>();
BlockID blockID3 = new BlockID(1, 2);
BlockID blockID3 = new BlockID(1, 103);
OmKeyLocationInfo omKeyLocationInfo3 = getOmKeyLocationInfo(blockID3,
pipeline);
@ -147,7 +149,7 @@ public void testGetKeysForContainer() throws Exception {
infoGroups.add(new OmKeyLocationInfoGroup(0,
omKeyLocationInfoListNew));
BlockID blockID4 = new BlockID(1, 3);
BlockID blockID4 = new BlockID(1, 104);
OmKeyLocationInfo omKeyLocationInfo4 = getOmKeyLocationInfo(blockID4,
pipeline);
@ -186,12 +188,20 @@ public void testGetKeysForContainer() throws Exception {
KeyMetadata keyMetadata = iterator.next();
assertTrue(keyMetadata.getKey().equals("key_one"));
assertTrue(keyMetadata.getVersions().size() == 1);
assertTrue(keyMetadata.getBlockIds().size() == 1);
Map<Long, List<KeyMetadata.ContainerBlockMetadata>> blockIds =
keyMetadata.getBlockIds();
assertTrue(blockIds.get(0L).iterator().next().getLocalID() == 101);
keyMetadata = iterator.next();
assertTrue(keyMetadata.getKey().equals("key_two"));
assertTrue(keyMetadata.getVersions().size() == 2);
assertTrue(keyMetadata.getVersions().contains(0L) && keyMetadata
.getVersions().contains(1L));
assertTrue(keyMetadata.getBlockIds().size() == 2);
blockIds = keyMetadata.getBlockIds();
assertTrue(blockIds.get(0L).iterator().next().getLocalID() == 103);
assertTrue(blockIds.get(1L).iterator().next().getLocalID() == 104);
response = containerKeyService.getKeysForContainer(3L);
keyMetadataList = (Collection<KeyMetadata>) response.getEntity();

View File

@ -55,6 +55,22 @@
<groupId>org.glassfish.jersey.inject</groupId>
<artifactId>jersey-hk2</artifactId>
<version>2.27</version>
<!-- The versions of these excluded dependencies are 2.5.0-beta. The
newer and release version 2.5.0 is being brought in by ozone-recon -->
<exclusions>
<exclusion>
<artifactId>hk2-api</artifactId>
<groupId>org.glassfish.hk2</groupId>
</exclusion>
<exclusion>
<groupId>org.glassfish.hk2.external</groupId>
<artifactId>aopalliance-repackaged</artifactId>
</exclusion>
<exclusion>
<groupId>org.glassfish.hk2</groupId>
<artifactId>hk2-utils</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>