HDDS-1358 : Recon Server REST API not working as expected. (#668)

hadoop-ozone/ozone-recon/pom.xml

@@ -27,6 +27,20 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-ozone-common</artifactId>
+      <exclusions>
+        <exclusion>
+          <artifactId>jersey-server</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jersey-core</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>jersey-servlet</artifactId>
+          <groupId>com.sun.jersey</groupId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -40,20 +54,63 @@
     <dependency>
       <groupId>com.google.inject.extensions</groupId>
       <artifactId>guice-servlet</artifactId>
-      <version>4.1.0</version>
+      <version>${guice.version}</version>
-      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.glassfish.jersey.containers</groupId>
+      <artifactId>jersey-container-servlet</artifactId>
+      <version>2.27</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.glassfish.hk2</groupId>
+          <artifactId>hk2-api</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.glassfish.jersey.containers</groupId>
       <artifactId>jersey-container-servlet-core</artifactId>
       <version>2.27</version>
-      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.glassfish.hk2</groupId>
+      <artifactId>guice-bridge</artifactId>
+      <version>2.5.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.glassfish.jersey.core</groupId>
+      <artifactId>jersey-server</artifactId>
+      <version>2.27</version>
+    </dependency>
+    <dependency>
+      <groupId>org.glassfish.jersey.media</groupId>
+      <artifactId>jersey-media-json-jackson</artifactId>
+      <version>2.27</version>
     </dependency>
     <dependency>
       <groupId>com.google.inject.extensions</groupId>
       <artifactId>guice-assistedinject</artifactId>
       <version>4.1.0</version>
     </dependency>
+    <dependency>
+      <groupId>org.glassfish.jersey.inject</groupId>
+      <artifactId>jersey-hk2</artifactId>
+      <version>2.27</version>
+      <exclusions>
+        <exclusion>
+          <artifactId>hk2-api</artifactId>
+          <groupId>org.glassfish.hk2</groupId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.glassfish.hk2.external</groupId>
+          <artifactId>aopalliance-repackaged</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.glassfish.hk2</groupId>
+          <artifactId>hk2-utils</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
@@ -70,6 +127,12 @@
       <artifactId>powermock-module-junit4</artifactId>
       <version>1.7.4</version>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.javassist</groupId>
+          <artifactId>javassist</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.powermock</groupId>

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconGuiceServletContextListener.java

@@ -15,15 +15,26 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.apache.hadoop.ozone.recon;
 
-import org.glassfish.jersey.server.ResourceConfig;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
 
 /**
- * JaxRS resource definition.
+ * Servlet Context Listener that provides the Guice injector.
  */
-public class ReconApplication extends ResourceConfig {
+public class ReconGuiceServletContextListener
-  public ReconApplication() {
+    extends GuiceServletContextListener {
-    packages("org.apache.hadoop.ozone.recon.api");
+
+  private static Injector injector;
+
+  @Override
+  public Injector getInjector() {
+    return injector;
+  }
+
+  static void setInjector(Injector inj) {
+    injector = inj;
   }
 }

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconRestServletModule.java

@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon;
+
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.glassfish.hk2.api.ServiceLocator;
+import org.glassfish.jersey.internal.inject.InjectionManager;
+import org.glassfish.jersey.server.ResourceConfig;
+import org.glassfish.jersey.server.spi.Container;
+import org.glassfish.jersey.server.spi.ContainerLifecycleListener;
+import org.glassfish.jersey.servlet.ServletContainer;
+import org.jvnet.hk2.guice.bridge.api.GuiceBridge;
+import org.jvnet.hk2.guice.bridge.api.GuiceIntoHK2Bridge;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Injector;
+import com.google.inject.Scopes;
+import com.google.inject.servlet.ServletModule;
+
+/**
+ * Class to scan API Service classes and bind them to the injector.
+ */
+public abstract class ReconRestServletModule extends ServletModule {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ReconRestServletModule.class);
+
+  @Override
+  abstract protected void configureServlets();
+
+  /**
+   * Interface to provide packages for scanning.
+   */
+  public interface RestKeyBindingBuilder {
+    void packages(String... packages);
+  }
+
+  protected RestKeyBindingBuilder rest(String... urlPatterns) {
+    return new RestKeyBindingBuilderImpl(Arrays.asList(urlPatterns));
+  }
+
+  private class RestKeyBindingBuilderImpl implements RestKeyBindingBuilder {
+    private List<String> paths;
+
+    RestKeyBindingBuilderImpl(List<String> paths) {
+      this.paths = paths;
+    }
+
+    private void checkIfPackageExistsAndLog(String pkg) {
+      String resourcePath = pkg.replace(".", "/");
+      URL resource = getClass().getClassLoader().getResource(resourcePath);
+      if (resource != null) {
+        LOG.info("rest(" + paths + ").packages(" + pkg + ")");
+      } else {
+        LOG.info("No Beans in '" + pkg + "' found. Requests " + paths
+            + " will fail.");
+      }
+    }
+
+    @Override
+    public void packages(String... packages) {
+      StringBuilder sb = new StringBuilder();
+
+      for (String pkg : packages) {
+        if (sb.length() > 0) {
+          sb.append(',');
+        }
+        checkIfPackageExistsAndLog(pkg);
+        sb.append(pkg);
+      }
+      Map<String, String> params = new HashMap<>();
+      params.put("javax.ws.rs.Application",
+          GuiceResourceConfig.class.getCanonicalName());
+      if (sb.length() > 0) {
+        params.put("jersey.config.server.provider.packages", sb.toString());
+      }
+      bind(ServletContainer.class).in(Scopes.SINGLETON);
+      for (String path : paths) {
+        serve(path).with(ServletContainer.class, params);
+      }
+    }
+  }
+}
+
+/**
+ * Class to bridge Guice bindings to Jersey hk2 bindings.
+ */
+class GuiceResourceConfig extends ResourceConfig {
+  GuiceResourceConfig() {
+    register(new ContainerLifecycleListener() {
+      public void onStartup(Container container) {
+        ServletContainer servletContainer = (ServletContainer) container;
+        InjectionManager injectionManager = container.getApplicationHandler()
+            .getInjectionManager();
+        ServiceLocator serviceLocator = injectionManager
+            .getInstance(ServiceLocator.class);
+        GuiceBridge.getGuiceBridge().initializeGuiceBridge(serviceLocator);
+        GuiceIntoHK2Bridge guiceBridge = serviceLocator
+            .getService(GuiceIntoHK2Bridge.class);
+        Injector injector = (Injector) servletContainer.getServletContext()
+            .getAttribute(Injector.class.getName());
+        guiceBridge.bridgeGuiceInjector(injector);
+      }
+
+      public void onReload(Container container) {
+      }
+
+      public void onShutdown(Container container) {
+      }
+    });
+  }
+}

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java

@@ -62,7 +62,17 @@ public Void call() throws Exception {
     OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
     OzoneConfigurationProvider.setConfiguration(ozoneConfiguration);
 
-    injector = Guice.createInjector(new ReconControllerModule());
+    injector =  Guice.createInjector(new
+        ReconControllerModule(), new ReconRestServletModule() {
+          @Override
+          protected void configureServlets() {
+            rest("/api/*")
+                .packages("org.apache.hadoop.ozone.recon.api");
+          }
+        });
+
+    //Pass on injector to listener that does the Guice - Jersey HK2 bridging.
+    ReconGuiceServletContextListener.setInjector(injector);
 
     httpServer = injector.getInstance(ReconHttpServer.class);
     LOG.info("Starting Recon server");

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java

@@ -19,32 +19,45 @@
 
 import java.io.IOException;
 import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 import java.util.stream.Collectors;
 
+import javax.inject.Inject;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
 import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.recon.ReconServer;
 import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
 import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
+import org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
 import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
 import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-import com.google.inject.Inject;
 
 /**
  * Endpoint for querying keys that belong to a container.
  */
 @Path("/containers")
+@Produces(MediaType.APPLICATION_JSON)
 public class ContainerKeyService {
 
+  private static final Logger LOG = LoggerFactory.getLogger(ReconServer.class);
+
   @Inject
   private ContainerDBServiceProvider containerDBServiceProvider;
 
@@ -59,7 +72,7 @@ public class ContainerKeyService {
    * @return {@link Response}
    */
   @GET
-  @Path("{id}")
+  @Path("/{id}")
   public Response getKeysForContainer(@PathParam("id") Long containerId) {
     Map<String, KeyMetadata> keyMetadataMap = new HashMap<>();
     try {
@@ -80,19 +93,36 @@ public Response getKeysForContainer(@PathParam("id") Long containerId) {
         }
 
         // Filter keys by version.
-        List<Long> matchedVersions = omKeyInfo.getKeyLocationVersions()
+        List<OmKeyLocationInfoGroup> matchedKeys = omKeyInfo
+            .getKeyLocationVersions()
             .stream()
             .filter(k -> (k.getVersion() == containerKeyPrefix.getKeyVersion()))
-            .mapToLong(OmKeyLocationInfoGroup::getVersion)
-            .boxed()
             .collect(Collectors.toList());
 
+        List<ContainerBlockMetadata> blockIds = new ArrayList<>();
+        for (OmKeyLocationInfoGroup omKeyLocationInfoGroup : matchedKeys) {
+          List<OmKeyLocationInfo> omKeyLocationInfos = omKeyLocationInfoGroup
+              .getLocationList()
+              .stream()
+              .filter(c -> c.getContainerID() == containerId)
+              .collect(Collectors.toList());
+          for (OmKeyLocationInfo omKeyLocationInfo : omKeyLocationInfos) {
+            blockIds.add(new ContainerBlockMetadata(omKeyLocationInfo
+                .getContainerID(), omKeyLocationInfo.getLocalID()));
+          }
+        }
+
         String ozoneKey = omMetadataManager.getOzoneKey(
             omKeyInfo.getVolumeName(),
             omKeyInfo.getBucketName(),
             omKeyInfo.getKeyName());
         if (keyMetadataMap.containsKey(ozoneKey)) {
-          keyMetadataMap.get(ozoneKey).getVersions().addAll(matchedVersions);
+          keyMetadataMap.get(ozoneKey).getVersions()
+              .add(containerKeyPrefix.getKeyVersion());
+
+          keyMetadataMap.get(ozoneKey).getBlockIds().putAll(
+              Collections.singletonMap(containerKeyPrefix.getKeyVersion(),
+                  blockIds));
         } else {
           KeyMetadata keyMetadata = new KeyMetadata();
           keyMetadata.setBucket(omKeyInfo.getBucketName());
@@ -103,8 +133,14 @@ public Response getKeysForContainer(@PathParam("id") Long containerId) {
           keyMetadata.setModificationTime(
               Instant.ofEpochMilli(omKeyInfo.getModificationTime()));
           keyMetadata.setDataSize(omKeyInfo.getDataSize());
-          keyMetadata.setVersions(matchedVersions);
+          keyMetadata.setVersions(new ArrayList<Long>() {{
+              add(containerKeyPrefix.getKeyVersion());
+            }});
           keyMetadataMap.put(ozoneKey, keyMetadata);
+          keyMetadata.setBlockIds(new TreeMap<Long,
+              List<ContainerBlockMetadata>>() {{
+              put(containerKeyPrefix.getKeyVersion(), blockIds);
+            }});
         }
       }
     } catch (IOException ioEx) {

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyMetadata.java

@@ -19,15 +19,18 @@
 
 import java.time.Instant;
 import java.util.List;
+import java.util.Map;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 
 /**
  * Metadata object represents one key in the object store.
  */
+@XmlRootElement (name = "KeyMetadata")
 @XmlAccessorType(XmlAccessType.FIELD)
 public class KeyMetadata {
 
@@ -46,6 +49,9 @@ public class KeyMetadata {
   @XmlElement(name = "Versions")
   private List<Long> versions;
 
+  @XmlElement(name = "Blocks")
+  private Map<Long, List<ContainerBlockMetadata>> blockIds;
+
   @XmlJavaTypeAdapter(IsoDateAdapter.class)
   @XmlElement(name = "CreationTime")
   private Instant creationTime;
@@ -109,4 +115,33 @@ public List<Long> getVersions() {
   public void setVersions(List<Long> versions) {
     this.versions = versions;
   }
+
+  public Map<Long, List<ContainerBlockMetadata>> getBlockIds() {
+    return blockIds;
+  }
+
+  public void setBlockIds(Map<Long, List<ContainerBlockMetadata>> blockIds) {
+    this.blockIds = blockIds;
+  }
+
+  /**
+   * Class to hold ContainerID and BlockID.
+   */
+  public static class ContainerBlockMetadata {
+    private long containerID;
+    private long localID;
+
+    public ContainerBlockMetadata(long containerID, long localID) {
+      this.containerID = containerID;
+      this.localID = localID;
+    }
+
+    public long getContainerID() {
+      return containerID;
+    }
+
+    public long getLocalID() {
+      return localID;
+    }
+  }
 }

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java

@@ -88,7 +88,7 @@ public void updateOmDB(File newDbLocation) throws IOException {
       if (oldDBLocation.exists()) {
         LOG.info("Cleaning up old OM snapshot db at {}.",
             oldDBLocation.getAbsolutePath());
-        FileUtils.deleteQuietly(oldDBLocation);
+        FileUtils.deleteDirectory(oldDBLocation);
       }
     }
     initializeNewRdbStore(newDbLocation);

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java

@@ -88,7 +88,7 @@ public void initNewContainerDB(Map<ContainerKeyPrefix, Integer>
     if (oldDBLocation.exists()) {
       LOG.info("Cleaning up old Recon Container DB at {}.",
           oldDBLocation.getAbsolutePath());
-      FileUtils.deleteQuietly(oldDBLocation);
+      FileUtils.deleteDirectory(oldDBLocation);
     }
     for (Map.Entry<ContainerKeyPrefix, Integer> entry :
         containerKeyPrefixCounts.entrySet()) {

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java

@@ -85,8 +85,13 @@ public void run() {
               long containerId = omKeyLocationInfo.getContainerID();
               ContainerKeyPrefix containerKeyPrefix = new ContainerKeyPrefix(
                   containerId, key.toString(), keyVersion);
+              if (containerDBServiceProvider.getCountForForContainerKeyPrefix(
+                  containerKeyPrefix) == 0) {
+                // Save on writes. No need to save same container-key prefix
+                // mapping again.
                 containerDBServiceProvider.storeContainerKeyMapping(
                     containerKeyPrefix, 1);
+              }
               containerCount++;
             }
           }

hadoop-ozone/ozone-recon/src/main/resources/webapps.recon.WEB-INF/web.xml

@@ -14,20 +14,15 @@
 <web-app version="3.0" xmlns="http://java.sun.com/xml/ns/javaee"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd">
-
+  <listener>
-  <servlet>
+    <listener-class>org.apache.hadoop.ozone.recon.ReconGuiceServletContextListener</listener-class>
-    <servlet-name>jaxrs</servlet-name>
+  </listener>
-    <servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
+  <filter>
-    <init-param>
+    <filter-name>guiceFilter</filter-name>
-      <param-name>javax.ws.rs.Application</param-name>
+    <filter-class>com.google.inject.servlet.GuiceFilter</filter-class>
-      <param-value>org.apache.hadoop.ozone.recon.ReconApplication</param-value>
+  </filter>
-    </init-param>
+  <filter-mapping>
-    <load-on-startup>1</load-on-startup>
+    <filter-name>guiceFilter</filter-name>
-  </servlet>
-  <servlet-mapping>
-    <servlet-name>jaxrs</servlet-name>
     <url-pattern>/*</url-pattern>
-  </servlet-mapping>
+  </filter-mapping>
-
-
 </web-app>

hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java

@@ -31,6 +31,7 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 
 import javax.ws.rs.core.Response;
 
@@ -81,7 +82,7 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
   private ReconOMMetadataManager reconOMMetadataManager;
   private Injector injector;
   private OzoneManagerServiceProviderImpl ozoneManagerServiceProvider;
-  private ContainerKeyService containerKeyService = new ContainerKeyService();
+  private ContainerKeyService containerKeyService;
 
   @Before
   public void setUp() throws Exception {
@@ -98,11 +99,12 @@ protected void configure() {
               in(Singleton.class);
           bind(ContainerDBServiceProvider.class).to(
               ContainerDBServiceProviderImpl.class).in(Singleton.class);
-          bind(ContainerKeyService.class).toInstance(containerKeyService);
           ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(
               getTestOzoneConfiguration());
           bind(OzoneManagerServiceProvider.class)
               .toInstance(ozoneManagerServiceProvider);
+          containerKeyService = new ContainerKeyService();
+          bind(ContainerKeyService.class).toInstance(containerKeyService);
         } catch (IOException e) {
           Assert.fail();
         }
@@ -119,12 +121,12 @@ public void testGetKeysForContainer() throws Exception {
     Pipeline pipeline = getRandomPipeline();
 
     List<OmKeyLocationInfo> omKeyLocationInfoList = new ArrayList<>();
-    BlockID blockID1 = new BlockID(1, 1);
+    BlockID blockID1 = new BlockID(1, 101);
     OmKeyLocationInfo omKeyLocationInfo1 = getOmKeyLocationInfo(blockID1,
         pipeline);
     omKeyLocationInfoList.add(omKeyLocationInfo1);
 
-    BlockID blockID2 = new BlockID(2, 1);
+    BlockID blockID2 = new BlockID(2, 102);
     OmKeyLocationInfo omKeyLocationInfo2 = getOmKeyLocationInfo(blockID2,
         pipeline);
     omKeyLocationInfoList.add(omKeyLocationInfo2);
@@ -138,7 +140,7 @@ public void testGetKeysForContainer() throws Exception {
         Collections.singletonList(omKeyLocationInfoGroup));
 
     List<OmKeyLocationInfoGroup> infoGroups = new ArrayList<>();
-    BlockID blockID3 = new BlockID(1, 2);
+    BlockID blockID3 = new BlockID(1, 103);
     OmKeyLocationInfo omKeyLocationInfo3 = getOmKeyLocationInfo(blockID3,
         pipeline);
 
@@ -147,7 +149,7 @@ public void testGetKeysForContainer() throws Exception {
     infoGroups.add(new OmKeyLocationInfoGroup(0,
         omKeyLocationInfoListNew));
 
-    BlockID blockID4 = new BlockID(1, 3);
+    BlockID blockID4 = new BlockID(1, 104);
     OmKeyLocationInfo omKeyLocationInfo4 = getOmKeyLocationInfo(blockID4,
         pipeline);
 
@@ -186,12 +188,20 @@ public void testGetKeysForContainer() throws Exception {
     KeyMetadata keyMetadata = iterator.next();
     assertTrue(keyMetadata.getKey().equals("key_one"));
     assertTrue(keyMetadata.getVersions().size() == 1);
+    assertTrue(keyMetadata.getBlockIds().size() == 1);
+    Map<Long, List<KeyMetadata.ContainerBlockMetadata>> blockIds =
+        keyMetadata.getBlockIds();
+    assertTrue(blockIds.get(0L).iterator().next().getLocalID() == 101);
 
     keyMetadata = iterator.next();
     assertTrue(keyMetadata.getKey().equals("key_two"));
     assertTrue(keyMetadata.getVersions().size() == 2);
     assertTrue(keyMetadata.getVersions().contains(0L) && keyMetadata
         .getVersions().contains(1L));
+    assertTrue(keyMetadata.getBlockIds().size() == 2);
+    blockIds = keyMetadata.getBlockIds();
+    assertTrue(blockIds.get(0L).iterator().next().getLocalID() == 103);
+    assertTrue(blockIds.get(1L).iterator().next().getLocalID() == 104);
 
     response = containerKeyService.getKeysForContainer(3L);
     keyMetadataList = (Collection<KeyMetadata>) response.getEntity();

hadoop-ozone/s3gateway/pom.xml

@@ -55,6 +55,22 @@
       <groupId>org.glassfish.jersey.inject</groupId>
       <artifactId>jersey-hk2</artifactId>
       <version>2.27</version>
+      <!-- The versions of these excluded dependencies are 2.5.0-beta. The
+      newer and release version 2.5.0 is being brought in by ozone-recon -->
+      <exclusions>
+        <exclusion>
+          <artifactId>hk2-api</artifactId>
+          <groupId>org.glassfish.hk2</groupId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.glassfish.hk2.external</groupId>
+          <artifactId>aopalliance-repackaged</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.glassfish.hk2</groupId>
+          <artifactId>hk2-utils</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.dataformat</groupId>