From c7ff34f8dcca3a2024230c5383abd9299daa1b20 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Fri, 2 Dec 2016 11:10:09 -0800 Subject: [PATCH] HDFS-11156. Add new op GETFILEBLOCKLOCATIONS to WebHDFS REST API. Contributed by Weiwei Yang --- .../hadoop/hdfs/web/JsonUtilClient.java | 32 ++++++++++++ .../hadoop/hdfs/web/WebHdfsFileSystem.java | 13 +++-- .../hadoop/hdfs/web/resources/GetOpParam.java | 12 ++++- .../web/resources/NamenodeWebHdfsMethods.java | 17 +++++++ .../org/apache/hadoop/hdfs/web/JsonUtil.java | 30 +++++++++++ .../apache/hadoop/hdfs/web/TestWebHDFS.java | 51 +++++++++++++++++++ 6 files changed, 151 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index a75f4f12de..12899f415a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.ContentSummary.Builder; import org.apache.hadoop.fs.FileChecksum; @@ -588,4 +589,35 @@ static LocatedBlocks toLocatedBlocks( lastLocatedBlock, isLastBlockComplete, null, null); } + /** Convert a Json map to BlockLocation. **/ + static BlockLocation toBlockLocation(Map m) + throws IOException{ + long length = ((Number) m.get("length")).longValue(); + long offset = ((Number) m.get("offset")).longValue(); + boolean corrupt = Boolean. + getBoolean(m.get("corrupt").toString()); + String[] storageIds = toStringArray(getList(m, "storageIds")); + String[] cachedHosts = toStringArray(getList(m, "cachedHosts")); + String[] hosts = toStringArray(getList(m, "hosts")); + String[] names = toStringArray(getList(m, "names")); + String[] topologyPaths = toStringArray(getList(m, "topologyPaths")); + StorageType[] storageTypes = toStorageTypeArray( + getList(m, "storageTypes")); + return new BlockLocation(names, hosts, cachedHosts, + topologyPaths, storageIds, storageTypes, + offset, length, corrupt); + } + + static String[] toStringArray(List list) { + if (list == null) { + return null; + } else { + final String[] array = new String[list.size()]; + int i = 0; + for (Object object : list) { + array[i++] = object.toString(); + } + return array; + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 23804b7dce..e82e9f69af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -1610,13 +1610,20 @@ public BlockLocation[] getFileBlockLocations(final Path p, statistics.incrementReadOps(1); storageStatistics.incrementOpCounter(OpType.GET_FILE_BLOCK_LOCATIONS); - final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS; + final HttpOpParam.Op op = GetOpParam.Op.GETFILEBLOCKLOCATIONS; return new FsPathResponseRunner(op, p, new OffsetParam(offset), new LengthParam(length)) { @Override + @SuppressWarnings("unchecked") BlockLocation[] decodeResponse(Map json) throws IOException { - return DFSUtilClient.locatedBlocks2Locations( - JsonUtilClient.toLocatedBlocks(json)); + List list = JsonUtilClient.getList(json, "BlockLocations"); + BlockLocation[] locations = new BlockLocation[list.size()]; + for(int i=0; i) list.get(i)); + locations[i] = bl; + } + return locations; } }.run(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java index 635e6d7e8e..ccb0bb3dce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java @@ -33,8 +33,18 @@ public enum Op implements HttpOpParam.Op { GETHOMEDIRECTORY(false, HttpURLConnection.HTTP_OK), GETDELEGATIONTOKEN(false, HttpURLConnection.HTTP_OK, true), - /** GET_BLOCK_LOCATIONS is a private unstable op. */ + /** + * GET_BLOCK_LOCATIONS is a private/stable API op. It returns a + * {@link org.apache.hadoop.hdfs.protocol.LocatedBlocks} + * json object. + */ GET_BLOCK_LOCATIONS(false, HttpURLConnection.HTTP_OK), + /** + * GETFILEBLOCKLOCATIONS is the public op that complies with + * {@link org.apache.hadoop.fs.FileSystem#getFileBlockLocations} + * interface. + */ + GETFILEBLOCKLOCATIONS(false, HttpURLConnection.HTTP_OK), GETACLSTATUS(false, HttpURLConnection.HTTP_OK), GETXATTRS(false, HttpURLConnection.HTTP_OK), GETTRASHROOT(false, HttpURLConnection.HTTP_OK), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 5d9b12ae88..107d4ed081 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -54,6 +54,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -975,6 +976,22 @@ private Response get( return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } } + case GETFILEBLOCKLOCATIONS: + { + final long offsetValue = offset.getValue(); + final Long lengthValue = length.getValue(); + + try (final FileSystem fs = FileSystem.get(conf != null ? + conf : new Configuration())) { + BlockLocation[] locations = fs.getFileBlockLocations( + new org.apache.hadoop.fs.Path(fullpath), + offsetValue, + lengthValue != null? lengthValue: Long.MAX_VALUE); + final String js = JsonUtil.toJsonString("BlockLocations", + JsonUtil.toJsonArray(locations)); + return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + } + } case GET_BLOCK_LOCATIONS: { final long offsetValue = offset.getValue(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index a0dadbd8ab..affa8616c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -436,4 +436,34 @@ public static String toJsonString(Object obj) throws IOException { return MAPPER.writeValueAsString(obj); } + public static Object[] toJsonArray(BlockLocation[] locations) + throws IOException { + if(locations == null) { + return null; + } + Object[] blockLocations = new Object[locations.length]; + for(int i=0; i toJsonMap( + final BlockLocation blockLocation) throws IOException { + if (blockLocation == null) { + return null; + } + + final Map m = new TreeMap(); + m.put("length", blockLocation.getLength()); + m.put("offset", blockLocation.getOffset()); + m.put("corrupt", blockLocation.isCorrupt()); + m.put("storageTypes", toJsonArray(blockLocation.getStorageTypes())); + m.put("storageIds", blockLocation.getStorageIds()); + m.put("cachedHosts", blockLocation.getCachedHosts()); + m.put("hosts", blockLocation.getHosts()); + m.put("names", blockLocation.getNames()); + m.put("topologyPaths", blockLocation.getTopologyPaths()); + return m; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 5386a45ead..82b708a6cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -37,6 +37,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.security.PrivilegedExceptionAction; +import java.util.Map; import java.util.Random; import org.apache.commons.io.IOUtils; @@ -88,6 +89,8 @@ import org.junit.Test; import org.mockito.internal.util.reflection.Whitebox; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.type.MapType; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyInt; import static org.mockito.Mockito.doReturn; @@ -852,6 +855,46 @@ public void testWebHdfsGetBlockLocationsWithStorageType() throws Exception{ Assert.assertTrue(storageTypes != null && storageTypes.length > 0 && storageTypes[0] == StorageType.DISK); } + + // Query webhdfs REST API to get block locations + InetSocketAddress addr = cluster.getNameNode().getHttpAddress(); + URL url = new URL("http", addr.getHostString(), addr.getPort(), + WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS"); + LOG.info("Sending GETFILEBLOCKLOCATIONS request " + url); + + String response = getResponse(url, "GET"); + LOG.info("The output of GETFILEBLOCKLOCATIONS request " + response); + // Expected output from rest API + // { "BlockLoactions" : [{Block_Loation_Json}, ...] } + ObjectMapper mapper = new ObjectMapper(); + MapType jsonType = mapper.getTypeFactory().constructMapType( + Map.class, + String.class, + BlockLocation[].class); + Map jsonMap = mapper.readValue(response, + jsonType); + BlockLocation[] array = jsonMap.get("BlockLocations"); + + for(int i=0; i