From 9a082fbe6e302df7139b65a23be9a39acd87715d Mon Sep 17 00:00:00 2001 From: Hanisha Koneru Date: Fri, 9 Mar 2018 13:20:32 -0800 Subject: [PATCH] HDFS-11394. Support for getting erasure coding policy through WebHDFS#FileStatus. --- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 3 + .../apache/hadoop/hdfs/web/TestWebHDFS.java | 111 ++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 095b9ac342..83fbc6e807 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -134,6 +134,9 @@ private static Map toJsonMap(HdfsFileStatus status) { } if (status.isErasureCoded()) { m.put("ecBit", true); + if (status.getErasureCodingPolicy() != null) { + m.put("ecPolicy", status.getErasureCodingPolicy().getName()); + } } if (status.isSnapshotEnabled()) { m.put("snapshotEnabled", status.isSnapshotEnabled()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index c94122eb45..8571d82341 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -44,6 +44,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Random; @@ -81,6 +82,7 @@ import org.apache.hadoop.hdfs.TestFileCreation; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; @@ -109,6 +111,7 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.DataChecksum; import org.apache.log4j.Level; +import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.Assert; @@ -1578,6 +1581,114 @@ public void testExceptionPropogationInAbstractRunner() throws Exception{ } } + /** + * Tests that the LISTSTATUS ang GETFILESTATUS WebHDFS calls return the + * ecPolicy for EC files. + */ + @Test(timeout=300000) + public void testECPolicyInFileStatus() throws Exception { + final Configuration conf = WebHdfsTestUtil.createConf(); + final ErasureCodingPolicy ecPolicy = SystemErasureCodingPolicies + .getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID); + final String ecPolicyName = ecPolicy.getName(); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(5) + .build(); + cluster.waitActive(); + final DistributedFileSystem fs = cluster.getFileSystem(); + + // Create an EC dir and write a test file in it + final Path ecDir = new Path("/ec"); + Path ecFile = new Path(ecDir, "ec_file.txt"); + Path nonEcFile = new Path(ecDir, "non_ec_file.txt"); + fs.mkdirs(ecDir); + + // Create a non-EC file before enabling ec policy + DFSTestUtil.createFile(fs, nonEcFile, 1024, (short) 1, 0); + + fs.enableErasureCodingPolicy(ecPolicyName); + fs.setErasureCodingPolicy(ecDir, ecPolicyName); + + // Create a EC file + DFSTestUtil.createFile(fs, ecFile, 1024, (short) 1, 0); + + // Query webhdfs REST API to list statuses of files/directories in ecDir + InetSocketAddress addr = cluster.getNameNode().getHttpAddress(); + URL listStatusUrl = new URL("http", addr.getHostString(), addr.getPort(), + WebHdfsFileSystem.PATH_PREFIX + ecDir.toString() + "?op=LISTSTATUS"); + + HttpURLConnection conn = (HttpURLConnection) listStatusUrl.openConnection(); + conn.setRequestMethod("GET"); + conn.setInstanceFollowRedirects(false); + String listStatusResponse = IOUtils.toString(conn.getInputStream(), + StandardCharsets.UTF_8); + Assert.assertEquals("Response wasn't " + HttpURLConnection.HTTP_OK, + HttpURLConnection.HTTP_OK, conn.getResponseCode()); + + // Verify that ecPolicy is set in the ListStatus response for ec file + String ecpolicyForECfile = getECPolicyFromFileStatusJson( + getFileStatusJson(listStatusResponse, ecFile.getName())); + assertEquals("EC policy for ecFile should match the set EC policy", + ecpolicyForECfile, ecPolicyName); + + // Verify that ecPolicy is not set in the ListStatus response for non-ec + // file + String ecPolicyForNonECfile = getECPolicyFromFileStatusJson( + getFileStatusJson(listStatusResponse, nonEcFile.getName())); + assertEquals("EC policy for nonEcFile should be null (not set)", + ecPolicyForNonECfile, null); + + // Query webhdfs REST API to get fileStatus for ecFile + URL getFileStatusUrl = new URL("http", addr.getHostString(), addr.getPort(), + WebHdfsFileSystem.PATH_PREFIX + ecFile.toString() + + "?op=GETFILESTATUS"); + + conn = (HttpURLConnection) getFileStatusUrl.openConnection(); + conn.setRequestMethod("GET"); + conn.setInstanceFollowRedirects(false); + String getFileStatusResponse = IOUtils.toString(conn.getInputStream(), + StandardCharsets.UTF_8); + Assert.assertEquals("Response wasn't " + HttpURLConnection.HTTP_OK, + HttpURLConnection.HTTP_OK, conn.getResponseCode()); + + // Verify that ecPolicy is set in getFileStatus response for ecFile + JSONObject fileStatusObject = new JSONObject(getFileStatusResponse) + .getJSONObject("FileStatus"); + ecpolicyForECfile = getECPolicyFromFileStatusJson(fileStatusObject); + assertEquals("EC policy for ecFile should match the set EC policy", + ecpolicyForECfile, ecPolicyName); + } + + /** + * Get FileStatus JSONObject from ListStatus response. + */ + private JSONObject getFileStatusJson(String response, String fileName) + throws JSONException { + JSONObject listStatusResponseJson = new JSONObject(response); + JSONArray fileStatusArray = listStatusResponseJson + .getJSONObject("FileStatuses") + .getJSONArray("FileStatus"); + for (int i = 0; i < fileStatusArray.length(); i++) { + JSONObject fileStatusJsonObject = fileStatusArray.getJSONObject(i); + if (fileName.equals(fileStatusJsonObject.get("pathSuffix"))) { + return fileStatusJsonObject; + } + } + return null; + } + + /** + * Get ECPolicy name from FileStatus JSONObject. + */ + private String getECPolicyFromFileStatusJson(JSONObject fileStatusJsonObject) + throws JSONException { + if (fileStatusJsonObject.has("ecPolicy")) { + return fileStatusJsonObject.getString("ecPolicy"); + } else { + return null; + } + } + final static class DummyThrowable extends Throwable { } }