big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11394. Support for getting erasure coding policy through WebHDFS#FileStatus.

Browse Source
This commit is contained in:
Hanisha Koneru 2018-03-09 13:20:32 -08:00
parent 99ab511cba
commit 9a082fbe6e
2 changed files with 114 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
View File

@ -134,6 +134,9 @@ private static Map<String, Object> toJsonMap(HdfsFileStatus status) {
}
if (status.isErasureCoded()) {
m.put("ecBit", true);
if (status.getErasureCodingPolicy() != null) {
m.put("ecPolicy", status.getErasureCodingPolicy().getName());
}
}
if (status.isSnapshotEnabled()) {
m.put("snapshotEnabled", status.isSnapshotEnabled());

111
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
View File

@ -44,6 +44,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Random;
@ -81,6 +82,7 @@
import org.apache.hadoop.hdfs.TestFileCreation;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
@ -109,6 +111,7 @@
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.DataChecksum;
import org.apache.log4j.Level;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Assert;
@ -1578,6 +1581,114 @@ public void testExceptionPropogationInAbstractRunner() throws Exception{
}
}
/**
* Tests that the LISTSTATUS ang GETFILESTATUS WebHDFS calls return the
* ecPolicy for EC files.
*/
@Test(timeout=300000)
public void testECPolicyInFileStatus() throws Exception {
final Configuration conf = WebHdfsTestUtil.createConf();
final ErasureCodingPolicy ecPolicy = SystemErasureCodingPolicies
.getByID(SystemErasureCodingPolicies.RS_3_2_POLICY_ID);
final String ecPolicyName = ecPolicy.getName();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(5)
.build();
cluster.waitActive();
final DistributedFileSystem fs = cluster.getFileSystem();
// Create an EC dir and write a test file in it
final Path ecDir = new Path("/ec");
Path ecFile = new Path(ecDir, "ec_file.txt");
Path nonEcFile = new Path(ecDir, "non_ec_file.txt");
fs.mkdirs(ecDir);
// Create a non-EC file before enabling ec policy
DFSTestUtil.createFile(fs, nonEcFile, 1024, (short) 1, 0);
fs.enableErasureCodingPolicy(ecPolicyName);
fs.setErasureCodingPolicy(ecDir, ecPolicyName);
// Create a EC file
DFSTestUtil.createFile(fs, ecFile, 1024, (short) 1, 0);
// Query webhdfs REST API to list statuses of files/directories in ecDir
InetSocketAddress addr = cluster.getNameNode().getHttpAddress();
URL listStatusUrl = new URL("http", addr.getHostString(), addr.getPort(),
WebHdfsFileSystem.PATH_PREFIX + ecDir.toString() + "?op=LISTSTATUS");
HttpURLConnection conn = (HttpURLConnection) listStatusUrl.openConnection();
conn.setRequestMethod("GET");
conn.setInstanceFollowRedirects(false);
String listStatusResponse = IOUtils.toString(conn.getInputStream(),
StandardCharsets.UTF_8);
Assert.assertEquals("Response wasn't " + HttpURLConnection.HTTP_OK,
HttpURLConnection.HTTP_OK, conn.getResponseCode());
// Verify that ecPolicy is set in the ListStatus response for ec file
String ecpolicyForECfile = getECPolicyFromFileStatusJson(
getFileStatusJson(listStatusResponse, ecFile.getName()));
assertEquals("EC policy for ecFile should match the set EC policy",
ecpolicyForECfile, ecPolicyName);
// Verify that ecPolicy is not set in the ListStatus response for non-ec
// file
String ecPolicyForNonECfile = getECPolicyFromFileStatusJson(
getFileStatusJson(listStatusResponse, nonEcFile.getName()));
assertEquals("EC policy for nonEcFile should be null (not set)",
ecPolicyForNonECfile, null);
// Query webhdfs REST API to get fileStatus for ecFile
URL getFileStatusUrl = new URL("http", addr.getHostString(), addr.getPort(),
WebHdfsFileSystem.PATH_PREFIX + ecFile.toString() +
"?op=GETFILESTATUS");
conn = (HttpURLConnection) getFileStatusUrl.openConnection();
conn.setRequestMethod("GET");
conn.setInstanceFollowRedirects(false);
String getFileStatusResponse = IOUtils.toString(conn.getInputStream(),
StandardCharsets.UTF_8);
Assert.assertEquals("Response wasn't " + HttpURLConnection.HTTP_OK,
HttpURLConnection.HTTP_OK, conn.getResponseCode());
// Verify that ecPolicy is set in getFileStatus response for ecFile
JSONObject fileStatusObject = new JSONObject(getFileStatusResponse)
.getJSONObject("FileStatus");
ecpolicyForECfile = getECPolicyFromFileStatusJson(fileStatusObject);
assertEquals("EC policy for ecFile should match the set EC policy",
ecpolicyForECfile, ecPolicyName);
}
/**
* Get FileStatus JSONObject from ListStatus response.
*/
private JSONObject getFileStatusJson(String response, String fileName)
throws JSONException {
JSONObject listStatusResponseJson = new JSONObject(response);
JSONArray fileStatusArray = listStatusResponseJson
.getJSONObject("FileStatuses")
.getJSONArray("FileStatus");
for (int i = 0; i < fileStatusArray.length(); i++) {
JSONObject fileStatusJsonObject = fileStatusArray.getJSONObject(i);
if (fileName.equals(fileStatusJsonObject.get("pathSuffix"))) {
return fileStatusJsonObject;
}
}
return null;
}
/**
* Get ECPolicy name from FileStatus JSONObject.
*/
private String getECPolicyFromFileStatusJson(JSONObject fileStatusJsonObject)
throws JSONException {
if (fileStatusJsonObject.has("ecPolicy")) {
return fileStatusJsonObject.getString("ecPolicy");
} else {
return null;
}
}
final static class DummyThrowable extends Throwable {
}
}