big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-2404. webhdfs liststatus json response is not correct. Contributed by Suresh Srinivas.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1180757 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Suresh Srinivas 2011-10-10 01:49:37 +00:00
parent a09a2b48cd
commit 676f488eff
5 changed files with 40 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -86,11 +86,11 @@ Trunk (unreleased changes)
there are two SC_START_IN_CTOR findbugs warnings. (szetszwo) there are two SC_START_IN_CTOR findbugs warnings. (szetszwo)
HDFS-2330. In NNStorage and FSImagePreTransactionalStorageInspector, HDFS-2330. In NNStorage and FSImagePreTransactionalStorageInspector,
IOExceptions of stream closures can mask root exceptions. (Uma Maheswara IOExceptions of stream closures can mask root exceptions. (Uma Maheswara
Rao G via szetszwo) Rao G via szetszwo)
HDFS-46. Change default namespace quota of root directory from HDFS-46. Change default namespace quota of root directory from
Integer.MAX_VALUE to Long.MAX_VALUE. (Uma Maheswara Rao G via szetszwo) Integer.MAX_VALUE to Long.MAX_VALUE. (Uma Maheswara Rao G via szetszwo)
HDFS-2366. Initialize WebHdfsFileSystem.ugi in object construction. HDFS-2366. Initialize WebHdfsFileSystem.ugi in object construction.
(szetszwo) (szetszwo)
@ -104,10 +104,12 @@ Trunk (unreleased changes)
not include multiple methods of the same name. (cutting) not include multiple methods of the same name. (cutting)
HDFS-2403. NamenodeWebHdfsMethods.generateDelegationToken(..) does not use HDFS-2403. NamenodeWebHdfsMethods.generateDelegationToken(..) does not use
the renewer parameter. (szetszwo) the renewer parameter. (szetszwo)
HDFS-2409. _HOST in dfs.web.authentication.kerberos.principal. (jitendra) HDFS-2409. _HOST in dfs.web.authentication.kerberos.principal. (jitendra)
HDFS-2404. webhdfs liststatus json response is not correct. (suresh)
Release 0.23.0 - Unreleased Release 0.23.0 - Unreleased
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
View File

@ -424,7 +424,7 @@ public Response run() throws IOException, URISyntaxException {
case GETFILESTATUS: case GETFILESTATUS:
{ {
final HdfsFileStatus status = np.getFileInfo(fullpath); final HdfsFileStatus status = np.getFileInfo(fullpath);
final String js = JsonUtil.toJsonString(status); final String js = JsonUtil.toJsonString(status, true);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
} }
case LISTSTATUS: case LISTSTATUS:
@ -480,22 +480,22 @@ private static StreamingOutput getListingStream(final NamenodeProtocols np,
@Override @Override
public void write(final OutputStream outstream) throws IOException { public void write(final OutputStream outstream) throws IOException {
final PrintStream out = new PrintStream(outstream); final PrintStream out = new PrintStream(outstream);
out.println("{\"" + HdfsFileStatus[].class.getSimpleName() + "\":["); out.println("{\"" + HdfsFileStatus.class.getSimpleName() + "\":[");
final HdfsFileStatus[] partial = first.getPartialListing(); final HdfsFileStatus[] partial = first.getPartialListing();
if (partial.length > 0) { if (partial.length > 0) {
out.print(JsonUtil.toJsonString(partial[0])); out.print(JsonUtil.toJsonString(partial[0], false));
} }
for(int i = 1; i < partial.length; i++) { for(int i = 1; i < partial.length; i++) {
out.println(','); out.println(',');
out.print(JsonUtil.toJsonString(partial[i])); out.print(JsonUtil.toJsonString(partial[i], false));
} }
for(DirectoryListing curr = first; curr.hasMore(); ) { for(DirectoryListing curr = first; curr.hasMore(); ) {
curr = getDirectoryListing(np, p, curr.getLastName()); curr = getDirectoryListing(np, p, curr.getLastName());
for(HdfsFileStatus s : curr.getPartialListing()) { for(HdfsFileStatus s : curr.getPartialListing()) {
out.println(','); out.println(',');
out.print(JsonUtil.toJsonString(s)); out.print(JsonUtil.toJsonString(s, false));
} }
} }

46
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
View File

@ -133,37 +133,39 @@ private static FsPermission toFsPermission(final String s) {
} }
/** Convert a HdfsFileStatus object to a Json string. */ /** Convert a HdfsFileStatus object to a Json string. */
public static String toJsonString(final HdfsFileStatus status) { public static String toJsonString(final HdfsFileStatus status,
boolean includeType) {
if (status == null) { if (status == null) {
return null; return null;
} else {
final Map<String, Object> m = new TreeMap<String, Object>();
m.put("localName", status.getLocalName());
m.put("isDir", status.isDir());
m.put("isSymlink", status.isSymlink());
if (status.isSymlink()) {
m.put("symlink", status.getSymlink());
}
m.put("len", status.getLen());
m.put("owner", status.getOwner());
m.put("group", status.getGroup());
m.put("permission", toString(status.getPermission()));
m.put("accessTime", status.getAccessTime());
m.put("modificationTime", status.getModificationTime());
m.put("blockSize", status.getBlockSize());
m.put("replication", status.getReplication());
return toJsonString(HdfsFileStatus.class, m);
} }
final Map<String, Object> m = new TreeMap<String, Object>();
m.put("localName", status.getLocalName());
m.put("isDir", status.isDir());
m.put("isSymlink", status.isSymlink());
if (status.isSymlink()) {
m.put("symlink", status.getSymlink());
}
m.put("len", status.getLen());
m.put("owner", status.getOwner());
m.put("group", status.getGroup());
m.put("permission", toString(status.getPermission()));
m.put("accessTime", status.getAccessTime());
m.put("modificationTime", status.getModificationTime());
m.put("blockSize", status.getBlockSize());
m.put("replication", status.getReplication());
return includeType ? toJsonString(HdfsFileStatus.class, m) :
JSON.toString(m);
} }
/** Convert a Json map to a HdfsFileStatus object. */ /** Convert a Json map to a HdfsFileStatus object. */
public static HdfsFileStatus toFileStatus(final Map<?, ?> json) { public static HdfsFileStatus toFileStatus(final Map<?, ?> json, boolean includesType) {
if (json == null) { if (json == null) {
return null; return null;
} }
final Map<?, ?> m = (Map<?, ?>)json.get(HdfsFileStatus.class.getSimpleName()); final Map<?, ?> m = includesType ?
(Map<?, ?>)json.get(HdfsFileStatus.class.getSimpleName()) : json;
final String localName = (String) m.get("localName"); final String localName = (String) m.get("localName");
final boolean isDir = (Boolean) m.get("isDir"); final boolean isDir = (Boolean) m.get("isDir");
final boolean isSymlink = (Boolean) m.get("isSymlink"); final boolean isSymlink = (Boolean) m.get("isSymlink");
@ -287,7 +289,7 @@ private static DatanodeInfo[] toDatanodeInfoArray(final Object[] objects) {
return array; return array;
} }
} }
/** Convert a LocatedBlock to a Json map. */ /** Convert a LocatedBlock to a Json map. */
private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock
) throws IOException { ) throws IOException {

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
View File

@ -253,7 +253,7 @@ private FsPermission applyUMask(FsPermission permission) {
private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException { private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
final HttpOpParam.Op op = GetOpParam.Op.GETFILESTATUS; final HttpOpParam.Op op = GetOpParam.Op.GETFILESTATUS;
final Map<String, Object> json = run(op, f); final Map<String, Object> json = run(op, f);
final HdfsFileStatus status = JsonUtil.toFileStatus(json); final HdfsFileStatus status = JsonUtil.toFileStatus(json, true);
if (status == null) { if (status == null) {
throw new FileNotFoundException("File does not exist: " + f); throw new FileNotFoundException("File does not exist: " + f);
} }
@ -405,14 +405,14 @@ public FileStatus[] listStatus(final Path f) throws IOException {
final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS; final HttpOpParam.Op op = GetOpParam.Op.LISTSTATUS;
final Map<?, ?> json = run(op, f); final Map<?, ?> json = run(op, f);
final Object[] array = (Object[])json.get( final Object[] array = (Object[])json.get(
HdfsFileStatus[].class.getSimpleName()); HdfsFileStatus.class.getSimpleName());
//convert FileStatus //convert FileStatus
final FileStatus[] statuses = new FileStatus[array.length]; final FileStatus[] statuses = new FileStatus[array.length];
for(int i = 0; i < array.length; i++) { for(int i = 0; i < array.length; i++) {
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
final Map<String, Object> m = (Map<String, Object>)array[i]; final Map<String, Object> m = (Map<String, Object>)array[i];
statuses[i] = makeQualified(JsonUtil.toFileStatus(m), f); statuses[i] = makeQualified(JsonUtil.toFileStatus(m, false), f);
} }
return statuses; return statuses;
} }
@ -472,4 +472,4 @@ public MD5MD5CRC32FileChecksum getFileChecksum(final Path p
final Map<String, Object> m = run(op, p); final Map<String, Object> m = run(op, p);
return JsonUtil.toMD5MD5CRC32FileChecksum(m); return JsonUtil.toMD5MD5CRC32FileChecksum(m);
} }
} }

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
View File

@ -46,9 +46,9 @@ public void testHdfsFileStatus() {
final FileStatus fstatus = toFileStatus(status, parent); final FileStatus fstatus = toFileStatus(status, parent);
System.out.println("status = " + status); System.out.println("status = " + status);
System.out.println("fstatus = " + fstatus); System.out.println("fstatus = " + fstatus);
final String json = JsonUtil.toJsonString(status); final String json = JsonUtil.toJsonString(status, true);
System.out.println("json = " + json.replace(",", ",\n ")); System.out.println("json = " + json.replace(",", ",\n "));
final HdfsFileStatus s2 = JsonUtil.toFileStatus((Map<?, ?>)JSON.parse(json)); final HdfsFileStatus s2 = JsonUtil.toFileStatus((Map<?, ?>)JSON.parse(json), true);
final FileStatus fs2 = toFileStatus(s2, parent); final FileStatus fs2 = toFileStatus(s2, parent);
System.out.println("s2 = " + s2); System.out.println("s2 = " + s2);
System.out.println("fs2 = " + fs2); System.out.println("fs2 = " + fs2);