big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12139. HTTPFS liststatus returns incorrect pathSuffix for path of file. Contributed by Yongjun Zhang.

Browse Source
This commit is contained in:
Yongjun Zhang 2017-07-19 10:54:13 -07:00
parent 413b23eb04
commit 3556e36be3
2 changed files with 34 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java
View File

@ -75,15 +75,17 @@ private static Map<String, Object> toJson(FileStatus fileStatus) {
/**
* @param fileStatuses list of FileStatus objects
* @param isFile is the fileStatuses from a file path
* @return JSON map suitable for wire transport
*/
@SuppressWarnings({"unchecked"})
private static Map<String, Object> toJson(FileStatus[] fileStatuses) {
private static Map<String, Object> toJson(FileStatus[] fileStatuses,
boolean isFile) {
Map<String, Object> json = new LinkedHashMap<>();
Map<String, Object> inner = new LinkedHashMap<>();
JSONArray statuses = new JSONArray();
for (FileStatus f : fileStatuses) {
statuses.add(toJsonInner(f, false));
statuses.add(toJsonInner(f, isFile));
}
inner.put(HttpFSFileSystem.FILE_STATUS_JSON, statuses);
json.put(HttpFSFileSystem.FILE_STATUSES_JSON, inner);
@ -129,13 +131,14 @@ private static Map<String, Object> toJsonInner(FileStatus fileStatus,
* These two classes are slightly different, due to the impedance
* mismatches between the WebHDFS and FileSystem APIs.
* @param entries
* @param isFile is the entries from a file path
* @return json
*/
private static Map<String, Object> toJson(FileSystem.DirectoryEntries
entries) {
entries, boolean isFile) {
Map<String, Object> json = new LinkedHashMap<>();
Map<String, Object> inner = new LinkedHashMap<>();
Map<String, Object> fileStatuses = toJson(entries.getEntries());
Map<String, Object> fileStatuses = toJson(entries.getEntries(), isFile);
inner.put(HttpFSFileSystem.PARTIAL_LISTING_JSON, fileStatuses);
inner.put(HttpFSFileSystem.REMAINING_ENTRIES_JSON, entries.hasMore() ? 1
: 0);
@ -690,7 +693,7 @@ public FSListStatus(String path, String filter) throws IOException {
@Override
public Map execute(FileSystem fs) throws IOException {
FileStatus[] fileStatuses = fs.listStatus(path, filter);
return toJson(fileStatuses);
return toJson(fileStatuses, fs.getFileStatus(path).isFile());
}
@Override
@ -735,7 +738,7 @@ public Map execute(FileSystem fs) throws IOException {
WrappedFileSystem wrappedFS = new WrappedFileSystem(fs);
FileSystem.DirectoryEntries entries =
wrappedFS.listStatusBatch(path, token);
return toJson(entries);
return toJson(entries, wrappedFS.getFileStatus(path).isFile());
}
}

26
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
View File

@ -364,8 +364,15 @@ private void testListStatus() throws Exception {
assertEquals(status2.getLen(), status1.getLen());
FileStatus[] stati = fs.listStatus(path.getParent());
assertEquals(stati.length, 1);
assertEquals(1, stati.length);
assertEquals(stati[0].getPath().getName(), path.getName());
// The full path should be the path to the file. See HDFS-12139
FileStatus[] statl = fs.listStatus(path);
Assert.assertEquals(1, statl.length);
Assert.assertEquals(status2.getPath(), statl[0].getPath());
Assert.assertEquals(statl[0].getPath().getName(), path.getName());
Assert.assertEquals(stati[0].getPath(), statl[0].getPath());
}
private static void assertSameListing(FileSystem expected, FileSystem
@ -411,6 +418,23 @@ private void testListStatusBatch() throws Exception {
proxyFs.create(new Path(dir, "file" + i)).close();
assertSameListing(proxyFs, httpFs, dir);
}
// Test for HDFS-12139
Path dir1 = new Path(getProxiedFSTestDir(), "dir1");
proxyFs.mkdirs(dir1);
Path file1 = new Path(dir1, "file1");
proxyFs.create(file1).close();
RemoteIterator<FileStatus> si = proxyFs.listStatusIterator(dir1);
FileStatus statusl = si.next();
FileStatus status = proxyFs.getFileStatus(file1);
Assert.assertEquals(file1.getName(), statusl.getPath().getName());
Assert.assertEquals(status.getPath(), statusl.getPath());
si = proxyFs.listStatusIterator(file1);
statusl = si.next();
Assert.assertEquals(file1.getName(), statusl.getPath().getName());
Assert.assertEquals(status.getPath(), statusl.getPath());
}
private void testWorkingdirectory() throws Exception {