big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17168. Support getTrashRoots API in WebHDFS (#5992) Contributed by Hualong Zhang.

Browse Source 
Reviewed-by: Shilun Fan <slfan1989@apache.org>
Signed-off-by: Shilun Fan <slfan1989@apache.org>
This commit is contained in:
zhtttylz 2023-09-02 15:24:52 +08:00 committed by GitHub
parent b6d06c8928
commit 9857fbcd61
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 260 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
View File

@ -75,6 +75,7 @@ public enum OpType {
GET_STORAGE_POLICIES("op_get_storage_policies"),
GET_STORAGE_POLICY("op_get_storage_policy"),
GET_TRASH_ROOT("op_get_trash_root"),
GET_TRASH_ROOTS("op_get_trash_roots"),
GET_XATTR("op_get_xattr"),
LIST_CACHE_DIRECTIVE("op_list_cache_directive"),
LIST_CACHE_POOL("op_list_cache_pool"),

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -3612,6 +3612,8 @@ public Path getTrashRoot(Path path) {
*/
@Override
public Collection<FileStatus> getTrashRoots(boolean allUsers) {
statistics.incrementReadOps(1);
storageStatistics.incrementOpCounter(OpType.GET_TRASH_ROOTS);
Set<FileStatus> ret = new HashSet<>();
// Get normal trash roots
ret.addAll(super.getTrashRoots(allUsers));

35
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
View File

@ -22,6 +22,7 @@
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Preconditions;
import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
import org.apache.hadoop.fs.ContentSummary;
@ -877,6 +878,40 @@ public static Map<String, String> getErasureCodeCodecs(Map<?, ?> json) {
return map;
}
public static Collection<FileStatus> getTrashRoots(Map<?, ?> json) {
List<?> objs = (List<?>) json.get("Paths");
if (objs != null) {
FileStatus[] trashRoots = new FileStatus[objs.size()];
for (int i = 0; i < objs.size(); i++) {
Map<?, ?> m = (Map<?, ?>) objs.get(i);
trashRoots[i] = toFileStatus(m);
}
return Arrays.asList(trashRoots);
}
return new ArrayList<FileStatus>(0);
}
public static FileStatus toFileStatus(Map<?, ?> json) {
Path path = new Path(getString(json, "path", ""));
long length = getLong(json, "length", 0);
boolean isdir = getBoolean(json, "isdir", false);
short replication = (short) getInt(json, "block_replication", -1);
long blockSize = getLong(json, "blocksize", 256);
long modificationTime = getLong(json, "modification_time", 0);
long accessTime = getLong(json, "access_time", 0);
String permString = getString(json, "permission", null);
FsPermission permission = toFsPermission(permString);
String owner = getString(json, "owner", null);
String group = getString(json, "group", null);
if (json.get("symlink") != null) {
Path symlink = new Path((String) json.get("symlink"));
return new FileStatus(length, isdir, replication, blockSize, modificationTime,
accessTime, permission, owner, group, symlink, path);
}
return new FileStatus(length, isdir, replication, blockSize, modificationTime,
accessTime, permission, owner, group, path);
}
private static List<SnapshotDiffReport.DiffReportEntry> toDiffList(
List<?> objs) {
if (objs == null) {

20
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
View File

@ -1956,6 +1956,26 @@ String decodeResponse(Map<?, ?> json) throws IOException {
}
}
public Collection<FileStatus> getTrashRoots(boolean allUsers) {
statistics.incrementReadOps(1);
storageStatistics.incrementOpCounter(OpType.GET_TRASH_ROOTS);
final HttpOpParam.Op op = GetOpParam.Op.GETTRASHROOTS;
try {
Collection<FileStatus> trashRoots =
new FsPathResponseRunner<Collection<FileStatus>>(op, null,
new AllUsersParam(allUsers)) {
@Override
Collection<FileStatus> decodeResponse(Map<?, ?> json) throws IOException {
return JsonUtilClient.getTrashRoots(json);
}
}.run();
return trashRoots;
} catch (IOException e) {
return super.getTrashRoots(allUsers);
}
}
@Override
public void access(final Path path, final FsAction mode) throws IOException {
final HttpOpParam.Op op = GetOpParam.Op.CHECKACCESS;

49
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/AllUsersParam.java Normal file
View File

@ -0,0 +1,49 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.web.resources;
/** AllUsers parameter. */
public class AllUsersParam extends BooleanParam {
/** Parameter name. */
public static final String NAME = "allusers";
/** Default parameter value. */
public static final String DEFAULT = FALSE;
private static final Domain DOMAIN = new Domain(NAME);
/**
* Constructor.
* @param value the parameter value.
*/
public AllUsersParam(final Boolean value) {
super(DOMAIN, value);
}
/**
* Constructor.
* @param str a string representation of the parameter value.
*/
public AllUsersParam(final String str) {
this(DOMAIN.parse(str));
}
@Override
public String getName() {
return NAME;
}
}

1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
View File

@ -69,6 +69,7 @@ public enum Op implements HttpOpParam.Op {
GETSTATUS(false, HttpURLConnection.HTTP_OK),
GETECPOLICIES(false, HttpURLConnection.HTTP_OK),
GETECCODECS(false, HttpURLConnection.HTTP_OK),
GETTRASHROOTS(false, HttpURLConnection.HTTP_OK),
GETSNAPSHOTLIST(false, HttpURLConnection.HTTP_OK);
final boolean redirect;

6
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java
View File

@ -43,6 +43,7 @@
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
import org.apache.hadoop.hdfs.web.resources.AclPermissionParam;
import org.apache.hadoop.hdfs.web.resources.AllUsersParam;
import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
import org.apache.hadoop.hdfs.web.resources.ConcatSourcesParam;
@ -344,7 +345,8 @@ protected Response get(
final TokenKindParam tokenKind,
final TokenServiceParam tokenService,
final NoRedirectParam noredirectParam,
final StartAfterParam startAfter
final StartAfterParam startAfter,
final AllUsersParam allUsers
) throws IOException, URISyntaxException {
try {
final Router router = getRouter();
@ -393,7 +395,7 @@ protected Response get(
offset, length, renewer, bufferSize, xattrNames, xattrEncoding,
excludeDatanodes, fsAction, snapshotName, oldSnapshotName,
snapshotDiffStartPath, snapshotDiffIndex,
tokenKind, tokenService, noredirectParam, startAfter);
tokenKind, tokenService, noredirectParam, startAfter, allUsers);
}
default:
throw new UnsupportedOperationException(op + " is not supported");

31
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
View File

@ -30,6 +30,7 @@
import java.security.PrivilegedExceptionAction;
import java.util.Base64;
import java.util.Base64.Encoder;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
@ -56,6 +57,7 @@
import javax.ws.rs.core.Response.ResponseBuilder;
import javax.ws.rs.core.Response.Status;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.QuotaUsage;
import org.apache.hadoop.fs.StorageType;
@ -1067,14 +1069,16 @@ public Response getRoot(
@QueryParam(NoRedirectParam.NAME) @DefaultValue(NoRedirectParam.DEFAULT)
final NoRedirectParam noredirect,
@QueryParam(StartAfterParam.NAME) @DefaultValue(StartAfterParam.DEFAULT)
final StartAfterParam startAfter
final StartAfterParam startAfter,
@QueryParam(AllUsersParam.NAME) @DefaultValue(AllUsersParam.DEFAULT)
final AllUsersParam allUsers
) throws IOException, InterruptedException {
return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length,
renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes,
fsAction, snapshotName, oldSnapshotName,
snapshotDiffStartPath, snapshotDiffIndex,
tokenKind, tokenService,
noredirect, startAfter);
noredirect, startAfter, allUsers);
}
/** Handle HTTP GET request. */
@ -1124,12 +1128,14 @@ public Response get(
@QueryParam(NoRedirectParam.NAME) @DefaultValue(NoRedirectParam.DEFAULT)
final NoRedirectParam noredirect,
@QueryParam(StartAfterParam.NAME) @DefaultValue(StartAfterParam.DEFAULT)
final StartAfterParam startAfter
final StartAfterParam startAfter,
@QueryParam(AllUsersParam.NAME) @DefaultValue(AllUsersParam.DEFAULT)
final AllUsersParam allUsers
) throws IOException, InterruptedException {
init(ugi, delegation, username, doAsUser, path, op, offset, length,
renewer, bufferSize, xattrEncoding, excludeDatanodes, fsAction,
snapshotName, oldSnapshotName, tokenKind, tokenService, startAfter);
snapshotName, oldSnapshotName, tokenKind, tokenService, startAfter, allUsers);
return doAs(ugi, new PrivilegedExceptionAction<Response>() {
@Override
@ -1138,7 +1144,7 @@ public Response run() throws IOException, URISyntaxException {
op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding,
excludeDatanodes, fsAction, snapshotName, oldSnapshotName,
snapshotDiffStartPath, snapshotDiffIndex,
tokenKind, tokenService, noredirect, startAfter);
tokenKind, tokenService, noredirect, startAfter, allUsers);
}
});
}
@ -1172,7 +1178,8 @@ protected Response get(
final TokenKindParam tokenKind,
final TokenServiceParam tokenService,
final NoRedirectParam noredirectParam,
final StartAfterParam startAfter
final StartAfterParam startAfter,
final AllUsersParam allUsers
) throws IOException, URISyntaxException {
final Configuration conf = (Configuration) context
.getAttribute(JspHelper.CURRENT_CONF);
@ -1322,6 +1329,12 @@ protected Response get(
final String jsonStr = JsonUtil.toJsonString("Path", trashPath);
return Response.ok(jsonStr).type(MediaType.APPLICATION_JSON).build();
}
case GETTRASHROOTS: {
Boolean value = allUsers.getValue();
final Collection<FileStatus> trashPaths = getTrashRoots(conf, value);
final String js = JsonUtil.toJsonString(trashPaths);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
case LISTSTATUS_BATCH:
{
byte[] start = HdfsFileStatus.EMPTY_NAME;
@ -1558,6 +1571,12 @@ public Void run() throws IOException {
};
}
private Collection<FileStatus> getTrashRoots(Configuration conf, boolean allUsers)
throws IOException {
FileSystem fs = FileSystem.get(conf != null ? conf : new Configuration());
return fs.getTrashRoots(allUsers);
}
/** Handle HTTP DELETE request for the root. */
@DELETE

33
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
View File

@ -752,6 +752,27 @@ public static Map<String, Object> toJsonMap(ErasureCodingPolicyInfo ecPolicyInfo
return m;
}
public static Map<String, Object> toJsonMap(FileStatus status) throws IOException {
Map<String, Object> m = new HashMap<>();
m.put("path", status.getPath().toUri().getPath());
m.put("length", status.getLen());
m.put("isdir", status.isDirectory());
m.put("block_replication", status.getReplication());
m.put("blocksize", status.getBlockSize());
m.put("modification_time", status.getModificationTime());
m.put("access_time", status.getAccessTime());
FsPermission perm = status.getPermission();
if (perm != null) {
m.put("permission", toString(perm));
}
m.put("owner", status.getOwner());
m.put("group", status.getGroup());
if (status.isSymlink()) {
m.put("symlink", status.getSymlink());
}
return m;
}
public static String toJsonString(ErasureCodingPolicyInfo[] ecPolicyInfos) {
final Map<String, Object> erasureCodingPolicies = new HashMap<>();
Object[] erasureCodingPolicyInfos = null;
@ -764,4 +785,16 @@ public static String toJsonString(ErasureCodingPolicyInfo[] ecPolicyInfos) {
erasureCodingPolicies.put("ErasureCodingPolicyInfo", erasureCodingPolicyInfos);
return toJsonString("ErasureCodingPolicies", erasureCodingPolicies);
}
public static String toJsonString(Collection<FileStatus> trashPaths) throws IOException {
List<FileStatus> list = new ArrayList<>(trashPaths);
Object[] paths = null;
if(list.size() > 0){
paths = new Object[list.size()];
for(int i = 0; i < list.size(); i++){
paths[i] = toJsonMap(list.get(i));
}
}
return toJsonString("Paths", paths);
}
}

61
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
View File

@ -63,6 +63,7 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop
* [`GETSTATUS`](#Get_Status) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getStatus)
* [`GETECPOLICIES`](#Get_EC_Policies)
* [`GETECCODECS`](#Get_EC_Codecs)
* [`GETTRASHROOTS`](#Get_Trash_Roots) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getTrashRoots)
* HTTP PUT
* [`CREATE`](#Create_and_Write_to_a_File) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).create)
* [`MKDIRS`](#Make_a_Directory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).mkdirs)
@ -1273,6 +1274,47 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getStatus
}
}
### Get Trash Roots
* Submit a HTTP GET request.
curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETTRASHROOTS
&allusers=<true|false>"
The client receives a response with a [`Paths` JSON object](#Paths_JSON_Schema):
HTTP/1.1 200 OK
Content-Type: application/json
Transfer-Encoding: chunked
{
"Paths": [{
"blocksize": 0,
"owner": "hadoop",
"path": "/user/user0/.Trash",
"length": 0,
"permission": "755",
"modification_time": 1693050205747,
"isdir": true,
"block_replication": 0,
"access_time": 0,
"group": "supergroup"
}, {
"blocksize": 0,
"owner": "hadoop",
"path": "/user/user1/.Trash",
"length": 0,
"permission": "755",
"modification_time": 1693049382962,
"isdir": true,
"block_replication": 0,
"access_time": 0,
"group": "supergroup"
}]
}
See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getTrashRoots
Storage Policy Operations
-------------------------
@ -3277,6 +3319,25 @@ var blockLocationProperties =
}
```
### Paths JSON Schema
```json
{
"Paths": [{
"blocksize": 0,
"owner": "hadoop",
"path": "/user/user0/.Trash",
"length": 0,
"permission": "755",
"modification_time": 1693050205747,
"isdir": true,
"block_replication": 0,
"access_time": 0,
"group": "supergroup"
}]
}
```
HTTP Query Parameter Dictionary
-------------------------------

29
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
View File

@ -60,6 +60,7 @@
import java.util.NoSuchElementException;
import java.util.Random;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.fs.QuotaUsage;
@ -2342,6 +2343,34 @@ public void getAllErasureCodingCodecs() throws Exception {
}
}
@Test
public void testGetTrashRoots() throws Exception {
final Configuration conf = WebHdfsTestUtil.createConf();
cluster = new MiniDFSCluster.Builder(conf).build();
final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
WebHdfsConstants.WEBHDFS_SCHEME);
// Create user trash
Path currUserHome = webFS.getHomeDirectory();
Path currUserTrash = new Path(currUserHome, FileSystem.TRASH_PREFIX);
webFS.mkdirs(currUserTrash);
Collection<FileStatus> webTrashRoots = webFS.getTrashRoots(true);
assertEquals(1, webTrashRoots.size());
// Create trash root for user0
UserGroupInformation ugi = UserGroupInformation.createRemoteUser("user0");
String user0HomeStr = DFSUtilClient.getHomeDirectory(conf, ugi);
Path user0Trash = new Path(user0HomeStr, FileSystem.TRASH_PREFIX);
webFS.mkdirs(user0Trash);
webTrashRoots = webFS.getTrashRoots(true);
assertEquals(2, webTrashRoots.size());
webTrashRoots = webFS.getTrashRoots(false);
assertEquals(1, webTrashRoots.size());
}
/**
* Get FileStatus JSONObject from ListStatus response.
*/