From c9e0af99617dfd5ff817776635dddb052f9acce9 Mon Sep 17 00:00:00 2001 From: zhtttylz Date: Tue, 25 Apr 2023 06:00:56 +0800 Subject: [PATCH] HDFS-16981. Support getFileLinkStatus API in WebHDFS (#5572). Contributed by Hualong Zhang. Reviewed-by: Simbarashe Dzinamarira Signed-off-by: Ayush Saxena --- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 18 ++++++++++ .../hadoop/hdfs/web/resources/GetOpParam.java | 1 + .../router/RouterWebHdfsMethods.java | 1 + .../web/resources/NamenodeWebHdfsMethods.java | 8 +++++ .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 34 +++++++++++++++++++ .../apache/hadoop/hdfs/web/TestWebHDFS.java | 25 ++++++++++++++ 6 files changed, 87 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 615cf3bd7c..f5a54dd9be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -2160,6 +2160,24 @@ Path decodeResponse(Map json) { }.run(); } + @Override + public FileStatus getFileLinkStatus(Path f) throws IOException { + statistics.incrementReadOps(1); + storageStatistics.incrementOpCounter(OpType.GET_FILE_LINK_STATUS); + final HttpOpParam.Op op = GetOpParam.Op.GETFILELINKSTATUS; + HdfsFileStatus status = + new FsPathResponseRunner(op, f) { + @Override + HdfsFileStatus decodeResponse(Map json) { + return JsonUtilClient.toFileStatus(json, true); + } + }.run(); + if (status == null) { + throw new FileNotFoundException("File does not exist: " + f); + } + return status.makeQualified(getUri(), f); + } + @VisibleForTesting InetSocketAddress[] getResolvedNNAddr() { return nnAddrs; diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java index 89979295c7..c1dcd76b15 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java @@ -65,6 +65,7 @@ public enum Op implements HttpOpParam.Op { GETSNAPSHOTDIFFLISTING(false, HttpURLConnection.HTTP_OK), GETSNAPSHOTTABLEDIRECTORYLIST(false, HttpURLConnection.HTTP_OK), GETLINKTARGET(false, HttpURLConnection.HTTP_OK), + GETFILELINKSTATUS(false, HttpURLConnection.HTTP_OK), GETSNAPSHOTLIST(false, HttpURLConnection.HTTP_OK); final boolean redirect; diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java index 477a59941f..888def5e63 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterWebHdfsMethods.java @@ -386,6 +386,7 @@ protected Response get( case LISTXATTRS: case CHECKACCESS: case GETLINKTARGET: + case GETFILELINKSTATUS: { return super.get(ugi, delegation, username, doAsUser, fullpath, op, offset, length, renewer, bufferSize, xattrNames, xattrEncoding, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 4b3b53731e..cd69f4ebd6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -1388,6 +1388,14 @@ protected Response get( final String js = JsonUtil.toJsonString("Path", target); return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } + case GETFILELINKSTATUS: { + HdfsFileStatus status = cp.getFileLinkInfo(fullpath); + if (status == null) { + throw new FileNotFoundException("File does not exist: " + fullpath); + } + final String js = JsonUtil.toJsonString(status, true); + return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); + } default: throw new UnsupportedOperationException(op + " is not supported"); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md index f84018ae82..3d22876946 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md @@ -59,6 +59,7 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop * [`GETECPOLICY`](#Get_EC_Policy) (see [HDFSErasureCoding](./HDFSErasureCoding.html#Administrative_commands).getErasureCodingPolicy) * [`GETSERVERDEFAULTS`](#Get_Server_Defaults) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getServerDefaults) * [`GETLINKTARGET`](#Get_Link_Target) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getLinkTarget) + * [`GETFILELINKSTATUS`](#Get_File_Link_Status) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileLinkStatus) * HTTP PUT * [`CREATE`](#Create_and_Write_to_a_File) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).create) * [`MKDIRS`](#Make_a_Directory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).mkdirs) @@ -1156,6 +1157,39 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getServer See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getLinkTarget +### Get File Link Status + +* Submit a HTTP GET request. + + curl -i "http://:/webhdfs/v1/?op=GETFILELINKSTATUS" + + The client receives a response with a [`FileStatus` JSON object](#FileStatuses_JSON_Schema): + + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + + { + "FileStatus": { + "accessTime": 0, + "blockSize": 0, + "childrenNum":0, + "fileId": 16388, + "group": "supergroup", + "length": 0, + "modificationTime": 1681916788427, + "owner": "hadoop", + "pathSuffix": "", + "permission": "777", + "replication": 0, + "storagePolicy": 0, + "symlink": "/webHdfsTest/file", + "type": "SYMLINK" + } + } + +See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileLinkInfo + Storage Policy Operations ------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 8f4759d8e3..aec7545fab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -2230,6 +2230,31 @@ public void testLinkTarget() throws Exception { } } + @Test + public void testFileLinkStatus() throws Exception { + final Configuration conf = WebHdfsTestUtil.createConf(); + try { + cluster = new MiniDFSCluster.Builder(conf).build(); + cluster.waitActive(); + + final WebHdfsFileSystem webHdfs = + WebHdfsTestUtil.getWebHdfsFileSystem(conf, + WebHdfsConstants.WEBHDFS_SCHEME); + // Symbolic link + Path root = new Path("/webHdfsTest/"); + Path file = new Path(root, "file"); + FileSystemTestHelper.createFile(webHdfs, file); + + Path linkToFile = new Path(root, "linkToFile"); + + webHdfs.createSymlink(file, linkToFile, false); + assertFalse(webHdfs.getFileLinkStatus(file).isSymlink()); + assertTrue(webHdfs.getFileLinkStatus(linkToFile).isSymlink()); + } finally { + cluster.shutdown(); + } + } + /** * Get FileStatus JSONObject from ListStatus response. */