From bf5c94899537011465350d5d999fad9ffaeb605d Mon Sep 17 00:00:00 2001 From: Yiqun Lin Date: Sat, 6 Jan 2018 14:31:08 +0800 Subject: [PATCH] HDFS-11848. Enhance dfsadmin listOpenFiles command to list files under a given path. Contributed by Yiqun Lin. --- .../org/apache/hadoop/hdfs/DFSClient.java | 36 +++++++++- .../hadoop/hdfs/DistributedFileSystem.java | 4 +- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 4 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 5 +- .../hdfs/protocol/OpenFilesIterator.java | 10 ++- .../ClientNamenodeProtocolTranslatorPB.java | 8 ++- .../main/proto/ClientNamenodeProtocol.proto | 1 + ...amenodeProtocolServerSideTranslatorPB.java | 2 +- .../federation/router/RouterRpcServer.java | 6 +- .../hdfs/server/namenode/FSNamesystem.java | 27 +++++--- .../hdfs/server/namenode/LeaseManager.java | 26 +++++-- .../server/namenode/NameNodeRpcServer.java | 8 ++- .../apache/hadoop/hdfs/tools/DFSAdmin.java | 23 +++++-- .../src/site/markdown/HDFSCommands.md | 4 +- .../apache/hadoop/hdfs/TestDecommission.java | 38 +++++++++- .../org/apache/hadoop/hdfs/TestHdfsAdmin.java | 4 +- .../server/namenode/TestLeaseManager.java | 8 ++- .../server/namenode/TestListOpenFiles.java | 20 ++++-- .../hadoop/hdfs/tools/TestDFSAdmin.java | 69 ++++++++++++++++++- 19 files changed, 249 insertions(+), 54 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index c774132cf3..f0769c1237 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -3088,11 +3088,26 @@ Tracer getTracer() { @Deprecated public RemoteIterator listOpenFiles() throws IOException { checkOpen(); - return listOpenFiles(EnumSet.of(OpenFilesType.ALL_OPEN_FILES)); + return listOpenFiles(EnumSet.of(OpenFilesType.ALL_OPEN_FILES), + OpenFilesIterator.FILTER_PATH_DEFAULT); } /** - * Get a remote iterator to the open files list by type, managed by NameNode. + * Get a remote iterator to the open files list by path, + * managed by NameNode. + * + * @param path + * @throws IOException + */ + public RemoteIterator listOpenFiles(String path) + throws IOException { + checkOpen(); + return listOpenFiles(EnumSet.of(OpenFilesType.ALL_OPEN_FILES), path); + } + + /** + * Get a remote iterator to the open files list by type, + * managed by NameNode. * * @param openFilesTypes * @throws IOException @@ -3100,6 +3115,21 @@ public RemoteIterator listOpenFiles() throws IOException { public RemoteIterator listOpenFiles( EnumSet openFilesTypes) throws IOException { checkOpen(); - return new OpenFilesIterator(namenode, tracer, openFilesTypes); + return listOpenFiles(openFilesTypes, + OpenFilesIterator.FILTER_PATH_DEFAULT); + } + + /** + * Get a remote iterator to the open files list by type and path, + * managed by NameNode. + * + * @param openFilesTypes + * @param path + * @throws IOException + */ + public RemoteIterator listOpenFiles( + EnumSet openFilesTypes, String path) throws IOException { + checkOpen(); + return new OpenFilesIterator(namenode, tracer, openFilesTypes, path); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 85e5964b75..3883f2f849 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -3086,8 +3086,8 @@ public RemoteIterator listOpenFiles() throws IOException { } public RemoteIterator listOpenFiles( - EnumSet openFilesTypes) throws IOException { - return dfs.listOpenFiles(openFilesTypes); + EnumSet openFilesTypes, String path) throws IOException { + return dfs.listOpenFiles(openFilesTypes, path); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java index e620039786..2c0659a206 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java @@ -659,8 +659,8 @@ public RemoteIterator listOpenFiles() throws IOException { } public RemoteIterator listOpenFiles( - EnumSet openFilesTypes) throws IOException { - return dfs.listOpenFiles(openFilesTypes); + EnumSet openFilesTypes, String path) throws IOException { + return dfs.listOpenFiles(openFilesTypes, path); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 38c242ab3d..fbef037361 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -1724,10 +1724,11 @@ AddErasureCodingPolicyResponse[] addErasureCodingPolicies( * the the list across batches are not atomic. * * @param prevId the cursor INode id. - * @param openFilesTypes types to filter the open files + * @param openFilesTypes types to filter the open files. + * @param path path to filter the open files. * @throws IOException */ @Idempotent BatchedEntries listOpenFiles(long prevId, - EnumSet openFilesTypes) throws IOException; + EnumSet openFilesTypes, String path) throws IOException; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/OpenFilesIterator.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/OpenFilesIterator.java index d113d65c09..c2b3781601 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/OpenFilesIterator.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/OpenFilesIterator.java @@ -37,6 +37,9 @@ public class OpenFilesIterator extends BatchedRemoteIterator { + /** No path to be filtered by default. */ + public static final String FILTER_PATH_DEFAULT = "/"; + /** * Open file types to filter the results. */ @@ -67,20 +70,23 @@ public static OpenFilesType valueOf(short num) { private final ClientProtocol namenode; private final Tracer tracer; private final EnumSet types; + /** List files filtered by given path. */ + private String path; public OpenFilesIterator(ClientProtocol namenode, Tracer tracer, - EnumSet types) { + EnumSet types, String path) { super(HdfsConstants.GRANDFATHER_INODE_ID); this.namenode = namenode; this.tracer = tracer; this.types = types; + this.path = path; } @Override public BatchedEntries makeRequest(Long prevId) throws IOException { try (TraceScope ignored = tracer.newScope("listOpenFiles")) { - return namenode.listOpenFiles(prevId, types); + return namenode.listOpenFiles(prevId, types, path); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index ea5c951734..4a22da9cf6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -78,6 +78,7 @@ import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType; import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats; import org.apache.hadoop.hdfs.protocol.OpenFileEntry; +import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus; import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport; @@ -1898,17 +1899,20 @@ public QuotaUsage getQuotaUsage(String path) throws IOException { @Override public BatchedEntries listOpenFiles(long prevId) throws IOException { - return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES)); + return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES), + OpenFilesIterator.FILTER_PATH_DEFAULT); } @Override public BatchedEntries listOpenFiles(long prevId, - EnumSet openFilesTypes) throws IOException { + EnumSet openFilesTypes, String path) throws IOException { ListOpenFilesRequestProto.Builder req = ListOpenFilesRequestProto.newBuilder().setId(prevId); if (openFilesTypes != null) { req.addAllTypes(PBHelperClient.convertOpenFileTypes(openFilesTypes)); } + req.setPath(path); + try { ListOpenFilesResponseProto response = rpcProxy.listOpenFiles(null, req.build()); diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto index f247da8c04..db31e22e9a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto @@ -804,6 +804,7 @@ enum OpenFilesTypeProto { message ListOpenFilesRequestProto { required int64 id = 1; repeated OpenFilesTypeProto types = 2; + optional string path = 3; } message OpenFilesBatchResponseProto { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index a9d2d1ebd1..d68669f2b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -1856,7 +1856,7 @@ public ListOpenFilesResponseProto listOpenFiles(RpcController controller, EnumSet openFilesTypes = PBHelperClient.convertOpenFileTypes(req.getTypesList()); BatchedEntries entries = server.listOpenFiles(req.getId(), - openFilesTypes); + openFilesTypes, req.getPath()); ListOpenFilesResponseProto.Builder builder = ListOpenFilesResponseProto.newBuilder(); builder.setHasMore(entries.hasMore()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java index 537eaf4a2e..11f7fa6492 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java @@ -90,6 +90,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.OpenFileEntry; +import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType; import org.apache.hadoop.hdfs.protocol.ReplicatedBlockStats; import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo; @@ -1940,12 +1941,13 @@ public ReplicatedBlockStats getReplicatedBlockStats() throws IOException { @Override public BatchedEntries listOpenFiles(long prevId) throws IOException { - return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES)); + return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES), + OpenFilesIterator.FILTER_PATH_DEFAULT); } @Override public BatchedEntries listOpenFiles(long prevId, - EnumSet openFilesTypes) throws IOException { + EnumSet openFilesTypes, String path) throws IOException { checkOperation(OperationCategory.READ, false); return null; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 54decc8047..ece649d7e9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1767,11 +1767,12 @@ private void metaSave(PrintWriter out) { * TODO: HDFS-12969 - to report open files by type. * * @param prevId the cursor INode id. - * @param openFilesTypes + * @param openFilesTypes types to filter the open files. + * @param path path to filter the open files. * @throws IOException */ BatchedListEntries listOpenFiles(long prevId, - EnumSet openFilesTypes) throws IOException { + EnumSet openFilesTypes, String path) throws IOException { final String operationName = "listOpenFiles"; checkSuperuserPrivilege(); checkOperation(OperationCategory.READ); @@ -1780,10 +1781,11 @@ BatchedListEntries listOpenFiles(long prevId, try { checkOperation(OperationCategory.READ); if(openFilesTypes.contains(OpenFilesType.ALL_OPEN_FILES)) { - batchedListEntries = leaseManager.getUnderConstructionFiles(prevId); + batchedListEntries = leaseManager.getUnderConstructionFiles(prevId, + path); } else { if(openFilesTypes.contains(OpenFilesType.BLOCKING_DECOMMISSION)) { - batchedListEntries = getFilesBlockingDecom(prevId); + batchedListEntries = getFilesBlockingDecom(prevId, path); } else { throw new IllegalArgumentException("Unknown OpenFileType: " + openFilesTypes); @@ -1799,7 +1801,8 @@ BatchedListEntries listOpenFiles(long prevId, return batchedListEntries; } - public BatchedListEntries getFilesBlockingDecom(long prevId) { + public BatchedListEntries getFilesBlockingDecom(long prevId, + String path) { assert hasReadLock(); final List openFileEntries = Lists.newArrayList(); LightWeightHashSet openFileIds = new LightWeightHashSet<>(); @@ -1817,10 +1820,16 @@ public BatchedListEntries getFilesBlockingDecom(long prevId) { Preconditions.checkState(ucFile instanceof INodeFile); openFileIds.add(ucFileId); INodeFile inodeFile = ucFile.asFile(); - openFileEntries.add(new OpenFileEntry( - inodeFile.getId(), inodeFile.getFullPathName(), - inodeFile.getFileUnderConstructionFeature().getClientName(), - inodeFile.getFileUnderConstructionFeature().getClientMachine())); + + String fullPathName = inodeFile.getFullPathName(); + if (org.apache.commons.lang.StringUtils.isEmpty(path) + || fullPathName.startsWith(path)) { + openFileEntries.add(new OpenFileEntry(inodeFile.getId(), + inodeFile.getFullPathName(), + inodeFile.getFileUnderConstructionFeature().getClientName(), + inodeFile.getFileUnderConstructionFeature().getClientMachine())); + } + if (openFileIds.size() >= this.maxListOpenFilesResponses) { return new BatchedListEntries<>(openFileEntries, true); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java index 3746e1368a..1e7a17402f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java @@ -37,10 +37,13 @@ import java.util.concurrent.Future; import com.google.common.collect.Lists; + +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.OpenFileEntry; +import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.util.Daemon; @@ -258,6 +261,12 @@ public List call() { return iipSet; } + public BatchedListEntries getUnderConstructionFiles( + final long prevId) throws IOException { + return getUnderConstructionFiles(prevId, + OpenFilesIterator.FILTER_PATH_DEFAULT); + } + /** * Get a batch of under construction files from the currently active leases. * File INodeID is the cursor used to fetch new batch of results and the @@ -270,7 +279,7 @@ public List call() { * @throws IOException */ public BatchedListEntries getUnderConstructionFiles( - final long prevId) throws IOException { + final long prevId, final String path) throws IOException { assert fsnamesystem.hasReadLock(); SortedMap remainingLeases; synchronized (this) { @@ -283,6 +292,7 @@ public BatchedListEntries getUnderConstructionFiles( Lists.newArrayListWithExpectedSize(numResponses); int count = 0; + String fullPathName = null; for (Long inodeId: inodeIds) { final INodeFile inodeFile = fsnamesystem.getFSDirectory().getInode(inodeId).asFile(); @@ -291,11 +301,15 @@ public BatchedListEntries getUnderConstructionFiles( inodeFile.getFullPathName()); continue; } - openFileEntries.add(new OpenFileEntry( - inodeFile.getId(), inodeFile.getFullPathName(), - inodeFile.getFileUnderConstructionFeature().getClientName(), - inodeFile.getFileUnderConstructionFeature().getClientMachine())); - count++; + + fullPathName = inodeFile.getFullPathName(); + if (StringUtils.isEmpty(path) || fullPathName.startsWith(path)) { + openFileEntries.add(new OpenFileEntry(inodeFile.getId(), fullPathName, + inodeFile.getFileUnderConstructionFeature().getClientName(), + inodeFile.getFileUnderConstructionFeature().getClientMachine())); + count++; + } + if (count >= numResponses) { break; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 80f1ba3707..121d17c7fb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -115,6 +115,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.OpenFileEntry; +import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType; import org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; @@ -1339,14 +1340,15 @@ public void metaSave(String filename) throws IOException { @Override // ClientProtocol public BatchedEntries listOpenFiles(long prevId) throws IOException { - return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES)); + return listOpenFiles(prevId, EnumSet.of(OpenFilesType.ALL_OPEN_FILES), + OpenFilesIterator.FILTER_PATH_DEFAULT); } @Override // ClientProtocol public BatchedEntries listOpenFiles(long prevId, - EnumSet openFilesTypes) throws IOException { + EnumSet openFilesTypes, String path) throws IOException { checkNNStartup(); - return namesystem.listOpenFiles(prevId, openFilesTypes); + return namesystem.listOpenFiles(prevId, openFilesTypes, path); } @Override // ClientProtocol diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index 73673090ec..1bedd824a2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -464,7 +464,7 @@ static int run(DistributedFileSystem dfs, String[] argv, int idx) throws IOExcep "\t[-getDatanodeInfo ]\n" + "\t[-metasave filename]\n" + "\t[-triggerBlockReport [-incremental] ]\n" + - "\t[-listOpenFiles [-blockingDecommission]]\n" + + "\t[-listOpenFiles [-blockingDecommission] [-path ]]\n" + "\t[-help [cmd]]\n"; /** @@ -918,16 +918,29 @@ public int refreshNodes() throws IOException { * @param argv */ public int listOpenFiles(String[] argv) throws IOException { + String path = null; List types = new ArrayList<>(); if (argv != null) { List args = new ArrayList<>(Arrays.asList(argv)); if (StringUtils.popOption("-blockingDecommission", args)) { types.add(OpenFilesType.BLOCKING_DECOMMISSION); } + + path = StringUtils.popOptionWithArgument("-path", args); } if (types.isEmpty()) { types.add(OpenFilesType.ALL_OPEN_FILES); } + + if (path != null) { + path = path.trim(); + if (path.length() == 0) { + path = OpenFilesIterator.FILTER_PATH_DEFAULT; + } + } else { + path = OpenFilesIterator.FILTER_PATH_DEFAULT; + } + EnumSet openFilesTypes = EnumSet.copyOf(types); DistributedFileSystem dfs = getDFS(); @@ -941,9 +954,9 @@ public int listOpenFiles(String[] argv) throws IOException { dfsConf, HAUtil.getAddressOfActive(getDFS()), ClientProtocol.class, UserGroupInformation.getCurrentUser(), false); openFilesRemoteIterator = new OpenFilesIterator(proxy.getProxy(), - FsTracer.get(dfsConf), openFilesTypes); + FsTracer.get(dfsConf), openFilesTypes, path); } else { - openFilesRemoteIterator = dfs.listOpenFiles(openFilesTypes); + openFilesRemoteIterator = dfs.listOpenFiles(openFilesTypes, path); } printOpenFiles(openFilesRemoteIterator); return 0; @@ -1982,7 +1995,7 @@ private static void printUsage(String cmd) { + " [-triggerBlockReport [-incremental] ]"); } else if ("-listOpenFiles".equals(cmd)) { System.err.println("Usage: hdfs dfsadmin" - + " [-listOpenFiles [-blockingDecommission]]"); + + " [-listOpenFiles [-blockingDecommission] [-path ]]"); } else { System.err.println("Usage: hdfs dfsadmin"); System.err.println("Note: Administrative commands can only be run as the HDFS superuser."); @@ -2137,7 +2150,7 @@ public int run(String[] argv) throws Exception { return exitCode; } } else if ("-listOpenFiles".equals(cmd)) { - if ((argv.length != 1) && (argv.length != 2)) { + if ((argv.length > 4)) { printUsage(cmd); return exitCode; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index a13116f4f3..58d9547773 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -372,7 +372,7 @@ Usage: hdfs dfsadmin [-getDatanodeInfo ] hdfs dfsadmin [-metasave filename] hdfs dfsadmin [-triggerBlockReport [-incremental] ] - hdfs dfsadmin [-listOpenFiles] + hdfs dfsadmin [-listOpenFiles [-blockingDecommission] [-path ]] hdfs dfsadmin [-help [cmd]] | COMMAND\_OPTION | Description | @@ -409,7 +409,7 @@ Usage: | `-getDatanodeInfo` \ | Get the information about the given datanode. See [Rolling Upgrade document](./HdfsRollingUpgrade.html#dfsadmin_-getDatanodeInfo) for the detail. | | `-metasave` filename | Save Namenode's primary data structures to *filename* in the directory specified by hadoop.log.dir property. *filename* is overwritten if it exists. *filename* will contain one line for each of the following
1. Datanodes heart beating with Namenode
2. Blocks waiting to be replicated
3. Blocks currently being replicated
4. Blocks waiting to be deleted | | `-triggerBlockReport` `[-incremental]` \ | Trigger a block report for the given datanode. If 'incremental' is specified, it will be otherwise, it will be a full block report. | -| `-listOpenFiles` `[-blockingDecommission]` | List all open files currently managed by the NameNode along with client name and client machine accessing them. | +| `-listOpenFiles` `[-blockingDecommission]` `[-path ]` | List all open files currently managed by the NameNode along with client name and client machine accessing them. Open files list will be filtered by given type and path. | | `-help` [cmd] | Displays help for the given command or all commands if none is specified. | Runs a HDFS dfsadmin client. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java index d82025c570..c0a595bcb7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java @@ -710,13 +710,49 @@ private void verifyOpenFilesBlockingDecommission(HashSet closedFileSet, @Override public Boolean get() { try { + boolean result1 = false; + boolean result2 = false; toolOut.reset(); assertEquals(0, ToolRunner.run(dfsAdmin, new String[]{"-listOpenFiles", "-blockingDecommission"})); toolOut.flush(); - return verifyOpenFilesListing( + result1 = verifyOpenFilesListing( "dfsadmin -listOpenFiles -blockingDecommission", closedFileSet, openFilesMap, toolOut, maxOpenFiles); + + // test -blockingDecommission with option -path + if (openFilesMap.size() > 0) { + String firstOpenFile = null; + // Construct a new open-file and close-file map. + // Pick the first open file into new open-file map, remaining + // open files move into close-files map. + HashMap newOpenFilesMap = + new HashMap<>(); + HashSet newClosedFileSet = new HashSet<>(); + for (Map.Entry entry : openFilesMap + .entrySet()) { + if (firstOpenFile == null) { + newOpenFilesMap.put(entry.getKey(), entry.getValue()); + firstOpenFile = entry.getKey().toString(); + } else { + newClosedFileSet.add(entry.getKey()); + } + } + + toolOut.reset(); + assertEquals(0, + ToolRunner.run(dfsAdmin, new String[] {"-listOpenFiles", + "-blockingDecommission", "-path", firstOpenFile})); + toolOut.flush(); + result2 = verifyOpenFilesListing( + "dfsadmin -listOpenFiles -blockingDecommission -path" + + firstOpenFile, + newClosedFileSet, newOpenFilesMap, toolOut, 1); + } else { + result2 = true; + } + + return result1 && result2; } catch (Exception e) { LOG.warn("Unexpected exception: " + e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java index 3cb10bf1e9..cc32a3cf44 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHdfsAdmin.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.client.HdfsAdmin; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.OpenFileEntry; +import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType; import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.junit.After; @@ -256,7 +257,8 @@ private void verifyOpenFiles(HashSet closedFiles, HdfsAdmin hdfsAdmin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf); HashSet openFiles = new HashSet<>(openFileMap.keySet()); RemoteIterator openFilesRemoteItr = - hdfsAdmin.listOpenFiles(EnumSet.of(OpenFilesType.ALL_OPEN_FILES)); + hdfsAdmin.listOpenFiles(EnumSet.of(OpenFilesType.ALL_OPEN_FILES), + OpenFilesIterator.FILTER_PATH_DEFAULT); while (openFilesRemoteItr.hasNext()) { String filePath = openFilesRemoteItr.next().getFilePath(); assertFalse(filePath + " should not be listed under open files!", diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java index 0a8da4b01b..ccd908b645 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestLeaseManager.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; +import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; @@ -405,8 +406,11 @@ private void verifyINodeLeaseCounts(FSNamesystem fsNamesystem, leaseManager.getINodeWithLeases(ancestorDirectory).size()); assertEquals(iNodeIdWithLeaseCount, leaseManager.getUnderConstructionFiles(0).size()); - assertEquals(0, (fsNamesystem.getFilesBlockingDecom(0) == null ? - 0 : fsNamesystem.getFilesBlockingDecom(0).size())); + assertEquals(0, + (fsNamesystem.getFilesBlockingDecom(0, + OpenFilesIterator.FILTER_PATH_DEFAULT) == null ? 0 + : fsNamesystem.getFilesBlockingDecom(0, + OpenFilesIterator.FILTER_PATH_DEFAULT).size())); } private Map createINodeTree(INodeDirectory parentDir, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListOpenFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListOpenFiles.java index cfee7ba46c..70550d5785 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListOpenFiles.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListOpenFiles.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.OpenFileEntry; +import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; @@ -97,11 +98,13 @@ public void testListOpenFilesViaNameNodeRPC() throws Exception { verifyOpenFiles(openFiles); BatchedEntries openFileEntryBatchedEntries = - nnRpc.listOpenFiles(0, EnumSet.of(OpenFilesType.ALL_OPEN_FILES)); + nnRpc.listOpenFiles(0, EnumSet.of(OpenFilesType.ALL_OPEN_FILES), + OpenFilesIterator.FILTER_PATH_DEFAULT); assertTrue("Open files list should be empty!", openFileEntryBatchedEntries.size() == 0); BatchedEntries openFilesBlockingDecomEntries = - nnRpc.listOpenFiles(0, EnumSet.of(OpenFilesType.BLOCKING_DECOMMISSION)); + nnRpc.listOpenFiles(0, EnumSet.of(OpenFilesType.BLOCKING_DECOMMISSION), + OpenFilesIterator.FILTER_PATH_DEFAULT); assertTrue("Open files list blocking decommission should be empty!", openFilesBlockingDecomEntries.size() == 0); @@ -128,15 +131,16 @@ public void testListOpenFilesViaNameNodeRPC() throws Exception { } private void verifyOpenFiles(Map openFiles, - EnumSet openFilesTypes) throws IOException { + EnumSet openFilesTypes, String path) throws IOException { HashSet remainingFiles = new HashSet<>(openFiles.keySet()); OpenFileEntry lastEntry = null; BatchedEntries batchedEntries; do { if (lastEntry == null) { - batchedEntries = nnRpc.listOpenFiles(0, openFilesTypes); + batchedEntries = nnRpc.listOpenFiles(0, openFilesTypes, path); } else { - batchedEntries = nnRpc.listOpenFiles(lastEntry.getId(), openFilesTypes); + batchedEntries = nnRpc.listOpenFiles(lastEntry.getId(), + openFilesTypes, path); } assertTrue("Incorrect open files list size!", batchedEntries.size() <= BATCH_SIZE); @@ -154,9 +158,11 @@ private void verifyOpenFiles(Map openFiles, private void verifyOpenFiles(Map openFiles) throws IOException { - verifyOpenFiles(openFiles, EnumSet.of(OpenFilesType.ALL_OPEN_FILES)); + verifyOpenFiles(openFiles, EnumSet.of(OpenFilesType.ALL_OPEN_FILES), + OpenFilesIterator.FILTER_PATH_DEFAULT); verifyOpenFiles(new HashMap<>(), - EnumSet.of(OpenFilesType.BLOCKING_DECOMMISSION)); + EnumSet.of(OpenFilesType.BLOCKING_DECOMMISSION), + OpenFilesIterator.FILTER_PATH_DEFAULT); } private Set createFiles(FileSystem fileSystem, String fileNamePrefix, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java index 6a01de2189..7237c884f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java @@ -725,6 +725,67 @@ public void testListOpenFiles() throws Exception { new String[]{"-listOpenFiles"})); verifyOpenFilesListing(closedFileSet, openFilesMap); } + + // test -listOpenFiles command with option + openFilesMap.clear(); + Path file; + HashMap openFiles1 = new HashMap<>(); + HashMap openFiles2 = new HashMap<>(); + for (int i = 0; i < numOpenFiles; i++) { + if (i % 2 == 0) { + file = new Path(new Path("/tmp/files/a"), "open-file-" + i); + } else { + file = new Path(new Path("/tmp/files/b"), "open-file-" + i); + } + + DFSTestUtil.createFile(fs, file, fileLength, replFactor, 12345L); + FSDataOutputStream outputStream = fs.append(file); + + if (i % 2 == 0) { + openFiles1.put(file, outputStream); + } else { + openFiles2.put(file, outputStream); + } + openFilesMap.put(file, outputStream); + } + + resetStream(); + // list all open files + assertEquals(0, + ToolRunner.run(dfsAdmin, new String[] {"-listOpenFiles"})); + verifyOpenFilesListing(null, openFilesMap); + + resetStream(); + // list open files under directory path /tmp/files/a + assertEquals(0, ToolRunner.run(dfsAdmin, + new String[] {"-listOpenFiles", "-path", "/tmp/files/a"})); + verifyOpenFilesListing(null, openFiles1); + + resetStream(); + // list open files without input path + assertEquals(-1, ToolRunner.run(dfsAdmin, + new String[] {"-listOpenFiles", "-path"})); + // verify the error + String outStr = scanIntoString(err); + assertTrue(outStr.contains("listOpenFiles: option" + + " -path requires 1 argument")); + + resetStream(); + // list open files with empty path + assertEquals(0, ToolRunner.run(dfsAdmin, + new String[] {"-listOpenFiles", "-path", ""})); + // all the open files will be listed + verifyOpenFilesListing(null, openFilesMap); + + resetStream(); + // list invalid path file + assertEquals(0, ToolRunner.run(dfsAdmin, + new String[] {"-listOpenFiles", "-path", "/invalid_path"})); + outStr = scanIntoString(out); + for (Path openFilePath : openFilesMap.keySet()) { + assertThat(outStr, not(containsString(openFilePath.toString()))); + } + DFSTestUtil.closeOpenFiles(openFilesMap, openFilesMap.size()); } } @@ -732,9 +793,13 @@ private void verifyOpenFilesListing(HashSet closedFileSet, HashMap openFilesMap) { final String outStr = scanIntoString(out); LOG.info("dfsadmin -listOpenFiles output: \n" + out); - for (Path closedFilePath : closedFileSet) { - assertThat(outStr, not(containsString(closedFilePath.toString() + "\n"))); + if (closedFileSet != null) { + for (Path closedFilePath : closedFileSet) { + assertThat(outStr, + not(containsString(closedFilePath.toString() + "\n"))); + } } + for (Path openFilePath : openFilesMap.keySet()) { assertThat(outStr, is(containsString(openFilePath.toString() + "\n"))); }