From 4fde5c01901b6acb4363747d01603664a0283fc4 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 23 Oct 2012 20:26:08 +0000 Subject: [PATCH] HDFS-4084. Provide CLI support to allow and disallow snapshot on a directory. Contributed by Brondon Li. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2802@1401444 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES.HDFS-2802.txt | 3 ++ .../org/apache/hadoop/hdfs/DFSClient.java | 18 +++++++ .../hadoop/hdfs/DistributedFileSystem.java | 22 ++++++++ .../hadoop/hdfs/protocol/ClientProtocol.java | 16 ++++++ ...amenodeProtocolServerSideTranslatorPB.java | 32 +++++++++++- .../ClientNamenodeProtocolTranslatorPB.java | 24 +++++++++ .../hdfs/server/namenode/FSNamesystem.java | 15 ++++++ .../server/namenode/NameNodeRpcServer.java | 12 +++++ .../apache/hadoop/hdfs/tools/DFSAdmin.java | 50 ++++++++++++++++++- .../main/proto/ClientNamenodeProtocol.proto | 18 +++++++ 10 files changed, 207 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt index f527507086..e435388974 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-2802.txt @@ -20,3 +20,6 @@ Branch-2802 Snapshot (Unreleased) directory to snapshottable and creating a snapshot. (szetszwo) HDFS-4078. Handle replication in snapshots. (szetszwo) + + HDFS-4084. Provide CLI support to allow and disallow snapshot + on a directory. (Brondon Li via suresh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index d29ab1d0a8..47033bca5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -1873,6 +1873,24 @@ public boolean setSafeMode(SafeModeAction action) throws IOException { return namenode.setSafeMode(action); } + /** + * Allow snapshot on a directory. + * + * @see ClientProtocol#allowSnapshot(String snapshotRoot) + */ + public void allowSnapshot(String snapshotRoot) throws IOException { + namenode.allowSnapshot(snapshotRoot); + } + + /** + * Disallow snapshot on a directory. + * + * @see ClientProtocol#disallowSnapshot(String snapshotRoot) + */ + public void disallowSnapshot(String snapshotRoot) throws IOException { + namenode.disallowSnapshot(snapshotRoot); + } + /** * Save namespace image. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 0e30245ce5..e11933d7ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -868,4 +868,26 @@ public String getCanonicalServiceName() { public boolean isInSafeMode() throws IOException { return setSafeMode(SafeModeAction.SAFEMODE_GET); } + + /** + * Allow snapshot on a directory. + * + * @param snapshotRoot the directory to be snapped + * @throws IOException + */ + public void allowSnapshot(String snapshotRoot) + throws IOException { + dfs.allowSnapshot(snapshotRoot); + } + + /** + * Disallow snapshot on a directory. + * + * @param snapshotRoot the directory to be snapped + * @throws IOException + */ + public void disallowSnapshot(String snapshotRoot) + throws IOException { + dfs.disallowSnapshot(snapshotRoot); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 9a159a48d8..19a17cbe79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -972,5 +972,21 @@ public void deleteSnapshot(String snapshotName, String snapshotRoot) */ public SnapshotInfo[] listSnapshots(String snapshotRoot) throws IOException; + + /** + * Allow snapshot on a directory. + * @param snapshotRoot the directory to be snapped + * @throws IOException + */ + public void allowSnapshot(String snapshotRoot) + throws IOException; + + /** + * Disallow snapshot on a directory. + * @param snapshotRoot the directory to disallow snapshot + * @throws IOException + */ + public void disallowSnapshot(String snapshotRoot) + throws IOException; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index f77994c974..ced35e0bdb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -37,6 +37,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AbandonBlockResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddBlockRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddBlockResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AllowSnapshotRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AllowSnapshotResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CancelDelegationTokenRequestProto; @@ -55,6 +57,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteSnapshotRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteSnapshotResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DisallowSnapshotRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DisallowSnapshotResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FinalizeUpgradeRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FinalizeUpgradeResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FsyncRequestProto; @@ -132,6 +136,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdateBlockForPipelineResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineResponseProto; + import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTokenIdentifierProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto; @@ -159,7 +164,10 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements DeleteSnapshotResponseProto.newBuilder().build(); static final CreateSnapshotResponseProto VOID_CREATE_SNAPSHOT_RESPONSE = CreateSnapshotResponseProto.newBuilder().build(); - + static final AllowSnapshotResponseProto VOID_ALLOW_SNAPSHOT_RESPONSE = + AllowSnapshotResponseProto.newBuilder().build(); + static final DisallowSnapshotResponseProto VOID_DISALLOW_SNAPSHOT_RESPONSE = + DisallowSnapshotResponseProto.newBuilder().build(); /** * Constructor @@ -904,4 +912,26 @@ public ListSnapshotsResponseProto listSnapshots(RpcController controller, throw new ServiceException(e); } } + + @Override + public AllowSnapshotResponseProto allowSnapshot(RpcController controller, + AllowSnapshotRequestProto req) throws ServiceException { + try { + server.allowSnapshot(req.getSnapshotRoot()); + return VOID_ALLOW_SNAPSHOT_RESPONSE; + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public DisallowSnapshotResponseProto disallowSnapshot(RpcController controller, + DisallowSnapshotRequestProto req) throws ServiceException { + try { + server.disallowSnapshot(req.getSnapshotRoot()); + return VOID_DISALLOW_SNAPSHOT_RESPONSE; + } catch (IOException e) { + throw new ServiceException(e); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 0bf0235e56..7d2a3ccc01 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AbandonBlockRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddBlockRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AllowSnapshotRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CancelDelegationTokenRequestProto; @@ -60,6 +61,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateSymlinkRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteSnapshotRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DisallowSnapshotRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FinalizeUpgradeRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FsyncRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetAdditionalDatanodeRequestProto; @@ -879,4 +881,26 @@ public SnapshotInfo[] listSnapshots(String snapshotRoot) throws IOException { throw ProtobufHelper.getRemoteException(e); } } + + @Override + public void allowSnapshot(String snapshotRoot) throws IOException { + AllowSnapshotRequestProto req = AllowSnapshotRequestProto.newBuilder() + .setSnapshotRoot(snapshotRoot).build(); + try { + rpcProxy.allowSnapshot(null, req); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + + @Override + public void disallowSnapshot(String snapshotRoot) throws IOException { + DisallowSnapshotRequestProto req = DisallowSnapshotRequestProto + .newBuilder().setSnapshotRoot(snapshotRoot).build(); + try { + rpcProxy.disallowSnapshot(null, req); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 2d110fd621..52e47bf071 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -140,6 +140,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; +import org.apache.hadoop.hdfs.protocol.SnapshotInfo; import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode; @@ -5556,4 +5557,18 @@ public boolean isAvoidingStaleDataNodesForWrite() { return this.blockManager.getDatanodeManager() .isAvoidingStaleDataNodesForWrite(); } + + // Allow snapshot on a directroy. + @VisibleForTesting + public void allowSnapshot(String snapshotRoot) + throws SafeModeException, IOException { + // TODO: implement + } + + // Disallow snapshot on a directory. + @VisibleForTesting + public void disallowSnapshot(String snapshotRoot) + throws SafeModeException, IOException { + // TODO: implement + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 2bc5b4c079..7579d99857 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -1093,4 +1093,16 @@ public SnapshotInfo[] listSnapshots(String snapshotRoot) throws IOException { si[0] = new SnapshotInfo(null, null, null, null, null, null); return si; } + + @Override + // Client Protocol + public void allowSnapshot(String snapshotRoot) throws IOException { + namesystem.allowSnapshot(snapshotRoot); + } + + @Override + // Client Protocol + public void disallowSnapshot(String snapshot) throws IOException { + namesystem.disallowSnapshot(snapshot); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index dd5a0e313d..3705df04b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -406,6 +406,30 @@ public void setSafeMode(String[] argv, int idx) throws IOException { System.out.println("Safe mode is " + (inSafeMode ? "ON" : "OFF")); } + /** + * Allow snapshot on a directory. + * Usage: java DFSAdmin -allowSnapshot + * @param argv List of of command line parameters. + * @exception IOException + */ + public void allowSnapshot(String[] argv) throws IOException { + DistributedFileSystem dfs = getDFS(); + dfs.allowSnapshot(argv[1]); + System.out.println("Allowing snaphot on " + argv[1] + " succeeded"); + } + + /** + * Allow snapshot on a directory. + * Usage: java DFSAdmin -disallowSnapshot + * @param argv List of of command line parameters. + * @exception IOException + */ + public void disallowSnapshot(String[] argv) throws IOException { + DistributedFileSystem dfs = getDFS(); + dfs.disallowSnapshot(argv[1]); + System.out.println("Disallowing snaphot on " + argv[1] + " succeeded"); + } + /** * Command to ask the namenode to save the namespace. * Usage: java DFSAdmin -saveNamespace @@ -879,7 +903,13 @@ private static void printUsage(String cmd) { + " [-report]"); } else if ("-safemode".equals(cmd)) { System.err.println("Usage: java DFSAdmin" - + " [-safemode enter | leave | get | wait]"); + + " [-safemode enter | leave | get | wait]"); + } else if ("-allowSnapshot".equalsIgnoreCase(cmd)) { + System.err.println("Usage: java DFSAdmin" + + " [-allowSnapshot ]"); + } else if ("-disallowsnapshot".equalsIgnoreCase(cmd)) { + System.err.println("Usage: java DFSAdmin" + + " [-disallowSnapshot ]"); } else if ("-saveNamespace".equals(cmd)) { System.err.println("Usage: java DFSAdmin" + " [-saveNamespace]"); @@ -938,7 +968,9 @@ private static void printUsage(String cmd) { System.err.println("Usage: java DFSAdmin"); System.err.println("Note: Administrative commands can only be run as the HDFS superuser."); System.err.println(" [-report]"); - System.err.println(" [-safemode enter | leave | get | wait]"); + System.err.println(" [-safemode enter | leave | get | wait]"); + System.err.println(" [-allowSnapshot ]"); + System.err.println(" [-disallowSnapshot ]"); System.err.println(" [-saveNamespace]"); System.err.println(" [-rollEdits]"); System.err.println(" [-restoreFailedStorage true|false|check]"); @@ -988,6 +1020,16 @@ public int run(String[] argv) throws Exception { printUsage(cmd); return exitCode; } + } else if ("-allowSnapshot".equalsIgnoreCase(cmd)) { + if (argv.length != 3) { + printUsage(cmd); + return exitCode; + } + } else if ("-disallowSnapshot".equalsIgnoreCase(cmd)) { + if (argv.length != 2) { + printUsage(cmd); + return exitCode; + } } else if ("-report".equals(cmd)) { if (argv.length != 1) { printUsage(cmd); @@ -1079,6 +1121,10 @@ public int run(String[] argv) throws Exception { report(); } else if ("-safemode".equals(cmd)) { setSafeMode(argv, i); + } else if ("-allowSnapshot".equalsIgnoreCase(cmd)) { + allowSnapshot(argv); + } else if ("-disallowSnapshot".equalsIgnoreCase(cmd)) { + disallowSnapshot(argv); } else if ("-saveNamespace".equals(cmd)) { exitCode = saveNamespace(); } else if ("-rollEdits".equals(cmd)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto index b381ded6d4..0594db3664 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto @@ -466,6 +466,20 @@ message ListSnapshotsResponseProto { repeated SnapshotInfoProto snapshots = 1; } +message AllowSnapshotRequestProto { + required string snapshotRoot = 1; +} + +message AllowSnapshotResponseProto { +} + +message DisallowSnapshotRequestProto { + required string snapshotRoot = 1; +} + +message DisallowSnapshotResponseProto { +} + service ClientNamenodeProtocol { rpc getBlockLocations(GetBlockLocationsRequestProto) returns(GetBlockLocationsResponseProto); @@ -545,4 +559,8 @@ service ClientNamenodeProtocol { returns(DeleteSnapshotResponseProto); rpc listSnapshots(ListSnapshotsRequestProto) returns(ListSnapshotsResponseProto); + rpc allowSnapshot(AllowSnapshotRequestProto) + returns(AllowSnapshotResponseProto); + rpc disallowSnapshot(DisallowSnapshotRequestProto) + returns(DisallowSnapshotResponseProto); }