big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-2793. Add an admin command to trigger an edit log roll. Contributed by Todd Lipcon.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1380982 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Todd Lipcon 2012-09-05 04:40:23 +00:00
parent 65b308f783
commit c334cc89a8
11 changed files with 128 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -243,6 +243,8 @@ Branch-2 ( Unreleased changes )
HDFS-3150. Add option for clients to contact DNs via hostname. (eli)
HDFS-2793. Add an admin command to trigger an edit log roll. (todd)
IMPROVEMENTS
HDFS-3390. DFSAdmin should print full stack traces of errors when DEBUG

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
View File

@ -1870,6 +1870,20 @@ void saveNamespace() throws AccessControlException, IOException {
throw re.unwrapRemoteException(AccessControlException.class);
}
}
/**
* Rolls the edit log on the active NameNode.
* @return the txid of the new log segment
*
* @see ClientProtocol#rollEdits()
*/
long rollEdits() throws AccessControlException, IOException {
try {
return namenode.rollEdits();
} catch(RemoteException re) {
throw re.unwrapRemoteException(AccessControlException.class);
}
}
/**
* enable/disable restore failed storage.

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
View File

@ -624,6 +624,16 @@ public boolean setSafeMode(HdfsConstants.SafeModeAction action)
public void saveNamespace() throws AccessControlException, IOException {
dfs.saveNamespace();
}
/**
* Rolls the edit log on the active NameNode.
* Requires super-user privileges.
* @see org.apache.hadoop.hdfs.protocol.ClientProtocol#rollEdits()
* @return the transaction ID of the newly created segment
*/
public long rollEdits() throws AccessControlException, IOException {
return dfs.rollEdits();
}
/**
* enable/disable/check restoreFaileStorage

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
View File

@ -667,6 +667,18 @@ public boolean setSafeMode(HdfsConstants.SafeModeAction action)
*/
public void saveNamespace() throws AccessControlException, IOException;
/**
* Roll the edit log.
* Requires superuser privileges.
*
* @throws AccessControlException if the superuser privilege is violated
* @throws IOException if log roll fails
* @return the txid of the new segment
*/
@Idempotent
public long rollEdits() throws AccessControlException, IOException;
/**
* Enable/Disable restore failed storage.
* <p>

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
View File

@ -103,6 +103,8 @@
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetBalancerBandwidthRequestProto;
@ -537,6 +539,20 @@ public SaveNamespaceResponseProto saveNamespace(RpcController controller,
}
}
@Override
public RollEditsResponseProto rollEdits(RpcController controller,
RollEditsRequestProto request) throws ServiceException {
try {
long txid = server.rollEdits();
return RollEditsResponseProto.newBuilder()
.setNewSegmentTxId(txid)
.build();
} catch (IOException e) {
throw new ServiceException(e);
}
}
static final RefreshNodesResponseProto VOID_REFRESHNODES_RESPONSE =
RefreshNodesResponseProto.newBuilder().build();

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
View File

@ -87,6 +87,8 @@
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewLeaseRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollEditsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SaveNamespaceRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetBalancerBandwidthRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetOwnerRequestProto;
@ -525,6 +527,17 @@ public void saveNamespace() throws AccessControlException, IOException {
throw ProtobufHelper.getRemoteException(e);
}
}
@Override
public long rollEdits() throws AccessControlException, IOException {
RollEditsRequestProto req = RollEditsRequestProto.getDefaultInstance();
try {
RollEditsResponseProto resp = rpcProxy.rollEdits(null, req);
return resp.getNewSegmentTxId();
} catch (ServiceException se) {
throw ProtobufHelper.getRemoteException(se);
}
}
@Override
public boolean restoreFailedStorage(String arg)

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -4415,6 +4415,7 @@ CheckpointSignature rollEditLog() throws IOException {
writeLock();
try {
checkOperation(OperationCategory.JOURNAL);
checkSuperuserPrivilege();
if (isInSafeMode()) {
throw new SafeModeException("Log not rolled", safeMode);
}

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
View File

@ -709,6 +709,13 @@ public void saveNamespace() throws IOException {
namesystem.checkOperation(OperationCategory.UNCHECKED);
namesystem.saveNamespace();
}
@Override // ClientProtocol
public long rollEdits() throws AccessControlException, IOException {
namesystem.checkOperation(OperationCategory.JOURNAL);
CheckpointSignature sig = namesystem.rollEditLog();
return sig.getCurSegmentTxId();
}
@Override // ClientProtocol
public void refreshNodes() throws IOException {

27
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
View File

@ -420,6 +420,14 @@ public int saveNamespace() throws IOException {
return exitCode;
}
public int rollEdits() throws IOException {
DistributedFileSystem dfs = getDFS();
long txid = dfs.rollEdits();
System.out.println("Successfully rolled edit logs.");
System.out.println("New segment starts at txid " + txid);
return 0;
}
/**
* Command to enable/disable/check restoring of failed storage replicas in the namenode.
* Usage: java DFSAdmin -restoreFailedStorage true|false|check
@ -516,6 +524,7 @@ private void printHelp(String cmd) {
"The full syntax is: \n\n" +
"hadoop dfsadmin [-report] [-safemode <enter | leave | get | wait>]\n" +
"\t[-saveNamespace]\n" +
"\t[-rollEdits]\n" +
"\t[-restoreFailedStorage true|false|check]\n" +
"\t[-refreshNodes]\n" +
"\t[" + SetQuotaCommand.USAGE + "]\n" +
@ -548,6 +557,10 @@ private void printHelp(String cmd) {
"Save current namespace into storage directories and reset edits log.\n" +
"\t\tRequires superuser permissions and safe mode.\n";
String rollEdits = "-rollEdits:\t" +
"Rolls the edit log.\n" +
"\t\tRequires superuser permissions.\n";
String restoreFailedStorage = "-restoreFailedStorage:\t" +
"Set/Unset/Check flag to attempt restore of failed storage replicas if they become available.\n" +
"\t\tRequires superuser permissions.\n";
@ -625,6 +638,8 @@ private void printHelp(String cmd) {
System.out.println(safemode);
} else if ("saveNamespace".equals(cmd)) {
System.out.println(saveNamespace);
} else if ("rollEdits".equals(cmd)) {
System.out.println(rollEdits);
} else if ("restoreFailedStorage".equals(cmd)) {
System.out.println(restoreFailedStorage);
} else if ("refreshNodes".equals(cmd)) {
@ -664,6 +679,7 @@ private void printHelp(String cmd) {
System.out.println(report);
System.out.println(safemode);
System.out.println(saveNamespace);
System.out.println(rollEdits);
System.out.println(restoreFailedStorage);
System.out.println(refreshNodes);
System.out.println(finalizeUpgrade);
@ -859,6 +875,9 @@ private static void printUsage(String cmd) {
} else if ("-saveNamespace".equals(cmd)) {
System.err.println("Usage: java DFSAdmin"
+ " [-saveNamespace]");
} else if ("-rollEdits".equals(cmd)) {
System.err.println("Usage: java DFSAdmin"
+ " [-rollEdits]");
} else if ("-restoreFailedStorage".equals(cmd)) {
System.err.println("Usage: java DFSAdmin"
+ " [-restoreFailedStorage true|false|check ]");
@ -913,6 +932,7 @@ private static void printUsage(String cmd) {
System.err.println(" [-report]");
System.err.println(" [-safemode enter | leave | get | wait]");
System.err.println(" [-saveNamespace]");
System.err.println(" [-rollEdits]");
System.err.println(" [-restoreFailedStorage true|false|check]");
System.err.println(" [-refreshNodes]");
System.err.println(" [-finalizeUpgrade]");
@ -970,6 +990,11 @@ public int run(String[] argv) throws Exception {
printUsage(cmd);
return exitCode;
}
} else if ("-rollEdits".equals(cmd)) {
if (argv.length != 1) {
printUsage(cmd);
return exitCode;
}
} else if ("-restoreFailedStorage".equals(cmd)) {
if (argv.length != 2) {
printUsage(cmd);
@ -1048,6 +1073,8 @@ public int run(String[] argv) throws Exception {
setSafeMode(argv, i);
} else if ("-saveNamespace".equals(cmd)) {
exitCode = saveNamespace();
} else if ("-rollEdits".equals(cmd)) {
exitCode = rollEdits();
} else if ("-restoreFailedStorage".equals(cmd)) {
exitCode = restoreFaileStorage(argv[i]);
} else if ("-refreshNodes".equals(cmd)) {

9
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
View File

@ -276,6 +276,13 @@ message SaveNamespaceRequestProto { // no parameters
message SaveNamespaceResponseProto { // void response
}
message RollEditsRequestProto { // no parameters
}
message RollEditsResponseProto { // response
required uint64 newSegmentTxId = 1;
}
message RestoreFailedStorageRequestProto {
required string arg = 1;
}
@ -472,6 +479,8 @@ service ClientNamenodeProtocol {
returns(SetSafeModeResponseProto);
rpc saveNamespace(SaveNamespaceRequestProto)
returns(SaveNamespaceResponseProto);
rpc rollEdits(RollEditsRequestProto)
returns(RollEditsResponseProto);
rpc restoreFailedStorage(RestoreFailedStorageRequestProto)
returns(RestoreFailedStorageResponseProto);
rpc refreshNodes(RefreshNodesRequestProto) returns(RefreshNodesResponseProto);

17
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
View File

@ -15885,6 +15885,23 @@
</comparators>
</test>
<!-- Test for rollEdits -->
<test> <!-- TESTED -->
<description>rollEdits: test rollEdits admin command</description>
<test-commands>
<dfs-admin-command>-fs NAMENODE -rollEdits</dfs-admin-command>
</test-commands>
<cleanup-commands>
<!-- no cleanup -->
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>New segment starts at txid \d+</expected-output>
</comparator>
</comparators>
</test>
<!-- Test for refreshNodes -->
<test> <!-- TESTED -->
<description>refreshNodes: to refresh the nodes</description>