big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11209. SNN can't checkpoint when rolling upgrade is not finalized. Contributed by Xiaoyu Yao.

Browse Source
This commit is contained in:
Xiaoyu Yao 2017-01-17 14:33:26 -08:00
parent 37468ebf6f
commit b1fce2b8b1
8 changed files with 69 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
View File

@ -37,6 +37,8 @@
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecentCheckpointTxIdResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsRollingUpgradeRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsRollingUpgradeResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterRequestProto;
@ -240,4 +242,18 @@ public IsUpgradeFinalizedResponseProto isUpgradeFinalized(
return IsUpgradeFinalizedResponseProto.newBuilder()
.setIsUpgradeFinalized(isUpgradeFinalized).build();
}
@Override
public IsRollingUpgradeResponseProto isRollingUpgrade(
RpcController controller, IsRollingUpgradeRequestProto request)
throws ServiceException {
boolean isRollingUpgrade;
try {
isRollingUpgrade = impl.isRollingUpgrade();
} catch (IOException e) {
throw new ServiceException(e);
}
return IsRollingUpgradeResponseProto.newBuilder()
.setIsRollingUpgrade(isRollingUpgrade).build();
}
}

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
View File

@ -34,6 +34,8 @@
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetEditLogManifestRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecentCheckpointTxIdRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsRollingUpgradeRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsRollingUpgradeResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterRequestProto;
@ -247,4 +249,17 @@ public boolean isUpgradeFinalized() throws IOException {
throw ProtobufHelper.getRemoteException(e);
}
}
@Override
public boolean isRollingUpgrade() throws IOException {
IsRollingUpgradeRequestProto req = IsRollingUpgradeRequestProto
.newBuilder().build();
try {
IsRollingUpgradeResponseProto response = rpcProxy.isRollingUpgrade(
NULL_CONTROLLER, req);
return response.getIsRollingUpgrade();
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
}
}

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
View File

@ -260,8 +260,8 @@ void doCheckpoint() throws IOException {
completeBlocksTotal);
}
bnImage.saveFSImageInAllDirs(backupNode.getNamesystem(), txid);
if (!backupNode.namesystem.isRollingUpgrade()) {
bnStorage.writeAll();
if (!backupNode.namenode.isRollingUpgrade()) {
bnImage.updateStorageVersion();
}
} finally {
backupNode.namesystem.writeUnlock("doCheckpoint");

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
View File

@ -539,7 +539,7 @@ void doImportCheckpoint(FSNamesystem target) throws IOException {
// and save it but keep the same checkpointTime
saveNamespace(target);
getStorage().writeAll();
updateStorageVersion();
}
void finalizeUpgrade(boolean finalizeEditLog) throws IOException {
@ -1052,7 +1052,7 @@ public synchronized void saveNamespace(FSNamesystem source, NameNodeFile nnf,
try {
saveFSImageInAllDirs(source, nnf, imageTxId, canceler);
if (!source.isRollingUpgrade()) {
storage.writeAll();
updateStorageVersion();
}
} finally {
if (editLogWasOpen) {

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
View File

@ -1250,6 +1250,13 @@ public boolean isUpgradeFinalized() throws IOException {
namesystem.checkSuperuserPrivilege();
return namesystem.isUpgradeFinalized();
}
@Override // NamenodeProtocol
public boolean isRollingUpgrade() throws IOException {
checkNNStartup();
namesystem.checkSuperuserPrivilege();
return namesystem.isRollingUpgrade();
}
@Override // ClientProtocol
public void finalizeUpgrade() throws IOException {

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
View File

@ -1064,7 +1064,7 @@ public boolean accept(File dir, String name) {
}
static void doMerge(
void doMerge(
CheckpointSignature sig, RemoteEditLogManifest manifest,
boolean loadImage, FSImage dstImage, FSNamesystem dstNamesystem)
throws IOException {
@ -1093,8 +1093,8 @@ static void doMerge(
Checkpointer.rollForwardByApplyingLogs(manifest, dstImage, dstNamesystem);
// The following has the side effect of purging old fsimages/edit logs.
dstImage.saveFSImageInAllDirs(dstNamesystem, dstImage.getLastAppliedTxId());
if (!dstNamesystem.isRollingUpgrade()) {
dstStorage.writeAll();
if (!namenode.isRollingUpgrade()) {
dstImage.updateStorageVersion();
}
}
}

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
View File

@ -193,5 +193,13 @@ public RemoteEditLogManifest getEditLogManifest(long sinceTxId)
@Idempotent
public boolean isUpgradeFinalized() throws IOException;
/**
* return whether the Namenode is rolling upgrade in progress (true) or
* not (false).
* @return
* @throws IOException
*/
@Idempotent
boolean isRollingUpgrade() throws IOException;
}

16
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
View File

@ -192,6 +192,16 @@ message IsUpgradeFinalizedResponseProto {
required bool isUpgradeFinalized = 1;
}
/**
* void request
*/
message IsRollingUpgradeRequestProto {
}
message IsRollingUpgradeResponseProto {
required bool isRollingUpgrade = 1;
}
/**
* Protocol used by the sub-ordinate namenode to send requests
* the active/primary namenode.
@ -267,4 +277,10 @@ service NamenodeProtocolService {
*/
rpc isUpgradeFinalized(IsUpgradeFinalizedRequestProto)
returns (IsUpgradeFinalizedResponseProto);
/**
* Return whether the NameNode is in rolling upgrade (true) or not (false).
*/
rpc isRollingUpgrade(IsRollingUpgradeRequestProto)
returns (IsRollingUpgradeResponseProto);
}