big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-2905. HA: Standby NN NPE when shared edits dir is deleted. Contributed by Bikas Saha.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-1623@1241757 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jitendra Nath Pandey 2012-02-08 03:17:09 +00:00
parent acacde55e6
commit b9e74da41b
4 changed files with 29 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.HDFS-1623.txt
View File

@ -180,3 +180,5 @@ HDFS-2733. Document HA configuration and CLI. (atm)
HDFS-2794. Active NN may purge edit log files before standby NN has a chance to read them (todd)
HDFS-2901. Improvements for SBN web UI - not show under-replicated/missing blocks. (Brandon Li via jitendra)
HDFS-2905. HA: Standby NN NPE when shared edits dir is deleted. (Bikas Saha via jitendra)

21
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
View File

@ -135,8 +135,7 @@ public void purgeLogsOlderThan(long minTxIdToKeep)
*/
List<RemoteEditLog> getRemoteEditLogs(long firstTxId) throws IOException {
File currentDir = sd.getCurrentDir();
List<EditLogFile> allLogFiles = matchEditLogs(
FileUtil.listFiles(currentDir));
List<EditLogFile> allLogFiles = matchEditLogs(currentDir);
List<RemoteEditLog> ret = Lists.newArrayListWithCapacity(
allLogFiles.size());
@ -155,6 +154,20 @@ List<RemoteEditLog> getRemoteEditLogs(long firstTxId) throws IOException {
return ret;
}
/**
* returns matching edit logs via the log directory. Simple helper function
* that lists the files in the logDir and calls matchEditLogs(File[])
*
* @param logDir
* directory to match edit logs in
* @return matched edit logs
* @throws IOException
* IOException thrown for invalid logDir
*/
static List<EditLogFile> matchEditLogs(File logDir) throws IOException {
return matchEditLogs(FileUtil.listFiles(logDir));
}
static List<EditLogFile> matchEditLogs(File[] filesInStorage) {
List<EditLogFile> ret = Lists.newArrayList();
for (File f : filesInStorage) {
@ -278,7 +291,7 @@ public long getNumberOfTransactions(long fromTxId, boolean inProgressOk)
synchronized public void recoverUnfinalizedSegments() throws IOException {
File currentDir = sd.getCurrentDir();
LOG.info("Recovering unfinalized segments in " + currentDir);
List<EditLogFile> allLogFiles = matchEditLogs(currentDir.listFiles());
List<EditLogFile> allLogFiles = matchEditLogs(currentDir);
for (EditLogFile elf : allLogFiles) {
if (elf.getFile().equals(currentInProgress)) {
@ -318,7 +331,7 @@ synchronized public void recoverUnfinalizedSegments() throws IOException {
private List<EditLogFile> getLogFiles(long fromTxId) throws IOException {
File currentDir = sd.getCurrentDir();
List<EditLogFile> allLogFiles = matchEditLogs(currentDir.listFiles());
List<EditLogFile> allLogFiles = matchEditLogs(currentDir);
List<EditLogFile> logFiles = Lists.newArrayList();
for (EditLogFile elf : allLogFiles) {

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
View File

@ -440,7 +440,7 @@ public static EditLogFile findLatestEditsLog(StorageDirectory sd)
throws IOException {
File currentDir = sd.getCurrentDir();
List<EditLogFile> foundEditLogs
= Lists.newArrayList(FileJournalManager.matchEditLogs(currentDir.listFiles()));
= Lists.newArrayList(FileJournalManager.matchEditLogs(currentDir));
return Collections.max(foundEditLogs, EditLogFile.COMPARE_BY_START_TXID);
}

9
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileJournalManager.java
View File

@ -315,6 +315,15 @@ public void testGetRemoteEditLog() throws IOException {
"", getLogsAsString(fjm, 9999));
}
/**
* tests that passing an invalid dir to matchEditLogs throws IOException
*/
@Test(expected = IOException.class)
public void testMatchEditLogInvalidDirThrowsIOException() throws IOException {
File badDir = new File("does not exist");
FileJournalManager.matchEditLogs(badDir);
}
/**
* Make sure that we starting reading the correct op when we request a stream
* with a txid in the middle of an edit log file.