big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-3415. Make sure all layout versions are the same for all storage directories in the Namenode. Contributed by Brandon Li

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1341676 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2012-05-22 23:18:04 +00:00
parent 23b96766ad
commit aa80e6895d
2 changed files with 25 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -732,6 +732,9 @@ Release 2.0.0-alpha - UNRELEASED
HDFS-860. fuse-dfs truncate behavior causes issues with scp.
(Brian Bockelman via eli)
HDFS-3415. Make sure all layout versions are the same for all storage
directories in the Namenode. (Brandon Li via szetszwo)
BREAKDOWN OF HDFS-1623 SUBTASKS
HDFS-2179. Add fencing framework and mechanisms for NameNode HA. (todd)

36
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
View File

@ -31,6 +31,7 @@
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.CopyOnWriteArrayList;
@ -58,6 +59,7 @@
import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
/**
* NNStorage is responsible for management of the StorageDirectories used by
@ -1076,13 +1078,14 @@ void inspectStorageDirs(FSImageStorageInspector inspector)
* inspected each directory.
*
* <b>Note:</b> this can mutate the storage info fields (ctime, version, etc).
* @throws IOException if no valid storage dirs are found
* @throws IOException if no valid storage dirs are found or no valid layout version
*/
FSImageStorageInspector readAndInspectDirs()
throws IOException {
int minLayoutVersion = Integer.MAX_VALUE; // the newest
int maxLayoutVersion = Integer.MIN_VALUE; // the oldest
Integer layoutVersion = null;
boolean multipleLV = false;
StringBuilder layoutVersions = new StringBuilder();
// First determine what range of layout versions we're going to inspect
for (Iterator<StorageDirectory> it = dirIterator();
it.hasNext();) {
@ -1092,24 +1095,29 @@ FSImageStorageInspector readAndInspectDirs()
continue;
}
readProperties(sd); // sets layoutVersion
minLayoutVersion = Math.min(minLayoutVersion, getLayoutVersion());
maxLayoutVersion = Math.max(maxLayoutVersion, getLayoutVersion());
int lv = getLayoutVersion();
if (layoutVersion == null) {
layoutVersion = Integer.valueOf(lv);
} else if (!layoutVersion.equals(lv)) {
multipleLV = true;
}
layoutVersions.append("(").append(sd.getRoot()).append(", ").append(lv).append(") ");
}
if (minLayoutVersion > maxLayoutVersion) {
if (layoutVersion == null) {
throw new IOException("No storage directories contained VERSION information");
}
assert minLayoutVersion <= maxLayoutVersion;
// If we have any storage directories with the new layout version
if (multipleLV) {
throw new IOException(
"Storage directories containe multiple layout versions: "
+ layoutVersions);
}
// If the storage directories are with the new layout version
// (ie edits_<txnid>) then use the new inspector, which will ignore
// the old format dirs.
FSImageStorageInspector inspector;
if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, minLayoutVersion)) {
if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, getLayoutVersion())) {
inspector = new FSImageTransactionalStorageInspector();
if (!LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, maxLayoutVersion)) {
FSImage.LOG.warn("Ignoring one or more storage directories with old layouts");
}
} else {
inspector = new FSImagePreTransactionalStorageInspector();
}