big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-4058. DirectoryScanner may fail with IOOB if the directory scanning threads return out of volume order. Contributed by Eli Collins

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1398612 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Eli Collins 2012-10-16 02:12:59 +00:00
parent 557ffe2101
commit a75673cbd8
2 changed files with 12 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -446,6 +446,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-3678. Edit log files are never being purged from 2NN. (atm)
HDFS-4058. DirectoryScanner may fail with IOOB if the directory
scanning threads return out of volume order. (eli)
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
View File

@ -431,16 +431,16 @@ private static boolean isValid(final FsDatasetSpi<?> dataset,
private Map<String, ScanInfo[]> getDiskReport() {
// First get list of data directories
final List<? extends FsVolumeSpi> volumes = dataset.getVolumes();
ArrayList<ScanInfoPerBlockPool> dirReports =
new ArrayList<ScanInfoPerBlockPool>(volumes.size());
// Use an array since the threads may return out of order and
// compilersInProgress#keySet may return out of order as well.
ScanInfoPerBlockPool[] dirReports = new ScanInfoPerBlockPool[volumes.size()];
Map<Integer, Future<ScanInfoPerBlockPool>> compilersInProgress =
new HashMap<Integer, Future<ScanInfoPerBlockPool>>();
for (int i = 0; i < volumes.size(); i++) {
if (!isValid(dataset, volumes.get(i))) {
// volume is invalid
dirReports.add(i, null);
} else {
if (isValid(dataset, volumes.get(i))) {
ReportCompiler reportCompiler =
new ReportCompiler(volumes.get(i));
Future<ScanInfoPerBlockPool> result =
@ -452,7 +452,7 @@ private Map<String, ScanInfo[]> getDiskReport() {
for (Entry<Integer, Future<ScanInfoPerBlockPool>> report :
compilersInProgress.entrySet()) {
try {
dirReports.add(report.getKey(), report.getValue().get());
dirReports[report.getKey()] = report.getValue().get();
} catch (Exception ex) {
LOG.error("Error compiling report", ex);
// Propagate ex to DataBlockScanner to deal with
@ -465,7 +465,7 @@ private Map<String, ScanInfo[]> getDiskReport() {
for (int i = 0; i < volumes.size(); i++) {
if (isValid(dataset, volumes.get(i))) {
// volume is still valid
list.addAll(dirReports.get(i));
list.addAll(dirReports[i]);
}
}