diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index dfa9701520..c451ea6d67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2567,6 +2567,9 @@ Release 2.8.0 - UNRELEASED HDFS-9605. Add links to failed volumes to explorer.html in HDFS Web UI. (Archana T via wheat9) + HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the + correct namenode (Wei-Chiu Chuang via vinayakumarb) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 4316854d4b..7a10379304 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -29,6 +29,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import javax.management.NotCompliantMBeanException; import javax.management.ObjectName; @@ -521,7 +522,7 @@ public class SimulatedFSDataset implements FsDatasetSpi { } private final Map> blockMap - = new HashMap>(); + = new ConcurrentHashMap>(); private final SimulatedStorage storage; private final SimulatedVolume volume; private final String datanodeUuid; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java index 8dc80d5796..dd7d239f6b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -38,6 +39,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory; import org.apache.hadoop.util.DataChecksum; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -338,4 +340,49 @@ public class TestSimulatedFSDataset { fsdataset.addBlockPool(bpid, conf); return fsdataset; } + + @Test + public void testConcurrentAddBlockPool() throws InterruptedException, + IOException { + final String[] bpids = {"BP-TEST1-", "BP-TEST2-"}; + final SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, conf); + class AddBlockPoolThread extends Thread { + private int id; + private IOException ioe; + public AddBlockPoolThread(int id) { + super(); + this.id = id; + } + public void test() throws InterruptedException, IOException { + this.join(); + if (ioe != null) { + throw ioe; + } + } + public void run() { + for (int i=0; i < 10000; i++) { + // add different block pools concurrently + String newbpid = bpids[id] + i; + fsdataset.addBlockPool(newbpid, conf); + // and then add a block into the pool + ExtendedBlock block = new ExtendedBlock(newbpid,1); + try { + // it will throw an exception if the block pool is not found + fsdataset.createTemporary(StorageType.DEFAULT, block); + } catch (IOException ioe) { + // JUnit does not capture exception in non-main thread, + // so cache it and then let main thread throw later. + this.ioe = ioe; + } + assert(fsdataset.getReplicaString(newbpid,1) != "null"); + } + } + }; + AddBlockPoolThread t1 = new AddBlockPoolThread(0); + AddBlockPoolThread t2 = new AddBlockPoolThread(1); + t1.start(); + t2.start(); + t1.test(); + t2.test(); + } }