From 80c3fec3a13c41051daaae42e5c9a9fedf5c7ee7 Mon Sep 17 00:00:00 2001 From: Virajith Jalaparti Date: Wed, 13 Dec 2017 13:39:21 -0800 Subject: [PATCH] HDFS-12912. [READ] Fix configuration and implementation of LevelDB-based alias maps --- .../server/aliasmap/InMemoryAliasMap.java | 42 +++++++++---------- .../InMemoryLevelDBAliasMapServer.java | 9 ++-- .../impl/LevelDBFileRegionAliasMap.java | 5 +++ .../src/site/markdown/HdfsProvidedStorage.md | 4 +- .../aliasmap/ITestInMemoryAliasMap.java | 9 ++-- .../server/aliasmap/TestInMemoryAliasMap.java | 2 +- .../TestInMemoryLevelDBAliasMapClient.java | 2 + .../impl/TestLevelDbMockAliasMapClient.java | 2 +- .../TestNameNodeProvidedImplementation.java | 2 + 9 files changed, 45 insertions(+), 32 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java index 3d9eeea1e3..142a040ca8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryAliasMap.java @@ -59,6 +59,7 @@ public class InMemoryAliasMap implements InMemoryAliasMapProtocol, private final DB levelDb; private Configuration conf; + private String blockPoolID; @Override public void setConf(Configuration conf) { @@ -79,32 +80,38 @@ static String createPathErrorMessage(String directory) { .toString(); } - public static @Nonnull InMemoryAliasMap init(Configuration conf) - throws IOException { + public static @Nonnull InMemoryAliasMap init(Configuration conf, + String blockPoolID) throws IOException { Options options = new Options(); options.createIfMissing(true); String directory = conf.get(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR); LOG.info("Attempting to load InMemoryAliasMap from \"{}\"", directory); - File path = new File(directory); - if (!path.exists()) { + File levelDBpath; + if (blockPoolID != null) { + levelDBpath = new File(directory, blockPoolID); + } else { + levelDBpath = new File(directory); + } + if (!levelDBpath.exists()) { String error = createPathErrorMessage(directory); throw new IOException(error); } - DB levelDb = JniDBFactory.factory.open(path, options); - InMemoryAliasMap aliasMap = new InMemoryAliasMap(levelDb); + DB levelDb = JniDBFactory.factory.open(levelDBpath, options); + InMemoryAliasMap aliasMap = new InMemoryAliasMap(levelDb, blockPoolID); aliasMap.setConf(conf); return aliasMap; } @VisibleForTesting - InMemoryAliasMap(DB levelDb) { + InMemoryAliasMap(DB levelDb, String blockPoolID) { this.levelDb = levelDb; + this.blockPoolID = blockPoolID; } @Override public IterationResult list(Optional marker) throws IOException { - return withIterator((DBIterator iterator) -> { + try (DBIterator iterator = levelDb.iterator()) { Integer batchSize = conf.getInt(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_BATCH_SIZE, DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_BATCH_SIZE_DEFAULT); @@ -130,8 +137,7 @@ public IterationResult list(Optional marker) throws IOException { } else { return new IterationResult(batch, Optional.empty()); } - - }); + } } public @Nonnull Optional read(@Nonnull Block block) @@ -159,7 +165,7 @@ public void write(@Nonnull Block block, @Override public String getBlockPoolId() { - return null; + return blockPoolID; } public void close() throws IOException { @@ -202,21 +208,15 @@ public static byte[] toProtoBufBytes(@Nonnull Block block) return blockOutputStream.toByteArray(); } - private IterationResult withIterator( - CheckedFunction func) throws IOException { - try (DBIterator iterator = levelDb.iterator()) { - return func.apply(iterator); - } - } - /** * CheckedFunction is akin to {@link java.util.function.Function} but * specifies an IOException. - * @param Argument type. + * @param First argument type. + * @param Second argument type. * @param Return type. */ @FunctionalInterface - public interface CheckedFunction { - R apply(T t) throws IOException; + public interface CheckedFunction2 { + R apply(T1 t1, T2 t2) throws IOException; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryLevelDBAliasMapServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryLevelDBAliasMapServer.java index a229ae7c5c..4edc9a2913 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryLevelDBAliasMapServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/aliasmap/InMemoryLevelDBAliasMapServer.java @@ -38,7 +38,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS_DEFAULT; import static org.apache.hadoop.hdfs.protocol.proto.AliasMapProtocolProtos.*; -import static org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap.CheckedFunction; +import static org.apache.hadoop.hdfs.server.aliasmap.InMemoryAliasMap.CheckedFunction2; /** * InMemoryLevelDBAliasMapServer is the entry point from the Namenode into @@ -51,14 +51,15 @@ public class InMemoryLevelDBAliasMapServer implements InMemoryAliasMapProtocol, private static final Logger LOG = LoggerFactory .getLogger(InMemoryLevelDBAliasMapServer.class); - private final CheckedFunction initFun; + private final CheckedFunction2 + initFun; private RPC.Server aliasMapServer; private Configuration conf; private InMemoryAliasMap aliasMap; private String blockPoolId; public InMemoryLevelDBAliasMapServer( - CheckedFunction initFun, + CheckedFunction2 initFun, String blockPoolId) { this.initFun = initFun; this.blockPoolId = blockPoolId; @@ -127,7 +128,7 @@ public String getBlockPoolId() { public void setConf(Configuration conf) { this.conf = conf; try { - this.aliasMap = initFun.apply(conf); + this.aliasMap = initFun.apply(conf, blockPoolId); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/LevelDBFileRegionAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/LevelDBFileRegionAliasMap.java index adad79abc2..6afe6bb41c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/LevelDBFileRegionAliasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/LevelDBFileRegionAliasMap.java @@ -113,6 +113,11 @@ private static DB createDB(String levelDBPath, boolean createIfMissing, } else { dbFile = new File(levelDBPath); } + if (createIfMissing && !dbFile.exists()) { + if (!dbFile.mkdirs()) { + throw new IOException("Unable to create " + dbFile); + } + } return factory.open(dbFile, options); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md index 7455044ed5..01e70766a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsProvidedStorage.md @@ -134,7 +134,7 @@ hadoop org.apache.hadoop.hdfs.server.namenode.FileSystemImage \ Assign ownership based on a custom `UGIResolver`, in LevelDB: ``` hadoop org.apache.hadoop.hdfs.server.namenode.FileSystemImage \ - -Ddfs.provided.aliasmap.leveldb.path=file:///path/to/leveldb/map/dingos.db \ + -Ddfs.provided.aliasmap.leveldb.path=/path/to/leveldb/map/dingos.db \ -b org.apache.hadoop.hdfs.server.common.blockaliasmap.impl.LevelDBFileRegionAliasMap \ -o file:///tmp/name \ -u CustomResolver \ @@ -180,7 +180,7 @@ Datanodes contact this alias map using the `org.apache.hadoop.hdfs.server.aliasm dfs.provided.aliasmap.inmemory.leveldb.dir - file:///path/to/leveldb/map/dingos.db + /path/to/leveldb/map/dingos.db The directory where the leveldb files will be kept diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/ITestInMemoryAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/ITestInMemoryAliasMap.java index 6f1ff3e333..432aefbc73 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/ITestInMemoryAliasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/ITestInMemoryAliasMap.java @@ -44,14 +44,17 @@ public class ITestInMemoryAliasMap { private InMemoryAliasMap aliasMap; private File tempDirectory; + private static String bpid = "bpid-0"; @Before public void setUp() throws Exception { Configuration conf = new Configuration(); - tempDirectory = Files.createTempDirectory("seagull").toFile(); + File temp = Files.createTempDirectory("seagull").toFile(); + tempDirectory = new File(temp, bpid); + tempDirectory.mkdirs(); conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, - tempDirectory.getAbsolutePath()); - aliasMap = InMemoryAliasMap.init(conf); + temp.getAbsolutePath()); + aliasMap = InMemoryAliasMap.init(conf, bpid); } @After diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/TestInMemoryAliasMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/TestInMemoryAliasMap.java index f699055562..0f57247cec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/TestInMemoryAliasMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/aliasmap/TestInMemoryAliasMap.java @@ -39,7 +39,7 @@ public void testInit() { nonExistingDirectory); assertThatExceptionOfType(IOException.class) - .isThrownBy(() -> InMemoryAliasMap.init(conf)).withMessage( + .isThrownBy(() -> InMemoryAliasMap.init(conf, "bpid")).withMessage( InMemoryAliasMap.createPathErrorMessage(nonExistingDirectory)); } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java index a3881999c6..61a1558978 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestInMemoryLevelDBAliasMapClient.java @@ -66,6 +66,8 @@ public void setUp() throws IOException { conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_RPC_ADDRESS, "localhost:" + port); tempDir = Files.createTempDir(); + File levelDBDir = new File(tempDir, BPID); + levelDBDir.mkdirs(); conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, tempDir.getAbsolutePath()); levelDBAliasMapServer = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java index 8212b28233..534bc36c65 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/blockaliasmap/impl/TestLevelDbMockAliasMapClient.java @@ -54,7 +54,7 @@ public void setUp() throws IOException { aliasMapMock = mock(InMemoryAliasMap.class); when(aliasMapMock.getBlockPoolId()).thenReturn(bpid); levelDBAliasMapServer = new InMemoryLevelDBAliasMapServer( - config -> aliasMapMock, bpid); + (config, blockPoolID) -> aliasMapMock, bpid); conf = new Configuration(); int port = 9877; diff --git a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java index 2917a34033..1023616c38 100644 --- a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java +++ b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java @@ -784,6 +784,8 @@ public void testInMemoryAliasMap() throws Exception { "localhost:32445"); File tempDirectory = Files.createTempDirectory("in-memory-alias-map").toFile(); + File leveDBPath = new File(tempDirectory, bpid); + leveDBPath.mkdirs(); conf.set(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_LEVELDB_DIR, tempDirectory.getAbsolutePath()); conf.setBoolean(DFSConfigKeys.DFS_PROVIDED_ALIASMAP_INMEMORY_ENABLED, true);