big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7537 [Atsv2] load hbase configuration from filesystem rather than URL. Contributed by Rohith Sharma

Browse Source
This commit is contained in:
Vrushali C 2018-01-19 15:34:40 -08:00
parent cce71dceef
commit ec8f47e7fa
2 changed files with 29 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -2455,7 +2455,7 @@
</property> </property>
<property> <property>
<description> Optional URL to an hbase-site.xml configuration file to be <description> Optional FS path to an hbase-site.xml configuration file to be
used to connect to the timeline-service hbase cluster. If empty or not used to connect to the timeline-service hbase cluster. If empty or not
specified, then the HBase configuration will be loaded from the classpath. specified, then the HBase configuration will be loaded from the classpath.
When specified the values in the specified configuration file will override When specified the values in the specified configuration file will override

38
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
View File

@ -18,13 +18,14 @@
package org.apache.hadoop.yarn.server.timelineservice.storage.common; package org.apache.hadoop.yarn.server.timelineservice.storage.common;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseConfiguration;
@ -269,28 +270,43 @@ public static String convertApplicationIdToString(ApplicationId appId) {
* @return a configuration with the HBase configuration from the classpath, * @return a configuration with the HBase configuration from the classpath,
* optionally overwritten by the timeline service configuration URL if * optionally overwritten by the timeline service configuration URL if
* specified. * specified.
* @throws MalformedURLException if a timeline service HBase configuration URL * @throws IOException if a timeline service HBase configuration path
* is specified but is a malformed URL. * is specified but unable to read it.
*/ */
public static Configuration getTimelineServiceHBaseConf(Configuration conf) public static Configuration getTimelineServiceHBaseConf(Configuration conf)
throws MalformedURLException { throws IOException {
if (conf == null) { if (conf == null) {
throw new NullPointerException(); throw new NullPointerException();
} }
Configuration hbaseConf; Configuration hbaseConf;
String timelineServiceHBaseConfFileURL = String timelineServiceHBaseConfFilePath =
conf.get(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE); conf.get(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE);
if (timelineServiceHBaseConfFileURL != null if (timelineServiceHBaseConfFilePath != null
&& timelineServiceHBaseConfFileURL.length() > 0) { && timelineServiceHBaseConfFilePath.length() > 0) {
LOG.info("Using hbase configuration at " + LOG.info("Using hbase configuration at " +
timelineServiceHBaseConfFileURL); timelineServiceHBaseConfFilePath);
// create a clone so that we don't mess with out input one // create a clone so that we don't mess with out input one
hbaseConf = new Configuration(conf); hbaseConf = new Configuration(conf);
Configuration plainHBaseConf = new Configuration(false); Configuration plainHBaseConf = new Configuration(false);
URL hbaseSiteXML = new URL(timelineServiceHBaseConfFileURL); FileSystem fs = null;
plainHBaseConf.addResource(hbaseSiteXML); FSDataInputStream in = null;
try {
Path hbaseConfigPath = new Path(timelineServiceHBaseConfFilePath);
fs = FileSystem.newInstance(conf);
in = fs.open(hbaseConfigPath);
plainHBaseConf.addResource(in);
HBaseConfiguration.merge(hbaseConf, plainHBaseConf); HBaseConfiguration.merge(hbaseConf, plainHBaseConf);
} catch (IOException e) {
throw e;
} finally {
if (in != null) {
in.close();
}
if (fs != null) {
fs.close();
}
}
} else { } else {
// default to what is on the classpath // default to what is on the classpath
hbaseConf = HBaseConfiguration.create(conf); hbaseConf = HBaseConfiguration.create(conf);