big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6910. MapReduceTrackingUriPlugin can not return the right URI of history server with HTTPS. Contributed by Lantao Jin

Browse Source
This commit is contained in:
Ravi Prakash 2017-07-13 16:16:45 -07:00
parent ebc048cc05
commit 43f0503286
2 changed files with 29 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRWebAppUtil.java
View File

@ -29,7 +29,6 @@
import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.InetSocketAddress; import java.net.InetSocketAddress;
@ -76,7 +75,9 @@ public static String getYARNWebappScheme() {
: "http://"; : "http://";
} }
public static String getJHSWebappScheme() { public static String getJHSWebappScheme(Configuration conf) {
setHttpPolicyInJHS(conf.get(JHAdminConfig.MR_HS_HTTP_POLICY,
JHAdminConfig.DEFAULT_MR_HS_HTTP_POLICY));
return httpPolicyInJHS == HttpConfig.Policy.HTTPS_ONLY ? "https://" return httpPolicyInJHS == HttpConfig.Policy.HTTPS_ONLY ? "https://"
: "http://"; : "http://";
} }
@ -101,7 +102,7 @@ public static String getJHSWebappURLWithoutScheme(Configuration conf) {
} }
public static String getJHSWebappURLWithScheme(Configuration conf) { public static String getJHSWebappURLWithScheme(Configuration conf) {
return getJHSWebappScheme() + getJHSWebappURLWithoutScheme(conf); return getJHSWebappScheme(conf) + getJHSWebappURLWithoutScheme(conf);
} }
public static InetSocketAddress getJHSWebBindAddress(Configuration conf) { public static InetSocketAddress getJHSWebBindAddress(Configuration conf) {
@ -153,7 +154,7 @@ hsAddress, getDefaultJHSWebappPort(),
public static String getApplicationWebURLOnJHSWithScheme(Configuration conf, public static String getApplicationWebURLOnJHSWithScheme(Configuration conf,
ApplicationId appId) throws UnknownHostException { ApplicationId appId) throws UnknownHostException {
return getJHSWebappScheme() return getJHSWebappScheme(conf)
+ getApplicationWebURLOnJHSWithoutScheme(conf, appId); + getApplicationWebURLOnJHSWithoutScheme(conf, appId);
} }

26
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestMapReduceTrackingUriPlugin.java
View File

@ -23,6 +23,7 @@
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -30,17 +31,38 @@
public class TestMapReduceTrackingUriPlugin { public class TestMapReduceTrackingUriPlugin {
@Test @Test
public void testProducesHistoryServerUriForAppId() throws URISyntaxException { public void testProducesHistoryServerUriForAppId()
throws URISyntaxException {
final String historyAddress = "example.net:424242"; final String historyAddress = "example.net:424242";
YarnConfiguration conf = new YarnConfiguration(); YarnConfiguration conf = new YarnConfiguration();
conf.set(JHAdminConfig.MR_HS_HTTP_POLICY,
HttpConfig.Policy.HTTP_ONLY.name());
conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, historyAddress); conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, historyAddress);
MapReduceTrackingUriPlugin plugin = new MapReduceTrackingUriPlugin(); MapReduceTrackingUriPlugin plugin = new MapReduceTrackingUriPlugin();
plugin.setConf(conf); plugin.setConf(conf);
ApplicationId id = ApplicationId.newInstance(6384623l, 5); ApplicationId id = ApplicationId.newInstance(6384623L, 5);
String jobSuffix = id.toString().replaceFirst("^application_", "job_"); String jobSuffix = id.toString().replaceFirst("^application_", "job_");
URI expected = URI expected =
new URI("http://" + historyAddress + "/jobhistory/job/" + jobSuffix); new URI("http://" + historyAddress + "/jobhistory/job/" + jobSuffix);
URI actual = plugin.getTrackingUri(id); URI actual = plugin.getTrackingUri(id);
assertEquals(expected, actual); assertEquals(expected, actual);
} }
@Test
public void testProducesHistoryServerUriWithHTTPS()
throws URISyntaxException {
final String historyAddress = "example.net:404040";
YarnConfiguration conf = new YarnConfiguration();
conf.set(JHAdminConfig.MR_HS_HTTP_POLICY,
HttpConfig.Policy.HTTPS_ONLY.name());
conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS, historyAddress);
MapReduceTrackingUriPlugin plugin = new MapReduceTrackingUriPlugin();
plugin.setConf(conf);
ApplicationId id = ApplicationId.newInstance(6384623L, 5);
String jobSuffix = id.toString().replaceFirst("^application_", "job_");
URI expected =
new URI("https://" + historyAddress + "/jobhistory/job/" + jobSuffix);
URI actual = plugin.getTrackingUri(id);
assertEquals(expected, actual);
}
} }