MAPREDUCE-4899. Implemented a MR specific plugin for tracking finished applications that YARN's ResourceManager doesn't keep track off anymore. Contributed by Derek Dagit.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1425225 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli
parent
04db3ce177
commit
dae6f0b7d9
@ -188,6 +188,10 @@ Release 2.0.3-alpha - Unreleased
|
||||
MAPREDUCE-4845. ClusterStatus.getMaxMemory() and getUsedMemory() exist in
|
||||
MR1 but not MR2. (Sandy Ryza via tomwhite)
|
||||
|
||||
MAPREDUCE-4899. Implemented a MR specific plugin for tracking finished
|
||||
applications that YARN's ResourceManager doesn't keep track off anymore
|
||||
(Derek Dagit via vinodkv)
|
||||
|
||||
OPTIMIZATIONS
|
||||
|
||||
BUG FIXES
|
||||
|
||||
@ -0,0 +1,46 @@
|
||||
<?xml version="1.0"?>
|
||||
<!--
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. See accompanying LICENSE file.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
|
||||
http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>hadoop-mapreduce-client</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<version>3.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-mapreduce-client-hs-plugins</artifactId>
|
||||
<version>3.0.0-SNAPSHOT</version>
|
||||
<name>hadoop-mapreduce-client-hs-plugins</name>
|
||||
|
||||
<properties>
|
||||
<!-- Needed for generating FindBugs warnings using parent pom -->
|
||||
<mr.basedir>${project.parent.basedir}/../</mr.basedir>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-common</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-mapreduce-client-common</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.mapreduce.v2.hs.webapp;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
|
||||
import org.apache.hadoop.conf.Configurable;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.mapred.JobConf;
|
||||
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.util.TrackingUriPlugin;
|
||||
|
||||
|
||||
public class MapReduceTrackingUriPlugin extends TrackingUriPlugin implements
|
||||
Configurable {
|
||||
|
||||
@Override
|
||||
public void setConf(Configuration conf) {
|
||||
Configuration jobConf = null;
|
||||
// Force loading of mapred configuration.
|
||||
if (conf != null) {
|
||||
jobConf = new JobConf(conf);
|
||||
} else {
|
||||
jobConf = new JobConf();
|
||||
}
|
||||
super.setConf(jobConf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the URI to access the given application on MapReduce history server
|
||||
* @param id the ID for which a URI is returned
|
||||
* @return the tracking URI
|
||||
* @throws URISyntaxException
|
||||
*/
|
||||
@Override
|
||||
public URI getTrackingUri(ApplicationId id) throws URISyntaxException {
|
||||
String jobSuffix = id.toString().replaceFirst("^application_", "job_");
|
||||
String historyServerAddress =
|
||||
this.getConf().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS);
|
||||
return new URI("http://" + historyServerAddress + "/jobhistory/job/"
|
||||
+ jobSuffix);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,47 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.mapreduce.v2.hs.webapp;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
|
||||
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.util.BuilderUtils;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestMapReduceTrackingUriPlugin {
|
||||
@Test
|
||||
public void testProducesHistoryServerUriForAppId() throws URISyntaxException {
|
||||
final String historyAddress = "example.net:424242";
|
||||
YarnConfiguration conf = new YarnConfiguration();
|
||||
conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, historyAddress);
|
||||
MapReduceTrackingUriPlugin plugin = new MapReduceTrackingUriPlugin();
|
||||
plugin.setConf(conf);
|
||||
ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
|
||||
String jobSuffix = id.toString().replaceFirst("^application_", "job_");
|
||||
URI expected =
|
||||
new URI("http://" + historyAddress + "/jobhistory/job/" + jobSuffix);
|
||||
URI actual = plugin.getTrackingUri(id);
|
||||
assertEquals(expected, actual);
|
||||
}
|
||||
}
|
||||
@ -194,5 +194,6 @@
|
||||
<module>hadoop-mapreduce-client-app</module>
|
||||
<module>hadoop-mapreduce-client-jobclient</module>
|
||||
<module>hadoop-mapreduce-client-hs</module>
|
||||
<module>hadoop-mapreduce-client-hs-plugins</module>
|
||||
</modules>
|
||||
</project>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user