big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-3387. Fixed AM's tracking URL to always go through the proxy, even before the job started, so that it works properly with oozie throughout the job execution. Contributed by Robert Joseph Evans.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1215345 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2011-12-16 23:41:36 +00:00
parent a238f931ea
commit feb3a23c71
6 changed files with 43 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/CHANGES.txt
View File

@ -327,6 +327,10 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3366. Mapreduce component should use consistent directory structure
layout as HDFS/common (Eric Yang via mahadev)
MAPREDUCE-3387. Fixed AM's tracking URL to always go through the proxy, even
before the job started, so that it works properly with oozie throughout
the job execution. (Robert Joseph Evans via vinodkv)
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES

9
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -273,6 +273,8 @@ public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
this.proxiedTrackingUrl = generateProxyUriWithoutScheme();
this.stateMachine = stateMachineFactory.make(this);
}
@ -358,11 +360,16 @@ public String getWebProxyBase() {
}
}
private String generateProxyUriWithoutScheme() {
return generateProxyUriWithoutScheme(null);
}
private String generateProxyUriWithoutScheme(
final String trackingUriWithoutScheme) {
this.readLock.lock();
try {
URI trackingUri = ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme);
URI trackingUri = trackingUriWithoutScheme == null ? null :
ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme);
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
applicationAttemptId.getApplicationId());

5
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
View File

@ -80,8 +80,10 @@ public AppInfo(RMApp app, Boolean hasAccess) {
if (app != null) {
String trackingUrl = app.getTrackingUrl();
this.state = app.getState();
this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
|| "N/A".equalsIgnoreCase(trackingUrl);
|| RMAppState.NEW == this.state || RMAppState.SUBMITTED == this.state
|| RMAppState.ACCEPTED == this.state;
this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
.getFinishTime() == 0 ? "ApplicationMaster" : "History");
if (!trackingUrlIsNotReady) {
@ -95,7 +97,6 @@ public AppInfo(RMApp app, Boolean hasAccess) {
this.user = app.getUser().toString();
this.name = app.getName().toString();
this.queue = app.getQueue().toString();
this.state = app.getState();
this.progress = app.getProgress() * 100;
this.diagnostics = app.getDiagnostics().toString();
if (diagnostics == null || diagnostics.isEmpty()) {

5
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
View File

@ -17,8 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.*;
import static org.mockito.Matchers.*;
import static org.mockito.Mockito.*;
@ -203,6 +202,8 @@ private void testAppAttemptNewState() {
assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001);
assertEquals(0, applicationAttempt.getRanNodes().size());
assertNull(applicationAttempt.getFinalApplicationStatus());
assertNotNull(applicationAttempt.getTrackingUrl());
assertFalse("N/A".equals(applicationAttempt.getTrackingUrl()));
}
/**

8
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
View File

@ -114,7 +114,8 @@ private static boolean appendQuery(StringBuilder builder, String query,
/**
* Get a proxied URI for the original URI.
* @param originalUri the original URI to go through the proxy
* @param originalUri the original URI to go through the proxy, or null if
* a default path "/" can be used.
* @param proxyUri the URI of the proxy itself, scheme, host and port are used.
* @param id the id of the application
* @return the proxied URI
@ -122,9 +123,10 @@ private static boolean appendQuery(StringBuilder builder, String query,
public static URI getProxyUri(URI originalUri, URI proxyUri,
ApplicationId id) {
try {
String path = getPath(id, originalUri.getPath());
String path = getPath(id, originalUri == null ? "/" : originalUri.getPath());
return new URI(proxyUri.getScheme(), proxyUri.getAuthority(), path,
originalUri.getQuery(), originalUri.getFragment());
originalUri == null ? null : originalUri.getQuery(),
originalUri == null ? null : originalUri.getFragment());
} catch (URISyntaxException e) {
throw new RuntimeException("Could not proxify "+originalUri,e);
}

58
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestProxyUriUtils.java
View File

@ -23,44 +23,16 @@
import java.net.URI;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
public class TestProxyUriUtils {
public static class TestAppId extends ApplicationId {
private long timestamp;
private int id;
public TestAppId(int id, long timestamp) {
setId(id);
setClusterTimestamp(timestamp);
}
@Override
public int getId() {
return id;
}
@Override
public void setId(int id) {
this.id = id;
}
@Override
public long getClusterTimestamp() {
return timestamp;
}
@Override
public void setClusterTimestamp(long clusterTimestamp) {
this.timestamp = clusterTimestamp;
}
}
@Test
public void testGetPathApplicationId() {
assertEquals("/proxy/application_100_0001",
ProxyUriUtils.getPath(new TestAppId(1, 100l)));
ProxyUriUtils.getPath(BuilderUtils.newApplicationId(100l, 1)));
assertEquals("/proxy/application_6384623_0005",
ProxyUriUtils.getPath(new TestAppId(5, 6384623l)));
ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5)));
}
@Test(expected = IllegalArgumentException.class)
@ -71,23 +43,23 @@ public void testGetPathApplicationIdBad() {
@Test
public void testGetPathApplicationIdString() {
assertEquals("/proxy/application_6384623_0005",
ProxyUriUtils.getPath(new TestAppId(5, 6384623l), null));
ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), null));
assertEquals("/proxy/application_6384623_0005/static/app",
ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "/static/app"));
ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/static/app"));
assertEquals("/proxy/application_6384623_0005/",
ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "/"));
ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "/"));
assertEquals("/proxy/application_6384623_0005/some/path",
ProxyUriUtils.getPath(new TestAppId(5, 6384623l), "some/path"));
ProxyUriUtils.getPath(BuilderUtils.newApplicationId(6384623l, 5), "some/path"));
}
@Test
public void testGetPathAndQuery() {
assertEquals("/proxy/application_6384623_0005/static/app?foo=bar",
ProxyUriUtils.getPathAndQuery(new TestAppId(5, 6384623l), "/static/app",
ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app",
"?foo=bar", false));
assertEquals("/proxy/application_6384623_0005/static/app?foo=bar&bad=good&proxyapproved=true",
ProxyUriUtils.getPathAndQuery(new TestAppId(5, 6384623l), "/static/app",
ProxyUriUtils.getPathAndQuery(BuilderUtils.newApplicationId(6384623l, 5), "/static/app",
"foo=bar&bad=good", true));
}
@ -95,10 +67,20 @@ public void testGetPathAndQuery() {
public void testGetProxyUri() throws Exception {
URI originalUri = new URI("http://host.com/static/foo?bar=bar");
URI proxyUri = new URI("http://proxy.net:8080/");
TestAppId id = new TestAppId(5, 6384623l);
ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
URI expected = new URI("http://proxy.net:8080/proxy/application_6384623_0005/static/foo?bar=bar");
URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id);
assertEquals(expected, result);
}
@Test
public void testGetProxyUriNull() throws Exception {
URI originalUri = null;
URI proxyUri = new URI("http://proxy.net:8080/");
ApplicationId id = BuilderUtils.newApplicationId(6384623l, 5);
URI expected = new URI("http://proxy.net:8080/proxy/application_6384623_0005/");
URI result = ProxyUriUtils.getProxyUri(originalUri, proxyUri, id);
assertEquals(expected, result);
}
}