big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-1157. Fixed ResourceManager UI to behave correctly when apps like distributed-shell do not set tracking urls. Contributed by Xuan Gong.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1526371 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-09-26 04:18:00 +00:00
parent f2e6091bb2
commit db5322695a
6 changed files with 115 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -83,6 +83,9 @@ Release 2.1.2 - UNRELEASED
YARN-49. Improve distributed shell application to work on a secure cluster.
(Vinod Kumar Vavilapalli via hitesh)
YARN-1157. Fixed ResourceManager UI to behave correctly when apps like
distributed-shell do not set tracking urls. (Xuan Gong via vinodkv)
Release 2.1.1-beta - 2013-09-23
INCOMPATIBLE CHANGES

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
View File

@ -100,11 +100,22 @@ public static FinishApplicationMasterRequest newInstance(
public abstract String getTrackingUrl();
/**
* Set the <em>tracking URL</em>for the <code>ApplicationMaster</code>
* This url if contains scheme then that will be used by resource manager
* web application proxy otherwise it will default to http.
* @param url <em>tracking URL</em>for the
* <code>ApplicationMaster</code>
* Set the <em>final tracking URL</em>for the <code>ApplicationMaster</code>.
* This is the web-URL to which ResourceManager or web-application proxy will
* redirect client/users once the application is finished and the
* <code>ApplicationMaster</code> is gone.
* <p>
* If the passed url has a scheme then that will be used by the
* ResourceManager and web-application proxy, otherwise the scheme will
* default to http.
* </p>
* <p>
* Empty, null, "N/A" strings are all valid besides a real URL. In case an url
* isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
* <p>
*
* @param url
* <em>tracking URL</em>for the <code>ApplicationMaster</code>
*/
@Public
@Stable

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
View File

@ -112,11 +112,22 @@ public static RegisterApplicationMasterRequest newInstance(String host,
public abstract String getTrackingUrl();
/**
* Set the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
* This url if contains scheme then that will be used by resource manager
* web application proxy otherwise it will default to http.
* @param trackingUrl <em>tracking URL</em> for the
* <code>ApplicationMaster</code>
* Set the <em>tracking URL</em>for the <code>ApplicationMaster</code> while
* it is running. This is the web-URL to which ResourceManager or
* web-application proxy will redirect client/users while the application and
* the <code>ApplicationMaster</code> are still running.
* <p>
* If the passed url has a scheme then that will be used by the
* ResourceManager and web-application proxy, otherwise the scheme will
* default to http.
* </p>
* <p>
* Empty, null, "N/A" strings are all valid besides a real URL. In case an url
* isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
* <p>
*
* @param trackingUrl
* <em>tracking URL</em>for the <code>ApplicationMaster</code>
*/
@Public
@Stable

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -994,7 +994,7 @@ public void transition(RMAppAttemptImpl appAttempt,
}
}
static final class AMRegisteredTransition extends BaseTransition {
private static final class AMRegisteredTransition extends BaseTransition {
@Override
public void transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
@ -1003,7 +1003,8 @@ public void transition(RMAppAttemptImpl appAttempt,
= (RMAppAttemptRegistrationEvent) event;
appAttempt.host = registrationEvent.getHost();
appAttempt.rpcPort = registrationEvent.getRpcport();
appAttempt.origTrackingUrl = registrationEvent.getTrackingurl();
appAttempt.origTrackingUrl =
sanitizeTrackingUrl(registrationEvent.getTrackingurl());
appAttempt.proxiedTrackingUrl =
appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
@ -1138,7 +1139,8 @@ public RMAppAttemptState transition(RMAppAttemptImpl appAttempt,
RMAppAttemptUnregistrationEvent unregisterEvent
= (RMAppAttemptUnregistrationEvent) event;
appAttempt.diagnostics.append(unregisterEvent.getDiagnostics());
appAttempt.origTrackingUrl = unregisterEvent.getTrackingUrl();
appAttempt.origTrackingUrl =
sanitizeTrackingUrl(unregisterEvent.getTrackingUrl());
appAttempt.proxiedTrackingUrl =
appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
appAttempt.finalStatus = unregisterEvent.getFinalApplicationStatus();
@ -1292,4 +1294,8 @@ private void removeCredentials(RMAppAttemptImpl appAttempt) {
appAttempt.rmContext.getAMRMTokenSecretManager()
.applicationMasterFinished(appAttempt.getAppAttemptId());
}
private static String sanitizeTrackingUrl(String url) {
return (url == null || url.trim().isEmpty()) ? "N/A" : url;
}
}

77
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java
View File

@ -1,77 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
import junit.framework.Assert;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.junit.Test;
import org.mockito.Mockito;
public class TestRMAppAttemptImpl {
private void testTrackingUrl(String url, boolean unmanaged) {
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance
(ApplicationId.newInstance(1, 2), 1);
EventHandler handler = Mockito.mock(EventHandler.class);
Dispatcher dispatcher = Mockito.mock(Dispatcher.class);
Mockito.when(dispatcher.getEventHandler()).thenReturn(handler);
RMContext rmContext = Mockito.mock(RMContext.class);
Mockito.when(rmContext.getDispatcher()).thenReturn(dispatcher);
ApplicationSubmissionContext appContext =
Mockito.mock(ApplicationSubmissionContext.class);
Mockito.when(appContext.getUnmanagedAM()).thenReturn(unmanaged);
RMAppAttemptImpl attempt = new RMAppAttemptImpl(attemptId, rmContext, null,
null, appContext, new YarnConfiguration(), null);
RMAppAttemptRegistrationEvent event =
Mockito.mock(RMAppAttemptRegistrationEvent.class);
Mockito.when(event.getHost()).thenReturn("h");
Mockito.when(event.getRpcport()).thenReturn(0);
Mockito.when(event.getTrackingurl()).thenReturn(url);
new RMAppAttemptImpl.AMRegisteredTransition().transition(attempt, event);
if (unmanaged) {
Assert.assertEquals(url, attempt.getTrackingUrl());
} else {
Assert.assertNotSame(url, attempt.getTrackingUrl());
Assert.assertTrue(attempt.getTrackingUrl().contains(
ProxyUriUtils.PROXY_SERVLET_NAME));
Assert.assertTrue(attempt.getTrackingUrl().contains(
attemptId.getApplicationId().toString()));
}
}
@Test
public void testTrackingUrlUnmanagedAM() {
testTrackingUrl("http://foo:8000/x", true);
}
@Test
public void testTrackingUrlManagedAM() {
testTrackingUrl("bar:8000/x", false);
}
}

87
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
View File

@ -30,14 +30,18 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@ -85,8 +89,10 @@
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -261,8 +267,22 @@ public void tearDown() throws Exception {
private String getProxyUrl(RMAppAttempt appAttempt) {
return pjoin(RM_WEBAPP_ADDR, "proxy",
appAttempt.getAppAttemptId().getApplicationId(), "");
String url = null;
try {
URI trackingUri =
StringUtils.isEmpty(appAttempt.getOriginalTrackingUrl()) ? null :
ProxyUriUtils
.getUriFromAMUrl(appAttempt.getOriginalTrackingUrl());
String proxy = WebAppUtils.getProxyHostAndPort(conf);
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
appAttempt.getAppAttemptId().getApplicationId());
url = result.toASCIIString().substring(
HttpConfig.getSchemePrefix().length());
} catch (URISyntaxException ex) {
Assert.fail();
}
return url;
}
/**
@ -448,9 +468,9 @@ private void testAppAttemptRunningState(Container container,
assertEquals(container, applicationAttempt.getMasterContainer());
assertEquals(host, applicationAttempt.getHost());
assertEquals(rpcPort, applicationAttempt.getRpcPort());
assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
if (unmanagedAM) {
assertEquals("oldtrackingurl", applicationAttempt.getTrackingUrl());
verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
} else {
assertEquals(getProxyUrl(applicationAttempt),
applicationAttempt.getTrackingUrl());
@ -468,7 +488,7 @@ private void testAppAttemptFinishingState(Container container,
assertEquals(RMAppAttemptState.FINISHING,
applicationAttempt.getAppAttemptState());
assertEquals(diagnostics, applicationAttempt.getDiagnostics());
assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
assertEquals(getProxyUrl(applicationAttempt),
applicationAttempt.getTrackingUrl());
assertEquals(container, applicationAttempt.getMasterContainer());
@ -487,9 +507,9 @@ private void testAppAttemptFinishedState(Container container,
assertEquals(RMAppAttemptState.FINISHED,
applicationAttempt.getAppAttemptState());
assertEquals(diagnostics, applicationAttempt.getDiagnostics());
assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
if (unmanagedAM) {
assertEquals("mytrackingurl", applicationAttempt.getTrackingUrl());
verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
} else {
assertEquals(getProxyUrl(applicationAttempt),
@ -603,9 +623,7 @@ private void unregisterApplicationAttempt(Container container,
trackingUrl, diagnostics);
}
@Test
public void testUnmanagedAMSuccess() {
private void testUnmanagedAMSuccess(String url) {
unmanagedAM = true;
when(submissionContext.getUnmanagedAM()).thenReturn(true);
// submit AM and check it goes to LAUNCHED state
@ -615,7 +633,7 @@ public void testUnmanagedAMSuccess() {
applicationAttempt.getAppAttemptId());
// launch AM
runApplicationAttempt(null, "host", 8042, "oldtrackingurl", true);
runApplicationAttempt(null, "host", 8042, url, true);
// complete a container
applicationAttempt.handle(new RMAppAttemptContainerAcquiredEvent(
@ -623,13 +641,12 @@ public void testUnmanagedAMSuccess() {
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class)));
// complete AM
String trackingUrl = "mytrackingurl";
String diagnostics = "Successful";
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
applicationAttempt.getAppAttemptId(), url, finalStatus,
diagnostics));
testAppAttemptFinishedState(null, finalStatus, trackingUrl, diagnostics, 1,
testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1,
true);
}
@ -824,12 +841,42 @@ public void testUnregisterToKilledFinishing() {
"Killed by user");
}
@Test
public void testTrackingUrlUnmanagedAM() {
testUnmanagedAMSuccess("oldTrackingUrl");
}
@Test
public void testNoTrackingUrl() {
public void testEmptyTrackingUrlUnmanagedAM() {
testUnmanagedAMSuccess("");
}
@Test
public void testNullTrackingUrlUnmanagedAM() {
testUnmanagedAMSuccess(null);
}
@Test
public void testManagedAMWithTrackingUrl() {
testTrackingUrlManagedAM("theTrackingUrl");
}
@Test
public void testManagedAMWithEmptyTrackingUrl() {
testTrackingUrlManagedAM("");
}
@Test
public void testManagedAMWithNullTrackingUrl() {
testTrackingUrlManagedAM(null);
}
private void testTrackingUrlManagedAM(String url) {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
runApplicationAttempt(amContainer, "host", 8042, "", false);
runApplicationAttempt(amContainer, "host", 8042, url, false);
unregisterApplicationAttempt(amContainer,
FinalApplicationStatus.SUCCEEDED, url, "Successful");
}
@Test
@ -927,4 +974,12 @@ private void verifyTokenCount(ApplicationAttemptId appAttemptId, int count) {
}
}
}
private void verifyUrl(String url1, String url2) {
if (url1 == null || url1.trim().isEmpty()) {
assertEquals("N/A", url2);
} else {
assertEquals(url1, url2);
}
}
}