big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2414. RM web UI: app page will crash if app is failed before any attempt has been created. Contributed by Wangda Tan

Browse Source
This commit is contained in:
Jason Lowe 2014-11-17 21:13:22 +00:00
parent bd8196e85e
commit 81c9d17af8
5 changed files with 130 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -99,6 +99,9 @@ Release 2.7.0 - UNRELEASED
YARN-1703. Fixed ResourceManager web-proxy to close connections correctly.
(Rohith Sharma via vinodkv)
YARN-2414. RM web UI: app page will crash if app is failed before any
attempt has been created (Wangda Tan via jlowe)
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
View File

@ -35,6 +35,8 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.AdminACLsManager;
import com.google.common.annotations.VisibleForTesting;
@InterfaceAudience.Private
public class ApplicationACLsManager {
@ -48,6 +50,11 @@ public class ApplicationACLsManager {
private final ConcurrentMap<ApplicationId, Map<ApplicationAccessType, AccessControlList>> applicationACLS
= new ConcurrentHashMap<ApplicationId, Map<ApplicationAccessType, AccessControlList>>();
@VisibleForTesting
public ApplicationACLsManager() {
this(new Configuration());
}
public ApplicationACLsManager(Configuration conf) {
this.conf = conf;
this.adminAclsManager = new AdminACLsManager(this.conf);

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
View File

@ -24,10 +24,17 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import com.google.common.annotations.VisibleForTesting;
public class QueueACLsManager {
private ResourceScheduler scheduler;
private boolean isACLsEnable;
@VisibleForTesting
public QueueACLsManager() {
this(null, new Configuration());
}
public QueueACLsManager(ResourceScheduler scheduler, Configuration conf) {
this.scheduler = scheduler;
this.isACLsEnable = conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE,

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
View File

@ -33,6 +33,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@ -45,6 +46,7 @@
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
@ -113,8 +115,23 @@ protected void render(Block html) {
setTitle(join("Application ", aid));
RMAppMetrics appMerics = rmApp.getRMAppMetrics();
RMAppAttemptMetrics attemptMetrics =
rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
// Get attempt metrics and fields, it is possible currentAttempt of RMApp is
// null. In that case, we will assume resource preempted and number of Non
// AM container preempted on that attempt is 0
RMAppAttemptMetrics attemptMetrics;
if (null == rmApp.getCurrentAppAttempt()) {
attemptMetrics = null;
} else {
attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
}
Resource attemptResourcePreempted =
attemptMetrics == null ? Resources.none() : attemptMetrics
.getResourcePreempted();
int attemptNumNonAMContainerPreempted =
attemptMetrics == null ? 0 : attemptMetrics
.getNumNonAMContainersPreempted();
info("Application Overview")
._("User:", app.getUser())
._("Name:", app.getName())
@ -143,13 +160,12 @@ protected void render(Block html) {
._("Total Number of AM Containers Preempted:",
String.valueOf(appMerics.getNumAMContainersPreempted()))
._("Resource Preempted from Current Attempt:",
attemptMetrics.getResourcePreempted())
attemptResourcePreempted)
._("Number of Non-AM Containers Preempted from Current Attempt:",
String.valueOf(attemptMetrics
.getNumNonAMContainersPreempted()))
attemptNumNonAMContainerPreempted)
._("Aggregate Resource Allocation:",
String.format("%d MB-seconds, %d vcore-seconds",
appMerics.getMemorySeconds(), appMerics.getVcoreSeconds()));
String.format("%d MB-seconds, %d vcore-seconds",
appMerics.getMemorySeconds(), appMerics.getVcoreSeconds()));
pdiv._();
Collection<RMAppAttempt> attempts = rmApp.getAppAttempts().values();

90
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java Normal file
View File

@ -0,0 +1,90 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.junit.Test;
import com.google.inject.Binder;
import com.google.inject.Injector;
import com.google.inject.Module;
public class TestAppPage {
@Test
public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception {
final ApplicationId APP_ID = ApplicationId.newInstance(1234L, 0);
Injector injector;
// init app
RMApp app = mock(RMApp.class);
when(app.getTrackingUrl()).thenReturn("http://host:123");
when(app.getState()).thenReturn(RMAppState.FAILED);
when(app.getApplicationId()).thenReturn(APP_ID);
when(app.getApplicationType()).thenReturn("Type");
when(app.getUser()).thenReturn("user");
when(app.getName()).thenReturn("Name");
when(app.getQueue()).thenReturn("queue");
when(app.getDiagnostics()).thenReturn(new StringBuilder());
when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
when(app.getStartTime()).thenReturn(0L);
when(app.getFinishTime()).thenReturn(0L);
when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED);
RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
when(app.getRMAppMetrics()).thenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt
final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8);
rmContext.getRMApps().put(APP_ID, app);
injector =
WebAppTests.createMockInjector(RMContext.class, rmContext,
new Module() {
@Override
public void configure(Binder binder) {
try {
binder.bind(ResourceManager.class).toInstance(
TestRMWebApp.mockRm(rmContext));
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
});
AppBlock instance = injector.getInstance(AppBlock.class);
instance.set(YarnWebParams.APPLICATION_ID, APP_ID.toString());
instance.render();
}
}