From 5de6f72054d4911fcaf12132dccb05662f36a82d Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Tue, 10 Jun 2014 19:03:06 +0000 Subject: [PATCH] YARN-1424. RMAppAttemptImpl should return the DummyApplicationResourceUsageReport for all invalid accesses. (Ray Chiang via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1601742 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 +++ .../ApplicationResourceUsageReport.java | 10 ++++---- .../server/resourcemanager/RMServerUtils.java | 12 +++++++++ .../resourcemanager/rmapp/RMAppImpl.java | 8 ++---- .../rmapp/attempt/RMAppAttemptImpl.java | 4 +-- .../resourcemanager/TestClientRMService.java | 25 +++++++++++++++++++ .../rmapp/TestRMAppTransitions.java | 2 ++ 7 files changed, 51 insertions(+), 14 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 08bb50c838..280a2cccf2 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -151,6 +151,10 @@ Release 2.5.0 - UNRELEASED YARN-2030. Augmented RMStateStore with state machine.(Binglin Chang via jianhe) + YARN-1424. RMAppAttemptImpl should return the + DummyApplicationResourceUsageReport for all invalid accesses. + (Ray Chiang via kasha) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java index 8de9ff3baf..6e9c76fb01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java @@ -47,7 +47,7 @@ public static ApplicationResourceUsageReport newInstance( } /** - * Get the number of used containers + * Get the number of used containers. -1 for invalid/inaccessible reports. * @return the number of used containers */ @Public @@ -63,7 +63,7 @@ public static ApplicationResourceUsageReport newInstance( public abstract void setNumUsedContainers(int num_containers); /** - * Get the number of reserved containers + * Get the number of reserved containers. -1 for invalid/inaccessible reports. * @return the number of reserved containers */ @Private @@ -79,7 +79,7 @@ public static ApplicationResourceUsageReport newInstance( public abstract void setNumReservedContainers(int num_reserved_containers); /** - * Get the used Resource + * Get the used Resource. -1 for invalid/inaccessible reports. * @return the used Resource */ @Public @@ -91,7 +91,7 @@ public static ApplicationResourceUsageReport newInstance( public abstract void setUsedResources(Resource resources); /** - * Get the reserved Resource + * Get the reserved Resource. -1 for invalid/inaccessible reports. * @return the reserved Resource */ @Public @@ -103,7 +103,7 @@ public static ApplicationResourceUsageReport newInstance( public abstract void setReservedResources(Resource reserved_resources); /** - * Get the needed Resource + * Get the needed Resource. -1 for invalid/inaccessible reports. * @return the needed Resource */ @Public diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java index e884d29e30..d93c45d0d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java @@ -28,6 +28,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; @@ -43,6 +44,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.resource.Resources; /** * Utility methods to aid serving RM data through the REST and RPC APIs @@ -225,4 +228,13 @@ public static YarnApplicationAttemptState createApplicationAttemptState( } } + /** + * Statically defined dummy ApplicationResourceUsageREport. Used as + * a return value when a valid report cannot be found. + */ + public static final ApplicationResourceUsageReport + DUMMY_APPLICATION_RESOURCE_USAGE_REPORT = + BuilderUtils.newApplicationResourceUsageReport(-1, -1, + Resources.createResource(-1, -1), Resources.createResource(-1, -1), + Resources.createResource(-1, -1)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index bbd135b990..3318f1582a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -71,6 +71,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.state.InvalidStateTransitonException; import org.apache.hadoop.yarn.state.MultipleArcTransition; @@ -293,11 +294,6 @@ RMAppEventType.KILL, new KillAttemptTransition()) private final StateMachine stateMachine; - private static final ApplicationResourceUsageReport - DUMMY_APPLICATION_RESOURCE_USAGE_REPORT = - BuilderUtils.newApplicationResourceUsageReport(-1, -1, - Resources.createResource(-1, -1), Resources.createResource(-1, -1), - Resources.createResource(-1, -1)); private static final int DUMMY_APPLICATION_ATTEMPT_NUMBER = -1; public RMAppImpl(ApplicationId applicationId, RMContext rmContext, @@ -498,7 +494,7 @@ public ApplicationReport createAndGetApplicationReport(String clientUserName, String origTrackingUrl = UNAVAILABLE; int rpcPort = -1; ApplicationResourceUsageReport appUsageReport = - DUMMY_APPLICATION_RESOURCE_USAGE_REPORT; + RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT; FinalApplicationStatus finishState = getFinalApplicationStatus(); String diags = UNAVAILABLE; float progress = 0.0f; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 5b1a17d930..2a1170d41d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -675,9 +675,7 @@ public ApplicationResourceUsageReport getApplicationResourceUsageReport() { ApplicationResourceUsageReport report = scheduler.getAppResourceUsageReport(this.getAppAttemptId()); if (report == null) { - Resource none = Resource.newInstance(0, 0); - report = ApplicationResourceUsageReport.newInstance(0, 0, none, none, - none); + report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT; } return report; } finally { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 045b5b4ae9..4b1f59c303 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -77,6 +77,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -258,6 +259,28 @@ public void testGetApplicationAttemptReport() throws YarnException, } } + @Test + public void testGetApplicationResourceUsageReportDummy() throws YarnException, + IOException { + ApplicationAttemptId attemptId = getApplicationAttemptId(1); + YarnScheduler yarnScheduler = mockYarnScheduler(); + RMContext rmContext = mock(RMContext.class); + mockRMContext(yarnScheduler, rmContext); + when(rmContext.getDispatcher().getEventHandler()).thenReturn( + new EventHandler() { + public void handle(Event event) { + } + }); + ApplicationSubmissionContext asContext = + mock(ApplicationSubmissionContext.class); + YarnConfiguration config = new YarnConfiguration(); + RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId, + rmContext, yarnScheduler, null, asContext, config, false); + ApplicationResourceUsageReport report = rmAppAttemptImpl + .getApplicationResourceUsageReport(); + assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT); + } + @Test public void testGetApplicationAttempts() throws YarnException, IOException { ClientRMService rmService = createRMService(); @@ -964,6 +987,8 @@ private static YarnScheduler mockYarnScheduler() { Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102))); when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn( Arrays.asList(getApplicationAttemptId(103))); + ApplicationAttemptId attemptId = getApplicationAttemptId(1); + when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null); return yarnScheduler; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index e89b71b410..0fd3c3c5c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; @@ -921,6 +922,7 @@ public void testGetAppReport() { assertAppState(RMAppState.NEW, app); ApplicationReport report = app.createAndGetApplicationReport(null, true); Assert.assertNotNull(report.getApplicationResourceUsageReport()); + Assert.assertEquals(report.getApplicationResourceUsageReport(),RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT); report = app.createAndGetApplicationReport("clientuser", true); Assert.assertNotNull(report.getApplicationResourceUsageReport()); }