From 2aaed1032719e978660d1f14a8ed299e83b5f65e Mon Sep 17 00:00:00 2001 From: Zhihai Xu Date: Tue, 15 Dec 2015 00:17:21 -0800 Subject: [PATCH] YARN-4440. FSAppAttempt#getAllowedLocalityLevelByTime should init the lastScheduler time. Contributed by Lin Yiqun --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/fair/FSAppAttempt.java | 7 +++ .../scheduler/fair/TestFairScheduler.java | 60 +++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 886ac7c737..ee8ba4443b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1135,6 +1135,9 @@ Release 2.8.0 - UNRELEASED YARN-4402. TestNodeManagerShutdown And TestNodeManagerResync fails with bind exception. (Brahma Reddy Battula via jianhe) + YARN-4440. FSAppAttempt#getAllowedLocalityLevelByTime should init the + lastScheduler time. (Lin Yiqun via zxu) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index abbf77a465..3778cbaa44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -286,6 +286,13 @@ public synchronized NodeType getAllowedLocalityLevelByTime(Priority priority, // default level is NODE_LOCAL if (! allowedLocalityLevel.containsKey(priority)) { + // add the initial time of priority to prevent comparing with FsApp + // startTime and allowedLocalityLevel degrade + lastScheduledContainer.put(priority, currentTimeMs); + if (LOG.isDebugEnabled()) { + LOG.debug("Init the lastScheduledContainer time, priority: " + priority + + ", time: " + currentTimeMs); + } allowedLocalityLevel.put(priority, NodeType.NODE_LOCAL); return NodeType.NODE_LOCAL; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 4909e094bf..2f48380c77 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -5151,4 +5151,64 @@ public void testUserAsDefaultQueueWithLeadingTrailingSpaceUserName() assertEquals("root.user1", resourceManager.getRMContext().getRMApps() .get(attId3.getApplicationId()).getQueue()); } + + @Test + public void testFairSchedulerContinuousSchedulingInitTime() throws Exception { + int DELAY_THRESHOLD_TIME_MS = 1000; + conf.set(FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_ENABLED, "true"); + conf.set(FairSchedulerConfiguration.LOCALITY_DELAY_NODE_MS, + String.valueOf(DELAY_THRESHOLD_TIME_MS)); + conf.set(FairSchedulerConfiguration.LOCALITY_DELAY_RACK_MS, + String.valueOf(DELAY_THRESHOLD_TIME_MS)); + + ControlledClock clock = new ControlledClock(); + scheduler.setClock(clock); + scheduler.init(conf); + scheduler.start(); + + int priorityValue; + Priority priority; + FSAppAttempt fsAppAttempt; + ResourceRequest request1; + ResourceRequest request2; + ApplicationAttemptId id11; + + priorityValue = 1; + id11 = createAppAttemptId(1, 1); + createMockRMApp(id11); + priority = Priority.newInstance(priorityValue); + scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1", + false); + scheduler.addApplicationAttempt(id11, false, false); + fsAppAttempt = scheduler.getApplicationAttempt(id11); + + String hostName = "127.0.0.1"; + RMNode node1 = + MockNodes.newNodeInfo(1, Resources.createResource(16 * 1024, 16), 1, + hostName); + List ask1 = new ArrayList<>(); + request1 = + createResourceRequest(1024, 8, node1.getRackName(), priorityValue, 1, + true); + request2 = + createResourceRequest(1024, 8, ResourceRequest.ANY, priorityValue, 1, + true); + ask1.add(request1); + ask1.add(request2); + scheduler.allocate(id11, ask1, new ArrayList(), null, null, + null, null); + + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + FSSchedulerNode node = + (FSSchedulerNode) scheduler.getSchedulerNode(node1.getNodeID()); + // Tick the time and let the fsApp startTime different from initScheduler + // time + clock.tickSec(DELAY_THRESHOLD_TIME_MS / 1000); + scheduler.attemptScheduling(node); + Map lastScheduledContainer = + fsAppAttempt.getLastScheduledContainer(); + long initSchedulerTime = lastScheduledContainer.get(priority); + assertEquals(DELAY_THRESHOLD_TIME_MS, initSchedulerTime); + } }