From 688f76a82242d14f89f7c67932004a179113a2b6 Mon Sep 17 00:00:00 2001 From: Yang Jiandan Date: Mon, 31 Jul 2023 07:39:42 +0800 Subject: [PATCH] =?UTF-8?q?YARN-11538.=20CS=20UI:=20queue=20filter=20do=20?= =?UTF-8?q?not=20work=20as=20expected=20when=20submitti=E2=80=A6=20(#5890)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resourcemanager/ClientRMService.java | 14 ++++++++-- .../server/resourcemanager/RMAppManager.java | 19 +++++++++++++ .../scheduler/capacity/CapacityScheduler.java | 2 +- .../resourcemanager/TestAppManager.java | 27 +++++++++++++++++++ .../scheduler/TestAbstractYarnScheduler.java | 3 +-- .../webapp/TestRMWebServicesApps.java | 16 +++++------ .../TestRMWebServicesAppsModification.java | 9 +++---- 7 files changed, 69 insertions(+), 21 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 08c664472a..4517825c12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -889,6 +889,16 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request) final Set runningAppsFilteredByQueues = getRunningAppsFilteredByQueues(apps, queues); + Set queuePaths = new HashSet<>(); + for (String queue : queues) { + String queuePath = rmAppManager.getQueuePath(queue); + if (queuePath != null) { + queuePaths.add(queuePath); + } else { + queuePaths.add(queue); + } + } + Iterator appsIter = apps.values().iterator(); List reports = new ArrayList(); @@ -901,9 +911,9 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request) continue; } - if (queues != null && !queues.isEmpty()) { + if (queuePaths != null && !queuePaths.isEmpty()) { if (!runningAppsFilteredByQueues.contains(application.getApplicationId()) && - !queues.contains(application.getQueue())) { + !queuePaths.contains(application.getQueue())) { continue; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index 0efc81d6bf..f87fff346d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager; +import java.io.IOException; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -28,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.security.ConfiguredYarnAuthorizer; import org.apache.hadoop.yarn.security.Permission; @@ -552,6 +554,9 @@ private RMAppImpl createAndPopulateNewRMApp( } } + // Get full queue path for the application when submitting by short queue name + placementQueueName = getQueuePath(placementQueueName); + // Create RMApp RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf, @@ -582,6 +587,20 @@ private RMAppImpl createAndPopulateNewRMApp( return application; } + public String getQueuePath(String queueName) { + String queuePath = queueName; + try { + QueueInfo queueInfo = + scheduler.getQueueInfo(queueName, false, false); + if (queueInfo != null && queueInfo.getQueuePath() != null) { + queuePath = queueInfo.getQueuePath(); + } + } catch (IOException e) { + // if the queue does not exist, we just ignore here + } + return queuePath; + } + private boolean checkPermission(AccessRequest accessRequest, YarnAuthorizationProvider dynamicAuthorizer) { return authorizer.checkPermission(accessRequest) || diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 2c61fe61ad..6ebadcba1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2725,7 +2725,7 @@ public String moveApplication(ApplicationId appId, application.setQueue(dest); LOG.info("App: " + appId + " successfully moved from " + sourceQueueName + " to: " + destQueueName); - return targetQueueName; + return dest.getQueuePath(); } finally { writeLock.unlock(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index fe2a7bab9e..0369feb130 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -355,6 +355,33 @@ public void testQueueSubmitWithACLsEnabledWithQueueMapping() "test1"); } + @Test + public void testQueueSubmitWithLeafQueueName() + throws YarnException { + YarnConfiguration conf = createYarnACLEnabledConfiguration(); + CapacitySchedulerConfiguration csConf = new + CapacitySchedulerConfiguration(conf, false); + csConf.set(PREFIX + "root.queues", "default,test"); + + csConf.setCapacity("root.default", 50.0f); + csConf.setMaximumCapacity("root.default", 100.0f); + + csConf.setCapacity("root.test", 50.0f); + csConf.setMaximumCapacity("root.test", 100.0f); + + MockRM newMockRM = new MockRM(csConf); + RMContext newMockRMContext = newMockRM.getRMContext(); + TestRMAppManager newAppMonitor = createAppManager(newMockRMContext, conf); + + ApplicationSubmissionContext submission = createAppSubmissionContext(MockApps.newAppID(1)); + submission.setQueue("test"); + verifyAppSubmission(submission, + newAppMonitor, + newMockRMContext, + "test", + "root.test"); + } + @Test public void testQueueSubmitWithACLsEnabledWithQueueMappingForLegacyAutoCreatedQueue() throws IOException, YarnException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java index e53ac5aff2..67b3dee2b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java @@ -75,7 +75,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; @@ -1168,7 +1167,7 @@ public void testContainerRecoveredByNode() throws Exception { //verify queue name when rmContainer is recovered if (scheduler instanceof CapacityScheduler) { Assert.assertEquals( - CapacitySchedulerConfiguration.ROOT + "." + app1.getQueue(), + app1.getQueue(), rmContainer.getQueueName()); } else { Assert.assertEquals(app1.getQueue(), rmContainer.getQueueName()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 964aab6913..f03336e038 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -2041,13 +2041,11 @@ public void testAppsQueryByQueueShortname() throws Exception { appIds.contains(runningApp1.getApplicationId().toString())); assertTrue("Running app 2 should be in the result list!", appIds.contains(runningApp2.getApplicationId().toString())); - assertFalse("Finished app 1 should not be in the result list " + - "as it was submitted to 'root.default' but the query is for 'default'", + assertTrue("Running app 1 should be in the result list!", appIds.contains(finishedApp1.getApplicationId().toString())); - assertTrue("Finished app 2 should be in the result list " + - "as it was submitted to 'default' and the query is exactly for 'default'", + assertTrue("Running app 1 should be in the result list!", appIds.contains(finishedApp2.getApplicationId().toString())); - assertEquals("incorrect number of elements", 3, array.length()); + assertEquals("incorrect number of elements", 4, array.length()); rm.stop(); } @@ -2110,13 +2108,11 @@ public void testAppsQueryByQueueFullname() throws Exception { appIds.contains(runningApp1.getApplicationId().toString())); assertTrue("Running app 2 should be in the result list!", appIds.contains(runningApp2.getApplicationId().toString())); - assertTrue("Finished app 1 should be in the result list, " + - "as it was submitted to 'root.default' and the query is exactly for 'root.default'!", + assertTrue("Running app 2 should be in the result list!", appIds.contains(finishedApp1.getApplicationId().toString())); - assertFalse("Finished app 2 should not be in the result list, " + - "as it was submitted to 'default' but the query is for 'root.default'!", + assertTrue("Running app 2 should be in the result list!", appIds.contains(finishedApp2.getApplicationId().toString())); - assertEquals("incorrect number of elements", 3, array.length()); + assertEquals("incorrect number of elements", 4, array.length()); rm.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 0c8f742fae..be5494a43d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -1066,7 +1066,7 @@ public void testGetAppQueue() throws Exception { .constructWebResource("apps", app.getApplicationId().toString(), "queue").accept(contentType).get(ClientResponse.class); assertResponseStatusCode(Status.OK, response.getStatusInfo()); - String expectedQueue = "default"; + String expectedQueue = "root.default"; if(!isCapacityScheduler) { expectedQueue = "root." + webserviceUserName; } @@ -1230,10 +1230,7 @@ public void testAppMove() throws Exception { continue; } assertResponseStatusCode(Status.OK, response.getStatusInfo()); - String expectedQueue = "test"; - if(!isCapacityScheduler) { - expectedQueue = "root.test"; - } + String expectedQueue = "root.test"; if (mediaType.contains(MediaType.APPLICATION_JSON)) { verifyAppQueueJson(response, expectedQueue); } else { @@ -1256,7 +1253,7 @@ public void testAppMove() throws Exception { .put(ClientResponse.class); assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo()); if(isCapacityScheduler) { - Assert.assertEquals("default", app.getQueue()); + Assert.assertEquals("root.default", app.getQueue()); } else { Assert.assertEquals("root.someuser", app.getQueue());