big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11538. CS UI: queue filter do not work as expected when submitti… (#5890)

Browse Source
This commit is contained in:
Yang Jiandan 2023-07-31 07:39:42 +08:00 committed by GitHub
parent 5056c267e6
commit 688f76a822
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 69 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -889,6 +889,16 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request)
final Set<ApplicationId> runningAppsFilteredByQueues =
getRunningAppsFilteredByQueues(apps, queues);
Set<String> queuePaths = new HashSet<>();
for (String queue : queues) {
String queuePath = rmAppManager.getQueuePath(queue);
if (queuePath != null) {
queuePaths.add(queuePath);
} else {
queuePaths.add(queue);
}
}
Iterator<RMApp> appsIter = apps.values().iterator();
List<ApplicationReport> reports = new ArrayList<ApplicationReport>();
@ -901,9 +911,9 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request)
continue;
}
if (queues != null && !queues.isEmpty()) {
if (queuePaths != null && !queuePaths.isEmpty()) {
if (!runningAppsFilteredByQueues.contains(application.getApplicationId()) &&
!queues.contains(application.getQueue())) {
!queuePaths.contains(application.getQueue())) {
continue;
}
}

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
@ -28,6 +29,7 @@
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.security.ConfiguredYarnAuthorizer;
import org.apache.hadoop.yarn.security.Permission;
@ -552,6 +554,9 @@ private RMAppImpl createAndPopulateNewRMApp(
}
}
// Get full queue path for the application when submitting by short queue name
placementQueueName = getQueuePath(placementQueueName);
// Create RMApp
RMAppImpl application =
new RMAppImpl(applicationId, rmContext, this.conf,
@ -582,6 +587,20 @@ private RMAppImpl createAndPopulateNewRMApp(
return application;
}
public String getQueuePath(String queueName) {
String queuePath = queueName;
try {
QueueInfo queueInfo =
scheduler.getQueueInfo(queueName, false, false);
if (queueInfo != null && queueInfo.getQueuePath() != null) {
queuePath = queueInfo.getQueuePath();
}
} catch (IOException e) {
// if the queue does not exist, we just ignore here
}
return queuePath;
}
private boolean checkPermission(AccessRequest accessRequest,
YarnAuthorizationProvider dynamicAuthorizer) {
return authorizer.checkPermission(accessRequest) ||

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
View File

@ -2725,7 +2725,7 @@ public String moveApplication(ApplicationId appId,
application.setQueue(dest);
LOG.info("App: " + appId + " successfully moved from " + sourceQueueName
+ " to: " + destQueueName);
return targetQueueName;
return dest.getQueuePath();
} finally {
writeLock.unlock();
}

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
View File

@ -355,6 +355,33 @@ public void testQueueSubmitWithACLsEnabledWithQueueMapping()
"test1");
}
@Test
public void testQueueSubmitWithLeafQueueName()
throws YarnException {
YarnConfiguration conf = createYarnACLEnabledConfiguration();
CapacitySchedulerConfiguration csConf = new
CapacitySchedulerConfiguration(conf, false);
csConf.set(PREFIX + "root.queues", "default,test");
csConf.setCapacity("root.default", 50.0f);
csConf.setMaximumCapacity("root.default", 100.0f);
csConf.setCapacity("root.test", 50.0f);
csConf.setMaximumCapacity("root.test", 100.0f);
MockRM newMockRM = new MockRM(csConf);
RMContext newMockRMContext = newMockRM.getRMContext();
TestRMAppManager newAppMonitor = createAppManager(newMockRMContext, conf);
ApplicationSubmissionContext submission = createAppSubmissionContext(MockApps.newAppID(1));
submission.setQueue("test");
verifyAppSubmission(submission,
newAppMonitor,
newMockRMContext,
"test",
"root.test");
}
@Test
public void testQueueSubmitWithACLsEnabledWithQueueMappingForLegacyAutoCreatedQueue()
throws IOException, YarnException {

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
View File

@ -75,7 +75,6 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
@ -1168,7 +1167,7 @@ public void testContainerRecoveredByNode() throws Exception {
//verify queue name when rmContainer is recovered
if (scheduler instanceof CapacityScheduler) {
Assert.assertEquals(
CapacitySchedulerConfiguration.ROOT + "." + app1.getQueue(),
app1.getQueue(),
rmContainer.getQueueName());
} else {
Assert.assertEquals(app1.getQueue(), rmContainer.getQueueName());

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
View File

@ -2041,13 +2041,11 @@ public void testAppsQueryByQueueShortname() throws Exception {
appIds.contains(runningApp1.getApplicationId().toString()));
assertTrue("Running app 2 should be in the result list!",
appIds.contains(runningApp2.getApplicationId().toString()));
assertFalse("Finished app 1 should not be in the result list " +
"as it was submitted to 'root.default' but the query is for 'default'",
assertTrue("Running app 1 should be in the result list!",
appIds.contains(finishedApp1.getApplicationId().toString()));
assertTrue("Finished app 2 should be in the result list " +
"as it was submitted to 'default' and the query is exactly for 'default'",
assertTrue("Running app 1 should be in the result list!",
appIds.contains(finishedApp2.getApplicationId().toString()));
assertEquals("incorrect number of elements", 3, array.length());
assertEquals("incorrect number of elements", 4, array.length());
rm.stop();
}
@ -2110,13 +2108,11 @@ public void testAppsQueryByQueueFullname() throws Exception {
appIds.contains(runningApp1.getApplicationId().toString()));
assertTrue("Running app 2 should be in the result list!",
appIds.contains(runningApp2.getApplicationId().toString()));
assertTrue("Finished app 1 should be in the result list, " +
"as it was submitted to 'root.default' and the query is exactly for 'root.default'!",
assertTrue("Running app 2 should be in the result list!",
appIds.contains(finishedApp1.getApplicationId().toString()));
assertFalse("Finished app 2 should not be in the result list, " +
"as it was submitted to 'default' but the query is for 'root.default'!",
assertTrue("Running app 2 should be in the result list!",
appIds.contains(finishedApp2.getApplicationId().toString()));
assertEquals("incorrect number of elements", 3, array.length());
assertEquals("incorrect number of elements", 4, array.length());
rm.stop();
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
View File

@ -1066,7 +1066,7 @@ public void testGetAppQueue() throws Exception {
.constructWebResource("apps", app.getApplicationId().toString(),
"queue").accept(contentType).get(ClientResponse.class);
assertResponseStatusCode(Status.OK, response.getStatusInfo());
String expectedQueue = "default";
String expectedQueue = "root.default";
if(!isCapacityScheduler) {
expectedQueue = "root." + webserviceUserName;
}
@ -1230,10 +1230,7 @@ public void testAppMove() throws Exception {
continue;
}
assertResponseStatusCode(Status.OK, response.getStatusInfo());
String expectedQueue = "test";
if(!isCapacityScheduler) {
expectedQueue = "root.test";
}
String expectedQueue = "root.test";
if (mediaType.contains(MediaType.APPLICATION_JSON)) {
verifyAppQueueJson(response, expectedQueue);
} else {
@ -1256,7 +1253,7 @@ public void testAppMove() throws Exception {
.put(ClientResponse.class);
assertResponseStatusCode(Status.FORBIDDEN, response.getStatusInfo());
if(isCapacityScheduler) {
Assert.assertEquals("default", app.getQueue());
Assert.assertEquals("root.default", app.getQueue());
}
else {
Assert.assertEquals("root.someuser", app.getQueue());