big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11202. Optimize ClientRMService.getApplications. Contributed by Tamas Domok.

Browse Source 
Change-Id: I55ddb46fd0e4cdb644747d6d43083215f10861b5
This commit is contained in:
Tamas Domok 2022-06-27 16:01:31 +02:00 committed by Benjamin Teke
parent 2d133a54ac
commit 3cad632709
1 changed files with 12 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -899,6 +899,9 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request)
String name = request.getName();
final Map<ApplicationId, RMApp> apps = rmContext.getRMApps();
final Set<ApplicationId> runningAppsFilteredByQueues =
getRunningAppsFilteredByQueues(apps, queues);
Iterator<RMApp> appsIter = apps.values().iterator();
List<ApplicationReport> reports = new ArrayList<ApplicationReport>();
@ -912,17 +915,8 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request)
}
if (queues != null && !queues.isEmpty()) {
Map<String, List<RMApp>> foundApps = queryApplicationsByQueues(apps, queues);
List<RMApp> runningAppsByQueues = foundApps.entrySet().stream()
.filter(e -> queues.contains(e.getKey()))
.map(Map.Entry::getValue)
.flatMap(Collection::stream)
.collect(Collectors.toList());
List<RMApp> runningAppsById = runningAppsByQueues.stream()
.filter(app -> app.getApplicationId().equals(application.getApplicationId()))
.collect(Collectors.toList());
if (runningAppsById.isEmpty() && !queues.contains(application.getQueue())) {
if (!runningAppsFilteredByQueues.contains(application.getApplicationId()) &&
!queues.contains(application.getQueue())) {
continue;
}
}
@ -1001,20 +995,21 @@ public GetApplicationsResponse getApplications(GetApplicationsRequest request)
return response;
}
private Map<String, List<RMApp>> queryApplicationsByQueues(
private Set<ApplicationId> getRunningAppsFilteredByQueues(
Map<ApplicationId, RMApp> apps, Set<String> queues) {
final Map<String, List<RMApp>> appsToQueues = new HashMap<>();
final Set<ApplicationId> runningApps = new HashSet<>();
for (String queue : queues) {
List<ApplicationAttemptId> appsInQueue = scheduler.getAppsInQueue(queue);
if (appsInQueue != null && !appsInQueue.isEmpty()) {
if (appsInQueue != null) {
for (ApplicationAttemptId appAttemptId : appsInQueue) {
RMApp rmApp = apps.get(appAttemptId.getApplicationId());
appsToQueues.putIfAbsent(queue, new ArrayList<>());
appsToQueues.get(queue).add(rmApp);
if (rmApp != null) {
runningApps.add(rmApp.getApplicationId());
}
}
}
}
return appsToQueues;
return runningApps;
}
private Set<String> getLowerCasedAppTypes(GetApplicationsRequest request) {