big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10829. Follow up: Adding null checks before merging ResourceUsage Report (#3252)

Browse Source
This commit is contained in:
Akshat Bordia 2021-09-08 23:06:56 +05:30 committed by GitHub
parent 2ff3fc50e4
commit a186460004
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 75 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterYarnClientUtils.java
View File

@ -133,43 +133,48 @@ private static void mergeAMWithUAM(ApplicationReport am,
ApplicationResourceUsageReport uamResourceReport = ApplicationResourceUsageReport uamResourceReport =
uam.getApplicationResourceUsageReport(); uam.getApplicationResourceUsageReport();
amResourceReport.setNumUsedContainers( if (amResourceReport == null) {
amResourceReport.getNumUsedContainers() + am.setApplicationResourceUsageReport(uamResourceReport);
uamResourceReport.getNumUsedContainers()); } else if (uamResourceReport != null) {
amResourceReport.setNumReservedContainers( amResourceReport.setNumUsedContainers(
amResourceReport.getNumReservedContainers() + amResourceReport.getNumUsedContainers() +
uamResourceReport.getNumReservedContainers()); uamResourceReport.getNumUsedContainers());
amResourceReport.setUsedResources(Resources.add( amResourceReport.setNumReservedContainers(
amResourceReport.getUsedResources(), amResourceReport.getNumReservedContainers() +
uamResourceReport.getUsedResources())); uamResourceReport.getNumReservedContainers());
amResourceReport.setReservedResources(Resources.add( amResourceReport.setUsedResources(Resources.add(
amResourceReport.getReservedResources(), amResourceReport.getUsedResources(),
uamResourceReport.getReservedResources())); uamResourceReport.getUsedResources()));
amResourceReport.setNeededResources(Resources.add( amResourceReport.setReservedResources(Resources.add(
amResourceReport.getNeededResources(), amResourceReport.getReservedResources(),
uamResourceReport.getNeededResources())); uamResourceReport.getReservedResources()));
amResourceReport.setMemorySeconds( amResourceReport.setNeededResources(Resources.add(
amResourceReport.getMemorySeconds() + amResourceReport.getNeededResources(),
uamResourceReport.getMemorySeconds()); uamResourceReport.getNeededResources()));
amResourceReport.setVcoreSeconds( amResourceReport.setMemorySeconds(
amResourceReport.getVcoreSeconds() + amResourceReport.getMemorySeconds() +
uamResourceReport.getVcoreSeconds()); uamResourceReport.getMemorySeconds());
amResourceReport.setQueueUsagePercentage( amResourceReport.setVcoreSeconds(
amResourceReport.getQueueUsagePercentage() + amResourceReport.getVcoreSeconds() +
uamResourceReport.getQueueUsagePercentage()); uamResourceReport.getVcoreSeconds());
amResourceReport.setClusterUsagePercentage( amResourceReport.setQueueUsagePercentage(
amResourceReport.getClusterUsagePercentage() + amResourceReport.getQueueUsagePercentage() +
uamResourceReport.getClusterUsagePercentage()); uamResourceReport.getQueueUsagePercentage());
am.setApplicationResourceUsageReport(amResourceReport); amResourceReport.setClusterUsagePercentage(
amResourceReport.getClusterUsagePercentage() +
uamResourceReport.getClusterUsagePercentage());
am.setApplicationResourceUsageReport(amResourceReport);
}
} }
/** /**

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/clientrm/TestRouterYarnClientUtils.java
View File

@ -124,6 +124,48 @@ public void testMergeUnmanagedApplications() {
Assert.assertTrue(result.getApplicationList().isEmpty()); Assert.assertTrue(result.getApplicationList().isEmpty());
} }
/**
* This test validates the correctness of
* RouterYarnClientUtils#mergeApplications when
* ApplicationResourceUsageReport might be null.
*/
@Test
public void testMergeApplicationsNullResourceUsage() {
ApplicationId appId = ApplicationId.newInstance(1234, 1);
ApplicationReport appReport = ApplicationReport.newInstance(
appId, ApplicationAttemptId.newInstance(appId, 1),
"user", "queue", "app1", "host",
124, null, YarnApplicationState.RUNNING,
"diagnostics", "url", 0, 0,
0, FinalApplicationStatus.SUCCEEDED, null, "N/A",
0.53789f, "YARN", null, null, false, null, null, null);
ApplicationReport uamAppReport = ApplicationReport.newInstance(
appId, ApplicationAttemptId.newInstance(appId, 1),
"user", "queue", "app1", "host",
124, null, YarnApplicationState.RUNNING,
"diagnostics", "url", 0, 0,
0, FinalApplicationStatus.SUCCEEDED, null, "N/A",
0.53789f, "YARN", null, null, true, null, null, null);
ArrayList<GetApplicationsResponse> responses = new ArrayList<>();
List<ApplicationReport> applications = new ArrayList<>();
applications.add(appReport);
applications.add(uamAppReport);
responses.add(GetApplicationsResponse.newInstance(applications));
GetApplicationsResponse result = RouterYarnClientUtils.
mergeApplications(responses, false);
Assert.assertNotNull(result);
Assert.assertEquals(1, result.getApplicationList().size());
String appName = result.getApplicationList().get(0).getName();
// Check that no Unmanaged applications are added to the result
Assert.assertFalse(appName.contains(UnmanagedApplicationManager.APP_NAME));
}
/** /**
* This generates a GetApplicationsResponse with 2 applications with * This generates a GetApplicationsResponse with 2 applications with
* same ApplicationId. * same ApplicationId.