big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10451. RM (v1) UI NodesPage can NPE when yarn.io/gpu resource type is defined. Contributed by Eric Payne

Browse Source
This commit is contained in:
Jim Brennan 2020-10-06 18:18:08 +00:00
parent c9ea344f98
commit b361f29dda
3 changed files with 48 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
View File

@ -175,7 +175,8 @@ protected void render(Block html) {
.get(ResourceInformation.GPU_URI);
long usedGPUs = 0;
long availableGPUs = 0;
if (gpuIndex != null) {
if (gpuIndex != null && info.getUsedResource() != null
&& info.getAvailableResource() != null) {
usedGPUs = info.getUsedResource().getResource()
.getResourceValue(ResourceInformation.GPU_URI);
availableGPUs = info.getAvailableResource().getResource()

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
View File

@ -19,12 +19,16 @@
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Map;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.NodesPage.NodesBlock;
import org.apache.hadoop.yarn.util.resource.CustomResourceTypesConfigurationProvider;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.junit.Before;
import org.junit.Test;
@ -56,6 +60,10 @@ public class TestNodesPage {
@Before
public void setUp() throws Exception {
setUpInternal(false);
}
private void setUpInternal(final boolean useDRC) throws Exception {
final RMContext mockRMContext =
TestRMWebApp.mockRMContext(3, numberOfRacks, numberOfNodesPerRack,
8 * TestRMWebApp.GiB);
@ -66,7 +74,7 @@ public void setUp() throws Exception {
public void configure(Binder binder) {
try {
binder.bind(ResourceManager.class).toInstance(
TestRMWebApp.mockRm(mockRMContext));
TestRMWebApp.mockRm(mockRMContext, useDRC));
} catch (IOException e) {
throw new IllegalStateException(e);
}
@ -101,7 +109,22 @@ public void testNodesBlockRenderForLostNodes() {
Mockito.verify(writer, Mockito.times(numberOfThInMetricsTable))
.print("<td");
}
@Test
public void testNodesBlockRenderForLostNodesWithGPUResources()
throws Exception {
Map<String, ResourceInformation> oldRtMap =
ResourceUtils.getResourceTypes();
CustomResourceTypesConfigurationProvider.
initResourceTypes(ResourceInformation.GPU_URI);
this.setUpInternal(true);
try {
this.testNodesBlockRenderForLostNodes();
} finally {
ResourceUtils.initializeResourcesFromResourceInformationMap(oldRtMap);
}
}
@Test
public void testNodesBlockRenderForNodeLabelFilterWithNonEmptyLabel() {
NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
View File

@ -247,8 +247,13 @@ public static ResourceManager mockRm(int apps, int racks, int nodes,
}
public static ResourceManager mockRm(RMContext rmContext) throws IOException {
return mockRm(rmContext, false);
}
public static ResourceManager mockRm(RMContext rmContext,
boolean useDRC) throws IOException {
ResourceManager rm = mock(ResourceManager.class);
ResourceScheduler rs = mockCapacityScheduler();
ResourceScheduler rs = mockCapacityScheduler(useDRC);
ApplicationACLsManager aclMgr = mockAppACLsManager();
ClientRMService clientRMService = mockClientRMService(rmContext);
when(rm.getResourceScheduler()).thenReturn(rs);
@ -259,9 +264,14 @@ public static ResourceManager mockRm(RMContext rmContext) throws IOException {
}
public static CapacityScheduler mockCapacityScheduler() throws IOException {
return mockCapacityScheduler(false);
}
public static CapacityScheduler mockCapacityScheduler(boolean useDRC)
throws IOException {
// stolen from TestCapacityScheduler
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf);
setupQueueConfiguration(conf, useDRC);
CapacityScheduler cs = new CapacityScheduler();
YarnConfiguration yarnConf = new YarnConfiguration();
@ -313,6 +323,11 @@ public static ClientRMService mockClientRMService(RMContext rmContext) {
static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
setupQueueConfiguration(conf, false);
}
static void setupQueueConfiguration(CapacitySchedulerConfiguration conf,
boolean useDRC) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
@ -358,6 +373,10 @@ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
conf.setCapacity(C11, 15);
conf.setCapacity(C12, 45);
conf.setCapacity(C13, 40);
if (useDRC) {
conf.set("yarn.scheduler.capacity.resource-calculator",
"org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
}
}
public static ResourceManager mockFifoRm(int apps, int racks, int nodes,