YARN-10451. RM (v1) UI NodesPage can NPE when yarn.io/gpu resource type is defined. Contributed by Eric Payne

This commit is contained in:
Jim Brennan 2020-10-06 18:18:08 +00:00
parent c9ea344f98
commit b361f29dda
3 changed files with 48 additions and 5 deletions

View File

@ -175,7 +175,8 @@ protected void render(Block html) {
.get(ResourceInformation.GPU_URI); .get(ResourceInformation.GPU_URI);
long usedGPUs = 0; long usedGPUs = 0;
long availableGPUs = 0; long availableGPUs = 0;
if (gpuIndex != null) { if (gpuIndex != null && info.getUsedResource() != null
&& info.getAvailableResource() != null) {
usedGPUs = info.getUsedResource().getResource() usedGPUs = info.getUsedResource().getResource()
.getResourceValue(ResourceInformation.GPU_URI); .getResourceValue(ResourceInformation.GPU_URI);
availableGPUs = info.getAvailableResource().getResource() availableGPUs = info.getAvailableResource().getResource()

View File

@ -19,12 +19,16 @@
import java.io.IOException; import java.io.IOException;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.util.Map;
import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.NodesPage.NodesBlock; import org.apache.hadoop.yarn.server.resourcemanager.webapp.NodesPage.NodesBlock;
import org.apache.hadoop.yarn.util.resource.CustomResourceTypesConfigurationProvider;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.webapp.test.WebAppTests; import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
@ -56,6 +60,10 @@ public class TestNodesPage {
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
setUpInternal(false);
}
private void setUpInternal(final boolean useDRC) throws Exception {
final RMContext mockRMContext = final RMContext mockRMContext =
TestRMWebApp.mockRMContext(3, numberOfRacks, numberOfNodesPerRack, TestRMWebApp.mockRMContext(3, numberOfRacks, numberOfNodesPerRack,
8 * TestRMWebApp.GiB); 8 * TestRMWebApp.GiB);
@ -66,7 +74,7 @@ public void setUp() throws Exception {
public void configure(Binder binder) { public void configure(Binder binder) {
try { try {
binder.bind(ResourceManager.class).toInstance( binder.bind(ResourceManager.class).toInstance(
TestRMWebApp.mockRm(mockRMContext)); TestRMWebApp.mockRm(mockRMContext, useDRC));
} catch (IOException e) { } catch (IOException e) {
throw new IllegalStateException(e); throw new IllegalStateException(e);
} }
@ -102,6 +110,21 @@ public void testNodesBlockRenderForLostNodes() {
.print("<td"); .print("<td");
} }
@Test
public void testNodesBlockRenderForLostNodesWithGPUResources()
throws Exception {
Map<String, ResourceInformation> oldRtMap =
ResourceUtils.getResourceTypes();
CustomResourceTypesConfigurationProvider.
initResourceTypes(ResourceInformation.GPU_URI);
this.setUpInternal(true);
try {
this.testNodesBlockRenderForLostNodes();
} finally {
ResourceUtils.initializeResourcesFromResourceInformationMap(oldRtMap);
}
}
@Test @Test
public void testNodesBlockRenderForNodeLabelFilterWithNonEmptyLabel() { public void testNodesBlockRenderForNodeLabelFilterWithNonEmptyLabel() {
NodesBlock nodesBlock = injector.getInstance(NodesBlock.class); NodesBlock nodesBlock = injector.getInstance(NodesBlock.class);

View File

@ -247,8 +247,13 @@ public static ResourceManager mockRm(int apps, int racks, int nodes,
} }
public static ResourceManager mockRm(RMContext rmContext) throws IOException { public static ResourceManager mockRm(RMContext rmContext) throws IOException {
return mockRm(rmContext, false);
}
public static ResourceManager mockRm(RMContext rmContext,
boolean useDRC) throws IOException {
ResourceManager rm = mock(ResourceManager.class); ResourceManager rm = mock(ResourceManager.class);
ResourceScheduler rs = mockCapacityScheduler(); ResourceScheduler rs = mockCapacityScheduler(useDRC);
ApplicationACLsManager aclMgr = mockAppACLsManager(); ApplicationACLsManager aclMgr = mockAppACLsManager();
ClientRMService clientRMService = mockClientRMService(rmContext); ClientRMService clientRMService = mockClientRMService(rmContext);
when(rm.getResourceScheduler()).thenReturn(rs); when(rm.getResourceScheduler()).thenReturn(rs);
@ -259,9 +264,14 @@ public static ResourceManager mockRm(RMContext rmContext) throws IOException {
} }
public static CapacityScheduler mockCapacityScheduler() throws IOException { public static CapacityScheduler mockCapacityScheduler() throws IOException {
return mockCapacityScheduler(false);
}
public static CapacityScheduler mockCapacityScheduler(boolean useDRC)
throws IOException {
// stolen from TestCapacityScheduler // stolen from TestCapacityScheduler
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf); setupQueueConfiguration(conf, useDRC);
CapacityScheduler cs = new CapacityScheduler(); CapacityScheduler cs = new CapacityScheduler();
YarnConfiguration yarnConf = new YarnConfiguration(); YarnConfiguration yarnConf = new YarnConfiguration();
@ -313,6 +323,11 @@ public static ClientRMService mockClientRMService(RMContext rmContext) {
static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
setupQueueConfiguration(conf, false);
}
static void setupQueueConfiguration(CapacitySchedulerConfiguration conf,
boolean useDRC) {
// Define top-level queues // Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
@ -358,6 +373,10 @@ static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
conf.setCapacity(C11, 15); conf.setCapacity(C11, 15);
conf.setCapacity(C12, 45); conf.setCapacity(C12, 45);
conf.setCapacity(C13, 40); conf.setCapacity(C13, 40);
if (useDRC) {
conf.set("yarn.scheduler.capacity.resource-calculator",
"org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
}
} }
public static ResourceManager mockFifoRm(int apps, int racks, int nodes, public static ResourceManager mockFifoRm(int apps, int racks, int nodes,