big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11036. Do not inherit from TestRMWebServicesCapacitySched. Contributed by Tamas Domok

Browse Source
This commit is contained in:
9uapaw 2022-03-10 13:11:19 +01:00
parent 9539ff108a
commit 383b73417d
3 changed files with 73 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -77,42 +77,37 @@
public class TestRMWebServicesCapacitySched extends JerseyTestBase {
protected static MockRM rm;
private MockRM rm;
public static class WebServletModule extends ServletModule {
private final MockRM rm;
WebServletModule(MockRM rm) {
this.rm = rm;
}
private static class WebServletModule extends ServletModule {
@Override
protected void configureServlets() {
bind(JAXBContextResolver.class);
bind(RMWebServices.class);
bind(GenericExceptionHandler.class);
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
new Configuration(false));
setupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
serve("/*").with(GuiceContainer.class);
}
}
public TestRMWebServicesCapacitySched() {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
super(createWebAppDescriptor());
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
rm = createMockRM(new CapacitySchedulerConfiguration(
new Configuration(false)));
GuiceServletConfig.setInjector(
Guice.createInjector(new WebServletModule()));
Guice.createInjector(new WebServletModule(rm)));
}
public static void setupQueueConfiguration(
@ -389,4 +384,22 @@ public static void updateTestDataAutomatically(String configFilename, String act
Assert.fail("overwrite should not fail " + e.getMessage());
}
}
public static WebAppDescriptor createWebAppDescriptor() {
return new WebAppDescriptor.Builder(
TestRMWebServicesCapacitySched.class.getPackage().getName())
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build();
}
public static MockRM createMockRM(CapacitySchedulerConfiguration csConf) {
setupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
return new MockRM(conf);
}
}

65
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java
View File

@ -19,10 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import java.io.IOException;
import java.util.HashMap;
@ -34,67 +31,28 @@
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.junit.Test;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.assertJsonResponse;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
public class TestRMWebServicesCapacitySchedDynamicConfig extends
JerseyTestBase {
private static final int GB = 1024;
private static MockRM rm;
private MockRM rm;
private CapacitySchedulerQueueManager autoQueueHandler;
private static class WebServletModule extends ServletModule {
private final Configuration conf;
WebServletModule(Configuration conf) {
this.conf = conf;
}
@Override
protected void configureServlets() {
bind(JAXBContextResolver.class);
bind(RMWebServices.class);
bind(GenericExceptionHandler.class);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
serve("/*").with(GuiceContainer.class);
}
}
private void initResourceManager(Configuration conf) throws IOException {
GuiceServletConfig.setInjector(
Guice.createInjector(new WebServletModule(conf)));
rm.start();
//Need to call reinitialize as
//MutableCSConfigurationProvider with InMemoryConfigurationStore
//somehow does not load the queues properly and falls back to default config.
//Therefore CS will think there's only the default queue there.
((CapacityScheduler) rm.getResourceScheduler()).reinitialize(conf,
rm.getRMContext(), true);
}
public TestRMWebServicesCapacitySchedDynamicConfig() {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
super(createWebAppDescriptor());
}
@Test
@ -327,4 +285,17 @@ public static Configuration createConfiguration(
return config;
}
}
private void initResourceManager(Configuration conf) throws IOException {
rm = createMockRM(new CapacitySchedulerConfiguration(conf));
GuiceServletConfig.setInjector(
Guice.createInjector(new TestRMWebServicesCapacitySched.WebServletModule(rm)));
rm.start();
//Need to call reinitialize as
//MutableCSConfigurationProvider with InMemoryConfigurationStore
//somehow does not load the queues properly and falls back to default config.
//Therefore CS will think there's only the default queue there.
((CapacityScheduler) rm.getResourceScheduler()).reinitialize(conf,
rm.getRMContext(), true);
}
}

31
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
View File

@ -18,15 +18,20 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import com.google.inject.Guice;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.core.util.MultivaluedMapImpl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityDiagnosticConstant;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivityState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.junit.Before;
import org.apache.hadoop.http.JettyUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@ -83,6 +88,8 @@
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyNumberOfNodes;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyQueueOrder;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyStateOfAllocations;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@ -91,11 +98,23 @@
/**
* Tests for scheduler/app activities.
*/
public class TestRMWebServicesSchedulerActivities
extends TestRMWebServicesCapacitySched {
public class TestRMWebServicesSchedulerActivities extends JerseyTestBase {
private static final Logger LOG = LoggerFactory.getLogger(
TestRMWebServicesSchedulerActivities.class);
private MockRM rm;
public TestRMWebServicesSchedulerActivities() {
super(createWebAppDescriptor());
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
rm = createMockRM(new CapacitySchedulerConfiguration(
new Configuration(false)));
GuiceServletConfig.setInjector(
Guice.createInjector(new TestRMWebServicesCapacitySched.WebServletModule(rm)));
}
@Test
public void testAssignMultipleContainersPerNodeHeartbeat()