YARN-8621. Add test coverage of custom Resource Types for the apps/<appId> REST API endpoint. (Contributed by Szilard Nemeth)
This commit is contained in:
Haibo Chen
parent
4eff629ab3
commit
d0ee6fbe28
@ -0,0 +1,397 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
|
||||
|
||||
import com.google.inject.Guice;
|
||||
import com.google.inject.servlet.ServletModule;
|
||||
import com.sun.jersey.api.client.ClientResponse;
|
||||
import com.sun.jersey.api.client.UniformInterfaceException;
|
||||
import com.sun.jersey.api.client.WebResource;
|
||||
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
|
||||
import com.sun.jersey.test.framework.WebAppDescriptor;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.http.JettyUtils;
|
||||
import org.apache.hadoop.yarn.api.records.ContainerState;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
|
||||
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
||||
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
|
||||
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
|
||||
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
|
||||
import org.codehaus.jettison.json.JSONArray;
|
||||
import org.codehaus.jettison.json.JSONObject;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.InputSource;
|
||||
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import java.io.StringReader;
|
||||
import java.util.Collection;
|
||||
|
||||
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
|
||||
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
public class TestRMWebServicesAppAttempts extends JerseyTestBase {
|
||||
|
||||
private static MockRM rm;
|
||||
|
||||
private static final int CONTAINER_MB = 1024;
|
||||
|
||||
private static class WebServletModule extends ServletModule {
|
||||
@Override
|
||||
protected void configureServlets() {
|
||||
bind(JAXBContextResolver.class);
|
||||
bind(RMWebServices.class);
|
||||
bind(GenericExceptionHandler.class);
|
||||
Configuration conf = new Configuration();
|
||||
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
|
||||
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
|
||||
conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
|
||||
ResourceScheduler.class);
|
||||
rm = new MockRM(conf);
|
||||
bind(ResourceManager.class).toInstance(rm);
|
||||
serve("/*").with(GuiceContainer.class);
|
||||
}
|
||||
}
|
||||
|
||||
static {
|
||||
GuiceServletConfig.setInjector(
|
||||
Guice.createInjector(new WebServletModule()));
|
||||
}
|
||||
|
||||
@Before
|
||||
@Override
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
GuiceServletConfig.setInjector(
|
||||
Guice.createInjector(new WebServletModule()));
|
||||
}
|
||||
|
||||
public TestRMWebServicesAppAttempts() {
|
||||
super(new WebAppDescriptor.Builder(
|
||||
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
|
||||
.contextListenerClass(GuiceServletConfig.class)
|
||||
.filterClass(com.google.inject.servlet.GuiceFilter.class)
|
||||
.contextPath("jersey-guice-filter").servletPath("/").build());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppAttempts() throws Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
testAppAttemptsHelper(app1.getApplicationId().toString(), app1,
|
||||
MediaType.APPLICATION_JSON);
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
@Test (timeout = 20000)
|
||||
public void testMultipleAppAttempts() throws Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
|
||||
MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
|
||||
int maxAppAttempts = rm.getConfig().getInt(
|
||||
YarnConfiguration.RM_AM_MAX_ATTEMPTS,
|
||||
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
|
||||
assertTrue(maxAppAttempts > 1);
|
||||
int numAttempt = 1;
|
||||
while (true) {
|
||||
// fail the AM by sending CONTAINER_FINISHED event without registering.
|
||||
amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1,
|
||||
ContainerState.COMPLETE);
|
||||
rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED);
|
||||
if (numAttempt == maxAppAttempts) {
|
||||
rm.waitForState(app1.getApplicationId(), RMAppState.FAILED);
|
||||
break;
|
||||
}
|
||||
// wait for app to start a new attempt.
|
||||
rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
|
||||
am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
|
||||
numAttempt++;
|
||||
}
|
||||
assertEquals("incorrect number of attempts", maxAppAttempts,
|
||||
app1.getAppAttempts().values().size());
|
||||
testAppAttemptsHelper(app1.getApplicationId().toString(), app1,
|
||||
MediaType.APPLICATION_JSON);
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppAttemptsSlash() throws Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB);
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1,
|
||||
MediaType.APPLICATION_JSON);
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppAttemptsDefault() throws Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB);
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, "");
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInvalidAppIdGetAttempts() throws Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app = rm.submitApp(CONTAINER_MB);
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
WebResource r = resource();
|
||||
|
||||
try {
|
||||
r.path("ws").path("v1").path("cluster").path("apps")
|
||||
.path("application_invalid_12").path("appattempts")
|
||||
.accept(MediaType.APPLICATION_JSON)
|
||||
.get(JSONObject.class);
|
||||
fail("should have thrown exception on invalid appAttempt");
|
||||
} catch (UniformInterfaceException ue) {
|
||||
ClientResponse response = ue.getResponse();
|
||||
|
||||
assertResponseStatusCode(ClientResponse.Status.BAD_REQUEST,
|
||||
response.getStatusInfo());
|
||||
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
JSONObject msg = response.getEntity(JSONObject.class);
|
||||
JSONObject exception = msg.getJSONObject("RemoteException");
|
||||
assertEquals("incorrect number of elements", 3, exception.length());
|
||||
String message = exception.getString("message");
|
||||
String type = exception.getString("exception");
|
||||
String classname = exception.getString("javaClassName");
|
||||
checkStringMatch("exception message",
|
||||
"java.lang.IllegalArgumentException: Invalid ApplicationId:"
|
||||
+ " application_invalid_12",
|
||||
message);
|
||||
checkStringMatch("exception type",
|
||||
"BadRequestException", type);
|
||||
checkStringMatch("exception classname",
|
||||
"org.apache.hadoop.yarn.webapp.BadRequestException", classname);
|
||||
|
||||
} finally {
|
||||
rm.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInvalidAppAttemptId() throws Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app = rm.submitApp(CONTAINER_MB);
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
WebResource r = resource();
|
||||
|
||||
try {
|
||||
r.path("ws").path("v1").path("cluster").path("apps")
|
||||
.path(app.getApplicationId().toString()).path("appattempts")
|
||||
.path("appattempt_invalid_12_000001")
|
||||
.accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
|
||||
fail("should have thrown exception on invalid appAttempt");
|
||||
} catch (UniformInterfaceException ue) {
|
||||
ClientResponse response = ue.getResponse();
|
||||
|
||||
assertResponseStatusCode(ClientResponse.Status.BAD_REQUEST,
|
||||
response.getStatusInfo());
|
||||
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
JSONObject msg = response.getEntity(JSONObject.class);
|
||||
JSONObject exception = msg.getJSONObject("RemoteException");
|
||||
assertEquals("incorrect number of elements", 3, exception.length());
|
||||
String message = exception.getString("message");
|
||||
String type = exception.getString("exception");
|
||||
String classname = exception.getString("javaClassName");
|
||||
checkStringMatch("exception message",
|
||||
"java.lang.IllegalArgumentException: Invalid AppAttemptId:"
|
||||
+ " appattempt_invalid_12_000001",
|
||||
message);
|
||||
checkStringMatch("exception type",
|
||||
"BadRequestException", type);
|
||||
checkStringMatch("exception classname",
|
||||
"org.apache.hadoop.yarn.webapp.BadRequestException", classname);
|
||||
|
||||
} finally {
|
||||
rm.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNonexistAppAttempts() throws Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
WebResource r = resource();
|
||||
|
||||
try {
|
||||
r.path("ws").path("v1").path("cluster").path("apps")
|
||||
.path("application_00000_0099").accept(MediaType.APPLICATION_JSON)
|
||||
.get(JSONObject.class);
|
||||
fail("should have thrown exception on invalid appid");
|
||||
} catch (UniformInterfaceException ue) {
|
||||
ClientResponse response = ue.getResponse();
|
||||
|
||||
assertResponseStatusCode(ClientResponse.Status.NOT_FOUND,
|
||||
response.getStatusInfo());
|
||||
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
|
||||
JSONObject msg = response.getEntity(JSONObject.class);
|
||||
JSONObject exception = msg.getJSONObject("RemoteException");
|
||||
assertEquals("incorrect number of elements", 3, exception.length());
|
||||
String message = exception.getString("message");
|
||||
String type = exception.getString("exception");
|
||||
String classname = exception.getString("javaClassName");
|
||||
checkStringMatch("exception message",
|
||||
"java.lang.Exception: app with id: application_00000_0099 not found",
|
||||
message);
|
||||
checkStringMatch("exception type",
|
||||
"NotFoundException", type);
|
||||
checkStringMatch("exception classname",
|
||||
"org.apache.hadoop.yarn.webapp.NotFoundException", classname);
|
||||
} finally {
|
||||
rm.stop();
|
||||
}
|
||||
}
|
||||
|
||||
private void testAppAttemptsHelper(String path, RMApp app, String media)
|
||||
throws Exception {
|
||||
WebResource r = resource();
|
||||
ClientResponse response = r.path("ws").path("v1").path("cluster")
|
||||
.path("apps").path(path).path("appattempts").accept(media)
|
||||
.get(ClientResponse.class);
|
||||
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
JSONObject json = response.getEntity(JSONObject.class);
|
||||
assertEquals("incorrect number of elements", 1, json.length());
|
||||
JSONObject jsonAppAttempts = json.getJSONObject("appAttempts");
|
||||
assertEquals("incorrect number of elements", 1, jsonAppAttempts.length());
|
||||
JSONArray jsonArray = jsonAppAttempts.getJSONArray("appAttempt");
|
||||
|
||||
Collection<RMAppAttempt> attempts = app.getAppAttempts().values();
|
||||
assertEquals("incorrect number of elements", attempts.size(),
|
||||
jsonArray.length());
|
||||
|
||||
// Verify these parallel arrays are the same
|
||||
int i = 0;
|
||||
for (RMAppAttempt attempt : attempts) {
|
||||
verifyAppAttemptsInfo(jsonArray.getJSONObject(i), attempt, app.getUser());
|
||||
++i;
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppAttemptsXML() throws Exception {
|
||||
rm.start();
|
||||
String user = "user1";
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", user);
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
WebResource r = resource();
|
||||
ClientResponse response = r.path("ws").path("v1").path("cluster")
|
||||
.path("apps").path(app1.getApplicationId().toString())
|
||||
.path("appattempts").accept(MediaType.APPLICATION_XML)
|
||||
.get(ClientResponse.class);
|
||||
assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
String xml = response.getEntity(String.class);
|
||||
|
||||
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
||||
DocumentBuilder db = dbf.newDocumentBuilder();
|
||||
InputSource is = new InputSource();
|
||||
is.setCharacterStream(new StringReader(xml));
|
||||
Document dom = db.parse(is);
|
||||
NodeList nodes = dom.getElementsByTagName("appAttempts");
|
||||
assertEquals("incorrect number of elements", 1, nodes.getLength());
|
||||
NodeList attempt = dom.getElementsByTagName("appAttempt");
|
||||
assertEquals("incorrect number of elements", 1, attempt.getLength());
|
||||
verifyAppAttemptsXML(attempt, app1.getCurrentAppAttempt(), user);
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
private void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt,
|
||||
String user) {
|
||||
|
||||
for (int i = 0; i < nodes.getLength(); i++) {
|
||||
Element element = (Element) nodes.item(i);
|
||||
|
||||
verifyAppAttemptInfoGeneric(appAttempt,
|
||||
WebServicesTestUtils.getXmlInt(element, "id"),
|
||||
WebServicesTestUtils.getXmlLong(element, "startTime"),
|
||||
WebServicesTestUtils.getXmlString(element, "containerId"),
|
||||
WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
|
||||
WebServicesTestUtils.getXmlString(element, "nodeId"),
|
||||
WebServicesTestUtils.getXmlString(element, "logsLink"), user);
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt,
|
||||
String user)
|
||||
throws Exception {
|
||||
|
||||
assertEquals("incorrect number of elements", 10, info.length());
|
||||
|
||||
verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"),
|
||||
info.getLong("startTime"), info.getString("containerId"),
|
||||
info.getString("nodeHttpAddress"), info.getString("nodeId"),
|
||||
info.getString("logsLink"), user);
|
||||
}
|
||||
|
||||
private void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id,
|
||||
long startTime, String containerId, String nodeHttpAddress, String
|
||||
nodeId,
|
||||
String logsLink, String user) {
|
||||
|
||||
assertEquals("id doesn't match", appAttempt.getAppAttemptId()
|
||||
.getAttemptId(), id);
|
||||
assertEquals("startedTime doesn't match", appAttempt.getStartTime(),
|
||||
startTime);
|
||||
checkStringMatch("containerId", appAttempt
|
||||
.getMasterContainer().getId().toString(), containerId);
|
||||
checkStringMatch("nodeHttpAddress", appAttempt
|
||||
.getMasterContainer().getNodeHttpAddress(), nodeHttpAddress);
|
||||
checkStringMatch("nodeId", appAttempt
|
||||
.getMasterContainer().getNodeId().toString(), nodeId);
|
||||
assertTrue("logsLink doesn't match ", logsLink.startsWith("http://"));
|
||||
assertTrue(
|
||||
"logsLink doesn't contain user info", logsLink.endsWith("/"
|
||||
+ user));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,170 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
|
||||
|
||||
import com.google.inject.Guice;
|
||||
import com.google.inject.servlet.ServletModule;
|
||||
import com.sun.jersey.api.client.ClientResponse;
|
||||
import com.sun.jersey.api.client.WebResource;
|
||||
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
|
||||
import com.sun.jersey.test.framework.WebAppDescriptor;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.CustomResourceTypesConfigurationProvider;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.BufferedClientResponse;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.JsonCustomResourceTypeTestcase;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase;
|
||||
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
||||
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
||||
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
|
||||
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
|
||||
import org.codehaus.jettison.json.JSONException;
|
||||
import org.codehaus.jettison.json.JSONObject;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.w3c.dom.NodeList;
|
||||
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCustomResourceTypesCommons.verifyAppInfoJson;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCustomResourceTypesCommons.verifyAppsXML;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
/**
|
||||
* This test verifies that custom resource types are correctly serialized to XML
|
||||
* and JSON when HTTP GET request is sent to the resource: ws/v1/cluster/apps.
|
||||
*/
|
||||
public class TestRMWebServicesAppCustomResourceTypes extends JerseyTestBase {
|
||||
|
||||
private static MockRM rm;
|
||||
private static final int CONTAINER_MB = 1024;
|
||||
|
||||
private static class WebServletModule extends ServletModule {
|
||||
@Override
|
||||
protected void configureServlets() {
|
||||
bind(JAXBContextResolver.class);
|
||||
bind(RMWebServices.class);
|
||||
bind(GenericExceptionHandler.class);
|
||||
Configuration conf = new Configuration();
|
||||
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
|
||||
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
|
||||
conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
|
||||
ResourceScheduler.class);
|
||||
initResourceTypes(conf);
|
||||
rm = new MockRM(conf);
|
||||
bind(ResourceManager.class).toInstance(rm);
|
||||
serve("/*").with(GuiceContainer.class);
|
||||
}
|
||||
|
||||
private void initResourceTypes(Configuration conf) {
|
||||
conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
|
||||
CustomResourceTypesConfigurationProvider.class.getName());
|
||||
ResourceUtils.resetResourceTypes(conf);
|
||||
}
|
||||
}
|
||||
|
||||
@Before
|
||||
@Override
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
createInjectorForWebServletModule();
|
||||
}
|
||||
|
||||
private void createInjectorForWebServletModule() {
|
||||
GuiceServletConfig
|
||||
.setInjector(Guice.createInjector(new WebServletModule()));
|
||||
}
|
||||
|
||||
public TestRMWebServicesAppCustomResourceTypes() {
|
||||
super(new WebAppDescriptor.Builder(
|
||||
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
|
||||
.contextListenerClass(GuiceServletConfig.class)
|
||||
.filterClass(com.google.inject.servlet.GuiceFilter.class)
|
||||
.contextPath("jersey-guice-filter").servletPath("/").build());
|
||||
}
|
||||
|
||||
private WebResource getWebResourcePathForApp(RMApp app1, WebResource r) {
|
||||
return r.path("ws").path("v1").path("cluster").path("apps")
|
||||
.path(String.valueOf(app1.getApplicationId().toString()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRunningAppXml() throws Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
|
||||
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
|
||||
am1.allocate("*", 2048, 1, new ArrayList<>());
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
|
||||
WebResource r = resource();
|
||||
WebResource path = getWebResourcePathForApp(app1, r);
|
||||
ClientResponse response =
|
||||
path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
|
||||
|
||||
XmlCustomResourceTypeTestCase testCase =
|
||||
new XmlCustomResourceTypeTestCase(path,
|
||||
new BufferedClientResponse(response));
|
||||
testCase.verify(document -> {
|
||||
NodeList appArray = document
|
||||
.getElementsByTagName("app");
|
||||
assertEquals("incorrect number of app elements", 1, appArray.getLength());
|
||||
|
||||
verifyAppsXML(appArray, app1, rm);
|
||||
});
|
||||
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRunningAppJson() throws Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
|
||||
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
|
||||
am1.allocate("*", 2048, 1, new ArrayList<>());
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
|
||||
WebResource r = resource();
|
||||
WebResource path = getWebResourcePathForApp(app1, r);
|
||||
ClientResponse response =
|
||||
path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
|
||||
|
||||
JsonCustomResourceTypeTestcase testCase =
|
||||
new JsonCustomResourceTypeTestcase(path,
|
||||
new BufferedClientResponse(response));
|
||||
testCase.verify(json -> {
|
||||
try {
|
||||
assertEquals("incorrect number of app elements", 1, json.length());
|
||||
JSONObject app = json.getJSONObject("app");
|
||||
verifyAppInfoJson(app, app1, rm);
|
||||
} catch (JSONException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
});
|
||||
|
||||
rm.stop();
|
||||
}
|
||||
}
|
||||
@ -1781,292 +1781,5 @@ public void verifyResourceRequestsGeneric(ResourceRequest request,
|
||||
enforceExecutionType);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppAttempts() throws JSONException, Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
testAppAttemptsHelper(app1.getApplicationId().toString(), app1,
|
||||
MediaType.APPLICATION_JSON);
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
@Test (timeout = 20000)
|
||||
public void testMultipleAppAttempts() throws JSONException, Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
|
||||
MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
|
||||
int maxAppAttempts = rm.getConfig().getInt(
|
||||
YarnConfiguration.RM_AM_MAX_ATTEMPTS,
|
||||
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
|
||||
assertTrue(maxAppAttempts > 1);
|
||||
int numAttempt = 1;
|
||||
while (true) {
|
||||
// fail the AM by sending CONTAINER_FINISHED event without registering.
|
||||
amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1,
|
||||
ContainerState.COMPLETE);
|
||||
rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED);
|
||||
if (numAttempt == maxAppAttempts) {
|
||||
rm.waitForState(app1.getApplicationId(), RMAppState.FAILED);
|
||||
break;
|
||||
}
|
||||
// wait for app to start a new attempt.
|
||||
rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
|
||||
am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
|
||||
numAttempt++;
|
||||
}
|
||||
assertEquals("incorrect number of attempts", maxAppAttempts,
|
||||
app1.getAppAttempts().values().size());
|
||||
testAppAttemptsHelper(app1.getApplicationId().toString(), app1,
|
||||
MediaType.APPLICATION_JSON);
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppAttemptsSlash() throws JSONException, Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB);
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1,
|
||||
MediaType.APPLICATION_JSON);
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppAttemtpsDefault() throws JSONException, Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB);
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, "");
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInvalidAppIdGetAttempts() throws JSONException, Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app = rm.submitApp(CONTAINER_MB);
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
WebResource r = resource();
|
||||
|
||||
try {
|
||||
r.path("ws").path("v1").path("cluster").path("apps")
|
||||
.path("application_invalid_12").path("appattempts")
|
||||
.accept(MediaType.APPLICATION_JSON)
|
||||
.get(JSONObject.class);
|
||||
fail("should have thrown exception on invalid appAttempt");
|
||||
} catch (UniformInterfaceException ue) {
|
||||
ClientResponse response = ue.getResponse();
|
||||
|
||||
assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo());
|
||||
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
JSONObject msg = response.getEntity(JSONObject.class);
|
||||
JSONObject exception = msg.getJSONObject("RemoteException");
|
||||
assertEquals("incorrect number of elements", 3, exception.length());
|
||||
String message = exception.getString("message");
|
||||
String type = exception.getString("exception");
|
||||
String classname = exception.getString("javaClassName");
|
||||
WebServicesTestUtils.checkStringMatch("exception message",
|
||||
"java.lang.IllegalArgumentException: Invalid ApplicationId:"
|
||||
+ " application_invalid_12",
|
||||
message);
|
||||
WebServicesTestUtils.checkStringMatch("exception type",
|
||||
"BadRequestException", type);
|
||||
WebServicesTestUtils.checkStringMatch("exception classname",
|
||||
"org.apache.hadoop.yarn.webapp.BadRequestException", classname);
|
||||
|
||||
} finally {
|
||||
rm.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInvalidAppAttemptId() throws JSONException, Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app = rm.submitApp(CONTAINER_MB);
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
WebResource r = resource();
|
||||
|
||||
try {
|
||||
r.path("ws").path("v1").path("cluster").path("apps")
|
||||
.path(app.getApplicationId().toString()).path("appattempts")
|
||||
.path("appattempt_invalid_12_000001")
|
||||
.accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
|
||||
fail("should have thrown exception on invalid appAttempt");
|
||||
} catch (UniformInterfaceException ue) {
|
||||
ClientResponse response = ue.getResponse();
|
||||
|
||||
assertResponseStatusCode(Status.BAD_REQUEST, response.getStatusInfo());
|
||||
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
JSONObject msg = response.getEntity(JSONObject.class);
|
||||
JSONObject exception = msg.getJSONObject("RemoteException");
|
||||
assertEquals("incorrect number of elements", 3, exception.length());
|
||||
String message = exception.getString("message");
|
||||
String type = exception.getString("exception");
|
||||
String classname = exception.getString("javaClassName");
|
||||
WebServicesTestUtils.checkStringMatch("exception message",
|
||||
"java.lang.IllegalArgumentException: Invalid AppAttemptId:"
|
||||
+ " appattempt_invalid_12_000001",
|
||||
message);
|
||||
WebServicesTestUtils.checkStringMatch("exception type",
|
||||
"BadRequestException", type);
|
||||
WebServicesTestUtils.checkStringMatch("exception classname",
|
||||
"org.apache.hadoop.yarn.webapp.BadRequestException", classname);
|
||||
|
||||
} finally {
|
||||
rm.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNonexistAppAttempts() throws JSONException, Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
WebResource r = resource();
|
||||
|
||||
try {
|
||||
r.path("ws").path("v1").path("cluster").path("apps")
|
||||
.path("application_00000_0099").accept(MediaType.APPLICATION_JSON)
|
||||
.get(JSONObject.class);
|
||||
fail("should have thrown exception on invalid appid");
|
||||
} catch (UniformInterfaceException ue) {
|
||||
ClientResponse response = ue.getResponse();
|
||||
|
||||
assertResponseStatusCode(Status.NOT_FOUND, response.getStatusInfo());
|
||||
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
|
||||
JSONObject msg = response.getEntity(JSONObject.class);
|
||||
JSONObject exception = msg.getJSONObject("RemoteException");
|
||||
assertEquals("incorrect number of elements", 3, exception.length());
|
||||
String message = exception.getString("message");
|
||||
String type = exception.getString("exception");
|
||||
String classname = exception.getString("javaClassName");
|
||||
WebServicesTestUtils.checkStringMatch("exception message",
|
||||
"java.lang.Exception: app with id: application_00000_0099 not found",
|
||||
message);
|
||||
WebServicesTestUtils.checkStringMatch("exception type",
|
||||
"NotFoundException", type);
|
||||
WebServicesTestUtils.checkStringMatch("exception classname",
|
||||
"org.apache.hadoop.yarn.webapp.NotFoundException", classname);
|
||||
} finally {
|
||||
rm.stop();
|
||||
}
|
||||
}
|
||||
|
||||
public void testAppAttemptsHelper(String path, RMApp app, String media)
|
||||
throws JSONException, Exception {
|
||||
WebResource r = resource();
|
||||
ClientResponse response = r.path("ws").path("v1").path("cluster")
|
||||
.path("apps").path(path).path("appattempts").accept(media)
|
||||
.get(ClientResponse.class);
|
||||
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
JSONObject json = response.getEntity(JSONObject.class);
|
||||
assertEquals("incorrect number of elements", 1, json.length());
|
||||
JSONObject jsonAppAttempts = json.getJSONObject("appAttempts");
|
||||
assertEquals("incorrect number of elements", 1, jsonAppAttempts.length());
|
||||
JSONArray jsonArray = jsonAppAttempts.getJSONArray("appAttempt");
|
||||
|
||||
Collection<RMAppAttempt> attempts = app.getAppAttempts().values();
|
||||
assertEquals("incorrect number of elements", attempts.size(),
|
||||
jsonArray.length());
|
||||
|
||||
// Verify these parallel arrays are the same
|
||||
int i = 0;
|
||||
for (RMAppAttempt attempt : attempts) {
|
||||
verifyAppAttemptsInfo(jsonArray.getJSONObject(i), attempt, app.getUser());
|
||||
++i;
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppAttemptsXML() throws JSONException, Exception {
|
||||
rm.start();
|
||||
String user = "user1";
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", user);
|
||||
amNodeManager.nodeHeartbeat(true);
|
||||
WebResource r = resource();
|
||||
ClientResponse response = r.path("ws").path("v1").path("cluster")
|
||||
.path("apps").path(app1.getApplicationId().toString())
|
||||
.path("appattempts").accept(MediaType.APPLICATION_XML)
|
||||
.get(ClientResponse.class);
|
||||
assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
String xml = response.getEntity(String.class);
|
||||
|
||||
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
|
||||
DocumentBuilder db = dbf.newDocumentBuilder();
|
||||
InputSource is = new InputSource();
|
||||
is.setCharacterStream(new StringReader(xml));
|
||||
Document dom = db.parse(is);
|
||||
NodeList nodes = dom.getElementsByTagName("appAttempts");
|
||||
assertEquals("incorrect number of elements", 1, nodes.getLength());
|
||||
NodeList attempt = dom.getElementsByTagName("appAttempt");
|
||||
assertEquals("incorrect number of elements", 1, attempt.getLength());
|
||||
verifyAppAttemptsXML(attempt, app1.getCurrentAppAttempt(), user);
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
public void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt,
|
||||
String user)
|
||||
throws JSONException, Exception {
|
||||
|
||||
for (int i = 0; i < nodes.getLength(); i++) {
|
||||
Element element = (Element) nodes.item(i);
|
||||
|
||||
verifyAppAttemptInfoGeneric(appAttempt,
|
||||
WebServicesTestUtils.getXmlInt(element, "id"),
|
||||
WebServicesTestUtils.getXmlLong(element, "startTime"),
|
||||
WebServicesTestUtils.getXmlString(element, "containerId"),
|
||||
WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
|
||||
WebServicesTestUtils.getXmlString(element, "nodeId"),
|
||||
WebServicesTestUtils.getXmlString(element, "logsLink"), user);
|
||||
}
|
||||
}
|
||||
|
||||
public void verifyAppAttemptsInfo(JSONObject info, RMAppAttempt appAttempt,
|
||||
String user)
|
||||
throws JSONException, Exception {
|
||||
|
||||
assertEquals("incorrect number of elements", 10, info.length());
|
||||
|
||||
verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"),
|
||||
info.getLong("startTime"), info.getString("containerId"),
|
||||
info.getString("nodeHttpAddress"), info.getString("nodeId"),
|
||||
info.getString("logsLink"), user);
|
||||
}
|
||||
|
||||
public void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id,
|
||||
long startTime, String containerId, String nodeHttpAddress, String nodeId,
|
||||
String logsLink, String user)
|
||||
throws JSONException, Exception {
|
||||
|
||||
assertEquals("id doesn't match", appAttempt.getAppAttemptId()
|
||||
.getAttemptId(), id);
|
||||
assertEquals("startedTime doesn't match", appAttempt.getStartTime(),
|
||||
startTime);
|
||||
WebServicesTestUtils.checkStringMatch("containerId", appAttempt
|
||||
.getMasterContainer().getId().toString(), containerId);
|
||||
WebServicesTestUtils.checkStringMatch("nodeHttpAddress", appAttempt
|
||||
.getMasterContainer().getNodeHttpAddress(), nodeHttpAddress);
|
||||
WebServicesTestUtils.checkStringMatch("nodeId", appAttempt
|
||||
.getMasterContainer().getNodeId().toString(), nodeId);
|
||||
assertTrue("logsLink doesn't match ", logsLink.startsWith("http://"));
|
||||
assertTrue(
|
||||
"logsLink doesn't contain user info", logsLink.endsWith("/"
|
||||
+ user));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -25,24 +25,18 @@
|
||||
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
|
||||
import com.sun.jersey.test.framework.WebAppDescriptor;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceRequest;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.CustomResourceTypesConfigurationProvider;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoJsonVerifications;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoXmlVerifications;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.BufferedClientResponse;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.JsonCustomResourceTypeTestcase;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsJsonVerifications;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsXmlVerifications;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase;
|
||||
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
||||
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
|
||||
@ -54,12 +48,15 @@
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.webapp
|
||||
.TestRMWebServicesCustomResourceTypesCommons.verifyAppInfoJson;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.webapp
|
||||
.TestRMWebServicesCustomResourceTypesCommons.verifyAppsXML;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
/**
|
||||
@ -116,7 +113,7 @@ public TestRMWebServicesAppsCustomResourceTypes() {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRunningAppXml() throws Exception {
|
||||
public void testRunningAppsXml() throws Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
|
||||
@ -140,14 +137,14 @@ public void testRunningAppXml() throws Exception {
|
||||
.getElementsByTagName("app");
|
||||
assertEquals("incorrect number of app elements", 1, appArray.getLength());
|
||||
|
||||
verifyAppsXML(appArray, app1);
|
||||
verifyAppsXML(appArray, app1, rm);
|
||||
});
|
||||
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRunningAppJson() throws Exception {
|
||||
public void testRunningAppsJson() throws Exception {
|
||||
rm.start();
|
||||
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
|
||||
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
|
||||
@ -171,7 +168,7 @@ public void testRunningAppJson() throws Exception {
|
||||
JSONArray array = apps.getJSONArray("app");
|
||||
assertEquals("incorrect count of app", 1, array.length());
|
||||
|
||||
verifyAppInfoJson(array.getJSONObject(0), app1);
|
||||
verifyAppInfoJson(array.getJSONObject(0), app1, rm);
|
||||
} catch (JSONException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
@ -179,64 +176,4 @@ public void testRunningAppJson() throws Exception {
|
||||
|
||||
rm.stop();
|
||||
}
|
||||
|
||||
private void verifyAppsXML(NodeList appArray, RMApp app) {
|
||||
for (int i = 0; i < appArray.getLength(); i++) {
|
||||
Element element = (Element) appArray.item(i);
|
||||
AppInfoXmlVerifications.verify(element, app);
|
||||
|
||||
NodeList resourceRequests =
|
||||
element.getElementsByTagName("resourceRequests");
|
||||
assertEquals(1, resourceRequests.getLength());
|
||||
Node resourceRequest = resourceRequests.item(0);
|
||||
ResourceRequest rr =
|
||||
((AbstractYarnScheduler) rm.getRMContext().getScheduler())
|
||||
.getApplicationAttempt(
|
||||
app.getCurrentAppAttempt().getAppAttemptId())
|
||||
.getAppSchedulingInfo().getAllResourceRequests().get(0);
|
||||
ResourceRequestsXmlVerifications.verifyWithCustomResourceTypes(
|
||||
(Element) resourceRequest, rr,
|
||||
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyAppInfoJson(JSONObject info, RMApp app) throws
|
||||
JSONException {
|
||||
int expectedNumberOfElements = getExpectedNumberOfElements(app);
|
||||
|
||||
assertEquals("incorrect number of elements", expectedNumberOfElements,
|
||||
info.length());
|
||||
|
||||
AppInfoJsonVerifications.verify(info, app);
|
||||
|
||||
JSONArray resourceRequests = info.getJSONArray("resourceRequests");
|
||||
JSONObject requestInfo = resourceRequests.getJSONObject(0);
|
||||
ResourceRequest rr =
|
||||
((AbstractYarnScheduler) rm.getRMContext().getScheduler())
|
||||
.getApplicationAttempt(app.getCurrentAppAttempt().getAppAttemptId())
|
||||
.getAppSchedulingInfo().getAllResourceRequests().get(0);
|
||||
|
||||
ResourceRequestsJsonVerifications.verifyWithCustomResourceTypes(
|
||||
requestInfo, rr,
|
||||
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
|
||||
}
|
||||
|
||||
private int getExpectedNumberOfElements(RMApp app) {
|
||||
int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests
|
||||
if (app.getApplicationSubmissionContext()
|
||||
.getNodeLabelExpression() != null) {
|
||||
expectedNumberOfElements++;
|
||||
}
|
||||
|
||||
if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) {
|
||||
expectedNumberOfElements++;
|
||||
}
|
||||
|
||||
if (AppInfo
|
||||
.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) {
|
||||
expectedNumberOfElements++;
|
||||
}
|
||||
return expectedNumberOfElements;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.ResourceRequest;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler
|
||||
.AbstractYarnScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler
|
||||
.CustomResourceTypesConfigurationProvider;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoJsonVerifications;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoXmlVerifications;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsJsonVerifications;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsXmlVerifications;
|
||||
import org.codehaus.jettison.json.JSONArray;
|
||||
import org.codehaus.jettison.json.JSONException;
|
||||
import org.codehaus.jettison.json.JSONObject;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.Node;
|
||||
import org.w3c.dom.NodeList;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
public class TestRMWebServicesCustomResourceTypesCommons {
|
||||
|
||||
static void verifyAppsXML(NodeList appArray, RMApp app, ResourceManager rm) {
|
||||
for (int i = 0; i < appArray.getLength(); i++) {
|
||||
Element element = (Element) appArray.item(i);
|
||||
AppInfoXmlVerifications.verify(element, app);
|
||||
|
||||
NodeList resourceRequests =
|
||||
element.getElementsByTagName("resourceRequests");
|
||||
assertEquals(1, resourceRequests.getLength());
|
||||
Node resourceRequest = resourceRequests.item(0);
|
||||
ResourceRequest rr =
|
||||
((AbstractYarnScheduler) rm.getRMContext().getScheduler())
|
||||
.getApplicationAttempt(
|
||||
app.getCurrentAppAttempt().getAppAttemptId())
|
||||
.getAppSchedulingInfo().getAllResourceRequests().get(0);
|
||||
ResourceRequestsXmlVerifications.verifyWithCustomResourceTypes(
|
||||
(Element) resourceRequest, rr,
|
||||
CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
|
||||
}
|
||||
}
|
||||
|
||||
static void verifyAppInfoJson(JSONObject info, RMApp app, ResourceManager rm)
|
||||
throws JSONException {
|
||||
int expectedNumberOfElements = getExpectedNumberOfElements(app);
|
||||
|
||||
assertEquals("incorrect number of elements", expectedNumberOfElements,
|
||||
info.length());
|
||||
|
||||
AppInfoJsonVerifications.verify(info, app);
|
||||
|
||||
JSONArray resourceRequests = info.getJSONArray("resourceRequests");
|
||||
JSONObject requestInfo = resourceRequests.getJSONObject(0);
|
||||
ResourceRequest rr =
|
||||
((AbstractYarnScheduler) rm.getRMContext().getScheduler())
|
||||
.getApplicationAttempt(app.getCurrentAppAttempt().getAppAttemptId())
|
||||
.getAppSchedulingInfo().getAllResourceRequests().get(0);
|
||||
|
||||
ResourceRequestsJsonVerifications.verifyWithCustomResourceTypes(requestInfo,
|
||||
rr, CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
|
||||
}
|
||||
|
||||
static int getExpectedNumberOfElements(RMApp app) {
|
||||
int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests
|
||||
if (app.getApplicationSubmissionContext()
|
||||
.getNodeLabelExpression() != null) {
|
||||
expectedNumberOfElements++;
|
||||
}
|
||||
|
||||
if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) {
|
||||
expectedNumberOfElements++;
|
||||
}
|
||||
|
||||
if (AppInfo
|
||||
.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) {
|
||||
expectedNumberOfElements++;
|
||||
}
|
||||
return expectedNumberOfElements;
|
||||
}
|
||||
}
|
||||
@ -24,6 +24,7 @@
|
||||
import com.sun.jersey.api.client.UniformInterfaceException;
|
||||
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import javax.ws.rs.core.Response;
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
@ -54,4 +55,12 @@ public <T> T getEntity(Class<T> clazz)
|
||||
public MediaType getType() {
|
||||
return response.getType();
|
||||
}
|
||||
|
||||
public int getStatus() {
|
||||
return response.getStatus();
|
||||
}
|
||||
|
||||
public Response.StatusType getStatusInfo() {
|
||||
return response.getStatusInfo();
|
||||
}
|
||||
}
|
||||
|
||||
@ -50,10 +50,19 @@ public class JsonCustomResourceTypeTestcase {
|
||||
public JsonCustomResourceTypeTestcase(WebResource path,
|
||||
BufferedClientResponse response) {
|
||||
this.path = path;
|
||||
verifyStatus(response);
|
||||
this.response = response;
|
||||
this.parsedResponse = response.getEntity(JSONObject.class);
|
||||
}
|
||||
|
||||
private void verifyStatus(BufferedClientResponse response) {
|
||||
String responseStr = response.getEntity(String.class);
|
||||
assertEquals("HTTP status should be 200, " +
|
||||
"status info: " + response.getStatusInfo() +
|
||||
" response as string: " + responseStr,
|
||||
200, response.getStatus());
|
||||
}
|
||||
|
||||
public void verify(Consumer<JSONObject> verifier) {
|
||||
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
|
||||
@ -59,9 +59,18 @@ public class XmlCustomResourceTypeTestCase {
|
||||
public XmlCustomResourceTypeTestCase(WebResource path,
|
||||
BufferedClientResponse response) {
|
||||
this.path = path;
|
||||
verifyStatus(response);
|
||||
this.response = response;
|
||||
}
|
||||
|
||||
private void verifyStatus(BufferedClientResponse response) {
|
||||
String responseStr = response.getEntity(String.class);
|
||||
assertEquals("HTTP status should be 200, " +
|
||||
"status info: " + response.getStatusInfo() +
|
||||
" response as string: " + responseStr,
|
||||
200, response.getStatus());
|
||||
}
|
||||
|
||||
public void verify(Consumer<Document> verifier) {
|
||||
assertEquals(MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8,
|
||||
response.getType().toString());
|
||||
|
||||
Loading…
Reference in New Issue
Block a user