big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-4709. NMWebServices produces incorrect JSON for containers. Contributed by Varun Saxena.

Browse Source
This commit is contained in:
Varun Vasudev 2016-02-23 12:29:25 +05:30
parent 27b77751c1
commit 140cb5d745
4 changed files with 65 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -1436,6 +1436,9 @@ Release 2.8.0 - UNRELEASED
YARN-4386. refreshNodesGracefully() should send recommission event to active
RMNodes only. (Kuhu Shukla via junping_du)
YARN-4709. NMWebServices produces incorrect JSON for containers.
(Varun Saxena via vvasudev)
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
View File

@ -20,6 +20,9 @@
import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.List;
import org.w3c.dom.Attr;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
@ -47,6 +50,31 @@ public static float getXmlFloat(Element element, String name) {
return Float.parseFloat(val);
}
public static List<String> getXmlStrings(Element element, String name) {
NodeList id = element.getElementsByTagName(name);
List<String> strings = new ArrayList<>();
int len = id.getLength();
if (id.getLength() == 0) {
return strings;
}
for (int i = 0; i < len; i++) {
Element line = (Element) id.item(i);
if (line == null) {
continue;
}
Node first = line.getFirstChild();
if (first == null) {
continue;
}
String val = first.getNodeValue();
if (val == null) {
continue;
}
strings.add(val);
}
return strings;
}
public static String getXmlString(Element element, String name) {
NodeList id = element.getElementsByTagName(name);
Element line = (Element) id.item(0);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
View File

@ -53,7 +53,6 @@ public class ContainerInfo {
@XmlTransient
protected String exitStatus;
@XmlElementWrapper
protected List<String> containerLogFiles;
public ContainerInfo() {

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
View File

@ -20,13 +20,16 @@
import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
@ -48,6 +51,7 @@
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@ -203,12 +207,27 @@ private HashMap<String, String> addAppContainers(Application app)
app.getAppId(), 1);
Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
app.getUser(), app.getAppId(), 1);
((MockContainer)container1).setState(ContainerState.RUNNING);
Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
app.getUser(), app.getAppId(), 2);
((MockContainer)container2).setState(ContainerState.RUNNING);
nmContext.getContainers()
.put(container1.getContainerId(), container1);
nmContext.getContainers()
.put(container2.getContainerId(), container2);
File appDir = new File(testLogDir + "/" + app.getAppId().toString());
appDir.mkdir();
File container1Dir =
new File(appDir + "/" + container1.getContainerId().toString());
container1Dir.mkdir();
// Create log files for containers.
new File(container1Dir + "/" + "syslog").createNewFile();
new File(container1Dir + "/" + "stdout").createNewFile();
File container2Dir =
new File(appDir + "/" + container2.getContainerId().toString());
container2Dir.mkdir();
new File(container2Dir + "/" + "syslog").createNewFile();
new File(container2Dir + "/" + "stdout").createNewFile();
app.getContainers().put(container1.getContainerId(), container1);
app.getContainers().put(container2.getContainerId(), container2);
@ -475,9 +494,13 @@ public void verifyContainersInfoXML(NodeList nodes, Container cont)
WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"),
WebServicesTestUtils.getXmlString(element, "containerLogsLink"));
// verify that the container log files element exists
assertTrue("containerLogFiles missing",
WebServicesTestUtils.getXmlString(element, "containerLogFiles")
!= null);
List<String> containerLogFiles =
WebServicesTestUtils.getXmlStrings(element, "containerLogFiles");
assertFalse("containerLogFiles missing",containerLogFiles.isEmpty());
assertEquals(2, containerLogFiles.size());
assertTrue("syslog and stdout expected",
containerLogFiles.contains("syslog") &&
containerLogFiles.contains("stdout"));
}
}
@ -492,8 +515,14 @@ public void verifyNodeContainerInfo(JSONObject info, Container cont)
info.getInt("totalVCoresNeeded"),
info.getString("containerLogsLink"));
// verify that the container log files element exists
assertTrue("containerLogFiles missing",
info.getJSONArray("containerLogFiles") != null);
JSONArray containerLogFilesArr = info.getJSONArray("containerLogFiles");
assertTrue("containerLogFiles missing", containerLogFilesArr != null);
assertEquals(2, containerLogFilesArr.length());
for (int i = 0; i < 2; i++) {
assertTrue("syslog and stdout expected",
containerLogFilesArr.get(i).equals("syslog") ||
containerLogFilesArr.get(i).equals("stdout"));
}
}
public void verifyNodeContainerInfoGeneric(Container cont, String id,