big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9728. Bugfix for escaping illegal xml characters for Resource Manager REST API.

Browse Source 
Contributed by Prabhu Joseph
This commit is contained in:
Eric Yang 2019-09-10 17:04:39 -04:00
parent dc9abd27d9
commit 10144a580e
5 changed files with 158 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -3960,6 +3960,10 @@ public static boolean areNodeLabelsEnabled(
public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER =
false;
public static final String FILTER_INVALID_XML_CHARS =
"yarn.webapp.filter-invalid-xml-chars";
public static final boolean DEFAULT_FILTER_INVALID_XML_CHARS = false;
// RM and NM CSRF props
public static final String REST_CSRF = "webapp.rest-csrf.";
public static final String RM_CSRF_PREFIX = RM_PREFIX + REST_CSRF;

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -3793,6 +3793,15 @@
</description>
</property>
<property>
<name>yarn.webapp.filter-invalid-xml-chars</name>
<value>false</value>
<description>
Flag to enable filter of invalid xml 1.0 characters present in the
value of diagnostics field of apps output from RM WebService.
</description>
</property>
<property>
<description>
The type of configuration store to use for scheduler configurations.

61
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -242,6 +242,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
@VisibleForTesting
boolean isCentralizedNodeLabelConfiguration = true;
private boolean filterAppsByUser = false;
private boolean filterInvalidXMLChars = false;
public final static String DELEGATION_TOKEN_HEADER =
"Hadoop-YARN-RM-Delegation-Token";
@ -257,6 +258,9 @@ public RMWebServices(final ResourceManager rm, Configuration conf) {
this.filterAppsByUser = conf.getBoolean(
YarnConfiguration.FILTER_ENTITY_LIST_BY_USER,
YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
this.filterInvalidXMLChars = conf.getBoolean(
YarnConfiguration.FILTER_INVALID_XML_CHARS,
YarnConfiguration.DEFAULT_FILTER_INVALID_XML_CHARS);
}
RMWebServices(ResourceManager rm, Configuration conf,
@ -551,6 +555,38 @@ private RMNode getRMNode(final String nodeId) {
return ni;
}
/**
* This method ensures that the output String has only
* valid XML unicode characters as specified by the
* XML 1.0 standard. For reference, please see
* <a href="http://www.w3.org/TR/2000/REC-xml-20001006#NT-Char">
* the standard</a>.
*
* @param str The String whose invalid xml characters we want to escape.
* @return The str String after escaping invalid xml characters.
*/
public static String escapeInvalidXMLCharacters(String str) {
StringBuffer out = new StringBuffer();
final int strlen = str.length();
final String substitute = "\uFFFD";
int idx = 0;
while (idx < strlen) {
final int cpt = str.codePointAt(idx);
idx += Character.isSupplementaryCodePoint(cpt) ? 2 : 1;
if ((cpt == 0x9) ||
(cpt == 0xA) ||
(cpt == 0xD) ||
((cpt >= 0x20) && (cpt <= 0xD7FF)) ||
((cpt >= 0xE000) && (cpt <= 0xFFFD)) ||
((cpt >= 0x10000) && (cpt <= 0x10FFFF))) {
out.append(Character.toChars(cpt));
} else {
out.append(substitute);
}
}
return out.toString();
}
@GET
@Path(RMWSConsts.APPS)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
@ -629,6 +665,17 @@ public AppsInfo getApps(@Context HttpServletRequest hsr,
WebAppUtils.getHttpSchemePrefix(conf), deSelectFields);
allApps.add(app);
}
if (filterInvalidXMLChars) {
final String format = hsr.getHeader(HttpHeaders.ACCEPT);
if (format != null &&
format.toLowerCase().contains(MediaType.APPLICATION_XML)) {
for (AppInfo appInfo : allApps.getApps()) {
appInfo.setNote(escapeInvalidXMLCharacters(appInfo.getNote()));
}
}
}
return allApps;
}
@ -985,8 +1032,18 @@ public AppInfo getApp(@Context HttpServletRequest hsr,
DeSelectFields deSelectFields = new DeSelectFields();
deSelectFields.initFields(unselectedFields);
return new AppInfo(rm, app, hasAccess(app, hsr), hsr.getScheme() + "://",
deSelectFields);
AppInfo appInfo = new AppInfo(rm, app, hasAccess(app, hsr),
hsr.getScheme() + "://", deSelectFields);
if (filterInvalidXMLChars) {
final String format = hsr.getHeader(HttpHeaders.ACCEPT);
if (format != null &&
format.toLowerCase().contains(MediaType.APPLICATION_XML)) {
appInfo.setNote(escapeInvalidXMLCharacters(appInfo.getNote()));
}
}
return appInfo;
}
@GET

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
View File

@ -390,6 +390,10 @@ public String getNote() {
return this.diagnostics;
}
public void setNote(String diagnosticsMsg) {
this.diagnostics = diagnosticsMsg;
}
public FinalApplicationStatus getFinalStatus() {
return this.finalStatus;
}

82
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
View File

@ -22,6 +22,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@ -30,11 +31,17 @@
import java.io.StringReader;
import java.security.Principal;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.HashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@ -49,12 +56,21 @@
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.*;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@ -870,6 +886,72 @@ public void testClusterUserInfo() throws JSONException, Exception {
verifyClusterUserInfo(userInfo, "yarn", "admin");
}
@Test
public void testInvalidXMLChars() throws Exception {
ResourceManager mockRM = mock(ResourceManager.class);
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
ApplicationReport appReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FAILED, "java.lang.Exception: \u0001", "url",
0, 0, 0, FinalApplicationStatus.FAILED, null, "N/A", 0.53789f, "YARN",
null, null, false, Priority.newInstance(0), "high-mem", "high-mem");
List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
appReports.add(appReport);
GetApplicationsResponse response = mock(GetApplicationsResponse.class);
when(response.getApplicationList()).thenReturn(appReports);
ClientRMService clientRMService = mock(ClientRMService.class);
when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
.thenReturn(response);
when(mockRM.getClientRMService()).thenReturn(clientRMService);
RMContext rmContext = mock(RMContext.class);
when(rmContext.getDispatcher()).thenReturn(mock(Dispatcher.class));
ApplicationSubmissionContext applicationSubmissionContext = mock(
ApplicationSubmissionContext.class);
when(applicationSubmissionContext.getUnmanagedAM()).thenReturn(true);
RMApp app = mock(RMApp.class);
RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0),
0, 0, new HashMap<>(), new HashMap<>());
when(app.getDiagnostics()).thenReturn(
new StringBuilder("java.lang.Exception: \u0001"));
when(app.getApplicationId()).thenReturn(applicationId);
when(app.getUser()).thenReturn("user");
when(app.getName()).thenReturn("appname");
when(app.getQueue()).thenReturn("queue");
when(app.getRMAppMetrics()).thenReturn(appMetrics);
when(app.getApplicationSubmissionContext()).thenReturn(
applicationSubmissionContext);
ConcurrentMap<ApplicationId, RMApp> applications =
new ConcurrentHashMap<>();
applications.put(applicationId, app);
when(rmContext.getRMApps()).thenReturn(applications);
when(mockRM.getRMContext()).thenReturn(rmContext);
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.FILTER_INVALID_XML_CHARS, true);
RMWebServices webSvc = new RMWebServices(mockRM, conf, mock(
HttpServletResponse.class));
HttpServletRequest mockHsr = mock(HttpServletRequest.class);
when(mockHsr.getHeader(HttpHeaders.ACCEPT)).
thenReturn(MediaType.APPLICATION_XML);
Set<String> emptySet = Collections.unmodifiableSet(Collections.emptySet());
AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
null, null, null, null, null, null, null, emptySet, emptySet, null);
assertEquals("Incorrect Number of Apps", 1, appsInfo.getApps().size());
assertEquals("Invalid XML Characters Present",
"java.lang.Exception: \uFFFD", appsInfo.getApps().get(0).getNote());
}
public void verifyClusterUserInfo(ClusterUserInfo userInfo,
String rmLoginUser, String requestedUser) {
assertEquals("rmLoginUser doesn't match: ",